Merge pull request #195 from marklogic/release/4.6.0

Merging release/4.6.0 into master

CODEOWNERS

@@ -0,0 +1,5 @@
+# Lines starting with '#' are comments.
+# Each line is a file pattern followed by one or more owners.
+
+# These owners will be the default owners for everything in the repo.
+* @anu3990 @billfarber @rjrudin

CONTRIBUTING.md

@@ -0,0 +1,9 @@
+To run the tests for ml-javaclient-util, clone this repository and deploy this project's test application first via 
+the following steps:
+
+1. Create a file named `gradle-local.properties`.
+2. Add `mlPassword=` followed by the password for your MarkLogic admin user.
+3. Verify that port 8070 is available on your computer - i.e. no other process is listening on it.
+4. Run `./gradlew -i mlDeploy`.
+
+You can then run `./gradlew -i test` to run all of the tests.

Jenkinsfile

@@ -21,6 +21,8 @@ pipeline{
           export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR
           export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH
           cd ml-javaclient-util
+          echo "mlPassword=admin" > gradle-local.properties
+          ./gradlew -i mlDeploy
           ./gradlew test  || true
         '''
         junit '**/build/**/*.xml'

build.gradle

@@ -7,35 +7,43 @@ plugins {
 	// This plugin requires Java 11; keeping it commented out so that this project can be built and tested via
 	// Java 8, 11 or 17; to use this plugin, uncomment this line and use java 11 or higher
 	//	id "com.github.jk1.dependency-license-report" version "2.1"
+
+	// For deploying the test app.
+	id 'net.saliman.properties' version '1.5.2'
+	id "com.marklogic.ml-gradle" version "4.5.2"
 }
 
 group = "com.marklogic"
-version = "4.5.1"
+version = "4.6.0"
 
 java {
 	sourceCompatibility = 1.8
 	targetCompatibility = 1.8
 }
 
 repositories {
+	mavenLocal()
+	maven {
+		url "https://nexus.marklogic.com/repository/maven-snapshots/"
+	}
 	mavenCentral()
 }
 
 dependencies {
-	api 'com.marklogic:marklogic-client-api:6.2.0'
-	api 'com.marklogic:marklogic-xcc:11.0.2'
-	api 'org.springframework:spring-context:5.3.27'
+	api 'com.marklogic:marklogic-client-api:6.3.0'
+	api 'com.marklogic:marklogic-xcc:11.0.3'
+	api 'org.springframework:spring-context:5.3.29'
 
 	implementation 'org.jdom:jdom2:2.0.6.1'
 
-	implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1'
+	implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2'
 
 	// This is currently an implementation dependency, though perhaps should be an api dependency due to its usage in
 	// LoggingObject; not clear yet on whether the protected Logger in that class should make this an api dependency or not
 	implementation 'org.slf4j:slf4j-api:1.7.36'
 
-	testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3'
-	testImplementation 'org.springframework:spring-test:5.3.27'
+	testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0'
+	testImplementation 'org.springframework:spring-test:5.3.29'
 	testImplementation 'org.mockito:mockito-core:4.11.0'
 
 	// Used for testing loading modules from the classpath
@@ -53,7 +61,7 @@ dependencies {
 
 	// Including the "new" JAXB libraries to verify that their presence doesn't cause the "old" JAXB libraries to fail
 	testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.0"
-	testImplementation "com.sun.xml.bind:jaxb-impl:4.0.1"
+	testImplementation "com.sun.xml.bind:jaxb-impl:4.0.3"
 }
 
 test {

gradle.properties

@@ -9,3 +9,11 @@ mavenCentralUrl=https://oss.sonatype.org/service/local/staging/deploy/maven2/
 #signing.keyId=YourKeyId
 #signing.password=YourPublicKeyPassword
 #signing.secretKeyRingFile=PathToYourKeyRingFile
+
+# For the test app
+mlHost=localhost
+mlAppName=ml-javaclient-util-test
+mlRestPort=8028
+mlUsername=admin
+mlPassword=change in gradle-local.properties
+mlConfigPaths=src/test/ml-config

pom.xml

@@ -12,7 +12,7 @@ It is not intended to be used to build this project.
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.marklogic</groupId>
   <artifactId>ml-javaclient-util</artifactId>
-  <version>4.5.1</version>
+  <version>4.6.0</version>
   <name>com.marklogic:ml-javaclient-util</name>
   <description>Library that adds functionality on top of the MarkLogic Java Client</description>
   <url>https://github.com/marklogic/ml-javaclient-util</url>
@@ -40,19 +40,19 @@ It is not intended to be used to build this project.
     <dependency>
       <groupId>com.marklogic</groupId>
       <artifactId>marklogic-client-api</artifactId>
-      <version>6.2.0</version>
+      <version>6.3.0</version>
       <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>com.marklogic</groupId>
       <artifactId>marklogic-xcc</artifactId>
-      <version>11.0.2</version>
+      <version>11.0.3</version>
       <scope>compile</scope>
     </dependency>
     <dependency>
       <groupId>org.springframework</groupId>
       <artifactId>spring-context</artifactId>
-      <version>5.3.27</version>
+      <version>5.3.29</version>
       <scope>compile</scope>
     </dependency>
     <dependency>
@@ -64,7 +64,7 @@ It is not intended to be used to build this project.
     <dependency>
       <groupId>com.fasterxml.jackson.core</groupId>
       <artifactId>jackson-databind</artifactId>
-      <version>2.14.1</version>
+      <version>2.15.2</version>
       <scope>runtime</scope>
     </dependency>
     <dependency>

src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java

@@ -0,0 +1,101 @@
+/*
+ * Copyright (c) 2023 MarkLogic Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.marklogic.client.ext.file;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+import java.util.Properties;
+import java.util.Stack;
+
+/**
+ * Adds a stack to store Properties objects while traversing a directory tree. Implements {@code FileVisitor} so that
+ * it can be informed when {@code DefaultDocumentFileReader} is entering and exiting a directory.
+ *
+ * To preserve backwards compatibility in subclasses, cascading is disabled by default. This will likely change in 5.0
+ * to be enabled by default.
+ *
+ * @since 4.6.0
+ */
+abstract class CascadingPropertiesDrivenDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor implements FileVisitor<Path> {
+
+	final private Stack<Properties> propertiesStack = new Stack<>();
+	private boolean cascadingEnabled = false;
+
+	protected CascadingPropertiesDrivenDocumentFileProcessor(String propertiesFilename) {
+		super(propertiesFilename);
+	}
+
+	@Override
+	public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+		// If cascading is disabled, we still use a stack to keep track of whether a directory has properties or not.
+		// We just never grab properties from the stack in case a directory doesn't have properties.
+		if (logger.isDebugEnabled()) {
+			logger.debug(format("Visiting directory: %s", dir.toFile().getAbsolutePath()));
+		}
+		File propertiesFile = new File(dir.toFile(), this.getPropertiesFilename());
+		if (propertiesFile.exists()) {
+			if (logger.isDebugEnabled()) {
+				logger.debug(format("Loading properties from file: %s", propertiesFile.getAbsolutePath()));
+			}
+			this.loadProperties(propertiesFile);
+		} else {
+			if (cascadingEnabled && !propertiesStack.isEmpty()) {
+				if (logger.isDebugEnabled()) {
+					logger.debug("No properties file, and cascading is enabled, so using properties from top of stack.");
+				}
+				this.setProperties(propertiesStack.peek());
+			} else {
+				if (logger.isDebugEnabled()) {
+					logger.debug("No properties file, or cascading is disabled, so using empty properties.");
+				}
+				this.setProperties(new Properties());
+			}
+		}
+		propertiesStack.push(this.getProperties());
+		return FileVisitResult.CONTINUE;
+	}
+
+	@Override
+	public FileVisitResult postVisitDirectory(Path dir, IOException exc) {
+		propertiesStack.pop();
+		if (!propertiesStack.isEmpty()) {
+			this.setProperties(propertiesStack.peek());
+		}
+		return FileVisitResult.CONTINUE;
+	}
+
+	@Override
+	public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) {
+		return FileVisitResult.CONTINUE;
+	}
+
+	@Override
+	public FileVisitResult visitFileFailed(Path file, IOException exc) {
+		return FileVisitResult.CONTINUE;
+	}
+
+	public boolean isCascadingEnabled() {
+		return cascadingEnabled;
+	}
+
+	public void setCascadingEnabled(boolean cascadingEnabled) {
+		this.cascadingEnabled = cascadingEnabled;
+	}
+}

src/main/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessor.java

@@ -22,7 +22,7 @@
  * key is the name of a file in the directory, and the value is a comma-delimited list of collections to load the file
  * into (which means you can't use a comma in any collection name).
  */
-public class CollectionsFileDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor {
+public class CollectionsFileDocumentFileProcessor extends CascadingPropertiesDrivenDocumentFileProcessor {
 
 	private String delimiter = ",";
 

src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java

@@ -20,7 +20,11 @@
 import java.io.IOException;
 import java.net.URI;
 import java.net.URISyntaxException;
-import java.nio.file.*;
+import java.nio.file.FileVisitResult;
+import java.nio.file.FileVisitor;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.List;
@@ -35,7 +39,8 @@ public class DefaultDocumentFileReader extends AbstractDocumentFileReader implem
 	private List<DocumentFile> documentFiles;
 	private String uriPrefix = "/";
 
-	// Each of these are eagerly instantiated, and we retain a reference in case a client wants to modify them
+	// As of 4.6.0, these no longer need to be class fields but are being kept for backwards compatibility.
+	// They should be removed in 5.0.0.
 	private CollectionsFileDocumentFileProcessor collectionsFileDocumentFileProcessor;
 	private PermissionsFileDocumentFileProcessor permissionsFileDocumentFileProcessor;
 
@@ -105,6 +110,11 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 			if (logger.isDebugEnabled()) {
 				logger.debug("Visiting directory: " + dir);
 			}
+			for (DocumentFileProcessor processor : getDocumentFileProcessors()) {
+				if (processor instanceof FileVisitor) {
+					((FileVisitor) processor).preVisitDirectory(dir, attrs);
+				}
+			}
 			return FileVisitResult.CONTINUE;
 		} else {
 			if (logger.isDebugEnabled()) {
@@ -115,7 +125,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th
 	}
 
 	@Override
-	public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
+	public FileVisitResult visitFileFailed(Path file, IOException exc) {
 		if (exc != null) {
 			logger.warn("Failed visiting file: " + exc.getMessage(), exc);
 		}
@@ -127,11 +137,16 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx
 		if (exc != null) {
 			logger.warn("Error in postVisitDirectory: " + exc.getMessage(), exc);
 		}
+		for (DocumentFileProcessor processor : getDocumentFileProcessors()) {
+			if (processor instanceof FileVisitor) {
+				((FileVisitor) processor).postVisitDirectory(dir, exc);
+			}
+		}
 		return FileVisitResult.CONTINUE;
 	}
 
 	@Override
-	public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
+	public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) {
 		if (acceptPath(path, attrs)) {
 			DocumentFile documentFile = buildDocumentFile(path, currentRootPath);
 			documentFile = processDocumentFile(documentFile);
@@ -202,10 +217,21 @@ public void setUriPrefix(String uriPrefix) {
 		this.uriPrefix = uriPrefix;
 	}
 
+	/**
+	 *
+	 * @return
+	 * @deprecated since 4.6.0, will be removed in 5.0.0
+	 */
+	@Deprecated
 	public CollectionsFileDocumentFileProcessor getCollectionsFileDocumentFileProcessor() {
 		return collectionsFileDocumentFileProcessor;
 	}
 
+	/**
+	 *
+	 * @return
+	 * @deprecated since 4.6.0, will be removed in 5.0.0
+	 */
 	public PermissionsFileDocumentFileProcessor getPermissionsFileDocumentFileProcessor() {
 		return permissionsFileDocumentFileProcessor;
 	}

src/main/java/com/marklogic/client/ext/file/FormatDocumentFileProcessor.java

@@ -15,13 +15,17 @@
  */
 package com.marklogic.client.ext.file;
 
+import com.marklogic.client.ext.helper.LoggingObject;
 import com.marklogic.client.io.Format;
 
+import java.util.Arrays;
+
 /**
  * Delegates to DefaultDocumentFormatGetter by default for determining what Format to use for the File in a given
  * DocumentFile.
  */
-public class FormatDocumentFileProcessor implements DocumentFileProcessor {
+public class FormatDocumentFileProcessor extends LoggingObject
+	implements DocumentFileProcessor, SupportsAdditionalBinaryExtensions {
 
 	private FormatGetter formatGetter;
 
@@ -35,14 +39,26 @@ public FormatDocumentFileProcessor(FormatGetter formatGetter) {
 
 	@Override
 	public DocumentFile processDocumentFile(DocumentFile documentFile) {
-
 		Format format = formatGetter.getFormat(documentFile.getResource());
 		if (format != null) {
 			documentFile.setFormat(format);
 		}
 		return documentFile;
 	}
 
+	@Override
+	public void setAdditionalBinaryExtensions(String[] extensions) {
+		if (formatGetter instanceof DefaultDocumentFormatGetter) {
+			DefaultDocumentFormatGetter ddfg = (DefaultDocumentFormatGetter) formatGetter;
+			for (String ext : extensions) {
+				ddfg.getBinaryExtensions().add(ext);
+			}
+		} else {
+			logger.warn("FormatGetter is not an instanceof DefaultDocumentFormatGetter, " +
+				"so unable to add additionalBinaryExtensions: " + Arrays.asList(extensions));
+		}
+	}
+
 	public FormatGetter getFormatGetter() {
 		return formatGetter;
 	}