From 87b8206fd9c4a9aed80be508db4d6ee9ee9fb155 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 8 May 2023 15:10:42 -0400 Subject: [PATCH 01/20] Bumped dev to 4.6-SNAPSHOT --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index 8fde06b..fa7b46f 100644 --- a/build.gradle +++ b/build.gradle @@ -10,7 +10,7 @@ plugins { } group = "com.marklogic" -version = "4.5.1" +version = "4.6-SNAPSHOT" java { sourceCompatibility = 1.8 From 5404ebbf5fa129cae327c19af057f2a49a6737a1 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 25 May 2023 09:38:25 -0400 Subject: [PATCH 02/20] Added test for loading modules with properties files --- .../client/ext/AbstractIntegrationTest.java | 9 ------ .../modulesloader/impl/LoadModulesTest.java | 28 +++++++++++++++++++ .../root/collections.properties | 1 + .../root/lib/collections.properties | 1 + .../root/lib/lib.sjs | 1 + .../root/lib/permissions.properties | 1 + .../root/permissions.properties | 1 + .../root/root.sjs | 1 + 8 files changed, 34 insertions(+), 9 deletions(-) create mode 100644 src/test/resources/base-dir-with-properties-files/root/collections.properties create mode 100644 src/test/resources/base-dir-with-properties-files/root/lib/collections.properties create mode 100644 src/test/resources/base-dir-with-properties-files/root/lib/lib.sjs create mode 100644 src/test/resources/base-dir-with-properties-files/root/lib/permissions.properties create mode 100644 src/test/resources/base-dir-with-properties-files/root/permissions.properties create mode 100644 src/test/resources/base-dir-with-properties-files/root/root.sjs diff --git a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java index 315e357..e4a1083 100644 --- a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java +++ b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java @@ -19,8 +19,6 @@ import com.marklogic.client.ext.spring.SpringDatabaseClientConfig; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -34,19 +32,12 @@ @ContextConfiguration(classes = {TestConfig.class}) public abstract class AbstractIntegrationTest { - protected Logger logger = LoggerFactory.getLogger(getClass()); - @Autowired protected DatabaseClientConfig clientConfig; protected DatabaseClient client; protected ConfiguredDatabaseClientFactory configuredDatabaseClientFactory = new DefaultConfiguredDatabaseClientFactory(); - protected DatabaseClient newClient() { - client = configuredDatabaseClientFactory.newDatabaseClient(clientConfig); - return client; - } - @AfterEach public void releaseClientOnTearDown() { if (client != null) { diff --git a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java index 6bde32c..edcaf81 100644 --- a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java +++ b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java @@ -22,6 +22,7 @@ import com.marklogic.client.ext.modulesloader.ModulesFinder; import com.marklogic.client.ext.tokenreplacer.DefaultTokenReplacer; import com.marklogic.client.io.BytesHandle; +import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.io.StringHandle; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -258,6 +259,33 @@ public void loadModulesAndAccountForHost() { assertEquals(0, files.size(), "No files should have been loaded since none were new or modified"); } + @Test + public void withPropertiesFiles() { + AssetFileLoader fileLoader = new AssetFileLoader(modulesClient); + fileLoader.setPermissions("rest-extension-user,read,rest-extension-user,update,rest-extension-user,execute"); + modulesLoader.setAssetFileLoader(fileLoader); + + String dir = Paths.get("src", "test", "resources", "base-dir-with-properties-files").toString(); + Set files = modulesLoader.loadModules(dir, new DefaultModulesFinder(), client); + assertEquals(2, files.size()); + + DocumentMetadataHandle metadata = new DocumentMetadataHandle(); + + modulesClient.newDocumentManager().readMetadata("/root.sjs", metadata); + assertEquals(1, metadata.getCollections().size()); + assertEquals("parent", metadata.getCollections().iterator().next()); + DocumentMetadataHandle.DocumentPermissions perms = metadata.getPermissions(); + assertEquals(DocumentMetadataHandle.Capability.READ, perms.get("qconsole-user").iterator().next()); + assertEquals(3, perms.get("rest-extension-user").size()); + + modulesClient.newDocumentManager().readMetadata("/lib/lib.sjs", metadata); + assertEquals(1, metadata.getCollections().size()); + assertEquals("lib", metadata.getCollections().iterator().next()); + perms = metadata.getPermissions(); + assertEquals(DocumentMetadataHandle.Capability.UPDATE, perms.get("app-user").iterator().next()); + assertEquals(3, perms.get("rest-extension-user").size()); + } + @Test public void test() { String dir = Paths.get("src", "test", "resources", "sample-base-dir").toString(); diff --git a/src/test/resources/base-dir-with-properties-files/root/collections.properties b/src/test/resources/base-dir-with-properties-files/root/collections.properties new file mode 100644 index 0000000..a59bc8b --- /dev/null +++ b/src/test/resources/base-dir-with-properties-files/root/collections.properties @@ -0,0 +1 @@ +*=parent diff --git a/src/test/resources/base-dir-with-properties-files/root/lib/collections.properties b/src/test/resources/base-dir-with-properties-files/root/lib/collections.properties new file mode 100644 index 0000000..e8b8650 --- /dev/null +++ b/src/test/resources/base-dir-with-properties-files/root/lib/collections.properties @@ -0,0 +1 @@ +*=lib diff --git a/src/test/resources/base-dir-with-properties-files/root/lib/lib.sjs b/src/test/resources/base-dir-with-properties-files/root/lib/lib.sjs new file mode 100644 index 0000000..21b35a8 --- /dev/null +++ b/src/test/resources/base-dir-with-properties-files/root/lib/lib.sjs @@ -0,0 +1 @@ +console.log("Hey"); diff --git a/src/test/resources/base-dir-with-properties-files/root/lib/permissions.properties b/src/test/resources/base-dir-with-properties-files/root/lib/permissions.properties new file mode 100644 index 0000000..384e04b --- /dev/null +++ b/src/test/resources/base-dir-with-properties-files/root/lib/permissions.properties @@ -0,0 +1 @@ +*=app-user,update diff --git a/src/test/resources/base-dir-with-properties-files/root/permissions.properties b/src/test/resources/base-dir-with-properties-files/root/permissions.properties new file mode 100644 index 0000000..76f93d5 --- /dev/null +++ b/src/test/resources/base-dir-with-properties-files/root/permissions.properties @@ -0,0 +1 @@ +*=qconsole-user,read diff --git a/src/test/resources/base-dir-with-properties-files/root/root.sjs b/src/test/resources/base-dir-with-properties-files/root/root.sjs new file mode 100644 index 0000000..b2341ef --- /dev/null +++ b/src/test/resources/base-dir-with-properties-files/root/root.sjs @@ -0,0 +1 @@ +console.log("Hi"); From fd5922ff6907a43e7d24014b6a7aa416b4a45123 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 8 May 2023 16:39:17 -0400 Subject: [PATCH 03/20] Changed a few references to marklogic-community --- README.md | 12 ++++++------ pom.xml | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index ac91d8c..503eb6b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ -![GitHub release](https://img.shields.io/github/release/marklogic-community/ml-javaclient-util.svg) -![GitHub last commit](https://img.shields.io/github/last-commit/marklogic-community/ml-javaclient-util.svg) +![GitHub release](https://img.shields.io/github/release/marklogic/ml-javaclient-util.svg) +![GitHub last commit](https://img.shields.io/github/last-commit/marklogic/ml-javaclient-util.svg) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) -[![Known Vulnerabilities](https://snyk.io/test/github/marklogic-community/ml-javaclient-util/badge.svg)](https://snyk.io/test/github/marklogic-community/ml-javaclient-util) +[![Known Vulnerabilities](https://snyk.io/test/github/marklogic/ml-javaclient-util/badge.svg)](https://snyk.io/test/github/marklogic/ml-javaclient-util) ml-javaclient-util is a library of Java classes that provide some useful functionality on top of the [MarkLogic Java Client API](http://docs.marklogic.com/guide/java). Those features include: @@ -18,6 +18,6 @@ and [ml-gradle](https://github.com/rjrudin/ml-gradle) and [ml-junit](https://git See the following Wiki pages for more information on some of the main features in this library: -1. [Loading files](https://github.com/marklogic-community/ml-javaclient-util/wiki/Loading-files), including modules -1. [DMSDK Support](https://github.com/marklogic-community/ml-javaclient-util/wiki/DMSDK-Support) -1. [Writing documents in batches](https://github.com/marklogic-community/ml-javaclient-util/wiki/Writing-documents-in-batches) +1. [Loading files](https://github.com/marklogic/ml-javaclient-util/wiki/Loading-files), including modules +1. [DMSDK Support](https://github.com/marklogic/ml-javaclient-util/wiki/DMSDK-Support) +1. [Writing documents in batches](https://github.com/marklogic/ml-javaclient-util/wiki/Writing-documents-in-batches) diff --git a/pom.xml b/pom.xml index 371b8dc..d1e3c42 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ It is not intended to be used to build this project. 4.5.1 com.marklogic:ml-javaclient-util Library that adds functionality on top of the MarkLogic Java Client - https://github.com/marklogic-community/ml-javaclient-util + https://github.com/marklogic/ml-javaclient-util The Apache License, Version 2.0 @@ -32,9 +32,9 @@ It is not intended to be used to build this project. - scm:git@github.com:marklogic-community/ml-javaclient-util.git - scm:git@github.com:marklogic-community/ml-javaclient-util.git - git@github.com:marklogic-community/ml-javaclient-util.git + scm:git@github.com:marklogic/ml-javaclient-util.git + scm:git@github.com:marklogic/ml-javaclient-util.git + git@github.com:marklogic/ml-javaclient-util.git From e2a2fbfe225bffcf4e6882aa9ea3caf3584a6c6a Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 15 Aug 2023 11:32:33 -0400 Subject: [PATCH 04/20] Refactored DefaultSchemasLoader Moved the logic for breaking up the list of files into a new private method with a new private class to capture the two lists. Also deprecated the constructor that receives a BatchWriter; this doesn't work when processors need access to a DatabaseClient for the schemas database. --- .../impl/DefaultSchemasLoader.java | 89 ++++++++++++------- 1 file changed, 56 insertions(+), 33 deletions(-) diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java index cebf430..3eee636 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java @@ -20,7 +20,6 @@ import com.marklogic.client.ext.batch.RestBatchWriter; import com.marklogic.client.ext.file.DocumentFile; import com.marklogic.client.ext.file.GenericFileLoader; -import com.marklogic.client.ext.helper.ClientHelper; import com.marklogic.client.ext.modulesloader.impl.DefaultFileFilter; import com.marklogic.client.ext.schemasloader.SchemasLoader; import com.marklogic.client.io.DocumentMetadataHandle; @@ -75,7 +74,10 @@ public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, String tdeVali * Assumes that the BatchWriter has already been initialized. * * @param batchWriter + * @deprecated Since 4.6.0; this class needs a DatabaseClient for the schemas database passed to it so that it can + * pass that client on to specific file processors. */ + @Deprecated public DefaultSchemasLoader(BatchWriter batchWriter) { super(batchWriter); initializeDefaultSchemasLoader(); @@ -100,46 +102,45 @@ protected void initializeDefaultSchemasLoader() { @Override public List loadSchemas(String... paths) { final List documentFiles = super.getDocumentFiles(paths); - if (documentFiles.isEmpty()) { - return documentFiles; - } - - if (TdeUtil.templateBatchInsertSupported(schemasDatabaseClient) && StringUtils.hasText(tdeValidationDatabase)) { - final List tdeFiles = new ArrayList<>(); - final List nonTdeFiles = new ArrayList<>(); - for (DocumentFile file : documentFiles) { - DocumentMetadataHandle metadata = file.getMetadata(); - if (metadata != null && metadata.getCollections().contains(TdeUtil.TDE_COLLECTION)) { - tdeFiles.add(file); - } else { - nonTdeFiles.add(file); + if (!documentFiles.isEmpty()) { + if (TdeUtil.templateBatchInsertSupported(schemasDatabaseClient) && StringUtils.hasText(tdeValidationDatabase)) { + SchemaFiles schemaFiles = readSchemaFiles(documentFiles); + if (!schemaFiles.tdeFiles.isEmpty()) { + loadTdeTemplatesViaBatchInsert(schemaFiles.tdeFiles); } - } - - if (!tdeFiles.isEmpty()) { - loadTdeTemplatesViaBatchInsert(tdeFiles); - } - if (!nonTdeFiles.isEmpty()) { - if (logger.isDebugEnabled()) { - logger.debug("Non-TDE files: " + nonTdeFiles); + if (!schemaFiles.nonTdeFiles.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("Non-TDE files: " + schemaFiles.nonTdeFiles); + } + super.writeDocumentFiles(schemaFiles.nonTdeFiles); } - super.writeDocumentFiles(nonTdeFiles); + } else { + writeDocumentFiles(documentFiles); } - - return documentFiles; } - writeDocumentFiles(documentFiles); return documentFiles; } - public String getTdeValidationDatabase() { - return tdeValidationDatabase; - } - - public void setTdeValidationDatabase(String tdeValidationDatabase) { - this.tdeValidationDatabase = tdeValidationDatabase; + /** + * @param documentFiles + * @return a SchemaFiles instance that captures a list of TDE files (if any) and a list of all other files found + * that are not TDEs (may also be empty). + */ + private SchemaFiles readSchemaFiles(List documentFiles) { + List tdeFiles = new ArrayList<>(); + List nonTdeFiles = new ArrayList<>(); + + for (DocumentFile file : documentFiles) { + DocumentMetadataHandle metadata = file.getMetadata(); + if (metadata != null && metadata.getCollections().contains(TdeUtil.TDE_COLLECTION)) { + tdeFiles.add(file); + } else { + nonTdeFiles.add(file); + } + } + return new SchemaFiles(tdeFiles, nonTdeFiles); } private void loadTdeTemplatesViaBatchInsert(List tdeFiles) { @@ -160,7 +161,6 @@ private void loadTdeTemplatesViaBatchInsert(List tdeFiles) { } /** - * * @param documentFiles * @return a JavaScript query that uses the tde.templateBatchInsert function introduced in ML 10.0-9 */ @@ -202,4 +202,27 @@ private String buildTdeBatchInsertQuery(List documentFiles) { .concat("]);"); return templateString; } + + private static class SchemaFiles { + private final List tdeFiles; + private final List nonTdeFiles; + + public SchemaFiles(List tdeFiles, List nonTdeFiles) { + this.tdeFiles = tdeFiles; + this.nonTdeFiles = nonTdeFiles; + } + } + + public String getTdeValidationDatabase() { + return tdeValidationDatabase; + } + + /** + * @param tdeValidationDatabase + * @deprecated Should be set via the constructor and not modified. + */ + @Deprecated + public void setTdeValidationDatabase(String tdeValidationDatabase) { + this.tdeValidationDatabase = tdeValidationDatabase; + } } From d4b6fe0d0f0db82068c55f3b7217855bbd91c227 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 15 Aug 2023 14:51:45 -0400 Subject: [PATCH 05/20] Added CODEOWNERS --- CODEOWNERS | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 CODEOWNERS diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..0a2d67e --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,5 @@ +# Lines starting with '#' are comments. +# Each line is a file pattern followed by one or more owners. + +# These owners will be the default owners for everything in the repo. +* @anu3990 @billfarber @rjrudin From edb6bbd480ef8158269989e154a144a31e887fe8 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 18 Aug 2023 16:11:45 -0400 Subject: [PATCH 06/20] Added test app This isn't perfect, and the CONTRIB needs a lot more work - but it's a start, and it lets us write more complete tests that don't depend on the admin user for everything. That'll be a future PR - trying to convert as many tests as possible to use a non-admin user. --- CONTRIBUTING.md | 9 +++++++++ Jenkinsfile | 2 ++ build.gradle | 4 ++++ gradle.properties | 7 +++++++ .../client/ext/AbstractIntegrationTest.java | 20 ++++++++++++++----- .../client/ext/NewDatabaseClientTest.java | 2 +- .../client/ext/batch/RestBatchWriterTest.java | 8 ++++---- .../client/ext/es/EntityServicesTest.java | 2 +- .../SetAdditionalBinaryExtensionsTest.java | 2 +- .../impl/LoadModulesFromClasspathTest.java | 2 +- .../modulesloader/impl/LoadModulesTest.java | 13 ++++++------ .../impl/StaticCheckModulesTest.java | 2 +- .../schemasloader/impl/LoadRulesetsTest.java | 2 +- .../schemasloader/impl/LoadSchemasTest.java | 2 +- .../impl/ValidateTdeTemplatesTest.java | 2 +- src/test/resources/test.properties | 4 ---- 16 files changed, 56 insertions(+), 27 deletions(-) create mode 100644 CONTRIBUTING.md delete mode 100644 src/test/resources/test.properties diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4e1acdc --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,9 @@ +To run the tests for ml-javaclient-util, clone this repository and deploy this project's test application first via +the following steps: + +1. Create a file named `gradle-local.properties`. +2. Add `mlPassword=` followed by the password for your MarkLogic admin user. +3. Verify that port 8070 is available on your computer - i.e. no other process is listening on it. +4. Run `./gradlew -i mlDeploy`. + +You can then run `./gradlew -i test` to run all of the tests. diff --git a/Jenkinsfile b/Jenkinsfile index 2c60fb8..576becf 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -21,6 +21,8 @@ pipeline{ export GRADLE_USER_HOME=$WORKSPACE/$GRADLE_DIR export PATH=$GRADLE_USER_HOME:$JAVA_HOME/bin:$PATH cd ml-javaclient-util + echo "mlPassword=admin" > gradle-local.properties + ./gradlew -i mlDeploy ./gradlew test || true ''' junit '**/build/**/*.xml' diff --git a/build.gradle b/build.gradle index fa7b46f..b73079a 100644 --- a/build.gradle +++ b/build.gradle @@ -7,6 +7,10 @@ plugins { // This plugin requires Java 11; keeping it commented out so that this project can be built and tested via // Java 8, 11 or 17; to use this plugin, uncomment this line and use java 11 or higher // id "com.github.jk1.dependency-license-report" version "2.1" + + // For deploying the test app. + id 'net.saliman.properties' version '1.5.2' + id "com.marklogic.ml-gradle" version "4.5.2" } group = "com.marklogic" diff --git a/gradle.properties b/gradle.properties index 1233b06..52bb5a4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,3 +9,10 @@ mavenCentralUrl=https://oss.sonatype.org/service/local/staging/deploy/maven2/ #signing.keyId=YourKeyId #signing.password=YourPublicKeyPassword #signing.secretKeyRingFile=PathToYourKeyRingFile + +# For the test app +mlHost=localhost +mlAppName=ml-javaclient-util-test +mlRestPort=8028 +mlUsername=admin +mlPassword=change in gradle-local.properties diff --git a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java index e4a1083..34368b2 100644 --- a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java +++ b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java @@ -16,13 +16,12 @@ package com.marklogic.client.ext; import com.marklogic.client.DatabaseClient; -import com.marklogic.client.ext.spring.SpringDatabaseClientConfig; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Import; import org.springframework.context.annotation.PropertySource; import org.springframework.context.support.PropertySourcesPlaceholderConfigurer; import org.springframework.test.context.ContextConfiguration; @@ -32,6 +31,9 @@ @ContextConfiguration(classes = {TestConfig.class}) public abstract class AbstractIntegrationTest { + protected final static String CONTENT_DATABASE = "ml-javaclient-util-test-content"; + protected final static String MODULES_DATABASE = "ml-javaclient-util-test-modules"; + @Autowired protected DatabaseClientConfig clientConfig; protected DatabaseClient client; @@ -59,9 +61,17 @@ protected DatabaseClient newClient(String database) { } @Configuration -@Import(value = {SpringDatabaseClientConfig.class}) -@PropertySource(value = { "classpath:test.properties", "classpath:user.properties" }, ignoreResourceNotFound = true) -class TestConfig { +@PropertySource(value = {"file:gradle.properties", "file:gradle-local.properties"}, ignoreResourceNotFound = true) +class TestConfig extends DatabaseClientConfig { + + @Autowired + public TestConfig( + @Value("${mlHost}") String host, + @Value("${mlRestPort}") int port, + @Value("${mlUsername}") String username, + @Value("${mlPassword}") String password) { + super(host, port, username, password); + } /** * Ensures that placeholders are replaced with property values diff --git a/src/test/java/com/marklogic/client/ext/NewDatabaseClientTest.java b/src/test/java/com/marklogic/client/ext/NewDatabaseClientTest.java index ca19361..5664fd7 100644 --- a/src/test/java/com/marklogic/client/ext/NewDatabaseClientTest.java +++ b/src/test/java/com/marklogic/client/ext/NewDatabaseClientTest.java @@ -29,7 +29,7 @@ public class NewDatabaseClientTest { @BeforeEach void setup() { - config = new DatabaseClientConfig("localhost", 8000); + config = new DatabaseClientConfig("localhost", 8028); } @Test diff --git a/src/test/java/com/marklogic/client/ext/batch/RestBatchWriterTest.java b/src/test/java/com/marklogic/client/ext/batch/RestBatchWriterTest.java index 8ea203c..1c5defa 100644 --- a/src/test/java/com/marklogic/client/ext/batch/RestBatchWriterTest.java +++ b/src/test/java/com/marklogic/client/ext/batch/RestBatchWriterTest.java @@ -42,7 +42,7 @@ public class RestBatchWriterTest extends AbstractIntegrationTest { @Test public void failureTest() { - RestBatchWriter writer = new RestBatchWriter(newClient("Documents")); + RestBatchWriter writer = new RestBatchWriter(newClient(CONTENT_DATABASE)); DocumentWriteOperation op = new DocumentWriteOperationImpl(DocumentWriteOperation.OperationType.DOCUMENT_WRITE, "/test.xml", null, new StringHandle("worldasdf")); @@ -60,7 +60,7 @@ public void failureTest() { @Test public void failureTestWithCustomListener() { - RestBatchWriter writer = new RestBatchWriter(newClient("Documents")); + RestBatchWriter writer = new RestBatchWriter(newClient(CONTENT_DATABASE)); TestWriteListener testWriteListener = new TestWriteListener(); writer.setWriteListener(testWriteListener); @@ -78,7 +78,7 @@ public void failureTestWithCustomListener() { @Test public void writeDocumentWithTransform() throws IOException { - DatabaseClient client = newClient("Documents"); + DatabaseClient client = newClient(CONTENT_DATABASE); Resource transform = new FileSystemResource(Paths.get("src", "test", "resources", "transform", "simple.xqy").toString()); TransformExtensionsManager transMgr = client.newServerConfigManager().newTransformExtensionsManager(); FileHandle fileHandle = new FileHandle(transform.getFile()); @@ -96,7 +96,7 @@ public void writeDocumentWithTransform() throws IOException { writer.write(Collections.singletonList(op)); writer.waitForCompletion(); - client = newClient("Documents"); + client = newClient(CONTENT_DATABASE); XMLDocumentManager docMgr = client.newXMLDocumentManager(); DocumentPage page = docMgr.read("/test.xml"); StringHandle handle = page.nextContent(new StringHandle()); diff --git a/src/test/java/com/marklogic/client/ext/es/EntityServicesTest.java b/src/test/java/com/marklogic/client/ext/es/EntityServicesTest.java index 027aad9..cd7e867 100644 --- a/src/test/java/com/marklogic/client/ext/es/EntityServicesTest.java +++ b/src/test/java/com/marklogic/client/ext/es/EntityServicesTest.java @@ -52,7 +52,7 @@ public class EntityServicesTest extends AbstractIntegrationTest { @BeforeEach public void setup() { - client = newClient("Documents"); + client = newClient(CONTENT_DATABASE); } @Test diff --git a/src/test/java/com/marklogic/client/ext/file/SetAdditionalBinaryExtensionsTest.java b/src/test/java/com/marklogic/client/ext/file/SetAdditionalBinaryExtensionsTest.java index bc44258..6222265 100644 --- a/src/test/java/com/marklogic/client/ext/file/SetAdditionalBinaryExtensionsTest.java +++ b/src/test/java/com/marklogic/client/ext/file/SetAdditionalBinaryExtensionsTest.java @@ -28,7 +28,7 @@ public class SetAdditionalBinaryExtensionsTest extends AbstractIntegrationTest { @Test public void test() { - client = newClient("Documents"); + client = newClient(CONTENT_DATABASE); client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval(); GenericFileLoader loader = new GenericFileLoader(client); diff --git a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesFromClasspathTest.java b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesFromClasspathTest.java index 733f385..8921bc2 100644 --- a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesFromClasspathTest.java +++ b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesFromClasspathTest.java @@ -36,7 +36,7 @@ public class LoadModulesFromClasspathTest extends AbstractIntegrationTest { @BeforeEach public void setup() { - client = newClient("Modules"); + client = newClient(MODULES_DATABASE); modulesClient = client; modulesClient.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval(); } diff --git a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java index edcaf81..ba5cd9c 100644 --- a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java +++ b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java @@ -45,14 +45,15 @@ public class LoadModulesTest extends AbstractIntegrationTest { @BeforeEach public void setup() { - client = newClient("Modules"); + client = newClient(MODULES_DATABASE); client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval(); modulesClient = client; assertEquals(0, getUriCountInModulesDatabase(), "No new modules should have been created"); /** * Odd - the Client REST API doesn't allow for loading namespaces when the DatabaseClient has a database - * specified, so we construct a DatabaseClient without a database and assume we get "Documents". + * specified, so we construct a DatabaseClient without a database and assume we get the expected content + * database. */ String currentDatabase = clientConfig.getDatabase(); clientConfig.setDatabase(null); @@ -88,7 +89,7 @@ public void jsonRestPropertiesFile() { assertTrue(StringUtils.isEmpty(mgr.getDefaultDocumentReadTransform())); assertTrue(mgr.getDefaultDocumentReadTransformAll()); } finally { - set8000RestPropertiesToMarkLogicDefaults(); + setRestPropertiesToMarkLogicDefaults(); } } @@ -112,11 +113,11 @@ public void xmlRestPropertiesFile() { assertTrue(StringUtils.isEmpty(mgr.getDefaultDocumentReadTransform())); assertTrue(mgr.getDefaultDocumentReadTransformAll()); } finally { - set8000RestPropertiesToMarkLogicDefaults(); + setRestPropertiesToMarkLogicDefaults(); } } - private void set8000RestPropertiesToMarkLogicDefaults() { + private void setRestPropertiesToMarkLogicDefaults() { ServerConfigurationManager mgr = client.newServerConfigManager(); mgr.setQueryValidation(false); mgr.setQueryOptionValidation(true); @@ -192,7 +193,7 @@ public void replaceTokens() { modulesLoader.loadModules(dir, new DefaultModulesFinder(), client); String optionsXml = modulesClient.newXMLDocumentManager().read( - "/Default/App-Services/rest-api/options/sample-options.xml", new StringHandle()).get(); + "/Default/ml-javaclient-util-test/rest-api/options/sample-options.xml", new StringHandle()).get(); assertTrue(optionsXml.contains("fn:collection('hello-world')")); String serviceText = new String(modulesClient.newDocumentManager().read( diff --git a/src/test/java/com/marklogic/client/ext/modulesloader/impl/StaticCheckModulesTest.java b/src/test/java/com/marklogic/client/ext/modulesloader/impl/StaticCheckModulesTest.java index 6246371..0a06df5 100644 --- a/src/test/java/com/marklogic/client/ext/modulesloader/impl/StaticCheckModulesTest.java +++ b/src/test/java/com/marklogic/client/ext/modulesloader/impl/StaticCheckModulesTest.java @@ -35,7 +35,7 @@ public class StaticCheckModulesTest extends AbstractIntegrationTest { private DefaultModulesLoader modulesLoader; private XccStaticChecker staticChecker; - private String database = "Modules"; + private String database = MODULES_DATABASE; private String dir = Paths.get("src", "test", "resources", "static-check").toString(); @BeforeEach diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java index aa6b6a1..4c7d0aa 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java @@ -30,7 +30,7 @@ public class LoadRulesetsTest extends AbstractSchemasTest { @Test public void test() { // Pass in a TDE validation database to ensure that TDE validation doesn't happen for these files - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); List files = loader.loadSchemas(Paths.get("src", "test", "resources", "rulesets", "collection-test").toString()); assertEquals(2, files.size()); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java index efa0fbc..3408bc5 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java @@ -50,7 +50,7 @@ public void test() { @Test public void testTemplateBatchInsert() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); List files = loader.loadSchemas(Paths.get("src", "test", "resources", "good-schemas", "originals").toString()); assertEquals(2, files.size()); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java index ef504b9..d5fce05 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java @@ -37,7 +37,7 @@ public class ValidateTdeTemplatesTest extends AbstractSchemasTest { public void setup() { super.setup(); // Assumes that Documents points to Schemas as its schemas database - loader = new DefaultSchemasLoader(client, "Documents"); + loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); } @Test diff --git a/src/test/resources/test.properties b/src/test/resources/test.properties deleted file mode 100644 index f859f31..0000000 --- a/src/test/resources/test.properties +++ /dev/null @@ -1,4 +0,0 @@ -marklogic.host=localhost -marklogic.port=8000 -marklogic.username=admin -marklogic.password=admin From 8ce60f4e2acd7d77e1213f05180ddeeb31e9932b Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Mon, 21 Aug 2023 10:16:56 -0400 Subject: [PATCH 07/20] Load QBV via script Load QBV via script Load QBV via script --- .../impl/DefaultSchemasLoader.java | 4 + .../impl/QbvDocumentFileProcessor.java | 148 ++++++++++++++++++ .../schemasloader/impl/GenerateQbvTest.java | 94 +++++++++++ .../qbv-bad-schemas/qbv/bad-authors.sjs | 1 + .../qbv-bad-schemas/tde/authors.json | 38 +++++ .../qbv-no-tde-schemas/qbv/books.sjs | 1 + src/test/resources/qbv-schemas/example.sjs | 0 .../resources/qbv-schemas/qbv/authors.sjs | 1 + .../qbv-schemas/qbv/collections.properties | 1 + .../qbv-schemas/qbv/permissions.properties | 1 + .../qbv-schemas/qbv/publications.xqy | 2 + .../resources/qbv-schemas/tde/authors.json | 38 +++++ .../qbv-schemas/tde/collections.properties | 1 + .../qbv-schemas/tde/publications.xml | 21 +++ 14 files changed, 351 insertions(+) create mode 100644 src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java create mode 100644 src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java create mode 100644 src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs create mode 100644 src/test/resources/qbv-bad-schemas/tde/authors.json create mode 100644 src/test/resources/qbv-no-tde-schemas/qbv/books.sjs create mode 100644 src/test/resources/qbv-schemas/example.sjs create mode 100644 src/test/resources/qbv-schemas/qbv/authors.sjs create mode 100644 src/test/resources/qbv-schemas/qbv/collections.properties create mode 100644 src/test/resources/qbv-schemas/qbv/permissions.properties create mode 100644 src/test/resources/qbv-schemas/qbv/publications.xqy create mode 100644 src/test/resources/qbv-schemas/tde/authors.json create mode 100644 src/test/resources/qbv-schemas/tde/collections.properties create mode 100644 src/test/resources/qbv-schemas/tde/publications.xml diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java index 3eee636..e821a5f 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java @@ -35,6 +35,7 @@ public class DefaultSchemasLoader extends GenericFileLoader implements SchemasLo private DatabaseClient schemasDatabaseClient; private String tdeValidationDatabase; + protected QbvDocumentFileProcessor qbvDocumentFileProcessor; /** * Simplest constructor for using this class. Just provide a DatabaseClient, and this will use sensible defaults for @@ -88,7 +89,9 @@ public DefaultSchemasLoader(BatchWriter batchWriter) { * a DocumentFileReader by the parent class. */ protected void initializeDefaultSchemasLoader() { + this.qbvDocumentFileProcessor = new QbvDocumentFileProcessor(this.schemasDatabaseClient, this.tdeValidationDatabase); addDocumentFileProcessor(new TdeDocumentFileProcessor(this.schemasDatabaseClient, this.tdeValidationDatabase)); + addDocumentFileProcessor(this.qbvDocumentFileProcessor); addFileFilter(new DefaultFileFilter()); } @@ -119,6 +122,7 @@ public List loadSchemas(String... paths) { writeDocumentFiles(documentFiles); } } + this.qbvDocumentFileProcessor.processQbvFiles(); return documentFiles; } diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java new file mode 100644 index 0000000..f8e6700 --- /dev/null +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2023 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.client.ext.schemasloader.impl; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.document.XMLDocumentManager; +import com.marklogic.client.eval.ServerEvaluationCall; +import com.marklogic.client.ext.file.DocumentFile; +import com.marklogic.client.ext.file.DocumentFileProcessor; +import com.marklogic.client.ext.helper.LoggingObject; +import com.marklogic.client.extra.jdom.JDOMHandle; +import com.marklogic.client.io.Format; +import com.marklogic.client.io.StringHandle; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.input.SAXBuilder; +import org.springframework.util.FileCopyUtils; + +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.List; + +public class QbvDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor { + + public static final String QBV_COLLECTION = "http://marklogic.com/xdmp/qbv"; + private static final String QBV_XML_PLAN_NAMESPACE = "http://marklogic.com/plan"; + private static final String QBV_XML_ROOT_ELEMENT = "query-based-view"; + private static final String JAVASCRIPT_EVAL_TEMPLATE = "declareUpdate(); xdmp.invokeFunction(function() {'use strict'; const op = require('/MarkLogic/optic'); return %s }, {database: xdmp.database('%s')})"; + private static final String XQUERY_EVAL_TEMPLATE = "xquery version \"1.0-ml\"; import module namespace op=\"http://marklogic.com/optic\" at \"/MarkLogic/optic.xqy\"; xdmp:invoke-function(function() {%s},{xdmp:database('%s')})"; + + final private DatabaseClient databaseClient; + final private String qbvGeneratorDatabaseName; + final protected List qbvFiles = new ArrayList<>(); + final private XMLDocumentManager docMgr; + + + /** + * @param databaseClient - a MarkLogic DatabaseClient object for the Schemas database + */ + public QbvDocumentFileProcessor(DatabaseClient databaseClient, String qbvGeneratorDatabaseName) { + this.databaseClient = databaseClient; + this.qbvGeneratorDatabaseName = qbvGeneratorDatabaseName; + this.docMgr = databaseClient.newXMLDocumentManager(); + } + + @Override + public DocumentFile processDocumentFile(DocumentFile documentFile) { + if (isQbvQuery(documentFile)) { + // Defer processing until after all other schemas have been processed. + qbvFiles.add(documentFile); + return null; + } else { + return documentFile; + } + } + + private boolean isQbvQuery(DocumentFile documentFile) { + String uri = documentFile.getUri(); + if (uri != null && uri.startsWith("/qbv")) { + String extension = documentFile.getFileExtension(); + if (extension != null) { + extension = extension.toLowerCase(); + } + return + "sjs".equals(extension) + || "js".equals(extension) + || "xqy".equals(extension) + || "xq".equals(extension); + } else return false; + } + + public void processQbvFiles() { + qbvFiles.forEach(this::processQbvFile); + qbvFiles.clear(); + } + + private void processQbvFile(DocumentFile qbvFile) { + ServerEvaluationCall call = getServerEvaluationCall(qbvFile); + if (call != null) { + StringHandle handleString = new StringHandle(); + try { + call.eval(handleString); + } catch (Exception e) { + throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); + } + if (Format.XML.equals(handleString.getFormat())) { + Document xmlDocument; + try { + xmlDocument = new SAXBuilder().build(new StringReader(handleString.get())); + } catch (Exception e) { + throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); + } + Element root = xmlDocument.getRootElement(); + if (QBV_XML_ROOT_ELEMENT.equals(root.getName()) & (root.getNamespace() != null && root.getNamespace().getURI().equals(QBV_XML_PLAN_NAMESPACE))) { + qbvFile.getDocumentMetadata().getCollections().add(QBV_COLLECTION); + String uri = qbvFile.getUri() + ".xml"; + docMgr.write(uri, qbvFile.getDocumentMetadata(), new JDOMHandle(xmlDocument)); + } else { + throw new RuntimeException(format("Query-Based view generation failed for file: %s; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get())); + } + } else { + throw new RuntimeException(format("Query-Based View generation failed for file: %s; ensure your Optic script includes a call to generate a view; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get())); + } + } + } + + private ServerEvaluationCall getServerEvaluationCall(DocumentFile qbvFile) { + String fileContent; + try { + fileContent = new String(FileCopyUtils.copyToByteArray(qbvFile.getFile())); + } catch (IOException e) { + throw new RuntimeException(format("Unable to generate Query-Based View; could not read from file %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); + } + String extension = qbvFile.getFileExtension(); + if (extension != null) { + extension = extension.toLowerCase(); + } + if (("xqy".equals(extension)) || ("xq".equals(extension))) { + return buildXqueryCall(fileContent); + } else { + return buildJavascriptCall(fileContent); + } + } + + private ServerEvaluationCall buildJavascriptCall(String fileContent) { + String script = format(JAVASCRIPT_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName); + return databaseClient.newServerEval().javascript(script); + } + + private ServerEvaluationCall buildXqueryCall(String fileContent) { + String script = format(XQUERY_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName); + return databaseClient.newServerEval().xquery(script); + } +} diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java new file mode 100644 index 0000000..be8d7c4 --- /dev/null +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java @@ -0,0 +1,94 @@ +package com.marklogic.client.ext.schemasloader.impl; + +import com.marklogic.client.ext.file.DocumentFile; +import com.marklogic.client.ext.helper.ClientHelper; +import com.marklogic.client.io.DocumentMetadataHandle; +import org.junit.jupiter.api.Test; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.*; + +public class GenerateQbvTest extends AbstractSchemasTest { + + @Test + public void test() { + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); + Path path = Paths.get("src", "test", "resources", "qbv-schemas"); + List files = loader.loadSchemas(path.toString()); + assertEquals(3, files.size(), + "Only the TDE templates should be in this list"); + + ClientHelper helper = new ClientHelper(client); + List tdeUris = helper.getUrisInCollection(TdeUtil.TDE_COLLECTION); + assertEquals(2, tdeUris.size()); + List qbvUris = helper.getUrisInCollection(QbvDocumentFileProcessor.QBV_COLLECTION); + assertEquals(2, qbvUris.size(), "Both QBVs, and only the QBVs, should be in the QBV collection"); + assertTrue(qbvUris.contains("/qbv/authors.sjs.xml")); + assertTrue(qbvUris.contains("/qbv/publications.xqy.xml")); + + // TODO - Verify that a query works for the QBV view + // TODO - Waiting for a real test-app for ml-javaclient-util +// RowManager rowManager = client.newRowManager(); +// PlanBuilder.ModifyPlan plan = rowManager.newPlanBuilder().fromView("alternate", "authors"); +// rowManager.resultDoc(plan, new JacksonHandle()).get(); + + verifyMetadata(qbvUris.get(0), metadata -> { + DocumentMetadataHandle.DocumentPermissions perms = metadata.getPermissions(); + assertPermissionExists(perms, "rest-reader", DocumentMetadataHandle.Capability.READ); + assertPermissionExists(perms, "rest-writer", DocumentMetadataHandle.Capability.UPDATE); + assertEquals(2, perms.size()); + DocumentMetadataHandle.DocumentCollections colls = metadata.getCollections(); + assertTrue(colls.contains("http://marklogic.com/xdmp/qbv")); + assertTrue(colls.contains("col1")); + assertTrue(colls.contains("col3")); + assertEquals(3, colls.size()); + }); + + } + + @Test + public void loadBadOptic() { + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); + Path path = Paths.get("src", "test", "resources", "qbv-bad-schemas"); + RuntimeException ex = assertThrows(RuntimeException.class, () -> loader.loadSchemas(path.toString())); + assertTrue(ex.getMessage().contains("Query-Based View generation failed for file:"), "Unexpected message: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("/qbv/bad-authors.sjs"), "Unexpected message: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("ensure your Optic script includes a call to generate a view;"), "Unexpected message: " + ex.getMessage()); + } + + @Test + public void schemaViewDoesNotExist() { + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); + Path path = Paths.get("src", "test", "resources", "qbv-no-tde-schemas"); + RuntimeException ex = assertThrows(RuntimeException.class, () -> loader.loadSchemas(path.toString())); + assertTrue(ex.getMessage().contains("Query-Based View generation failed for file:"), "Unexpected message: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("/qbv/books.sjs"), "Unexpected message: " + ex.getMessage()); + assertTrue(ex.getMessage().contains("Server Message: SQL-TABLENOTFOUND: plan.generateView(plan.sparql(\"\"), \"alternate\", \"books\") -- Unknown table: Table 'Medical.Books' not found"), "Unexpected message: " + ex.getMessage()); + } + + @Test + public void emptyDirectories() { + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); + Path path = Paths.get("src", "test", "resources", "qbv-empty-schemas"); + List files = loader.loadSchemas(path.toString()); + assertEquals(0, files.size()); + ClientHelper helper = new ClientHelper(client); + List qbvUris = helper.getUrisInCollection(QbvDocumentFileProcessor.QBV_COLLECTION); + assertEquals(0, qbvUris.size()); + } + + private void verifyMetadata(String uri, Consumer verifier) { + verifier.accept(client.newJSONDocumentManager().readMetadata(uri, new DocumentMetadataHandle())); + } + + private void assertPermissionExists(DocumentMetadataHandle.DocumentPermissions perms, String role, + DocumentMetadataHandle.Capability capability) { + assertTrue(perms.containsKey(role), "No permissions for role: " + role); + assertTrue(perms.get(role).contains(capability), + "Capability " + capability + " for role " + role + " not found"); + } +} diff --git a/src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs b/src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs new file mode 100644 index 0000000..34b98dd --- /dev/null +++ b/src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs @@ -0,0 +1 @@ +op.fromView('Medical', 'Authors'); diff --git a/src/test/resources/qbv-bad-schemas/tde/authors.json b/src/test/resources/qbv-bad-schemas/tde/authors.json new file mode 100644 index 0000000..7819d99 --- /dev/null +++ b/src/test/resources/qbv-bad-schemas/tde/authors.json @@ -0,0 +1,38 @@ +{ + "template": { + "context": "/Citations/Citation/Article/AuthorList/Author", + "rows": [ + { + "schemaName": "Medical", + "viewName": "Authors", + "columns": [ + { + "name": "ID", + "scalarType": "long", + "val": "../../../ID" + }, + { + "name": "LastName", + "scalarType": "string", + "val": "LastName" + }, + { + "name": "ForeName", + "scalarType": "string", + "val": "ForeName" + }, + { + "name": "Date", + "scalarType": "date", + "val": "../../Journal/JournalIssue/PubDate/Year || '-' || ../../Journal/JournalIssue/PubDate/Month || '-' || ../../Journal/JournalIssue/PubDate/Day" + }, + { + "name": "DateTime", + "scalarType": "dateTime", + "val": "../../Journal/JournalIssue/PubDate/Year || '-' || ../../Journal/JournalIssue/PubDate/Month || '-' || ../../Journal/JournalIssue/PubDate/Day || 'T' || ../../Journal/JournalIssue/PubDate/Time" + } + ] + } + ] + } +} diff --git a/src/test/resources/qbv-no-tde-schemas/qbv/books.sjs b/src/test/resources/qbv-no-tde-schemas/qbv/books.sjs new file mode 100644 index 0000000..f6cadea --- /dev/null +++ b/src/test/resources/qbv-no-tde-schemas/qbv/books.sjs @@ -0,0 +1 @@ +op.fromView('Medical', 'Books').generateView('alternate', 'books'); diff --git a/src/test/resources/qbv-schemas/example.sjs b/src/test/resources/qbv-schemas/example.sjs new file mode 100644 index 0000000..e69de29 diff --git a/src/test/resources/qbv-schemas/qbv/authors.sjs b/src/test/resources/qbv-schemas/qbv/authors.sjs new file mode 100644 index 0000000..8c32416 --- /dev/null +++ b/src/test/resources/qbv-schemas/qbv/authors.sjs @@ -0,0 +1 @@ +op.fromView('Medical', 'Authors').generateView('alternate', 'authors'); \ No newline at end of file diff --git a/src/test/resources/qbv-schemas/qbv/collections.properties b/src/test/resources/qbv-schemas/qbv/collections.properties new file mode 100644 index 0000000..6c2a7d9 --- /dev/null +++ b/src/test/resources/qbv-schemas/qbv/collections.properties @@ -0,0 +1 @@ +*=col1,col3 diff --git a/src/test/resources/qbv-schemas/qbv/permissions.properties b/src/test/resources/qbv-schemas/qbv/permissions.properties new file mode 100644 index 0000000..c977854 --- /dev/null +++ b/src/test/resources/qbv-schemas/qbv/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read,rest-writer,update diff --git a/src/test/resources/qbv-schemas/qbv/publications.xqy b/src/test/resources/qbv-schemas/qbv/publications.xqy new file mode 100644 index 0000000..037803f --- /dev/null +++ b/src/test/resources/qbv-schemas/qbv/publications.xqy @@ -0,0 +1,2 @@ +op:from-view("Medical", "Publications") + => op:generate-view("alternate", "publications") \ No newline at end of file diff --git a/src/test/resources/qbv-schemas/tde/authors.json b/src/test/resources/qbv-schemas/tde/authors.json new file mode 100644 index 0000000..7819d99 --- /dev/null +++ b/src/test/resources/qbv-schemas/tde/authors.json @@ -0,0 +1,38 @@ +{ + "template": { + "context": "/Citations/Citation/Article/AuthorList/Author", + "rows": [ + { + "schemaName": "Medical", + "viewName": "Authors", + "columns": [ + { + "name": "ID", + "scalarType": "long", + "val": "../../../ID" + }, + { + "name": "LastName", + "scalarType": "string", + "val": "LastName" + }, + { + "name": "ForeName", + "scalarType": "string", + "val": "ForeName" + }, + { + "name": "Date", + "scalarType": "date", + "val": "../../Journal/JournalIssue/PubDate/Year || '-' || ../../Journal/JournalIssue/PubDate/Month || '-' || ../../Journal/JournalIssue/PubDate/Day" + }, + { + "name": "DateTime", + "scalarType": "dateTime", + "val": "../../Journal/JournalIssue/PubDate/Year || '-' || ../../Journal/JournalIssue/PubDate/Month || '-' || ../../Journal/JournalIssue/PubDate/Day || 'T' || ../../Journal/JournalIssue/PubDate/Time" + } + ] + } + ] + } +} diff --git a/src/test/resources/qbv-schemas/tde/collections.properties b/src/test/resources/qbv-schemas/tde/collections.properties new file mode 100644 index 0000000..2c7505d --- /dev/null +++ b/src/test/resources/qbv-schemas/tde/collections.properties @@ -0,0 +1 @@ +*=col1,col2 diff --git a/src/test/resources/qbv-schemas/tde/publications.xml b/src/test/resources/qbv-schemas/tde/publications.xml new file mode 100644 index 0000000..08f8227 --- /dev/null +++ b/src/test/resources/qbv-schemas/tde/publications.xml @@ -0,0 +1,21 @@ + \ No newline at end of file From 5b75adfd49e362b8b7c702eee908ba8461c8ff19 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Mon, 21 Aug 2023 10:21:22 -0400 Subject: [PATCH 08/20] Bumping dependencies These will go up again when Java Client 6.3.0 is released. --- build.gradle | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index b73079a..d0b5780 100644 --- a/build.gradle +++ b/build.gradle @@ -26,20 +26,20 @@ repositories { } dependencies { - api 'com.marklogic:marklogic-client-api:6.2.0' - api 'com.marklogic:marklogic-xcc:11.0.2' - api 'org.springframework:spring-context:5.3.27' + api 'com.marklogic:marklogic-client-api:6.2.2' + api 'com.marklogic:marklogic-xcc:11.0.3' + api 'org.springframework:spring-context:5.3.29' implementation 'org.jdom:jdom2:2.0.6.1' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.1' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.3' // This is currently an implementation dependency, though perhaps should be an api dependency due to its usage in // LoggingObject; not clear yet on whether the protected Logger in that class should make this an api dependency or not implementation 'org.slf4j:slf4j-api:1.7.36' - testImplementation 'org.junit.jupiter:junit-jupiter:5.9.3' - testImplementation 'org.springframework:spring-test:5.3.27' + testImplementation 'org.junit.jupiter:junit-jupiter:5.10.0' + testImplementation 'org.springframework:spring-test:5.3.29' testImplementation 'org.mockito:mockito-core:4.11.0' // Used for testing loading modules from the classpath @@ -57,7 +57,7 @@ dependencies { // Including the "new" JAXB libraries to verify that their presence doesn't cause the "old" JAXB libraries to fail testImplementation "jakarta.xml.bind:jakarta.xml.bind-api:4.0.0" - testImplementation "com.sun.xml.bind:jaxb-impl:4.0.1" + testImplementation "com.sun.xml.bind:jaxb-impl:4.0.3" } test { From b66fba6fe0ed7ca886964911ca553bfca8f10f7e Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Tue, 22 Aug 2023 15:52:51 -0400 Subject: [PATCH 09/20] Bumped Java Client dependency to 6.3-SNAPSHOT --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index d0b5780..6d679b4 100644 --- a/build.gradle +++ b/build.gradle @@ -22,11 +22,12 @@ java { } repositories { + mavenLocal() mavenCentral() } dependencies { - api 'com.marklogic:marklogic-client-api:6.2.2' + api 'com.marklogic:marklogic-client-api:6.3-SNAPSHOT' api 'com.marklogic:marklogic-xcc:11.0.3' api 'org.springframework:spring-context:5.3.29' From 500452789be9c35bcbd272b465bb7a1863ec7d5a Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 24 Aug 2023 14:13:28 -0400 Subject: [PATCH 10/20] Added ML Maven repo --- build.gradle | 3 +++ 1 file changed, 3 insertions(+) diff --git a/build.gradle b/build.gradle index 6d679b4..b8b0829 100644 --- a/build.gradle +++ b/build.gradle @@ -23,6 +23,9 @@ java { repositories { mavenLocal() + maven { + url "https://nexus.marklogic.com/repository/maven-snapshots/" + } mavenCentral() } From 25b7b03783932b31b9c41e6c6b2ac2d2c9ba9317 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 25 Aug 2023 08:37:12 -0400 Subject: [PATCH 11/20] Refactoring of TDE/QBV processors Changes: - Renamed client to `schemasDatabaseClient` to make it clear the kind of client that's needed. - Using `FilenameUtil` - not the most glamorous class, but nice to have all the checks in one place. - Deprecated some getters/setters that I shouldn't have added. - Added a schemas database to the test app and changed schemas tests to use that. --- gradle.properties | 1 + .../client/ext/helper/FilenameUtil.java | 19 ++++- .../impl/QbvDocumentFileProcessor.java | 58 ++++++++------- .../impl/TdeDocumentFileProcessor.java | 71 ++++++++++++------- .../impl/AbstractSchemasTest.java | 2 +- .../schemasloader/impl/GenerateQbvTest.java | 12 ++-- .../schemasloader/impl/LoadSchemasTest.java | 7 +- .../ml-config/databases/content-database.json | 4 ++ .../ml-config/databases/schemas-database.json | 3 + 9 files changed, 114 insertions(+), 63 deletions(-) create mode 100644 src/test/ml-config/databases/content-database.json create mode 100644 src/test/ml-config/databases/schemas-database.json diff --git a/gradle.properties b/gradle.properties index 52bb5a4..3f5101e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -16,3 +16,4 @@ mlAppName=ml-javaclient-util-test mlRestPort=8028 mlUsername=admin mlPassword=change in gradle-local.properties +mlConfigPaths=src/test/ml-config diff --git a/src/main/java/com/marklogic/client/ext/helper/FilenameUtil.java b/src/main/java/com/marklogic/client/ext/helper/FilenameUtil.java index 574a393..2a7e2bb 100644 --- a/src/main/java/com/marklogic/client/ext/helper/FilenameUtil.java +++ b/src/main/java/com/marklogic/client/ext/helper/FilenameUtil.java @@ -20,17 +20,30 @@ public abstract class FilenameUtil { public static boolean isXslFile(String filename) { - return filename.endsWith(".xsl") || filename.endsWith(".xslt"); + return endsWithExtension(filename, ".xsl", ".xslt"); } public static boolean isXqueryFile(String filename) { - return filename.endsWith(".xqy") || filename.endsWith(".xq"); + return endsWithExtension(filename, ".xqy", ".xqy"); } public static boolean isJavascriptFile(String filename) { - return filename.endsWith(".sjs") || filename.endsWith(".js"); + return endsWithExtension(filename, ".sjs", ".js"); } + public static boolean endsWithExtension(String filename, String... extensions) { + if (filename == null || extensions == null) { + return false; + } + filename = filename.toLowerCase(); + for (String extension : extensions) { + if (extension != null && filename.endsWith(extension.toLowerCase())) { + return true; + } + } + return false; + } + public static String getFileExtension(File f) { String[] split = f.getName().split("\\."); if (split.length > 1) { diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java index f8e6700..6ee1706 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java @@ -20,6 +20,7 @@ import com.marklogic.client.eval.ServerEvaluationCall; import com.marklogic.client.ext.file.DocumentFile; import com.marklogic.client.ext.file.DocumentFileProcessor; +import com.marklogic.client.ext.helper.FilenameUtil; import com.marklogic.client.ext.helper.LoggingObject; import com.marklogic.client.extra.jdom.JDOMHandle; import com.marklogic.client.io.Format; @@ -29,32 +30,41 @@ import org.jdom2.input.SAXBuilder; import org.springframework.util.FileCopyUtils; +import java.io.File; import java.io.IOException; import java.io.StringReader; import java.util.ArrayList; import java.util.List; +/** + * @since 4.6.0 + */ public class QbvDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor { public static final String QBV_COLLECTION = "http://marklogic.com/xdmp/qbv"; private static final String QBV_XML_PLAN_NAMESPACE = "http://marklogic.com/plan"; private static final String QBV_XML_ROOT_ELEMENT = "query-based-view"; - private static final String JAVASCRIPT_EVAL_TEMPLATE = "declareUpdate(); xdmp.invokeFunction(function() {'use strict'; const op = require('/MarkLogic/optic'); return %s }, {database: xdmp.database('%s')})"; - private static final String XQUERY_EVAL_TEMPLATE = "xquery version \"1.0-ml\"; import module namespace op=\"http://marklogic.com/optic\" at \"/MarkLogic/optic.xqy\"; xdmp:invoke-function(function() {%s},{xdmp:database('%s')})"; + private static final String JAVASCRIPT_EVAL_TEMPLATE = "declareUpdate(); " + + "xdmp.invokeFunction(function() {'use strict'; const op = require('/MarkLogic/optic'); return %s }, " + + "{database: xdmp.database('%s')})"; + private static final String XQUERY_EVAL_TEMPLATE = "xquery version \"1.0-ml\"; " + + "import module namespace op=\"http://marklogic.com/optic\" at \"/MarkLogic/optic.xqy\"; " + + "xdmp:invoke-function(function() {%s},{xdmp:database('%s')})"; - final private DatabaseClient databaseClient; + final private DatabaseClient schemasDatabaseClient; final private String qbvGeneratorDatabaseName; - final protected List qbvFiles = new ArrayList<>(); + final private List qbvFiles = new ArrayList<>(); final private XMLDocumentManager docMgr; /** - * @param databaseClient - a MarkLogic DatabaseClient object for the Schemas database + * @param schemasDatabaseClient database client for the application's schemas database + * @param qbvGeneratorDatabaseName the database to run a script against for generating a QBV */ - public QbvDocumentFileProcessor(DatabaseClient databaseClient, String qbvGeneratorDatabaseName) { - this.databaseClient = databaseClient; + public QbvDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String qbvGeneratorDatabaseName) { + this.schemasDatabaseClient = schemasDatabaseClient; this.qbvGeneratorDatabaseName = qbvGeneratorDatabaseName; - this.docMgr = databaseClient.newXMLDocumentManager(); + this.docMgr = schemasDatabaseClient.newXMLDocumentManager(); } @Override @@ -70,17 +80,11 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) { private boolean isQbvQuery(DocumentFile documentFile) { String uri = documentFile.getUri(); - if (uri != null && uri.startsWith("/qbv")) { - String extension = documentFile.getFileExtension(); - if (extension != null) { - extension = extension.toLowerCase(); - } - return - "sjs".equals(extension) - || "js".equals(extension) - || "xqy".equals(extension) - || "xq".equals(extension); - } else return false; + File file = documentFile.getFile(); + return uri != null + && uri.startsWith("/qbv") + && file != null + && (FilenameUtil.isXqueryFile(file.getName()) || FilenameUtil.isJavascriptFile(file.getName())); } public void processQbvFiles() { @@ -125,24 +129,18 @@ private ServerEvaluationCall getServerEvaluationCall(DocumentFile qbvFile) { } catch (IOException e) { throw new RuntimeException(format("Unable to generate Query-Based View; could not read from file %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); } - String extension = qbvFile.getFileExtension(); - if (extension != null) { - extension = extension.toLowerCase(); - } - if (("xqy".equals(extension)) || ("xq".equals(extension))) { - return buildXqueryCall(fileContent); - } else { - return buildJavascriptCall(fileContent); - } + return FilenameUtil.isXqueryFile(qbvFile.getFile().getName()) ? + buildXqueryCall(fileContent) : + buildJavascriptCall(fileContent); } private ServerEvaluationCall buildJavascriptCall(String fileContent) { String script = format(JAVASCRIPT_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName); - return databaseClient.newServerEval().javascript(script); + return schemasDatabaseClient.newServerEval().javascript(script); } private ServerEvaluationCall buildXqueryCall(String fileContent) { String script = format(XQUERY_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName); - return databaseClient.newServerEval().xquery(script); + return schemasDatabaseClient.newServerEval().xquery(script); } } diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java index fb92a3e..33095f2 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java @@ -20,7 +20,7 @@ import com.marklogic.client.eval.ServerEvaluationCall; import com.marklogic.client.ext.file.DocumentFile; import com.marklogic.client.ext.file.DocumentFileProcessor; -import com.marklogic.client.ext.helper.ClientHelper; +import com.marklogic.client.ext.helper.FilenameUtil; import com.marklogic.client.ext.helper.LoggingObject; import com.marklogic.client.io.Format; import com.marklogic.client.io.JacksonHandle; @@ -32,7 +32,7 @@ public class TdeDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor { - private DatabaseClient databaseClient; + private DatabaseClient schemasDatabaseClient; private String tdeValidationDatabase; private Boolean templateBatchInsertSupported; @@ -45,31 +45,34 @@ public TdeDocumentFileProcessor() { /** * Use this constructor when you want to validate a TDE template before writing it to MarkLogic. * - * @param databaseClient + * @param schemasDatabaseClient database client for the application's schemas database + * @param tdeValidationDatabase the database to run a script against for validating a TDE */ - public TdeDocumentFileProcessor(DatabaseClient databaseClient, String tdeValidationDatabase) { - this.databaseClient = databaseClient; + public TdeDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String tdeValidationDatabase) { + this.schemasDatabaseClient = schemasDatabaseClient; this.tdeValidationDatabase = tdeValidationDatabase; } @Override public DocumentFile processDocumentFile(DocumentFile documentFile) { String uri = documentFile.getUri(); - String extension = documentFile.getFileExtension(); - if (extension != null) { - extension = extension.toLowerCase(); - } - - boolean isTdeTemplate = ("tdej".equals(extension) || "tdex".equals(extension)) || (uri != null && uri.startsWith("/tde")); - if (isTdeTemplate) { + String filename = documentFile.getFile() != null ? documentFile.getFile().getName() : null; + boolean isTdeUri = (uri != null && uri.startsWith("/tde")); + boolean isJsonTde = (isTdeUri && FilenameUtil.endsWithExtension(filename, ".json")) + || FilenameUtil.endsWithExtension(filename, ".tdej"); + boolean isXmlTde = (isTdeUri && FilenameUtil.endsWithExtension(filename, ".xml")) + || FilenameUtil.endsWithExtension(filename, ".tdex"); + + // We have a test suggesting that a TDE may not be JSON or XML; that doesn't seem likely, but it also does not + // appear to cause any issues. + if (isTdeUri || isJsonTde || isXmlTde) { documentFile.getDocumentMetadata().withCollections(TdeUtil.TDE_COLLECTION); validateTdeTemplate(documentFile); - } - - if ("tdej".equals(extension) || "json".equals(extension)) { - documentFile.setFormat(Format.JSON); - } else if ("tdex".equals(extension) || "xml".equals(extension)) { - documentFile.setFormat(Format.XML); + if (isJsonTde) { + documentFile.setFormat(Format.JSON); + } else if (isXmlTde) { + documentFile.setFormat(Format.XML); + } } return documentFile; @@ -79,14 +82,14 @@ private boolean isTemplateBatchInsertSupported() { if (this.templateBatchInsertSupported == null) { // Memoize this to avoid repeated calls; the result will always be the same unless the databaseClient is // modified, in which case templateBatchInsertSupported is set to null - this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(databaseClient); + this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(schemasDatabaseClient); } return this.templateBatchInsertSupported; } protected void validateTdeTemplate(DocumentFile documentFile) { final File file = documentFile.getFile(); - if (databaseClient == null) { + if (schemasDatabaseClient == null) { logger.info("No DatabaseClient provided for TDE validation, so will not validate TDE templates"); } else if (tdeValidationDatabase == null) { logger.info("No TDE validation database specified, so will not validate TDE templates"); @@ -127,7 +130,7 @@ protected ServerEvaluationCall buildJavascriptCall(DocumentFile documentFile, St "\nreturn tde.validate([xdmp.toJSON(template)], ['%s'])}, {database: xdmp.database('%s')})", documentFile.getUri(), tdeValidationDatabase )); - return databaseClient.newServerEval().javascript(script.toString()) + return schemasDatabaseClient.newServerEval().javascript(script.toString()) .addVariable("template", new StringHandle(fileContent).withFormat(Format.JSON)); } @@ -139,22 +142,42 @@ protected ServerEvaluationCall buildXqueryCall(DocumentFile documentFile, String "\ntde:validate($template, '%s')}, {xdmp:database('%s')})", documentFile.getUri(), tdeValidationDatabase )); - return databaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML)); + return schemasDatabaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML)); } + /** + * @return + * @deprecated since 4.6.0, will be removed in 5.0.0 + */ + @Deprecated public DatabaseClient getDatabaseClient() { - return databaseClient; + return schemasDatabaseClient; } + /** + * @param databaseClient + * @deprecated since 4.6.0, will be removed in 5.0.0 + */ + @Deprecated public void setDatabaseClient(DatabaseClient databaseClient) { - this.databaseClient = databaseClient; + this.schemasDatabaseClient = databaseClient; this.templateBatchInsertSupported = null; } + /** + * @return + * @deprecated since 4.6.0, will be removed in 5.0.0 + */ + @Deprecated public String getTdeValidationDatabase() { return tdeValidationDatabase; } + /** + * @param tdeValidationDatabase + * @deprecated since 4.6.0, will be removed in 5.0.0 + */ + @Deprecated public void setTdeValidationDatabase(String tdeValidationDatabase) { this.tdeValidationDatabase = tdeValidationDatabase; } diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/AbstractSchemasTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/AbstractSchemasTest.java index 1280636..86c5e05 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/AbstractSchemasTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/AbstractSchemasTest.java @@ -26,7 +26,7 @@ public abstract class AbstractSchemasTest extends AbstractIntegrationTest { */ @BeforeEach public void setup() { - client = newClient("Schemas"); + client = newClient("ml-javaclient-util-test-schemas"); client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval(); } diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java index be8d7c4..2e50f60 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java @@ -3,6 +3,7 @@ import com.marklogic.client.ext.file.DocumentFile; import com.marklogic.client.ext.helper.ClientHelper; import com.marklogic.client.io.DocumentMetadataHandle; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.nio.file.Path; @@ -14,9 +15,15 @@ public class GenerateQbvTest extends AbstractSchemasTest { + private DefaultSchemasLoader loader; + + @BeforeEach + void beforeEach() { + loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); + } + @Test public void test() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); Path path = Paths.get("src", "test", "resources", "qbv-schemas"); List files = loader.loadSchemas(path.toString()); assertEquals(3, files.size(), @@ -52,7 +59,6 @@ public void test() { @Test public void loadBadOptic() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); Path path = Paths.get("src", "test", "resources", "qbv-bad-schemas"); RuntimeException ex = assertThrows(RuntimeException.class, () -> loader.loadSchemas(path.toString())); assertTrue(ex.getMessage().contains("Query-Based View generation failed for file:"), "Unexpected message: " + ex.getMessage()); @@ -62,7 +68,6 @@ public void loadBadOptic() { @Test public void schemaViewDoesNotExist() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); Path path = Paths.get("src", "test", "resources", "qbv-no-tde-schemas"); RuntimeException ex = assertThrows(RuntimeException.class, () -> loader.loadSchemas(path.toString())); assertTrue(ex.getMessage().contains("Query-Based View generation failed for file:"), "Unexpected message: " + ex.getMessage()); @@ -72,7 +77,6 @@ public void schemaViewDoesNotExist() { @Test public void emptyDirectories() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, "Documents"); Path path = Paths.get("src", "test", "resources", "qbv-empty-schemas"); List files = loader.loadSchemas(path.toString()); assertEquals(0, files.size()); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java index 3408bc5..5c3bce1 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java @@ -25,7 +25,9 @@ import java.nio.file.Paths; import java.util.List; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class LoadSchemasTest extends AbstractSchemasTest { @@ -45,6 +47,9 @@ public void test() { assertTrue(uris.contains("/child/child.tdej")); assertTrue(uris.contains("/child/grandchild/grandchild.tdex")); assertTrue(uris.contains("/parent.tdex")); + + // This assertion seems a little off - a TDE should be either a JSON or XML file. This doesn't seem to cause + // any problems, but it also doesn't seem to make sense. assertTrue(uris.contains("/tde/ruleset.txt")); } diff --git a/src/test/ml-config/databases/content-database.json b/src/test/ml-config/databases/content-database.json new file mode 100644 index 0000000..b82b620 --- /dev/null +++ b/src/test/ml-config/databases/content-database.json @@ -0,0 +1,4 @@ +{ + "database-name": "%%DATABASE%%", + "schema-database": "%%SCHEMAS_DATABASE%%" +} diff --git a/src/test/ml-config/databases/schemas-database.json b/src/test/ml-config/databases/schemas-database.json new file mode 100644 index 0000000..f0f14c7 --- /dev/null +++ b/src/test/ml-config/databases/schemas-database.json @@ -0,0 +1,3 @@ +{ + "database-name": "%%SCHEMAS_DATABASE%%" +} \ No newline at end of file From 87b771d20a8b427c5959757120f1be8f46275511 Mon Sep 17 00:00:00 2001 From: Phil Barber Date: Fri, 25 Aug 2023 09:53:34 -0400 Subject: [PATCH 12/20] DEVEXP-527: Cascading collections and permissions. DEVEXP-527: Cascading collections and permissions. DEVEXP-527: Cascading collections and permissions. --- ...PropertiesDrivenDocumentFileProcessor.java | 54 +++++++++++++ .../CollectionsFileDocumentFileProcessor.java | 2 +- .../ext/file/DefaultDocumentFileReader.java | 11 ++- .../PermissionsFileDocumentFileProcessor.java | 2 +- ...PropertiesDrivenDocumentFileProcessor.java | 46 ++++------- .../client/ext/AbstractIntegrationTest.java | 37 +++++++++ .../CascadeCollectionsAndPermissionsTest.java | 78 +++++++++++++++++++ ...lectionsFileDocumentFileProcessorTest.java | 11 ++- ...missionsFileDocumentFileProcessorTest.java | 11 ++- .../schemasloader/impl/GenerateQbvTest.java | 32 ++------ .../parent1-withCP/child1/child1.json | 1 + .../parent1-withCP/child1_1-noCP/test.json | 3 + .../child1_2-withCP/collections.properties | 1 + .../child1_2-withCP/permissions.properties | 1 + .../parent1-withCP/child1_2-withCP/test.json | 3 + .../parent1-withCP/child2/child2.json | 1 + .../child2/collections.properties | 1 + .../child2/permissions.properties | 1 + .../child3_1-withCP/collections.properties | 1 + .../grandchild3_1_1-noCP/test.json | 3 + .../child3_1-withCP/permissions.properties | 1 + .../parent1-withCP/child3_1-withCP/test.json | 3 + .../parent1-withCP/collections.properties | 1 + .../parent1-withCP/parent.json | 1 + .../parent1-withCP/permissions.properties | 1 + .../parent1-withCP/test.json | 3 + .../child2_1-withCP/collections.properties | 1 + .../child2_1-withCP/permissions.properties | 1 + .../parent2-noCP/child2_1-withCP/test.json | 3 + .../parent2-noCP/child2_2-noCP/test.json | 3 + .../child2_3-withCnoP/collections.properties | 1 + .../permissions.properties | 1 + .../grandchild2_3_1-withPnoC/test.json | 3 + .../parent2-noCP/child2_3-withCnoP/test.json | 3 + .../parent2-noCP/test.json | 3 + 35 files changed, 263 insertions(+), 66 deletions(-) create mode 100644 src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java create mode 100644 src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1/child1.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_1-noCP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/permissions.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/child2.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/collections.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/permissions.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/grandchild3_1_1-noCP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/collections.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/parent.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/permissions.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/collections.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/permissions.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_2-noCP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/collections.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/permissions.properties create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/test.json create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent2-noCP/test.json diff --git a/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java new file mode 100644 index 0000000..a057dc9 --- /dev/null +++ b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java @@ -0,0 +1,54 @@ +/* + * Copyright (c) 2023 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.client.ext.file; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Path; +import java.util.Properties; +import java.util.Stack; + +/** + * Adds a stack to store Properties objects while traversing a directory tree. + */ +abstract class CascadingPropertiesDrivenDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor { + final private Stack propertiesStack = new Stack<>(); + + protected CascadingPropertiesDrivenDocumentFileProcessor(String propertiesFilename) { + super(propertiesFilename); + } + + protected void preVisitDirectory(Path dir) throws IOException { + File collectionsPropertiesFile = new File(dir.toFile(), this.getPropertiesFilename()); + if (collectionsPropertiesFile.exists()) { + this.loadProperties(collectionsPropertiesFile); + } else { + if (!propertiesStack.isEmpty()) { + this.setProperties(propertiesStack.peek()); + } else { + this.setProperties(new Properties()); + } + } + propertiesStack.push(this.getProperties()); + } + + protected void postVisitDirectory() { + propertiesStack.pop(); + if (!propertiesStack.isEmpty()) { + this.setProperties(propertiesStack.peek()); + } + } +} diff --git a/src/main/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessor.java index 2ecde19..28d09e2 100644 --- a/src/main/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessor.java @@ -22,7 +22,7 @@ * key is the name of a file in the directory, and the value is a comma-delimited list of collections to load the file * into (which means you can't use a comma in any collection name). */ -public class CollectionsFileDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor { +public class CollectionsFileDocumentFileProcessor extends CascadingPropertiesDrivenDocumentFileProcessor { private String delimiter = ","; diff --git a/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java b/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java index 74db1ed..ae0d4df 100644 --- a/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java +++ b/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java @@ -22,8 +22,7 @@ import java.net.URISyntaxException; import java.nio.file.*; import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.List; +import java.util.*; /** * Non-threadsafe implementation that implements FileVisitor as a way of descending one or more file paths. @@ -105,6 +104,10 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th if (logger.isDebugEnabled()) { logger.debug("Visiting directory: " + dir); } + + collectionsFileDocumentFileProcessor.preVisitDirectory(dir); + permissionsFileDocumentFileProcessor.preVisitDirectory(dir); + return FileVisitResult.CONTINUE; } else { if (logger.isDebugEnabled()) { @@ -127,6 +130,10 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx if (exc != null) { logger.warn("Error in postVisitDirectory: " + exc.getMessage(), exc); } + + collectionsFileDocumentFileProcessor.postVisitDirectory(); + permissionsFileDocumentFileProcessor.postVisitDirectory(); + return FileVisitResult.CONTINUE; } diff --git a/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java index e11adae..6889584 100644 --- a/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java @@ -24,7 +24,7 @@ * Looks for a special file in each directory - defaults to permissions.properties - that contains properties where the * key is the name of a file in the directory, and the value is a comma-delimited list of role,capability,role,capability,etc. */ -public class PermissionsFileDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor { +public class PermissionsFileDocumentFileProcessor extends CascadingPropertiesDrivenDocumentFileProcessor { private DocumentPermissionsParser documentPermissionsParser; diff --git a/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java index 5041035..296fec7 100644 --- a/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java @@ -22,8 +22,6 @@ import java.io.FileFilter; import java.io.FileReader; import java.io.IOException; -import java.util.HashMap; -import java.util.Map; import java.util.Properties; /** @@ -34,10 +32,9 @@ public abstract class PropertiesDrivenDocumentFileProcessor extends LoggingObjec protected final static String WILDCARD_KEY = "*"; - private String propertiesFilename; + private final String propertiesFilename; - // Used to avoid checking for and loading the properties for every file in a directory - private Map propertiesCache = new HashMap<>(); + private Properties properties; private TokenReplacer tokenReplacer; @@ -60,36 +57,17 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) { if (!accept(file)) { return null; } - - File propertiesFile = new File(file.getParentFile(), propertiesFilename); - if (propertiesFile.exists()) { - try { - Properties props = loadProperties(propertiesFile); - processProperties(documentFile, props); - } catch (IOException e) { - logger.warn("Unable to load properties from file: " + propertiesFile.getAbsolutePath(), e); - } - } - + processProperties(documentFile, properties); return documentFile; } protected abstract void processProperties(DocumentFile documentFile, Properties properties); protected Properties loadProperties(File propertiesFile) throws IOException { - Properties props = null; - if (propertiesCache.containsKey(propertiesFile)) { - props = propertiesCache.get(propertiesFile); - } - if (props != null) { - return props; - } - - props = new Properties(); + properties = new Properties(); try (FileReader reader = new FileReader(propertiesFile)) { - props.load(reader); - propertiesCache.put(propertiesFile, props); - return props; + properties.load(reader); + return properties; } } @@ -101,10 +79,6 @@ protected String getPropertyValue(Properties properties, String propertyName) { return tokenReplacer != null && value != null ? tokenReplacer.replaceTokens(value) : value; } - public Map getPropertiesCache() { - return propertiesCache; - } - public String getPropertiesFilename() { return propertiesFilename; } @@ -116,4 +90,12 @@ public void setTokenReplacer(TokenReplacer tokenReplacer) { protected TokenReplacer getTokenReplacer() { return tokenReplacer; } + + protected void setProperties(Properties properties) { + this.properties = properties; + } + + protected Properties getProperties() { + return this.properties; + } } diff --git a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java index 34368b2..bc6e755 100644 --- a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java +++ b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java @@ -16,6 +16,7 @@ package com.marklogic.client.ext; import com.marklogic.client.DatabaseClient; +import com.marklogic.client.io.DocumentMetadataHandle; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -27,6 +28,12 @@ import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit.jupiter.SpringExtension; +import java.util.Set; +import java.util.function.Consumer; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + @ExtendWith(SpringExtension.class) @ContextConfiguration(classes = {TestConfig.class}) public abstract class AbstractIntegrationTest { @@ -58,6 +65,36 @@ protected DatabaseClient newClient(String database) { clientConfig.setDatabase(currentDatabase); return client; } + + protected final void verifyMetadata(String uri, Consumer verifier) { + verifier.accept(client.newJSONDocumentManager().readMetadata(uri, new DocumentMetadataHandle())); + } + + protected final void verifyCollections(String uri, String... collections) { + verifyMetadata(uri, metadata -> { + assertEquals(collections.length, metadata.getCollections().size()); + for (String collection : collections) { + assertTrue(metadata.getCollections().contains(collection), "Did not find expected collection: " + + collection + "; actual collections: " + metadata.getCollections()); + } + }); + } + + protected final void verifyPermissions(String uri, String... permissionsRolesAndCapabilities) { + verifyMetadata(uri, metadata -> { + for (int i = 0; i < permissionsRolesAndCapabilities.length; i += 2) { + String role = permissionsRolesAndCapabilities[i]; + assertTrue(metadata.getPermissions().containsKey(role), "Did not find permissions with role: " + + role + "; actual permissions: " + metadata.getPermissions()); + + DocumentMetadataHandle.Capability capability = + DocumentMetadataHandle.Capability.valueOf(permissionsRolesAndCapabilities[i + 1].toUpperCase()); + Set capabilities = metadata.getPermissions().get(role); + assertTrue(capabilities.contains(capability), "Did not find permission for role: " + role + + " with capability: " + capability + "; actual capabilities: " + capabilities); + } + }); + } } @Configuration diff --git a/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java b/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java new file mode 100644 index 0000000..b91342b --- /dev/null +++ b/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2023 MarkLogic Corporation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.marklogic.client.ext.file; + +import com.marklogic.client.DatabaseClient; +import com.marklogic.client.ext.AbstractIntegrationTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +public class CascadeCollectionsAndPermissionsTest extends AbstractIntegrationTest { + + final private String PARENT_COLLECTION = "ParentCollection"; + final private String CHILD_COLLECTION = "ChildCollection"; + + @BeforeEach + public void setup() { + client = newClient(MODULES_DATABASE); + DatabaseClient modulesClient = client; + modulesClient.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval(); + } + + @Test + public void parentWithBothProperties() { + String directory = "src/test/resources/process-files/cascading-metadata-test/parent1-withCP"; + GenericFileLoader loader = new GenericFileLoader(client); + loader.loadFiles(directory); + + verifyCollections( "/child1_1-noCP/test.json", PARENT_COLLECTION); + verifyPermissions( "/child1_1-noCP/test.json", "rest-writer", "update"); + + verifyCollections( "/child1_2-withCP/test.json", CHILD_COLLECTION); + verifyPermissions( "/child1_2-withCP/test.json", "rest-reader", "read"); + + verifyCollections( "/child3_1-withCP/grandchild3_1_1-noCP/test.json", CHILD_COLLECTION); + verifyPermissions( "/child3_1-withCP/grandchild3_1_1-noCP/test.json", "rest-reader", "read"); + + verifyCollections("/child1/child1.json", "ParentCollection"); + verifyPermissions("/child1/child1.json", "rest-writer", "update"); + + verifyCollections("/child2/child2.json", "child2"); + verifyPermissions("/child2/child2.json", "app-user", "read"); + + verifyCollections("/parent.json", "ParentCollection"); + verifyPermissions("/parent.json", "rest-writer", "update"); + } + + @Test + public void parentWithNoProperties() { + String directory = "src/test/resources/process-files/cascading-metadata-test/parent2-noCP"; + GenericFileLoader loader = new GenericFileLoader(client); + loader.loadFiles(directory); + + verifyCollections( "/child2_1-withCP/test.json", CHILD_COLLECTION); + verifyPermissions( "/child2_1-withCP/test.json", "rest-reader", "read"); + + verifyCollections( "/child2_2-noCP/test.json"); + verifyPermissions( "/child2_2-noCP/test.json"); + + verifyCollections( "/child2_3-withCnoP/test.json", PARENT_COLLECTION); + verifyPermissions( "/child2_3-withCnoP/test.json"); + + verifyCollections( "/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", PARENT_COLLECTION); + verifyPermissions( "/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", "rest-reader", "read"); + } +} diff --git a/src/test/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessorTest.java b/src/test/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessorTest.java index 160f7d4..e33939f 100644 --- a/src/test/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessorTest.java +++ b/src/test/java/com/marklogic/client/ext/file/CollectionsFileDocumentFileProcessorTest.java @@ -19,6 +19,7 @@ import org.junit.jupiter.api.Test; import java.io.File; +import java.io.IOException; import java.util.Properties; import static org.junit.jupiter.api.Assertions.*; @@ -28,9 +29,12 @@ public class CollectionsFileDocumentFileProcessorTest { private CollectionsFileDocumentFileProcessor processor = new CollectionsFileDocumentFileProcessor(); @Test - public void wildcard() { + public void wildcard() throws IOException { File testDir = new File("src/test/resources/process-files/wildcard-test"); + File collectionsPropertiesFile = new File(testDir, processor.getPropertiesFilename()); + processor.loadProperties(collectionsPropertiesFile); + DocumentFile file = new DocumentFile("/test.json", new File(testDir, "test.json")); processor.processDocumentFile(file); assertTrue(file.getDocumentMetadata().getCollections().contains("json-data")); @@ -45,9 +49,12 @@ public void wildcard() { } @Test - public void replaceTokens() { + public void replaceTokens() throws IOException { File testDir = new File("src/test/resources/process-files/token-test"); + File collectionsPropertiesFile = new File(testDir, processor.getPropertiesFilename()); + processor.loadProperties(collectionsPropertiesFile); + DefaultTokenReplacer tokenReplacer = new DefaultTokenReplacer(); Properties props = new Properties(); props.setProperty("%%someCollection%%", "this-was-replaced"); diff --git a/src/test/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessorTest.java b/src/test/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessorTest.java index 1486882..64c9ade 100644 --- a/src/test/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessorTest.java +++ b/src/test/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessorTest.java @@ -20,6 +20,7 @@ import org.junit.jupiter.api.Test; import java.io.File; +import java.io.IOException; import java.util.Properties; import static org.junit.jupiter.api.Assertions.*; @@ -34,9 +35,12 @@ public class PermissionsFileDocumentFileProcessorTest { * test.xml=qconsole-user,update */ @Test - public void wildcard() { + public void wildcard() throws IOException { File testDir = new File("src/test/resources/process-files/wildcard-test"); + File collectionsPropertiesFile = new File(testDir, processor.getPropertiesFilename()); + processor.loadProperties(collectionsPropertiesFile); + DocumentFile file = new DocumentFile("/test.json", new File(testDir, "test.json")); processor.processDocumentFile(file); DocumentMetadataHandle.DocumentPermissions permissions = file.getDocumentMetadata().getPermissions(); @@ -55,9 +59,12 @@ public void wildcard() { } @Test - public void replaceTokens() { + public void replaceTokens() throws IOException { File testDir = new File("src/test/resources/process-files/token-test"); + File collectionsPropertiesFile = new File(testDir, processor.getPropertiesFilename()); + processor.loadProperties(collectionsPropertiesFile); + DefaultTokenReplacer tokenReplacer = new DefaultTokenReplacer(); Properties props = new Properties(); props.setProperty("%%roleName%%", "rest-admin"); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java index 2e50f60..5df2aef 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java @@ -2,16 +2,16 @@ import com.marklogic.client.ext.file.DocumentFile; import com.marklogic.client.ext.helper.ClientHelper; -import com.marklogic.client.io.DocumentMetadataHandle; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.nio.file.Path; import java.nio.file.Paths; import java.util.List; -import java.util.function.Consumer; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; public class GenerateQbvTest extends AbstractSchemasTest { @@ -43,18 +43,9 @@ public void test() { // PlanBuilder.ModifyPlan plan = rowManager.newPlanBuilder().fromView("alternate", "authors"); // rowManager.resultDoc(plan, new JacksonHandle()).get(); - verifyMetadata(qbvUris.get(0), metadata -> { - DocumentMetadataHandle.DocumentPermissions perms = metadata.getPermissions(); - assertPermissionExists(perms, "rest-reader", DocumentMetadataHandle.Capability.READ); - assertPermissionExists(perms, "rest-writer", DocumentMetadataHandle.Capability.UPDATE); - assertEquals(2, perms.size()); - DocumentMetadataHandle.DocumentCollections colls = metadata.getCollections(); - assertTrue(colls.contains("http://marklogic.com/xdmp/qbv")); - assertTrue(colls.contains("col1")); - assertTrue(colls.contains("col3")); - assertEquals(3, colls.size()); - }); - + String uri = qbvUris.get(0); + verifyCollections(uri, "http://marklogic.com/xdmp/qbv", "col1", "col3"); + verifyPermissions(uri, "rest-reader", "read", "rest-writer", "update"); } @Test @@ -84,15 +75,4 @@ public void emptyDirectories() { List qbvUris = helper.getUrisInCollection(QbvDocumentFileProcessor.QBV_COLLECTION); assertEquals(0, qbvUris.size()); } - - private void verifyMetadata(String uri, Consumer verifier) { - verifier.accept(client.newJSONDocumentManager().readMetadata(uri, new DocumentMetadataHandle())); - } - - private void assertPermissionExists(DocumentMetadataHandle.DocumentPermissions perms, String role, - DocumentMetadataHandle.Capability capability) { - assertTrue(perms.containsKey(role), "No permissions for role: " + role); - assertTrue(perms.get(role).contains(capability), - "Capability " + capability + " for role " + role + " not found"); - } } diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1/child1.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1/child1.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1/child1.json @@ -0,0 +1 @@ +{} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_1-noCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_1-noCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_1-noCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties new file mode 100644 index 0000000..2a20739 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties @@ -0,0 +1 @@ +*=ChildCollection diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/permissions.properties new file mode 100644 index 0000000..b56406b --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/child2.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/child2.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/child2.json @@ -0,0 +1 @@ +{} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/collections.properties new file mode 100644 index 0000000..66a0875 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/collections.properties @@ -0,0 +1 @@ +*=child2 diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/permissions.properties new file mode 100644 index 0000000..f6019e2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child2/permissions.properties @@ -0,0 +1 @@ +*=app-user,read diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties new file mode 100644 index 0000000..2a20739 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties @@ -0,0 +1 @@ +*=ChildCollection diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/grandchild3_1_1-noCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/grandchild3_1_1-noCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/grandchild3_1_1-noCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties new file mode 100644 index 0000000..b56406b --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/collections.properties new file mode 100644 index 0000000..ab5b0ee --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/collections.properties @@ -0,0 +1 @@ +*=ParentCollection diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/parent.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/parent.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/parent.json @@ -0,0 +1 @@ +{} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/permissions.properties new file mode 100644 index 0000000..d83899b --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/permissions.properties @@ -0,0 +1 @@ +*=rest-writer,update diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/collections.properties new file mode 100644 index 0000000..2a20739 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/collections.properties @@ -0,0 +1 @@ +*=ChildCollection diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/permissions.properties new file mode 100644 index 0000000..b56406b --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_1-withCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_2-noCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_2-noCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_2-noCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/collections.properties new file mode 100644 index 0000000..ab5b0ee --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/collections.properties @@ -0,0 +1 @@ +*=ParentCollection diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/permissions.properties new file mode 100644 index 0000000..b56406b --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/permissions.properties @@ -0,0 +1 @@ +*=rest-reader,read diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/child2_3-withCnoP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} diff --git a/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/test.json new file mode 100644 index 0000000..658d2f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent2-noCP/test.json @@ -0,0 +1,3 @@ +{ + "test": "test" +} From 9f1dae5ac09dc1e4805ae8cc05dd4f9c37f0958d Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 25 Aug 2023 10:36:28 -0400 Subject: [PATCH 13/20] Refactoring file loading Changes: 1. The Cascading processor now implements `FileVisitor` so that DDFR can bind to that instead of the concrete classes. 2. Added two `Supports` interfaces that are a little cheesy but allow for DDFR/GFL to not be bound directly to the collections/permissions processors (though I couldn't completely remove that binding without introducing breaking changes in the public API). --- ...PropertiesDrivenDocumentFileProcessor.java | 27 +++++++++-- .../ext/file/DefaultDocumentFileReader.java | 45 ++++++++++++------ .../ext/file/FormatDocumentFileProcessor.java | 20 +++++++- .../client/ext/file/GenericFileLoader.java | 46 ++++++------------- ...PropertiesDrivenDocumentFileProcessor.java | 6 ++- .../SupportsAdditionalBinaryExtensions.java | 11 +++++ .../ext/file/SupportsTokenReplacer.java | 12 +++++ 7 files changed, 113 insertions(+), 54 deletions(-) create mode 100644 src/main/java/com/marklogic/client/ext/file/SupportsAdditionalBinaryExtensions.java create mode 100644 src/main/java/com/marklogic/client/ext/file/SupportsTokenReplacer.java diff --git a/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java index a057dc9..e7403a0 100644 --- a/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java @@ -17,21 +17,27 @@ import java.io.File; import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Properties; import java.util.Stack; /** - * Adds a stack to store Properties objects while traversing a directory tree. + * Adds a stack to store Properties objects while traversing a directory tree. Implements {@code FileVisitor} so that + * it can be informed when {@code DefaultDocumentFileReader} is entering and exiting a directory. */ -abstract class CascadingPropertiesDrivenDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor { +abstract class CascadingPropertiesDrivenDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor implements FileVisitor { + final private Stack propertiesStack = new Stack<>(); protected CascadingPropertiesDrivenDocumentFileProcessor(String propertiesFilename) { super(propertiesFilename); } - protected void preVisitDirectory(Path dir) throws IOException { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { File collectionsPropertiesFile = new File(dir.toFile(), this.getPropertiesFilename()); if (collectionsPropertiesFile.exists()) { this.loadProperties(collectionsPropertiesFile); @@ -43,12 +49,25 @@ protected void preVisitDirectory(Path dir) throws IOException { } } propertiesStack.push(this.getProperties()); + return FileVisitResult.CONTINUE; } - protected void postVisitDirectory() { + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) { propertiesStack.pop(); if (!propertiesStack.isEmpty()) { this.setProperties(propertiesStack.peek()); } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) { + return FileVisitResult.CONTINUE; } } diff --git a/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java b/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java index ae0d4df..09b2255 100644 --- a/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java +++ b/src/main/java/com/marklogic/client/ext/file/DefaultDocumentFileReader.java @@ -20,9 +20,14 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; -import java.nio.file.*; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; -import java.util.*; +import java.util.ArrayList; +import java.util.List; /** * Non-threadsafe implementation that implements FileVisitor as a way of descending one or more file paths. @@ -34,7 +39,8 @@ public class DefaultDocumentFileReader extends AbstractDocumentFileReader implem private List documentFiles; private String uriPrefix = "/"; - // Each of these are eagerly instantiated, and we retain a reference in case a client wants to modify them + // As of 4.6.0, these no longer need to be class fields but are being kept for backwards compatibility. + // They should be removed in 5.0.0. private CollectionsFileDocumentFileProcessor collectionsFileDocumentFileProcessor; private PermissionsFileDocumentFileProcessor permissionsFileDocumentFileProcessor; @@ -104,10 +110,11 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th if (logger.isDebugEnabled()) { logger.debug("Visiting directory: " + dir); } - - collectionsFileDocumentFileProcessor.preVisitDirectory(dir); - permissionsFileDocumentFileProcessor.preVisitDirectory(dir); - + for (DocumentFileProcessor processor : getDocumentFileProcessors()) { + if (processor instanceof FileVisitor) { + ((FileVisitor) processor).preVisitDirectory(dir, attrs); + } + } return FileVisitResult.CONTINUE; } else { if (logger.isDebugEnabled()) { @@ -118,7 +125,7 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th } @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + public FileVisitResult visitFileFailed(Path file, IOException exc) { if (exc != null) { logger.warn("Failed visiting file: " + exc.getMessage(), exc); } @@ -130,15 +137,16 @@ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOEx if (exc != null) { logger.warn("Error in postVisitDirectory: " + exc.getMessage(), exc); } - - collectionsFileDocumentFileProcessor.postVisitDirectory(); - permissionsFileDocumentFileProcessor.postVisitDirectory(); - + for (DocumentFileProcessor processor : getDocumentFileProcessors()) { + if (processor instanceof FileVisitor) { + ((FileVisitor) processor).postVisitDirectory(dir, exc); + } + } return FileVisitResult.CONTINUE; } @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) { if (acceptPath(path, attrs)) { DocumentFile documentFile = buildDocumentFile(path, currentRootPath); documentFile = processDocumentFile(documentFile); @@ -209,10 +217,21 @@ public void setUriPrefix(String uriPrefix) { this.uriPrefix = uriPrefix; } + /** + * + * @return + * @deprecated since 4.6.0, will be removed in 5.0.0 + */ + @Deprecated public CollectionsFileDocumentFileProcessor getCollectionsFileDocumentFileProcessor() { return collectionsFileDocumentFileProcessor; } + /** + * + * @return + * @deprecated since 4.6.0, will be removed in 5.0.0 + */ public PermissionsFileDocumentFileProcessor getPermissionsFileDocumentFileProcessor() { return permissionsFileDocumentFileProcessor; } diff --git a/src/main/java/com/marklogic/client/ext/file/FormatDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/FormatDocumentFileProcessor.java index 14f4c15..b50a583 100644 --- a/src/main/java/com/marklogic/client/ext/file/FormatDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/FormatDocumentFileProcessor.java @@ -15,13 +15,17 @@ */ package com.marklogic.client.ext.file; +import com.marklogic.client.ext.helper.LoggingObject; import com.marklogic.client.io.Format; +import java.util.Arrays; + /** * Delegates to DefaultDocumentFormatGetter by default for determining what Format to use for the File in a given * DocumentFile. */ -public class FormatDocumentFileProcessor implements DocumentFileProcessor { +public class FormatDocumentFileProcessor extends LoggingObject + implements DocumentFileProcessor, SupportsAdditionalBinaryExtensions { private FormatGetter formatGetter; @@ -35,7 +39,6 @@ public FormatDocumentFileProcessor(FormatGetter formatGetter) { @Override public DocumentFile processDocumentFile(DocumentFile documentFile) { - Format format = formatGetter.getFormat(documentFile.getResource()); if (format != null) { documentFile.setFormat(format); @@ -43,6 +46,19 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) { return documentFile; } + @Override + public void setAdditionalBinaryExtensions(String[] extensions) { + if (formatGetter instanceof DefaultDocumentFormatGetter) { + DefaultDocumentFormatGetter ddfg = (DefaultDocumentFormatGetter) formatGetter; + for (String ext : extensions) { + ddfg.getBinaryExtensions().add(ext); + } + } else { + logger.warn("FormatGetter is not an instanceof DefaultDocumentFormatGetter, " + + "so unable to add additionalBinaryExtensions: " + Arrays.asList(extensions)); + } + } + public FormatGetter getFormatGetter() { return formatGetter; } diff --git a/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java b/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java index 1d89984..ce15dfc 100644 --- a/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java +++ b/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java @@ -24,7 +24,6 @@ import java.io.FileFilter; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.stream.Collectors; @@ -178,46 +177,27 @@ public void initializeDocumentFileReader() { * @param reader */ public void prepareAbstractDocumentFileReader(AbstractDocumentFileReader reader) { - for (DocumentFileProcessor processor : buildDocumentFileProcessors()) { - reader.addDocumentFileProcessor(processor); - } - - applyTokenReplacerOnKnownDocumentProcessors(reader); + buildDocumentFileProcessors().forEach(processor -> reader.addDocumentFileProcessor(processor)); - if (additionalBinaryExtensions != null) { - FormatDocumentFileProcessor processor = reader.getFormatDocumentFileProcessor(); - FormatGetter formatGetter = processor.getFormatGetter(); - if (formatGetter instanceof DefaultDocumentFormatGetter) { - DefaultDocumentFormatGetter ddfg = (DefaultDocumentFormatGetter) formatGetter; - for (String ext : additionalBinaryExtensions) { - ddfg.getBinaryExtensions().add(ext); - } - } else { - logger.warn("FormatGetter is not an instanceof DefaultDocumentFormatGetter, " + - "so unable to add additionalBinaryExtensions: " + Arrays.asList(additionalBinaryExtensions)); + reader.getDocumentFileProcessors().forEach(processor -> { + if (tokenReplacer != null && processor instanceof SupportsTokenReplacer) { + ((SupportsTokenReplacer) processor).setTokenReplacer(tokenReplacer); } - } + if (additionalBinaryExtensions != null && processor instanceof SupportsAdditionalBinaryExtensions) { + ((SupportsAdditionalBinaryExtensions) processor).setAdditionalBinaryExtensions(additionalBinaryExtensions); + } + }); } /** - * If this is an instance of DefaultDocumentFileReader and a TokenReplacer has been set on the instance of this - * class, then pass the TokenReplacer along to the known processors in the reader so that those processors - * can replace token occurrences in property values. - * * @param reader + * @deprecated since 4.6.0, will be removed in 5.0.0. */ + @Deprecated protected void applyTokenReplacerOnKnownDocumentProcessors(AbstractDocumentFileReader reader) { - if (reader instanceof DefaultDocumentFileReader && tokenReplacer != null) { - DefaultDocumentFileReader defaultReader = (DefaultDocumentFileReader) reader; - CollectionsFileDocumentFileProcessor cp = defaultReader.getCollectionsFileDocumentFileProcessor(); - if (cp != null) { - cp.setTokenReplacer(tokenReplacer); - } - PermissionsFileDocumentFileProcessor pp = defaultReader.getPermissionsFileDocumentFileProcessor(); - if (pp != null) { - pp.setTokenReplacer(tokenReplacer); - } - } + // The logic previously performed here is now handled via prepareAbstractDocumentFileReader . This is being + // kept here solely to avoid any compilation issues in case this class was extended and this method was + // overridden. } /** diff --git a/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java index 296fec7..30ffc4e 100644 --- a/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/PropertiesDrivenDocumentFileProcessor.java @@ -28,7 +28,8 @@ * Base class for processors that look for a special file in each directory and intend to perform some processing based * on the contents of that file. By default, that special file is NOT loaded into MarkLogic. */ -public abstract class PropertiesDrivenDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor, FileFilter { +public abstract class PropertiesDrivenDocumentFileProcessor extends LoggingObject + implements DocumentFileProcessor, FileFilter, SupportsTokenReplacer { protected final static String WILDCARD_KEY = "*"; @@ -83,6 +84,7 @@ public String getPropertiesFilename() { return propertiesFilename; } + @Override public void setTokenReplacer(TokenReplacer tokenReplacer) { this.tokenReplacer = tokenReplacer; } @@ -90,7 +92,7 @@ public void setTokenReplacer(TokenReplacer tokenReplacer) { protected TokenReplacer getTokenReplacer() { return tokenReplacer; } - + protected void setProperties(Properties properties) { this.properties = properties; } diff --git a/src/main/java/com/marklogic/client/ext/file/SupportsAdditionalBinaryExtensions.java b/src/main/java/com/marklogic/client/ext/file/SupportsAdditionalBinaryExtensions.java new file mode 100644 index 0000000..711d272 --- /dev/null +++ b/src/main/java/com/marklogic/client/ext/file/SupportsAdditionalBinaryExtensions.java @@ -0,0 +1,11 @@ +package com.marklogic.client.ext.file; + +/** + * Intended to allow for {@code DefaultDocumentFileReader} to pass along a user-supplied list of additional binary + * extensions without being coupled tightly to the {@code DocumentFileProcessor} that uses them. + */ +public interface SupportsAdditionalBinaryExtensions { + + void setAdditionalBinaryExtensions(String[] extensions); + +} diff --git a/src/main/java/com/marklogic/client/ext/file/SupportsTokenReplacer.java b/src/main/java/com/marklogic/client/ext/file/SupportsTokenReplacer.java new file mode 100644 index 0000000..74fcb95 --- /dev/null +++ b/src/main/java/com/marklogic/client/ext/file/SupportsTokenReplacer.java @@ -0,0 +1,12 @@ +package com.marklogic.client.ext.file; + +import com.marklogic.client.ext.tokenreplacer.TokenReplacer; + +/** + * Intended to be implemented by instances of {@code DocumentFileProcessor} that wish to use a + * {@code TokenReplacer} on the files that they read. + */ +public interface SupportsTokenReplacer { + + void setTokenReplacer(TokenReplacer tokenReplacer); +} From 0ab3bae5d35826e818462cf590d4de75b136b741 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Fri, 25 Aug 2023 15:59:51 -0400 Subject: [PATCH 14/20] Deprecating DocumentPermissionsParser This will work once https://github.com/marklogic/java-client-api/pull/1592/files is merged into the develop branch of the Java Client. --- .../PermissionsDocumentFileProcessor.java | 26 ++++++++++------ .../PermissionsFileDocumentFileProcessor.java | 23 +++++++++++--- .../DefaultDocumentPermissionsParser.java | 30 +++++-------------- .../ext/util/DocumentPermissionsParser.java | 20 ++++++++----- 4 files changed, 57 insertions(+), 42 deletions(-) diff --git a/src/main/java/com/marklogic/client/ext/file/PermissionsDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/PermissionsDocumentFileProcessor.java index f52a836..cccb374 100644 --- a/src/main/java/com/marklogic/client/ext/file/PermissionsDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/PermissionsDocumentFileProcessor.java @@ -15,9 +15,8 @@ */ package com.marklogic.client.ext.file; -import com.marklogic.client.ext.util.DefaultDocumentPermissionsParser; -import com.marklogic.client.io.DocumentMetadataHandle; import com.marklogic.client.ext.util.DocumentPermissionsParser; +import com.marklogic.client.io.DocumentMetadataHandle; /** * DocumentFileProcessor that uses a DocumentPermissionsParser to parse a string of permissions (typically, a delimited @@ -25,24 +24,33 @@ */ public class PermissionsDocumentFileProcessor implements DocumentFileProcessor { - private String permissions; + private String commaDelimitedRolesAndCapabilities; private DocumentPermissionsParser documentPermissionsParser; - public PermissionsDocumentFileProcessor(String permissions) { - this(permissions, new DefaultDocumentPermissionsParser()); + public PermissionsDocumentFileProcessor(String commaDelimitedRolesAndCapabilities) { + this.commaDelimitedRolesAndCapabilities = commaDelimitedRolesAndCapabilities; } - public PermissionsDocumentFileProcessor(String permissions, DocumentPermissionsParser documentPermissionsParser) { - this.permissions = permissions; + /** + * @param commaDelimitedRolesAndCapabilities + * @param documentPermissionsParser + * @deprecated since 4.6.0 + */ + public PermissionsDocumentFileProcessor(String commaDelimitedRolesAndCapabilities, DocumentPermissionsParser documentPermissionsParser) { + this.commaDelimitedRolesAndCapabilities = commaDelimitedRolesAndCapabilities; this.documentPermissionsParser = documentPermissionsParser; } @Override public DocumentFile processDocumentFile(DocumentFile documentFile) { - if (permissions != null && documentPermissionsParser != null) { + if (this.commaDelimitedRolesAndCapabilities != null) { DocumentMetadataHandle metadata = documentFile.getDocumentMetadata(); if (metadata != null) { - documentPermissionsParser.parsePermissions(permissions, metadata.getPermissions()); + if (this.documentPermissionsParser != null) { + this.documentPermissionsParser.parsePermissions(this.commaDelimitedRolesAndCapabilities, metadata.getPermissions()); + } else { + metadata.getPermissions().addFromDelimitedString(this.commaDelimitedRolesAndCapabilities); + } } } return documentFile; diff --git a/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java index 6889584..b92db8e 100644 --- a/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/PermissionsFileDocumentFileProcessor.java @@ -15,7 +15,6 @@ */ package com.marklogic.client.ext.file; -import com.marklogic.client.ext.util.DefaultDocumentPermissionsParser; import com.marklogic.client.ext.util.DocumentPermissionsParser; import java.util.Properties; @@ -33,9 +32,15 @@ public PermissionsFileDocumentFileProcessor() { } public PermissionsFileDocumentFileProcessor(String propertiesFilename) { - this(propertiesFilename, new DefaultDocumentPermissionsParser()); + super(propertiesFilename); } + /** + * @param propertiesFilename + * @param documentPermissionsParser + * @deprecated since 4.6.0 + */ + @Deprecated public PermissionsFileDocumentFileProcessor(String propertiesFilename, DocumentPermissionsParser documentPermissionsParser) { super(propertiesFilename); this.documentPermissionsParser = documentPermissionsParser; @@ -46,15 +51,25 @@ protected void processProperties(DocumentFile documentFile, Properties propertie String name = documentFile.getFile().getName(); if (properties.containsKey(name)) { String value = getPropertyValue(properties, name); - documentPermissionsParser.parsePermissions(value, documentFile.getDocumentMetadata().getPermissions()); + if (documentPermissionsParser != null) { + documentPermissionsParser.parsePermissions(value, documentFile.getDocumentMetadata().getPermissions()); + } else { + documentFile.getDocumentMetadata().getPermissions().addFromDelimitedString(value); + } + } if (properties.containsKey(WILDCARD_KEY)) { String value = getPropertyValue(properties, WILDCARD_KEY); - documentPermissionsParser.parsePermissions(value, documentFile.getDocumentMetadata().getPermissions()); + if (documentPermissionsParser != null) { + documentPermissionsParser.parsePermissions(value, documentFile.getDocumentMetadata().getPermissions()); + } else { + documentFile.getDocumentMetadata().getPermissions().addFromDelimitedString(value); + } } } + @Deprecated public void setDocumentPermissionsParser(DocumentPermissionsParser documentPermissionsParser) { this.documentPermissionsParser = documentPermissionsParser; } diff --git a/src/main/java/com/marklogic/client/ext/util/DefaultDocumentPermissionsParser.java b/src/main/java/com/marklogic/client/ext/util/DefaultDocumentPermissionsParser.java index 9128467..4d5fc1e 100644 --- a/src/main/java/com/marklogic/client/ext/util/DefaultDocumentPermissionsParser.java +++ b/src/main/java/com/marklogic/client/ext/util/DefaultDocumentPermissionsParser.java @@ -15,34 +15,20 @@ */ package com.marklogic.client.ext.util; -import com.marklogic.client.io.DocumentMetadataHandle.Capability; import com.marklogic.client.io.DocumentMetadataHandle.DocumentPermissions; +/** + * @deprecated since 4.6.0, will be removed in 5.0.0. Can use the new {@code addFromDelimitedString} method in + * the Java Client's {@code DocumentPermissions} class in Java Client 6.3.0. + */ +@Deprecated public class DefaultDocumentPermissionsParser implements DocumentPermissionsParser { @Override + @Deprecated public void parsePermissions(String str, DocumentPermissions permissions) { - if (str != null && str.trim().length() > 0) { - String[] tokens = str.split(","); - for (int i = 0; i < tokens.length; i += 2) { - String role = tokens[i]; - if (i + 1 >= tokens.length) { - throw new IllegalArgumentException("Unable to parse permissions string, which must be a comma-separated " + - "list of role names and capabilities - i.e. role1,read,role2,update,role3,execute; string: " + str); - } - Capability c; - try { - c = Capability.getValueOf(tokens[i + 1]); - } catch (Exception e) { - throw new IllegalArgumentException("Unable to parse permissions string: " + str + "; cause: " + e.getMessage()); - } - if (permissions.containsKey(role)) { - permissions.get(role).add(c); - } else { - permissions.add(role, c); - } - } - } + // As of Java Client 6.3.0, can just defer to the client. + permissions.addFromDelimitedString(str); } } diff --git a/src/main/java/com/marklogic/client/ext/util/DocumentPermissionsParser.java b/src/main/java/com/marklogic/client/ext/util/DocumentPermissionsParser.java index a746650..ad0af04 100644 --- a/src/main/java/com/marklogic/client/ext/util/DocumentPermissionsParser.java +++ b/src/main/java/com/marklogic/client/ext/util/DocumentPermissionsParser.java @@ -17,13 +17,19 @@ import com.marklogic.client.io.DocumentMetadataHandle.DocumentPermissions; +/** + * @deprecated since 4.6.0, will be removed in 5.0.0. Can use the new {@code addFromDelimitedString} method in + * the Java Client's {@code DocumentPermissions} class in Java Client 6.3.0. + */ +@Deprecated public interface DocumentPermissionsParser { - /** - * Parse the string and add role/capability sets to the given DocumentPermissions object. - * - * @param str - * @param permissions - */ - void parsePermissions(String str, DocumentPermissions permissions); + /** + * Parse the string and add role/capability sets to the given DocumentPermissions object. + * + * @param str + * @param permissions + */ + @Deprecated + void parsePermissions(String str, DocumentPermissions permissions); } From 1e42d7dacc1322cc57ed3211a33aabe342da2f9d Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 30 Aug 2023 12:25:53 -0400 Subject: [PATCH 15/20] Bumping Jackson to 2.15.2 This is to match Java Client 6.3.0 going to 2.15.2. --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b8b0829..6dc5716 100644 --- a/build.gradle +++ b/build.gradle @@ -36,7 +36,7 @@ dependencies { implementation 'org.jdom:jdom2:2.0.6.1' - implementation 'com.fasterxml.jackson.core:jackson-databind:2.14.3' + implementation 'com.fasterxml.jackson.core:jackson-databind:2.15.2' // This is currently an implementation dependency, though perhaps should be an api dependency due to its usage in // LoggingObject; not clear yet on whether the protected Logger in that class should make this an api dependency or not From 2e77d4feb13bb5f55ff3fe81ea0b1ac3695dc512 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 6 Sep 2023 13:40:18 -0400 Subject: [PATCH 16/20] Can now configure whether processors cascade ml-app-deployer will then introduce new properties to call the setters on GFL. --- ...PropertiesDrivenDocumentFileProcessor.java | 28 +++++++-- .../client/ext/file/GenericFileLoader.java | 43 +++++++++++++ .../client/ext/AbstractIntegrationTest.java | 5 ++ .../CascadeCollectionsAndPermissionsTest.java | 61 ++++++++++++------- .../modulesloader/impl/LoadModulesTest.java | 6 +- 5 files changed, 113 insertions(+), 30 deletions(-) diff --git a/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java index e7403a0..1671e56 100644 --- a/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java @@ -27,10 +27,16 @@ /** * Adds a stack to store Properties objects while traversing a directory tree. Implements {@code FileVisitor} so that * it can be informed when {@code DefaultDocumentFileReader} is entering and exiting a directory. + * + * To preserve backwards compatibility in subclasses, cascading is disabled by default. This will likely change in 5.0 + * to be enabled by default. + * + * @since 4.6.0 */ abstract class CascadingPropertiesDrivenDocumentFileProcessor extends PropertiesDrivenDocumentFileProcessor implements FileVisitor { final private Stack propertiesStack = new Stack<>(); + private boolean cascadingEnabled = false; protected CascadingPropertiesDrivenDocumentFileProcessor(String propertiesFilename) { super(propertiesFilename); @@ -42,21 +48,25 @@ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) th if (collectionsPropertiesFile.exists()) { this.loadProperties(collectionsPropertiesFile); } else { - if (!propertiesStack.isEmpty()) { + if (cascadingEnabled && !propertiesStack.isEmpty()) { this.setProperties(propertiesStack.peek()); } else { this.setProperties(new Properties()); } } - propertiesStack.push(this.getProperties()); + if (cascadingEnabled) { + propertiesStack.push(this.getProperties()); + } return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) { - propertiesStack.pop(); - if (!propertiesStack.isEmpty()) { - this.setProperties(propertiesStack.peek()); + if (cascadingEnabled) { + propertiesStack.pop(); + if (!propertiesStack.isEmpty()) { + this.setProperties(propertiesStack.peek()); + } } return FileVisitResult.CONTINUE; } @@ -70,4 +80,12 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { public FileVisitResult visitFileFailed(Path file, IOException exc) { return FileVisitResult.CONTINUE; } + + public boolean isCascadingEnabled() { + return cascadingEnabled; + } + + public void setCascadingEnabled(boolean cascadingEnabled) { + this.cascadingEnabled = cascadingEnabled; + } } diff --git a/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java b/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java index ce15dfc..b2d4956 100644 --- a/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java +++ b/src/main/java/com/marklogic/client/ext/file/GenericFileLoader.java @@ -53,6 +53,8 @@ public class GenericFileLoader extends LoggingObject implements FileLoader { private String[] collections; private TokenReplacer tokenReplacer; private String[] additionalBinaryExtensions; + private boolean cascadeCollections; + private boolean cascadePermissions; /** * The given DatabaseClient is used to construct a BatchWriter that writes to MarkLogic via the REST API. The @@ -186,6 +188,15 @@ public void prepareAbstractDocumentFileReader(AbstractDocumentFileReader reader) if (additionalBinaryExtensions != null && processor instanceof SupportsAdditionalBinaryExtensions) { ((SupportsAdditionalBinaryExtensions) processor).setAdditionalBinaryExtensions(additionalBinaryExtensions); } + + // Awful hack for 4.6.0. In 5.0, the hope is to replace the processor-specific fields on this class with + // a "Config"-type class that can be passed to each processor. + if (processor instanceof PermissionsFileDocumentFileProcessor) { + ((PermissionsFileDocumentFileProcessor)processor).setCascadingEnabled(this.cascadePermissions); + } + if (processor instanceof CollectionsFileDocumentFileProcessor) { + ((CollectionsFileDocumentFileProcessor)processor).setCascadingEnabled(this.cascadeCollections); + } }); } @@ -301,4 +312,36 @@ public void setBatchSize(Integer batchSize) { public void setBatchWriter(BatchWriter batchWriter) { this.batchWriter = batchWriter; } + + /** + * @param cascadeCollections + * @since 4.6.0 + */ + public void setCascadeCollections(boolean cascadeCollections) { + this.cascadeCollections = cascadeCollections; + } + + /** + * @param cascadePermissions + * @since 4.6.0 + */ + public void setCascadePermissions(boolean cascadePermissions) { + this.cascadePermissions = cascadePermissions; + } + + /** + * @return + * @since 4.6.0 + */ + public boolean isCascadeCollections() { + return cascadeCollections; + } + + /** + * @return + * @since 4.6.0 + */ + public boolean isCascadePermissions() { + return cascadePermissions; + } } diff --git a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java index bc6e755..88ff68e 100644 --- a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java +++ b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java @@ -82,6 +82,11 @@ protected final void verifyCollections(String uri, String... collections) { protected final void verifyPermissions(String uri, String... permissionsRolesAndCapabilities) { verifyMetadata(uri, metadata -> { + // TODO This will likely need to be modified once we shift the tests to not use an admin user, and thus + // the user will have to specify at least one update permission. + if (permissionsRolesAndCapabilities.length == 0) { + assertEquals(0, metadata.getPermissions().size()); + } for (int i = 0; i < permissionsRolesAndCapabilities.length; i += 2) { String role = permissionsRolesAndCapabilities[i]; assertTrue(metadata.getPermissions().containsKey(role), "Did not find permissions with role: " + diff --git a/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java b/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java index b91342b..8733a4c 100644 --- a/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java +++ b/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java @@ -25,27 +25,31 @@ public class CascadeCollectionsAndPermissionsTest extends AbstractIntegrationTes final private String PARENT_COLLECTION = "ParentCollection"; final private String CHILD_COLLECTION = "ChildCollection"; + + private GenericFileLoader loader; + @BeforeEach - public void setup() { + void setup() { client = newClient(MODULES_DATABASE); DatabaseClient modulesClient = client; modulesClient.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval(); + loader = new GenericFileLoader(client); + loader.setCascadeCollections(true); + loader.setCascadePermissions(true); } @Test - public void parentWithBothProperties() { - String directory = "src/test/resources/process-files/cascading-metadata-test/parent1-withCP"; - GenericFileLoader loader = new GenericFileLoader(client); - loader.loadFiles(directory); + void parentWithBothProperties() { + loader.loadFiles("src/test/resources/process-files/cascading-metadata-test/parent1-withCP"); - verifyCollections( "/child1_1-noCP/test.json", PARENT_COLLECTION); - verifyPermissions( "/child1_1-noCP/test.json", "rest-writer", "update"); + verifyCollections("/child1_1-noCP/test.json", PARENT_COLLECTION); + verifyPermissions("/child1_1-noCP/test.json", "rest-writer", "update"); - verifyCollections( "/child1_2-withCP/test.json", CHILD_COLLECTION); - verifyPermissions( "/child1_2-withCP/test.json", "rest-reader", "read"); + verifyCollections("/child1_2-withCP/test.json", CHILD_COLLECTION); + verifyPermissions("/child1_2-withCP/test.json", "rest-reader", "read"); - verifyCollections( "/child3_1-withCP/grandchild3_1_1-noCP/test.json", CHILD_COLLECTION); - verifyPermissions( "/child3_1-withCP/grandchild3_1_1-noCP/test.json", "rest-reader", "read"); + verifyCollections("/child3_1-withCP/grandchild3_1_1-noCP/test.json", CHILD_COLLECTION); + verifyPermissions("/child3_1-withCP/grandchild3_1_1-noCP/test.json", "rest-reader", "read"); verifyCollections("/child1/child1.json", "ParentCollection"); verifyPermissions("/child1/child1.json", "rest-writer", "update"); @@ -58,21 +62,32 @@ public void parentWithBothProperties() { } @Test - public void parentWithNoProperties() { - String directory = "src/test/resources/process-files/cascading-metadata-test/parent2-noCP"; - GenericFileLoader loader = new GenericFileLoader(client); - loader.loadFiles(directory); + void parentWithNoProperties() { + loader.loadFiles("src/test/resources/process-files/cascading-metadata-test/parent2-noCP"); + + verifyCollections("/child2_1-withCP/test.json", CHILD_COLLECTION); + verifyPermissions("/child2_1-withCP/test.json", "rest-reader", "read"); - verifyCollections( "/child2_1-withCP/test.json", CHILD_COLLECTION); - verifyPermissions( "/child2_1-withCP/test.json", "rest-reader", "read"); + verifyCollections("/child2_2-noCP/test.json"); + verifyPermissions("/child2_2-noCP/test.json"); - verifyCollections( "/child2_2-noCP/test.json"); - verifyPermissions( "/child2_2-noCP/test.json"); + verifyCollections("/child2_3-withCnoP/test.json", PARENT_COLLECTION); + verifyPermissions("/child2_3-withCnoP/test.json"); - verifyCollections( "/child2_3-withCnoP/test.json", PARENT_COLLECTION); - verifyPermissions( "/child2_3-withCnoP/test.json"); + verifyCollections("/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", PARENT_COLLECTION); + verifyPermissions("/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", "rest-reader", "read"); + } + + /** + * Verifies that by default, cascading is disabled. This is to preserve backwards compatibility in 4.x. We + * expect to change this for 5.0. + */ + @Test + void cascadingDisabled() { + loader = new GenericFileLoader(client); - verifyCollections( "/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", PARENT_COLLECTION); - verifyPermissions( "/child2_3-withCnoP/grandchild2_3_1-withPnoC/test.json", "rest-reader", "read"); + loader.loadFiles("src/test/resources/process-files/cascading-metadata-test/parent1-withCP"); + verifyCollections("/child1_1-noCP/test.json"); + verifyPermissions("/child1_1-noCP/test.json"); } } diff --git a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java index ba5cd9c..6acacd1 100644 --- a/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java +++ b/src/test/java/com/marklogic/client/ext/modulesloader/impl/LoadModulesTest.java @@ -36,7 +36,9 @@ import java.util.Set; import java.util.regex.Pattern; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; public class LoadModulesTest extends AbstractIntegrationTest { @@ -48,7 +50,7 @@ public void setup() { client = newClient(MODULES_DATABASE); client.newServerEval().xquery("cts:uris((), (), cts:true-query()) ! xdmp:document-delete(.)").eval(); modulesClient = client; - assertEquals(0, getUriCountInModulesDatabase(), "No new modules should have been created"); + assertEquals(0, getUriCountInModulesDatabase(), "No modules should exist"); /** * Odd - the Client REST API doesn't allow for loading namespaces when the DatabaseClient has a database From c1bd5556aa8fc051d566e75e045cb0eb08272a29 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 6 Sep 2023 18:38:38 -0400 Subject: [PATCH 17/20] Fixed how cascading is disabled The fix involves still using the propertiesStack, but never peeking at the top if cascading is disabled. I did some renaming in parent1-withCP; I was finding it easier to just do child1, child2, and child3, with files being named child1.json, child2.json, etc - as opposed to some being named "test.json" and some not being named that. I added child4 to capture a scenario of - parent has C/P; child does not; grandchild does. --- ...PropertiesDrivenDocumentFileProcessor.java | 32 ++++++---- .../CascadeCollectionsAndPermissionsTest.java | 60 +++++++++++++++---- src/test/resources/logback.xml | 4 -- .../child1_2-withCP/collections.properties | 1 - .../test.json => child3/child3.json} | 0 .../child3/collections.properties | 1 + .../grandchild3/grandchild3.json} | 0 .../permissions.properties | 0 .../child3_1-withCP/collections.properties | 1 - .../child3_1-withCP/permissions.properties | 1 - .../test.json => child4/child4.json} | 0 .../child4/grandchild4/collections.properties | 1 + .../grandchild4/grandchild4.json} | 0 .../child4/grandchild4/permissions.properties | 1 + .../parent1-withCP/test.json | 3 - 15 files changed, 71 insertions(+), 34 deletions(-) delete mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties rename src/test/resources/process-files/cascading-metadata-test/parent1-withCP/{child1_1-noCP/test.json => child3/child3.json} (100%) create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/collections.properties rename src/test/resources/process-files/cascading-metadata-test/parent1-withCP/{child1_2-withCP/test.json => child3/grandchild3/grandchild3.json} (100%) rename src/test/resources/process-files/cascading-metadata-test/parent1-withCP/{child1_2-withCP => child3}/permissions.properties (100%) delete mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties delete mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties rename src/test/resources/process-files/cascading-metadata-test/parent1-withCP/{child3_1-withCP/grandchild3_1_1-noCP/test.json => child4/child4.json} (100%) create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/collections.properties rename src/test/resources/process-files/cascading-metadata-test/parent1-withCP/{child3_1-withCP/test.json => child4/grandchild4/grandchild4.json} (100%) create mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/permissions.properties delete mode 100644 src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json diff --git a/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java index 1671e56..d171619 100644 --- a/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/file/CascadingPropertiesDrivenDocumentFileProcessor.java @@ -44,29 +44,39 @@ protected CascadingPropertiesDrivenDocumentFileProcessor(String propertiesFilena @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - File collectionsPropertiesFile = new File(dir.toFile(), this.getPropertiesFilename()); - if (collectionsPropertiesFile.exists()) { - this.loadProperties(collectionsPropertiesFile); + // If cascading is disabled, we still use a stack to keep track of whether a directory has properties or not. + // We just never grab properties from the stack in case a directory doesn't have properties. + if (logger.isDebugEnabled()) { + logger.debug(format("Visiting directory: %s", dir.toFile().getAbsolutePath())); + } + File propertiesFile = new File(dir.toFile(), this.getPropertiesFilename()); + if (propertiesFile.exists()) { + if (logger.isDebugEnabled()) { + logger.debug(format("Loading properties from file: %s", propertiesFile.getAbsolutePath())); + } + this.loadProperties(propertiesFile); } else { if (cascadingEnabled && !propertiesStack.isEmpty()) { + if (logger.isDebugEnabled()) { + logger.debug("No properties file, and cascading is enabled, so using properties from top of stack."); + } this.setProperties(propertiesStack.peek()); } else { + if (logger.isDebugEnabled()) { + logger.debug("No properties file, or cascading is disabled, so using empty properties."); + } this.setProperties(new Properties()); } } - if (cascadingEnabled) { - propertiesStack.push(this.getProperties()); - } + propertiesStack.push(this.getProperties()); return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) { - if (cascadingEnabled) { - propertiesStack.pop(); - if (!propertiesStack.isEmpty()) { - this.setProperties(propertiesStack.peek()); - } + propertiesStack.pop(); + if (!propertiesStack.isEmpty()) { + this.setProperties(propertiesStack.peek()); } return FileVisitResult.CONTINUE; } diff --git a/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java b/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java index 8733a4c..1045cee 100644 --- a/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java +++ b/src/test/java/com/marklogic/client/ext/file/CascadeCollectionsAndPermissionsTest.java @@ -42,23 +42,32 @@ void setup() { void parentWithBothProperties() { loader.loadFiles("src/test/resources/process-files/cascading-metadata-test/parent1-withCP"); - verifyCollections("/child1_1-noCP/test.json", PARENT_COLLECTION); - verifyPermissions("/child1_1-noCP/test.json", "rest-writer", "update"); - - verifyCollections("/child1_2-withCP/test.json", CHILD_COLLECTION); - verifyPermissions("/child1_2-withCP/test.json", "rest-reader", "read"); - - verifyCollections("/child3_1-withCP/grandchild3_1_1-noCP/test.json", CHILD_COLLECTION); - verifyPermissions("/child3_1-withCP/grandchild3_1_1-noCP/test.json", "rest-reader", "read"); + verifyCollections("/parent.json", PARENT_COLLECTION); + verifyPermissions("/parent.json", "rest-writer", "update"); - verifyCollections("/child1/child1.json", "ParentCollection"); + // Should be same as parent as it doesn't have C/P files. + verifyCollections("/child1/child1.json", PARENT_COLLECTION); verifyPermissions("/child1/child1.json", "rest-writer", "update"); + // Differs from parent because it has its own C/P files. verifyCollections("/child2/child2.json", "child2"); verifyPermissions("/child2/child2.json", "app-user", "read"); - verifyCollections("/parent.json", "ParentCollection"); - verifyPermissions("/parent.json", "rest-writer", "update"); + // Differs from parent because it has its own C/P files. + verifyCollections("/child3/child3.json", "child3"); + verifyPermissions("/child3/child3.json", "rest-reader", "read"); + + // Should inherit from child3, not parent. + verifyCollections("/child3/grandchild3/grandchild3.json", "child3"); + verifyPermissions("/child3/grandchild3/grandchild3.json", "rest-reader", "read"); + + // Should inherit from parent. + verifyCollections("/child4/child4.json", PARENT_COLLECTION); + verifyPermissions("/child4/child4.json", "rest-writer", "update"); + + // Should override parent. + verifyCollections("/child4/grandchild4/grandchild4.json", "grandchild4"); + verifyPermissions("/child4/grandchild4/grandchild4.json", "qconsole-user", "read"); } @Test @@ -87,7 +96,32 @@ void cascadingDisabled() { loader = new GenericFileLoader(client); loader.loadFiles("src/test/resources/process-files/cascading-metadata-test/parent1-withCP"); - verifyCollections("/child1_1-noCP/test.json"); - verifyPermissions("/child1_1-noCP/test.json"); + + verifyCollections("/parent.json", PARENT_COLLECTION); + verifyPermissions("/parent.json", "rest-writer", "update"); + + // Has no C/P files. + verifyCollections("/child1/child1.json"); + verifyPermissions("/child1/child1.json"); + + // Has C/P files. + verifyCollections("/child2/child2.json", "child2"); + verifyPermissions("/child2/child2.json", "app-user", "read"); + + // Has C/P files. + verifyCollections("/child3/child3.json", "child3"); + verifyPermissions("/child3/child3.json", "rest-reader", "read"); + + // Has no C/P files. + verifyCollections("/child3/grandchild3/grandchild3.json"); + verifyPermissions("/child3/grandchild3/grandchild3.json"); + + // Has no C/P files. + verifyCollections("/child4/child4.json"); + verifyPermissions("/child4/child4.json"); + + // Has C/P files. + verifyCollections("/child4/grandchild4/grandchild4.json", "grandchild4"); + verifyPermissions("/child4/grandchild4/grandchild4.json", "qconsole-user", "read"); } } diff --git a/src/test/resources/logback.xml b/src/test/resources/logback.xml index 1992bba..9af1326 100644 --- a/src/test/resources/logback.xml +++ b/src/test/resources/logback.xml @@ -24,10 +24,6 @@ - - - - diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties deleted file mode 100644 index 2a20739..0000000 --- a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/collections.properties +++ /dev/null @@ -1 +0,0 @@ -*=ChildCollection diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_1-noCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/child3.json similarity index 100% rename from src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_1-noCP/test.json rename to src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/child3.json diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/collections.properties new file mode 100644 index 0000000..a507d2c --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/collections.properties @@ -0,0 +1 @@ +*=child3 diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/grandchild3/grandchild3.json similarity index 100% rename from src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/test.json rename to src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/grandchild3/grandchild3.json diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/permissions.properties similarity index 100% rename from src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child1_2-withCP/permissions.properties rename to src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3/permissions.properties diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties deleted file mode 100644 index 2a20739..0000000 --- a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/collections.properties +++ /dev/null @@ -1 +0,0 @@ -*=ChildCollection diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties deleted file mode 100644 index b56406b..0000000 --- a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/permissions.properties +++ /dev/null @@ -1 +0,0 @@ -*=rest-reader,read diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/grandchild3_1_1-noCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/child4.json similarity index 100% rename from src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/grandchild3_1_1-noCP/test.json rename to src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/child4.json diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/collections.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/collections.properties new file mode 100644 index 0000000..017d0f2 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/collections.properties @@ -0,0 +1 @@ +*=grandchild4 diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/grandchild4.json similarity index 100% rename from src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child3_1-withCP/test.json rename to src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/grandchild4.json diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/permissions.properties b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/permissions.properties new file mode 100644 index 0000000..76f93d5 --- /dev/null +++ b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/child4/grandchild4/permissions.properties @@ -0,0 +1 @@ +*=qconsole-user,read diff --git a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json b/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json deleted file mode 100644 index 658d2f2..0000000 --- a/src/test/resources/process-files/cascading-metadata-test/parent1-withCP/test.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "test": "test" -} From f476b8e0574a352a15de7a0281a13649487f2953 Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 7 Sep 2023 10:08:13 -0400 Subject: [PATCH 18/20] Added some logging to QBV generation Found this helpful while testing ml-gradle --- .../ext/schemasloader/impl/QbvDocumentFileProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java index 6ee1706..078ba2b 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java @@ -93,6 +93,9 @@ public void processQbvFiles() { } private void processQbvFile(DocumentFile qbvFile) { + if (logger.isInfoEnabled()) { + logger.info(format("Generating Query-Based View for file: %s", qbvFile.getFile().getName())); + } ServerEvaluationCall call = getServerEvaluationCall(qbvFile); if (call != null) { StringHandle handleString = new StringHandle(); From f01ca0b93a2f45dc7a38195a601e92413d1f567a Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Thu, 7 Sep 2023 13:50:03 -0400 Subject: [PATCH 19/20] Reworking TDE/QBV processors Both are now using a content DatabaseClient so they don't have to do an additional eval/invokeFunction. This does involve significant changes to both TdeDocumentFileProcess and DefaultSchemasLoader, but those changes are based on an existing flaw, where a content database name was passed instead of an actual content DatabaseClient. --- .../impl/DefaultSchemasLoader.java | 90 ++++++----------- .../impl/QbvDocumentFileProcessor.java | 72 +++++--------- .../impl/TdeDocumentFileProcessor.java | 99 +++++-------------- .../client/ext/AbstractIntegrationTest.java | 8 ++ .../schemasloader/impl/GenerateQbvTest.java | 6 +- .../schemasloader/impl/LoadRulesetsTest.java | 2 +- .../schemasloader/impl/LoadSchemasTest.java | 8 +- .../impl/ValidateTdeTemplatesTest.java | 3 +- .../qbv-bad-schemas/qbv/bad-authors.sjs | 3 + .../qbv-no-tde-schemas/qbv/books.sjs | 3 + src/test/resources/qbv-schemas/example.sjs | 0 .../resources/qbv-schemas/qbv/authors.sjs | 5 +- .../qbv-schemas/qbv/publications.xqy | 6 +- 13 files changed, 114 insertions(+), 191 deletions(-) delete mode 100644 src/test/resources/qbv-schemas/example.sjs diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java index e821a5f..5893a0d 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/DefaultSchemasLoader.java @@ -23,7 +23,6 @@ import com.marklogic.client.ext.modulesloader.impl.DefaultFileFilter; import com.marklogic.client.ext.schemasloader.SchemasLoader; import com.marklogic.client.io.DocumentMetadataHandle; -import org.springframework.util.StringUtils; import java.util.ArrayList; import java.util.List; @@ -33,30 +32,25 @@ public class DefaultSchemasLoader extends GenericFileLoader implements SchemasLoader { - private DatabaseClient schemasDatabaseClient; - private String tdeValidationDatabase; - protected QbvDocumentFileProcessor qbvDocumentFileProcessor; + private final DatabaseClient schemasDatabaseClient; + private final DatabaseClient contentDatabaseClient; + private final boolean validateTdeTemplates; + private QbvDocumentFileProcessor qbvDocumentFileProcessor; /** - * Simplest constructor for using this class. Just provide a DatabaseClient, and this will use sensible defaults for - * how documents are read and written. Note that the DatabaseClient will not be released after this class is done - * with it, as this class wasn't the one that created it. - * - * @param schemasDatabaseClient + * @param schemasDatabaseClient for loading files into an application's schemas database + * @param contentDatabaseClient for validating TDEs and generating QBVs */ - public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient) { - this(schemasDatabaseClient, null); + public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, DatabaseClient contentDatabaseClient) { + this(schemasDatabaseClient, contentDatabaseClient, true); } /** - * If you want to validate TDE templates before they're loaded, you need to provide a second DatabaseClient that - * connects to the content database associated with the schemas database that schemas will be loaded into. This is - * because the "tde.validate" function must run against the content database. - * - * @param schemasDatabaseClient - * @param tdeValidationDatabase + * @param schemasDatabaseClient for loading files into an application's schemas database + * @param contentDatabaseClient for validating TDEs and generating QBVs + * @param validateTdeTemplates if false, TDEs will not be validated nor loaded via tde.templateBatchInsert */ - public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, String tdeValidationDatabase) { + public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, DatabaseClient contentDatabaseClient, boolean validateTdeTemplates) { super(((Supplier) () -> { RestBatchWriter writer = new RestBatchWriter(schemasDatabaseClient); // Default this to 1, as it's not typical to have such a large number of schemas to load that multiple threads @@ -67,31 +61,20 @@ public DefaultSchemasLoader(DatabaseClient schemasDatabaseClient, String tdeVali }).get()); this.schemasDatabaseClient = schemasDatabaseClient; - this.tdeValidationDatabase = tdeValidationDatabase; - initializeDefaultSchemasLoader(); - } + this.contentDatabaseClient = contentDatabaseClient; + this.validateTdeTemplates = validateTdeTemplates; - /** - * Assumes that the BatchWriter has already been initialized. - * - * @param batchWriter - * @deprecated Since 4.6.0; this class needs a DatabaseClient for the schemas database passed to it so that it can - * pass that client on to specific file processors. - */ - @Deprecated - public DefaultSchemasLoader(BatchWriter batchWriter) { - super(batchWriter); - initializeDefaultSchemasLoader(); - } + if (this.contentDatabaseClient != null) { + this.qbvDocumentFileProcessor = new QbvDocumentFileProcessor(this.schemasDatabaseClient, this.contentDatabaseClient); + addDocumentFileProcessor(this.qbvDocumentFileProcessor); + } + + if (this.validateTdeTemplates && this.contentDatabaseClient != null) { + addDocumentFileProcessor(new TdeDocumentFileProcessor(this.contentDatabaseClient)); + } else { + addDocumentFileProcessor(new TdeDocumentFileProcessor(null)); + } - /** - * Adds the DocumentFileProcessors and FileFilters specific to loading schemas, which will then be used to construct - * a DocumentFileReader by the parent class. - */ - protected void initializeDefaultSchemasLoader() { - this.qbvDocumentFileProcessor = new QbvDocumentFileProcessor(this.schemasDatabaseClient, this.tdeValidationDatabase); - addDocumentFileProcessor(new TdeDocumentFileProcessor(this.schemasDatabaseClient, this.tdeValidationDatabase)); - addDocumentFileProcessor(this.qbvDocumentFileProcessor); addFileFilter(new DefaultFileFilter()); } @@ -107,7 +90,7 @@ public List loadSchemas(String... paths) { final List documentFiles = super.getDocumentFiles(paths); if (!documentFiles.isEmpty()) { - if (TdeUtil.templateBatchInsertSupported(schemasDatabaseClient) && StringUtils.hasText(tdeValidationDatabase)) { + if (this.validateTdeTemplates && TdeUtil.templateBatchInsertSupported(schemasDatabaseClient) && contentDatabaseClient != null) { SchemaFiles schemaFiles = readSchemaFiles(documentFiles); if (!schemaFiles.tdeFiles.isEmpty()) { loadTdeTemplatesViaBatchInsert(schemaFiles.tdeFiles); @@ -122,7 +105,10 @@ public List loadSchemas(String... paths) { writeDocumentFiles(documentFiles); } } - this.qbvDocumentFileProcessor.processQbvFiles(); + + if (this.qbvDocumentFileProcessor != null) { + this.qbvDocumentFileProcessor.processQbvFiles(); + } return documentFiles; } @@ -152,11 +138,10 @@ private void loadTdeTemplatesViaBatchInsert(List tdeFiles) { tdeFiles.stream().map(documentFile -> documentFile.getFile().getName()).collect(Collectors.toList())); String query = buildTdeBatchInsertQuery(tdeFiles); - StringBuilder script = new StringBuilder("declareUpdate(); xdmp.invokeFunction(function() {var tde = require('/MarkLogic/tde.xqy');"); + StringBuilder script = new StringBuilder("declareUpdate(); const tde = require('/MarkLogic/tde.xqy'); "); script.append(query); - script.append(format("}, {database: xdmp.database('%s')})", tdeValidationDatabase)); try { - schemasDatabaseClient.newServerEval().javascript(script.toString()).eval().close(); + contentDatabaseClient.newServerEval().javascript(script.toString()).eval().close(); } catch (Exception ex) { throw new RuntimeException("Unable to load and validate TDE templates via tde.templateBatchInsert; " + "cause: " + ex.getMessage() + "; the following script can be run in Query Console against your content " + @@ -216,17 +201,4 @@ public SchemaFiles(List tdeFiles, List nonTdeFiles) this.nonTdeFiles = nonTdeFiles; } } - - public String getTdeValidationDatabase() { - return tdeValidationDatabase; - } - - /** - * @param tdeValidationDatabase - * @deprecated Should be set via the constructor and not modified. - */ - @Deprecated - public void setTdeValidationDatabase(String tdeValidationDatabase) { - this.tdeValidationDatabase = tdeValidationDatabase; - } } diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java index 078ba2b..79b6aef 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/QbvDocumentFileProcessor.java @@ -39,32 +39,24 @@ /** * @since 4.6.0 */ -public class QbvDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor { +class QbvDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor { public static final String QBV_COLLECTION = "http://marklogic.com/xdmp/qbv"; private static final String QBV_XML_PLAN_NAMESPACE = "http://marklogic.com/plan"; private static final String QBV_XML_ROOT_ELEMENT = "query-based-view"; - private static final String JAVASCRIPT_EVAL_TEMPLATE = "declareUpdate(); " + - "xdmp.invokeFunction(function() {'use strict'; const op = require('/MarkLogic/optic'); return %s }, " + - "{database: xdmp.database('%s')})"; - private static final String XQUERY_EVAL_TEMPLATE = "xquery version \"1.0-ml\"; " + - "import module namespace op=\"http://marklogic.com/optic\" at \"/MarkLogic/optic.xqy\"; " + - "xdmp:invoke-function(function() {%s},{xdmp:database('%s')})"; - final private DatabaseClient schemasDatabaseClient; - final private String qbvGeneratorDatabaseName; + final private DatabaseClient contentDatabaseClient; final private List qbvFiles = new ArrayList<>(); - final private XMLDocumentManager docMgr; + final private XMLDocumentManager schemasDocumentManager; /** - * @param schemasDatabaseClient database client for the application's schemas database - * @param qbvGeneratorDatabaseName the database to run a script against for generating a QBV + * @param schemasDatabaseClient used to write the QBV XML document to the application's schemas database + * @param contentDatabaseClient used to generate the QBV based on a user-provided script */ - public QbvDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String qbvGeneratorDatabaseName) { - this.schemasDatabaseClient = schemasDatabaseClient; - this.qbvGeneratorDatabaseName = qbvGeneratorDatabaseName; - this.docMgr = schemasDatabaseClient.newXMLDocumentManager(); + QbvDocumentFileProcessor(DatabaseClient schemasDatabaseClient, DatabaseClient contentDatabaseClient) { + this.schemasDocumentManager = schemasDatabaseClient.newXMLDocumentManager(); + this.contentDatabaseClient = contentDatabaseClient; } @Override @@ -99,29 +91,29 @@ private void processQbvFile(DocumentFile qbvFile) { ServerEvaluationCall call = getServerEvaluationCall(qbvFile); if (call != null) { StringHandle handleString = new StringHandle(); + try { + call.eval(handleString); + } catch (Exception e) { + throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); + } + if (Format.XML.equals(handleString.getFormat())) { + Document xmlDocument; try { - call.eval(handleString); + xmlDocument = new SAXBuilder().build(new StringReader(handleString.get())); } catch (Exception e) { throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); } - if (Format.XML.equals(handleString.getFormat())) { - Document xmlDocument; - try { - xmlDocument = new SAXBuilder().build(new StringReader(handleString.get())); - } catch (Exception e) { - throw new RuntimeException(format("Query-Based View generation failed for file: %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); - } - Element root = xmlDocument.getRootElement(); - if (QBV_XML_ROOT_ELEMENT.equals(root.getName()) & (root.getNamespace() != null && root.getNamespace().getURI().equals(QBV_XML_PLAN_NAMESPACE))) { - qbvFile.getDocumentMetadata().getCollections().add(QBV_COLLECTION); - String uri = qbvFile.getUri() + ".xml"; - docMgr.write(uri, qbvFile.getDocumentMetadata(), new JDOMHandle(xmlDocument)); - } else { - throw new RuntimeException(format("Query-Based view generation failed for file: %s; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get())); - } + Element root = xmlDocument.getRootElement(); + if (QBV_XML_ROOT_ELEMENT.equals(root.getName()) & (root.getNamespace() != null && root.getNamespace().getURI().equals(QBV_XML_PLAN_NAMESPACE))) { + qbvFile.getDocumentMetadata().getCollections().add(QBV_COLLECTION); + String uri = qbvFile.getUri() + ".xml"; + schemasDocumentManager.write(uri, qbvFile.getDocumentMetadata(), new JDOMHandle(xmlDocument)); } else { - throw new RuntimeException(format("Query-Based View generation failed for file: %s; ensure your Optic script includes a call to generate a view; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get())); + throw new RuntimeException(format("Query-Based view generation failed for file: %s; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get())); } + } else { + throw new RuntimeException(format("Query-Based View generation failed for file: %s; ensure your Optic script includes a call to generate a view; received unexpected response from server: %s", qbvFile.getFile().getAbsolutePath(), handleString.get())); + } } } @@ -133,17 +125,7 @@ private ServerEvaluationCall getServerEvaluationCall(DocumentFile qbvFile) { throw new RuntimeException(format("Unable to generate Query-Based View; could not read from file %s; cause: %s", qbvFile.getFile().getAbsolutePath(), e.getMessage())); } return FilenameUtil.isXqueryFile(qbvFile.getFile().getName()) ? - buildXqueryCall(fileContent) : - buildJavascriptCall(fileContent); - } - - private ServerEvaluationCall buildJavascriptCall(String fileContent) { - String script = format(JAVASCRIPT_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName); - return schemasDatabaseClient.newServerEval().javascript(script); - } - - private ServerEvaluationCall buildXqueryCall(String fileContent) { - String script = format(XQUERY_EVAL_TEMPLATE, fileContent, qbvGeneratorDatabaseName); - return schemasDatabaseClient.newServerEval().xquery(script); + contentDatabaseClient.newServerEval().xquery(fileContent) : + contentDatabaseClient.newServerEval().javascript(fileContent); } } diff --git a/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java b/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java index 33095f2..b337706 100644 --- a/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java +++ b/src/main/java/com/marklogic/client/ext/schemasloader/impl/TdeDocumentFileProcessor.java @@ -30,27 +30,17 @@ import java.io.File; import java.io.IOException; -public class TdeDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor { +class TdeDocumentFileProcessor extends LoggingObject implements DocumentFileProcessor { - private DatabaseClient schemasDatabaseClient; - private String tdeValidationDatabase; + private final DatabaseClient contentDatabaseClient; private Boolean templateBatchInsertSupported; /** - * Use this constructor when you don't want any TDE validation to occur. + * @param contentDatabaseClient the database to run a script against for validating a TDE. If null, TDE validation + * will not be performed. */ - public TdeDocumentFileProcessor() { - } - - /** - * Use this constructor when you want to validate a TDE template before writing it to MarkLogic. - * - * @param schemasDatabaseClient database client for the application's schemas database - * @param tdeValidationDatabase the database to run a script against for validating a TDE - */ - public TdeDocumentFileProcessor(DatabaseClient schemasDatabaseClient, String tdeValidationDatabase) { - this.schemasDatabaseClient = schemasDatabaseClient; - this.tdeValidationDatabase = tdeValidationDatabase; + TdeDocumentFileProcessor(DatabaseClient contentDatabaseClient) { + this.contentDatabaseClient = contentDatabaseClient; } @Override @@ -79,22 +69,25 @@ public DocumentFile processDocumentFile(DocumentFile documentFile) { } private boolean isTemplateBatchInsertSupported() { - if (this.templateBatchInsertSupported == null) { + if (this.templateBatchInsertSupported == null && contentDatabaseClient != null) { // Memoize this to avoid repeated calls; the result will always be the same unless the databaseClient is // modified, in which case templateBatchInsertSupported is set to null - this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(schemasDatabaseClient); + this.templateBatchInsertSupported = TdeUtil.templateBatchInsertSupported(contentDatabaseClient); } return this.templateBatchInsertSupported; } - protected void validateTdeTemplate(DocumentFile documentFile) { + /** + * This mechanism is only needed on older versions of MarkLogic that do not support tde.templateBatchInsert. + * + * @param documentFile + */ + private void validateTdeTemplate(DocumentFile documentFile) { final File file = documentFile.getFile(); - if (schemasDatabaseClient == null) { - logger.info("No DatabaseClient provided for TDE validation, so will not validate TDE templates"); - } else if (tdeValidationDatabase == null) { - logger.info("No TDE validation database specified, so will not validate TDE templates"); + if (contentDatabaseClient == null) { + logger.info("No content database client provided, so will not validate TDE templates."); } else if (isTemplateBatchInsertSupported()) { - logger.debug("Not performing TDE validation; it will be performed automatically via tde.templateBatchInsert"); + logger.debug("Not performing TDE validation; it will be performed automatically via tde.templateBatchInsert."); } else { String fileContent = null; try { @@ -124,61 +117,17 @@ protected void validateTdeTemplate(DocumentFile documentFile) { } } - protected ServerEvaluationCall buildJavascriptCall(DocumentFile documentFile, String fileContent) { - StringBuilder script = new StringBuilder("var template; xdmp.invokeFunction(function() {var tde = require('/MarkLogic/tde.xqy');"); - script.append(format( - "\nreturn tde.validate([xdmp.toJSON(template)], ['%s'])}, {database: xdmp.database('%s')})", - documentFile.getUri(), tdeValidationDatabase - )); - return schemasDatabaseClient.newServerEval().javascript(script.toString()) + private ServerEvaluationCall buildJavascriptCall(DocumentFile documentFile, String fileContent) { + StringBuilder script = new StringBuilder("const tde = require('/MarkLogic/tde.xqy'); var template; "); + script.append(format("\ntde.validate([xdmp.toJSON(template)], ['%s'])", documentFile.getUri())); + return contentDatabaseClient.newServerEval().javascript(script.toString()) .addVariable("template", new StringHandle(fileContent).withFormat(Format.JSON)); } - protected ServerEvaluationCall buildXqueryCall(DocumentFile documentFile, String fileContent) { + private ServerEvaluationCall buildXqueryCall(DocumentFile documentFile, String fileContent) { StringBuilder script = new StringBuilder("import module namespace tde = 'http://marklogic.com/xdmp/tde' at '/MarkLogic/tde.xqy'; "); script.append("\ndeclare variable $template external; "); - script.append("\nxdmp:invoke-function(function() { "); - script.append(format( - "\ntde:validate($template, '%s')}, {xdmp:database('%s')})", - documentFile.getUri(), tdeValidationDatabase - )); - return schemasDatabaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML)); - } - - /** - * @return - * @deprecated since 4.6.0, will be removed in 5.0.0 - */ - @Deprecated - public DatabaseClient getDatabaseClient() { - return schemasDatabaseClient; - } - - /** - * @param databaseClient - * @deprecated since 4.6.0, will be removed in 5.0.0 - */ - @Deprecated - public void setDatabaseClient(DatabaseClient databaseClient) { - this.schemasDatabaseClient = databaseClient; - this.templateBatchInsertSupported = null; - } - - /** - * @return - * @deprecated since 4.6.0, will be removed in 5.0.0 - */ - @Deprecated - public String getTdeValidationDatabase() { - return tdeValidationDatabase; - } - - /** - * @param tdeValidationDatabase - * @deprecated since 4.6.0, will be removed in 5.0.0 - */ - @Deprecated - public void setTdeValidationDatabase(String tdeValidationDatabase) { - this.tdeValidationDatabase = tdeValidationDatabase; + script.append(format("\ntde:validate($template, '%s')", documentFile.getUri())); + return contentDatabaseClient.newServerEval().xquery(script.toString()).addVariable("template", new StringHandle(fileContent).withFormat(Format.XML)); } } diff --git a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java index 88ff68e..e1ae276 100644 --- a/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java +++ b/src/test/java/com/marklogic/client/ext/AbstractIntegrationTest.java @@ -58,6 +58,14 @@ public void releaseClientOnTearDown() { } } + protected final DatabaseClient newContentClient() { + String currentDatabase = clientConfig.getDatabase(); + clientConfig.setDatabase(CONTENT_DATABASE); + DatabaseClient client = configuredDatabaseClientFactory.newDatabaseClient(clientConfig); + clientConfig.setDatabase(currentDatabase); + return client; + } + protected DatabaseClient newClient(String database) { String currentDatabase = clientConfig.getDatabase(); clientConfig.setDatabase(database); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java index 5df2aef..497c300 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/GenerateQbvTest.java @@ -19,15 +19,15 @@ public class GenerateQbvTest extends AbstractSchemasTest { @BeforeEach void beforeEach() { - loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); + // As a sanity check, verify that QBVs get generated when we tell DSL not to validate TDEs. + loader = new DefaultSchemasLoader(client, newContentClient(), false); } @Test public void test() { Path path = Paths.get("src", "test", "resources", "qbv-schemas"); List files = loader.loadSchemas(path.toString()); - assertEquals(3, files.size(), - "Only the TDE templates should be in this list"); + assertEquals(2, files.size(), "Only the TDE templates should be in this list"); ClientHelper helper = new ClientHelper(client); List tdeUris = helper.getUrisInCollection(TdeUtil.TDE_COLLECTION); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java index 4c7d0aa..8bac823 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadRulesetsTest.java @@ -30,7 +30,7 @@ public class LoadRulesetsTest extends AbstractSchemasTest { @Test public void test() { // Pass in a TDE validation database to ensure that TDE validation doesn't happen for these files - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, newContentClient()); List files = loader.loadSchemas(Paths.get("src", "test", "resources", "rulesets", "collection-test").toString()); assertEquals(2, files.size()); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java index 5c3bce1..0f8397c 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/LoadSchemasTest.java @@ -33,7 +33,7 @@ public class LoadSchemasTest extends AbstractSchemasTest { @Test public void test() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(client); + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, null); RestBatchWriter writer = (RestBatchWriter) loader.getBatchWriter(); assertEquals(1, writer.getThreadCount(), "Should default to 1 so that any error from loading a document " + "into a schemas database is immediately thrown to the client"); @@ -55,7 +55,7 @@ public void test() { @Test public void testTemplateBatchInsert() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); + DefaultSchemasLoader loader = new DefaultSchemasLoader(client, newContentClient()); List files = loader.loadSchemas(Paths.get("src", "test", "resources", "good-schemas", "originals").toString()); assertEquals(2, files.size()); @@ -80,7 +80,7 @@ public void testTemplateBatchInsert() { @Test public void invalidClientAndNoFilesToLoad() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(newClient("invalid-database-doesnt-exist")); + DefaultSchemasLoader loader = new DefaultSchemasLoader(newClient("invalid-database-doesnt-exist"), null); List files = loader.loadSchemas(Paths.get("src", "test", "resources", "no-schemas").toString()); assertEquals(0, files.size(), "When there aren't any files to load, then no error should be thrown when the client is invalid (which in " + @@ -89,7 +89,7 @@ public void invalidClientAndNoFilesToLoad() { @Test public void invalidClientWithFilesToLoad() { - DefaultSchemasLoader loader = new DefaultSchemasLoader(newClient("invalid-database-doesnt-exist")); + DefaultSchemasLoader loader = new DefaultSchemasLoader(newClient("invalid-database-doesnt-exist"), null); FailedRequestException ex = assertThrows(FailedRequestException.class, () -> loader.loadSchemas(Paths.get("src", "test", "resources", "good-schemas").toString())); diff --git a/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java b/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java index d5fce05..575af8e 100644 --- a/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java +++ b/src/test/java/com/marklogic/client/ext/schemasloader/impl/ValidateTdeTemplatesTest.java @@ -36,8 +36,7 @@ public class ValidateTdeTemplatesTest extends AbstractSchemasTest { @BeforeEach public void setup() { super.setup(); - // Assumes that Documents points to Schemas as its schemas database - loader = new DefaultSchemasLoader(client, CONTENT_DATABASE); + loader = new DefaultSchemasLoader(client, newContentClient()); } @Test diff --git a/src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs b/src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs index 34b98dd..eb0b509 100644 --- a/src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs +++ b/src/test/resources/qbv-bad-schemas/qbv/bad-authors.sjs @@ -1 +1,4 @@ +'use strict'; +const op = require('/MarkLogic/optic'); + op.fromView('Medical', 'Authors'); diff --git a/src/test/resources/qbv-no-tde-schemas/qbv/books.sjs b/src/test/resources/qbv-no-tde-schemas/qbv/books.sjs index f6cadea..8240e0f 100644 --- a/src/test/resources/qbv-no-tde-schemas/qbv/books.sjs +++ b/src/test/resources/qbv-no-tde-schemas/qbv/books.sjs @@ -1 +1,4 @@ +'use strict'; +const op = require('/MarkLogic/optic'); + op.fromView('Medical', 'Books').generateView('alternate', 'books'); diff --git a/src/test/resources/qbv-schemas/example.sjs b/src/test/resources/qbv-schemas/example.sjs deleted file mode 100644 index e69de29..0000000 diff --git a/src/test/resources/qbv-schemas/qbv/authors.sjs b/src/test/resources/qbv-schemas/qbv/authors.sjs index 8c32416..7806fae 100644 --- a/src/test/resources/qbv-schemas/qbv/authors.sjs +++ b/src/test/resources/qbv-schemas/qbv/authors.sjs @@ -1 +1,4 @@ -op.fromView('Medical', 'Authors').generateView('alternate', 'authors'); \ No newline at end of file +'use strict'; +const op = require('/MarkLogic/optic'); + +op.fromView('Medical', 'Authors').generateView('alternate', 'authors') diff --git a/src/test/resources/qbv-schemas/qbv/publications.xqy b/src/test/resources/qbv-schemas/qbv/publications.xqy index 037803f..e9f99e2 100644 --- a/src/test/resources/qbv-schemas/qbv/publications.xqy +++ b/src/test/resources/qbv-schemas/qbv/publications.xqy @@ -1,2 +1,6 @@ +xquery version "1.0-ml"; + +import module namespace op="http://marklogic.com/optic" at "/MarkLogic/optic.xqy"; + op:from-view("Medical", "Publications") - => op:generate-view("alternate", "publications") \ No newline at end of file + => op:generate-view("alternate", "publications") From 6622cfb6143c1a98015be4123e30e1c71da9d3cc Mon Sep 17 00:00:00 2001 From: Rob Rudin Date: Wed, 6 Sep 2023 11:31:20 -0400 Subject: [PATCH 20/20] Bumped to 4.6.0 Also updated pom.xml --- build.gradle | 4 ++-- pom.xml | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/build.gradle b/build.gradle index 6dc5716..1b80e34 100644 --- a/build.gradle +++ b/build.gradle @@ -14,7 +14,7 @@ plugins { } group = "com.marklogic" -version = "4.6-SNAPSHOT" +version = "4.6.0" java { sourceCompatibility = 1.8 @@ -30,7 +30,7 @@ repositories { } dependencies { - api 'com.marklogic:marklogic-client-api:6.3-SNAPSHOT' + api 'com.marklogic:marklogic-client-api:6.3.0' api 'com.marklogic:marklogic-xcc:11.0.3' api 'org.springframework:spring-context:5.3.29' diff --git a/pom.xml b/pom.xml index d1e3c42..c3c1e5e 100644 --- a/pom.xml +++ b/pom.xml @@ -12,7 +12,7 @@ It is not intended to be used to build this project. 4.0.0 com.marklogic ml-javaclient-util - 4.5.1 + 4.6.0 com.marklogic:ml-javaclient-util Library that adds functionality on top of the MarkLogic Java Client https://github.com/marklogic/ml-javaclient-util @@ -40,19 +40,19 @@ It is not intended to be used to build this project. com.marklogic marklogic-client-api - 6.2.0 + 6.3.0 compile com.marklogic marklogic-xcc - 11.0.2 + 11.0.3 compile org.springframework spring-context - 5.3.27 + 5.3.29 compile @@ -64,7 +64,7 @@ It is not intended to be used to build this project. com.fasterxml.jackson.core jackson-databind - 2.14.1 + 2.15.2 runtime