diff --git a/tools/content-generator/src/intTest/java/org/projectnessie/tools/contentgenerator/ITCopyContent.java b/tools/content-generator/src/intTest/java/org/projectnessie/tools/contentgenerator/ITCopyContent.java new file mode 100644 index 00000000000..7dd65689270 --- /dev/null +++ b/tools/content-generator/src/intTest/java/org/projectnessie/tools/contentgenerator/ITCopyContent.java @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2020 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.tools.contentgenerator; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.projectnessie.tools.contentgenerator.RunContentGenerator.runGeneratorCmd; + +import java.util.HashSet; +import java.util.Set; +import java.util.stream.Collectors; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.projectnessie.client.api.NessieApiV2; +import org.projectnessie.model.Branch; +import org.projectnessie.model.CommitMeta; +import org.projectnessie.model.ContentKey; +import org.projectnessie.model.EntriesResponse; +import org.projectnessie.model.IcebergTable; +import org.projectnessie.model.Operation; +import org.projectnessie.tools.contentgenerator.RunContentGenerator.ProcessResult; + +class ITCopyContent extends AbstractContentGeneratorTest { + + @ParameterizedTest + @ValueSource(strings = {"", "test1"}) + void basicCopyContent(String branchName) throws Exception { + int numCommits = 20; + + try (NessieApiV2 api = buildNessieApi()) { + + Branch branch = api.getDefaultBranch(); + if (!branchName.isEmpty()) { + Branch main = api.getDefaultBranch(); + branch = + (Branch) + api.createReference() + .sourceRefName(main.getName()) + .reference(Branch.of(branchName, main.getHash())) + .create(); + } + + ContentKey src = ContentKey.of("test", "key"); + branch = + api.commitMultipleOperations() + .branch(branch) + .commitMeta(CommitMeta.fromMessage("test initial")) + .operation(Operation.Put.of(src.getNamespace().toContentKey(), src.getNamespace())) + .operation(Operation.Put.of(src, IcebergTable.of("loc", 1, 2, 3, 4))) + .commit(); + + ProcessResult proc = + runGeneratorCmd( + "copy", + "--author", + "Test Author ABC", + "-n", + Integer.toString(numCommits), + "-u", + NESSIE_API_URI, + "--ref", + branch.getName(), + "--from", + "test", + "--from", + "key", + "--to", + "test", + "--to", + "copy-%d"); + + assertThat(proc).extracting(ProcessResult::getExitCode).isEqualTo(0); + + Set expectedKeys = new HashSet<>(); + expectedKeys.add(src.getNamespace().toCanonicalString()); + expectedKeys.add(src.toCanonicalString()); + expectedKeys.add(CONTENT_KEY.getNamespace().toCanonicalString()); + for (int i = 0; i < numCommits; i++) { + expectedKeys.add("test.copy-" + i); + } + + assertThat( + api.getEntries().refName(branch.getName()).stream() + .map(EntriesResponse.Entry::getName) + .map(ContentKey::toCanonicalString) + .collect(Collectors.toSet())) + .containsExactlyInAnyOrderElementsOf(expectedKeys); + } + } +} diff --git a/tools/content-generator/src/main/java/org/projectnessie/tools/contentgenerator/cli/ContentGenerator.java b/tools/content-generator/src/main/java/org/projectnessie/tools/contentgenerator/cli/ContentGenerator.java index 72630fad4ba..a1457e63e42 100644 --- a/tools/content-generator/src/main/java/org/projectnessie/tools/contentgenerator/cli/ContentGenerator.java +++ b/tools/content-generator/src/main/java/org/projectnessie/tools/contentgenerator/cli/ContentGenerator.java @@ -38,6 +38,7 @@ ReadContent.class, RefreshContent.class, DeleteContent.class, + CopyContent.class, CreateMissingNamespaces.class, CommandLine.HelpCommand.class }) diff --git a/tools/content-generator/src/main/java/org/projectnessie/tools/contentgenerator/cli/CopyContent.java b/tools/content-generator/src/main/java/org/projectnessie/tools/contentgenerator/cli/CopyContent.java new file mode 100644 index 00000000000..b50570fbe1a --- /dev/null +++ b/tools/content-generator/src/main/java/org/projectnessie/tools/contentgenerator/cli/CopyContent.java @@ -0,0 +1,94 @@ +/* + * Copyright (C) 2020 Dremio + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.projectnessie.tools.contentgenerator.cli; + +import jakarta.validation.constraints.Min; +import java.util.ArrayList; +import java.util.List; +import org.projectnessie.client.api.NessieApiV2; +import org.projectnessie.error.BaseNessieClientServerException; +import org.projectnessie.model.Content; +import org.projectnessie.model.ContentKey; +import org.projectnessie.model.Operation.Put; +import org.projectnessie.model.Reference; +import picocli.CommandLine.Command; +import picocli.CommandLine.Option; + +@Command(name = "copy", mixinStandardHelpOptions = true, description = "Repeatedly copy content") +public class CopyContent extends CommittingCommand { + + @Option( + names = {"-r", "--ref", "--branch"}, + description = "Branch name for making changes (defaults to the default branch if not set).") + private String ref; + + @Min(value = 1, message = "At least one copy is to be made.") + @Option( + names = {"-n", "--num-copies"}, + required = true, + defaultValue = "10000", + description = "Number of copies to make.") + private int numCopies; + + @Option( + names = {"-k", "--from"}, + description = "Content key to copy", + required = true) + private List fromKey; + + @Option( + names = {"-t", "--to"}, + description = + "Content key pattern for the copies. Each element of the pattern may contain one instance of '%d'," + + " which will be replaced with the copy counter.", + required = true) + private List keyPattern; + + @Override + public void execute() throws BaseNessieClientServerException { + try (NessieApiV2 api = createNessieApiInstance()) { + Reference branch; + if (ref == null) { + branch = api.getDefaultBranch(); + } else { + branch = api.getReference().refName(ref).get(); + } + + ContentKey key = ContentKey.of(fromKey); + Content content = api.getContent().reference(branch).getSingle(key).getContent(); + + for (int i = 0; i < numCopies; i++) { + List to = new ArrayList<>(keyPattern.size()); + for (String pattern : keyPattern) { + to.add(String.format(pattern, i)); + } + + ContentKey copyKey = ContentKey.of(to); + branch = + api.commitMultipleOperations() + .branchName(branch.getName()) + .hash(branch.getHash()) + .commitMeta(commitMetaFromMessage("Copy " + i)) + .operation(Put.of(copyKey, content.withId(null))) + .commit(); + + spec.commandLine() + .getOut() + .printf("Created %s at %s%n", copyKey.toCanonicalString(), branch.getHash()); + } + } + } +}