From 01552bc295f301f39d134a56166ceb3589736df8 Mon Sep 17 00:00:00 2001 From: marikaris Date: Mon, 19 Feb 2024 09:30:27 +0100 Subject: [PATCH] Revert "Feat: add views on tables" --- .../armadillo/audit/AuditEventPublisher.java | 1 - .../armadillo/controller/DataController.java | 121 ++----------- .../controller/LinkedObjectRequestBody.java | 16 -- .../controller/StorageController.java | 55 ------ .../exceptions/UnknownVariableException.java | 14 -- .../armadillo/storage/ArmadilloLinkFile.java | 124 -------------- .../storage/ArmadilloStorageService.java | 47 +---- .../storage/LocalStorageService.java | 11 -- .../armadillo/storage/ParquetUtils.java | 15 +- .../armadillo/storage/StorageService.java | 4 - .../controller/DataControllerTest.java | 160 +----------------- .../controller/StorageControllerTest.java | 31 ---- .../storage/ArmadilloLinkFileTest.java | 111 ------------ .../storage/ArmadilloStorageServiceTest.java | 80 +-------- .../storage/LocalStorageServiceTest.java | 31 ---- .../armadillo/storage/ParquetUtilsTest.java | 6 - scripts/release/release-test.R | 54 +----- 17 files changed, 39 insertions(+), 842 deletions(-) delete mode 100644 armadillo/src/main/java/org/molgenis/armadillo/controller/LinkedObjectRequestBody.java delete mode 100644 armadillo/src/main/java/org/molgenis/armadillo/exceptions/UnknownVariableException.java delete mode 100644 armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloLinkFile.java delete mode 100644 armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloLinkFileTest.java diff --git a/armadillo/src/main/java/org/molgenis/armadillo/audit/AuditEventPublisher.java b/armadillo/src/main/java/org/molgenis/armadillo/audit/AuditEventPublisher.java index ab812890f..1ed5b89cc 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/audit/AuditEventPublisher.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/audit/AuditEventPublisher.java @@ -69,7 +69,6 @@ public class AuditEventPublisher implements ApplicationEventPublisherAware { public static final String LIST_OBJECTS = "LIST_OBJECTS"; public static final String UPLOAD_OBJECT = "UPLOAD_OBJECT"; public static final String COPY_OBJECT = "COPY_OBJECT"; - public static final String CREATE_LINKED_OBJECT = "CREATE_LINKED_OBJECT"; public static final String MOVE_OBJECT = "MOVE_OBJECT"; public static final String GET_OBJECT = "GET_OBJECT"; public static final String PREVIEW_OBJECT = "PREVIEW_OBJECT"; diff --git a/armadillo/src/main/java/org/molgenis/armadillo/controller/DataController.java b/armadillo/src/main/java/org/molgenis/armadillo/controller/DataController.java index e08a3e429..3dee2888d 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/controller/DataController.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/controller/DataController.java @@ -6,9 +6,6 @@ import static java.util.concurrent.CompletableFuture.completedFuture; import static org.molgenis.armadillo.audit.AuditEventPublisher.*; import static org.molgenis.armadillo.controller.ArmadilloUtils.getLastCommandLocation; -import static org.molgenis.armadillo.security.RunAs.runAsSystem; -import static org.molgenis.armadillo.storage.ArmadilloStorageService.LINK_FILE; -import static org.molgenis.armadillo.storage.ArmadilloStorageService.PARQUET; import static org.obiba.datashield.core.DSMethodType.AGGREGATE; import static org.obiba.datashield.core.DSMethodType.ASSIGN; import static org.springframework.http.HttpStatus.*; @@ -18,15 +15,12 @@ import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.Parameter; -import io.swagger.v3.oas.annotations.media.Content; import io.swagger.v3.oas.annotations.responses.ApiResponse; -import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.security.SecurityRequirement; import io.swagger.v3.oas.annotations.tags.Tag; import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.Pattern; -import java.io.InputStream; import java.security.Principal; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -35,12 +29,9 @@ import org.molgenis.armadillo.command.ArmadilloCommandDTO; import org.molgenis.armadillo.command.Commands; import org.molgenis.armadillo.exceptions.ExpressionException; -import org.molgenis.armadillo.exceptions.UnknownObjectException; -import org.molgenis.armadillo.exceptions.UnknownVariableException; import org.molgenis.armadillo.model.Workspace; import org.molgenis.armadillo.service.DSEnvironmentCache; import org.molgenis.armadillo.service.ExpressionRewriter; -import org.molgenis.armadillo.storage.ArmadilloLinkFile; import org.molgenis.armadillo.storage.ArmadilloStorageService; import org.molgenis.r.RServerResult; import org.molgenis.r.model.RPackage; @@ -50,7 +41,6 @@ import org.springframework.security.access.prepost.PreAuthorize; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.*; -import org.springframework.web.server.ResponseStatusException; @Tag(name = "DataSHIELD", description = "Core API that interacts with the DataSHIELD environments") @SecurityRequirement(name = "bearerAuth") @@ -135,18 +125,6 @@ public ResponseEntity tableExists( summary = "Load table", description = "Load a table", security = {@SecurityRequirement(name = "jwt")}) - @ApiResponses( - value = { - @ApiResponse(responseCode = "200", description = "Object loaded successfully"), - @ApiResponse( - responseCode = "404", - description = "Unknown project or object", - content = @Content(mediaType = "application/json")), - @ApiResponse( - responseCode = "401", - description = "Unauthorized", - content = @Content(mediaType = "application/json")) - }) @PostMapping(value = "/load-table") public CompletableFuture> loadTable( Principal principal, @@ -154,24 +132,32 @@ public CompletableFuture> loadTable( @Valid @Pattern(regexp = TABLE_RESOURCE_REGEX) @RequestParam String table, @Valid @Pattern(regexp = SYMBOL_CSV_RE) @RequestParam(required = false) String variables, @RequestParam(defaultValue = "false") boolean async) { + java.util.regex.Pattern tableResourcePattern = java.util.regex.Pattern.compile(TABLE_RESOURCE_REGEX); HashMap data = getMatchedData(tableResourcePattern, table, TABLE); data.put(SYMBOL, symbol); - String project = (String) data.get(PROJECT); - String objectName = String.format(PATH_FORMAT, data.get(FOLDER), data.get(TABLE)); - if (storage.hasObject(project, objectName + LINK_FILE)) { - return loadTableFromLinkFile(project, objectName, variables, principal, data, symbol, async); - } else if (storage.hasObject(project, objectName + PARQUET)) { - var variableList = getVariableList(variables); - return doLoadTable(symbol, table, variableList, principal, data, async); - } else { + if (!storage.tableExists( + (String) data.get(PROJECT), + String.format(PATH_FORMAT, data.get(FOLDER), data.get(TABLE)))) { data = new HashMap<>(data); data.put(MESSAGE, "Table not found"); auditEventPublisher.audit(principal, LOAD_TABLE_FAILURE, data); - throw new ResponseStatusException( - NOT_FOUND, format("Project '%s' has no object '%s'", project, objectName)); + return completedFuture(notFound().build()); } + var variableList = + Optional.ofNullable(variables).map(it -> it.split(",")).stream() + .flatMap(Arrays::stream) + .map(String::trim) + .toList(); + var result = + auditEventPublisher.audit( + commands.loadTable(symbol, table, variableList), principal, LOAD_TABLE, data); + return async + ? completedFuture(created(getLastCommandLocation()).body(null)) + : result + .thenApply(ResponseEntity::ok) + .exceptionally(t -> status(INTERNAL_SERVER_ERROR).build()); } @Operation( @@ -474,75 +460,4 @@ HashMap getMatchedData( groups.put(resource, matcher.group(3)); return groups; } - - public List getVariableList(String variables) { - return Optional.ofNullable(variables).map(it -> it.split(",")).stream() - .flatMap(Arrays::stream) - .map(String::trim) - .toList(); - } - - private CompletableFuture> doLoadTable( - String symbol, - String table, - List variableList, - Principal principal, - Map data, - Boolean async) { - var result = - auditEventPublisher.audit( - commands.loadTable(symbol, table, variableList), principal, LOAD_TABLE, data); - return async - ? completedFuture(created(getLastCommandLocation()).body(null)) - : result - .thenApply(ResponseEntity::ok) - .exceptionally(t -> status(INTERNAL_SERVER_ERROR).build()); - } - - protected List getLinkedVariables(ArmadilloLinkFile linkFile, String variables) { - List allowedVariables = List.of(linkFile.getVariables().split(",")); - List variableList = getVariableList(variables); - var invalidVariables = - variableList.stream().filter(element -> !allowedVariables.contains(element)).toList(); - if (invalidVariables.size() > 0) { - String invalid = invalidVariables.toString(); - throw new UnknownVariableException(linkFile.getProject(), linkFile.getLinkObject(), invalid); - } - return variableList.size() == 0 - ? allowedVariables - : variableList.stream().filter(allowedVariables::contains).toList(); - } - - private CompletableFuture> loadTableFromLinkFile( - String project, - String objectName, - String variables, - Principal principal, - HashMap data, - String symbol, - Boolean async) { - InputStream armadilloLinkFileStream = storage.loadObject(project, objectName + LINK_FILE); - ArmadilloLinkFile linkFile = - storage.createArmadilloLinkFileFromStream(armadilloLinkFileStream, project, objectName); - String sourceProject = linkFile.getSourceProject(); - String sourceObject = linkFile.getSourceObject(); - if (runAsSystem(() -> storage.hasObject(sourceProject, sourceObject + PARQUET))) { - List variableList = getLinkedVariables(linkFile, variables); - HashMap finalData = data; - return runAsSystem( - () -> - doLoadTable( - symbol, - sourceProject + "/" + sourceObject, - variableList, - principal, - finalData, - async)); - } else { - data = new HashMap<>(data); - data.put(MESSAGE, "Object not found"); - auditEventPublisher.audit(principal, LOAD_TABLE_FAILURE, data); - throw new UnknownObjectException(sourceProject, sourceObject); - } - } } diff --git a/armadillo/src/main/java/org/molgenis/armadillo/controller/LinkedObjectRequestBody.java b/armadillo/src/main/java/org/molgenis/armadillo/controller/LinkedObjectRequestBody.java deleted file mode 100644 index 91ea26f4b..000000000 --- a/armadillo/src/main/java/org/molgenis/armadillo/controller/LinkedObjectRequestBody.java +++ /dev/null @@ -1,16 +0,0 @@ -package org.molgenis.armadillo.controller; - -import static org.molgenis.armadillo.controller.DataController.TABLE_RESOURCE_REGEX; - -import jakarta.validation.Valid; -import jakarta.validation.constraints.NotEmpty; -import jakarta.validation.constraints.NotNull; -import jakarta.validation.constraints.Pattern; -import org.springframework.web.bind.annotation.RequestParam; - -public record LinkedObjectRequestBody( - @Valid @Pattern(regexp = TABLE_RESOURCE_REGEX) @RequestParam(required = true) - String sourceObjectName, - @NotNull @NotEmpty String sourceProject, - @NotNull @NotEmpty String linkedObject, - @RequestParam(required = false) String variables) {} diff --git a/armadillo/src/main/java/org/molgenis/armadillo/controller/StorageController.java b/armadillo/src/main/java/org/molgenis/armadillo/controller/StorageController.java index 64f58c031..aea0277e7 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/controller/StorageController.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/controller/StorageController.java @@ -1,8 +1,6 @@ package org.molgenis.armadillo.controller; -import static org.apache.logging.log4j.util.Strings.concat; import static org.molgenis.armadillo.audit.AuditEventPublisher.*; -import static org.molgenis.armadillo.storage.ArmadilloStorageService.LINK_FILE; import static org.springframework.http.HttpStatus.NO_CONTENT; import static org.springframework.http.HttpStatus.OK; import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE; @@ -24,10 +22,8 @@ import jakarta.validation.constraints.NotEmpty; import java.io.IOException; import java.security.Principal; -import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.Optional; import org.molgenis.armadillo.audit.AuditEventPublisher; import org.molgenis.armadillo.exceptions.FileProcessingException; import org.molgenis.armadillo.exceptions.UnknownObjectException; @@ -151,57 +147,6 @@ public void copyObject( Map.of(PROJECT, project, "from", object, "to", requestBody.name())); } - @Operation( - summary = "Create a view from an existing table in another project", - description = - "The view you're creating will be a symbolic link to selected variables of an existing table. It will" - + "look and respond like a table, but it will not take up duplicated resources") - @ApiResponses( - value = { - @ApiResponse(responseCode = "204", description = "Link successfully created"), - @ApiResponse(responseCode = "404", description = "Unknown project or object"), - @ApiResponse(responseCode = "409", description = "Object already exists"), - @ApiResponse(responseCode = "401", description = "Unauthorized") - }) - @PostMapping( - value = "/projects/{project}/objects/link", - consumes = {APPLICATION_JSON_VALUE}) - @ResponseStatus(NO_CONTENT) - public void createLinkedObject( - Principal principal, - @PathVariable String project, - @RequestBody LinkedObjectRequestBody requestBody) { - var variableList = - Optional.ofNullable(requestBody.variables()).map(it -> it.split(",")).stream() - .flatMap(Arrays::stream) - .map(String::trim) - .toList(); - auditor.audit( - () -> { - try { - storage.createLinkedObject( - requestBody.sourceProject(), - requestBody.sourceObjectName(), - requestBody.linkedObject(), - project, - requestBody.variables()); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - principal, - CREATE_LINKED_OBJECT, - Map.of( - PROJECT, - project, - OBJECT, - requestBody.linkedObject() + LINK_FILE, - "source", - concat(concat(project, "/"), requestBody.linkedObject()), - "columns", - variableList)); - } - @Operation( summary = "Move an object within a project", description = diff --git a/armadillo/src/main/java/org/molgenis/armadillo/exceptions/UnknownVariableException.java b/armadillo/src/main/java/org/molgenis/armadillo/exceptions/UnknownVariableException.java deleted file mode 100644 index 5ce5bc89b..000000000 --- a/armadillo/src/main/java/org/molgenis/armadillo/exceptions/UnknownVariableException.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.molgenis.armadillo.exceptions; - -import static java.lang.String.format; -import static org.springframework.http.HttpStatus.NOT_FOUND; - -import org.springframework.web.bind.annotation.ResponseStatus; - -@ResponseStatus(NOT_FOUND) -public class UnknownVariableException extends RuntimeException { - - public UnknownVariableException(String project, String object, String variable) { - super(format("Variables '%s' do not exist in object '%s/%s'", variable, project, object)); - } -} diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloLinkFile.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloLinkFile.java deleted file mode 100644 index c56977c12..000000000 --- a/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloLinkFile.java +++ /dev/null @@ -1,124 +0,0 @@ -package org.molgenis.armadillo.storage; - -import static java.lang.String.format; -import static org.molgenis.armadillo.storage.ArmadilloStorageService.LINK_FILE; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import com.google.gson.JsonParser; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import java.io.InputStreamReader; -import org.molgenis.armadillo.exceptions.StorageException; - -public class ArmadilloLinkFile { - - private final String SOURCE_OBJECT = "sourceObject"; - - private final String SOURCE_PROJECT = "sourceProject"; - - private final String VARIABLES = "variables"; - private final String sourceProject; - private final String sourceObject; - private final String variables; - private final String linkObject; - private final String project; - - public ArmadilloLinkFile( - String sourceProject, - String sourceObject, - String variables, - String linkObject, - String project) { - this.linkObject = linkObject; - this.sourceProject = sourceProject; - this.sourceObject = sourceObject; - this.variables = variables; - this.project = project; - } - - public ArmadilloLinkFile(InputStream armadilloLinkStream, String linkObject, String linkProject) { - this.linkObject = linkObject; - this.project = linkProject; - JsonObject json; - try { - json = loadFromStream(armadilloLinkStream); - } catch (JsonParseException e) { - throw new JsonParseException( - format("Cannot load [%s/%s] because JSON is invalid", project, linkObject)); - } catch (Exception e) { - throw new StorageException( - format("Cannot load [%s/%s] for unknown reason", project, linkObject)); - } - try { - this.sourceObject = json.get(SOURCE_OBJECT).getAsString(); - } catch (NullPointerException e) { - throw new NullPointerException( - format("Source object is missing from [%s/%s]", project, linkObject)); - } - try { - this.sourceProject = json.get(SOURCE_PROJECT).getAsString(); - } catch (NullPointerException e) { - throw new NullPointerException( - format("Source project is missing from [%s/%s]", project, linkObject)); - } - try { - this.variables = json.get(VARIABLES).getAsString(); - } catch (NullPointerException e) { - throw new NullPointerException( - format("Variables are not defined on [%s/%s]", project, linkObject)); - } - } - - public String getSourceProject() { - return this.sourceProject; - } - ; - - public String getSourceObject() { - return this.sourceObject; - } - ; - - public String getVariables() { - return this.variables; - } - ; - - public String getLinkObject() { - return this.linkObject; - } - ; - - public String toString() { - return buildJson().toString(); - } - - public JsonObject buildJson() { - JsonObject json = new JsonObject(); - json.addProperty(SOURCE_OBJECT, sourceObject); - json.addProperty(SOURCE_PROJECT, sourceProject); - json.addProperty(VARIABLES, variables); - return json; - } - - public InputStream toStream() { - return new ByteArrayInputStream(toString().getBytes()); - } - - public String getExtension() { - return LINK_FILE; - } - - public String getFileName() { - return getLinkObject() + LINK_FILE; - } - - public String getProject() { - return this.project; - } - - public JsonObject loadFromStream(InputStream inputStream) { - return JsonParser.parseReader(new InputStreamReader(inputStream)).getAsJsonObject(); - } -} diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloStorageService.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloStorageService.java index d7eaf857a..e51ebc9b2 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloStorageService.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/storage/ArmadilloStorageService.java @@ -4,7 +4,6 @@ import static java.util.Objects.requireNonNull; import static java.util.stream.Collectors.toList; import static org.apache.commons.io.FilenameUtils.removeExtension; -import static org.springframework.http.MediaType.APPLICATION_JSON; import static org.springframework.http.MediaType.APPLICATION_OCTET_STREAM; import java.io.IOException; @@ -17,7 +16,10 @@ import java.util.regex.Pattern; import java.util.stream.Stream; import org.apache.commons.io.FilenameUtils; -import org.molgenis.armadillo.exceptions.*; +import org.molgenis.armadillo.exceptions.DuplicateObjectException; +import org.molgenis.armadillo.exceptions.InvalidProjectNameException; +import org.molgenis.armadillo.exceptions.UnknownObjectException; +import org.molgenis.armadillo.exceptions.UnknownProjectException; import org.molgenis.armadillo.model.Workspace; import org.springframework.http.MediaType; import org.springframework.security.access.prepost.PostFilter; @@ -31,7 +33,6 @@ public class ArmadilloStorageService { public static final String USER_PREFIX = "user-"; public static final String BUCKET_REGEX = "(?=^.{3,63}$)(?!xn--)([a-z0-9][a-z0-9-]*[a-z0-9])"; public static final String PARQUET = ".parquet"; - public static final String LINK_FILE = ".alf"; public static final String RDS = ".rds"; public static final String SYSTEM = "system"; private final StorageService storageService; @@ -83,46 +84,6 @@ public void copyObject(String project, String newObject, String oldObject) { storageService.save(inputStream, SHARED_PREFIX + project, newObject, APPLICATION_OCTET_STREAM); } - @PreAuthorize("hasRole('ROLE_SU')") - public void createLinkedObject( - String sourceProject, - String sourceObject, - String linkName, - String linkProject, - String variables) - throws IOException { - throwIfUnknown(sourceProject, sourceObject + PARQUET); - throwIfUnknown(linkProject); - throwIfDuplicate(linkProject, linkName + LINK_FILE); - // Save information in armadillo link file (alf) - List unavailableVariables = - storageService.getUnavailableVariables( - SHARED_PREFIX + sourceProject, sourceObject, variables); - if (unavailableVariables.size() > 0) { - throw new UnknownVariableException( - sourceProject, sourceObject, unavailableVariables.toString()); - } - ArmadilloLinkFile armadilloLinkFile = - createLinkFileFromSource(sourceProject, sourceObject, variables, linkName, linkProject); - InputStream is = armadilloLinkFile.toStream(); - storageService.save( - is, SHARED_PREFIX + linkProject, armadilloLinkFile.getFileName(), APPLICATION_JSON); - } - - public ArmadilloLinkFile createArmadilloLinkFileFromStream( - InputStream armadilloLinkFileStream, String project, String objectName) { - return new ArmadilloLinkFile(armadilloLinkFileStream, project, objectName); - } - - public ArmadilloLinkFile createLinkFileFromSource( - String sourceProject, - String sourceObject, - String variables, - String linkName, - String linkProject) { - return new ArmadilloLinkFile(sourceProject, sourceObject, variables, linkName, linkProject); - } - @PreAuthorize("hasRole('ROLE_SU')") public void deleteObject(String project, String object) { throwIfUnknown(project, object); diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/LocalStorageService.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/LocalStorageService.java index 7a7415468..4f7f6746a 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/storage/LocalStorageService.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/storage/LocalStorageService.java @@ -2,7 +2,6 @@ import static java.lang.String.format; import static java.util.Collections.emptyList; -import static org.molgenis.armadillo.storage.ArmadilloStorageService.PARQUET; import java.io.*; import java.nio.file.Files; @@ -40,16 +39,6 @@ public LocalStorageService(@Value("${" + ROOT_DIR_PROPERTY + "}") String rootDir LOGGER.info("Using local storage at {}", dir.getAbsolutePath()); } - @Override - public List getUnavailableVariables( - String bucketName, String objectName, String variables) throws IOException { - Path objectPath = getPathIfObjectExists(bucketName, objectName + PARQUET); - List availableColumns = ParquetUtils.getColumns(objectPath); - return Arrays.stream(variables.split(",")) - .filter(variable -> !availableColumns.contains(variable)) - .toList(); - } - @Override public boolean objectExists(String bucketName, String objectName) { Objects.requireNonNull(objectName); diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/ParquetUtils.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/ParquetUtils.java index e316590ef..b7d11f964 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/storage/ParquetUtils.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/storage/ParquetUtils.java @@ -6,9 +6,7 @@ import java.io.IOException; import java.nio.file.Path; import java.util.*; -import java.util.stream.IntStream; import org.apache.parquet.column.page.PageReadStore; -import org.apache.parquet.example.data.Group; import org.apache.parquet.example.data.simple.SimpleGroup; import org.apache.parquet.example.data.simple.convert.GroupRecordConverter; import org.apache.parquet.hadoop.ParquetFileReader; @@ -26,8 +24,7 @@ public static List> previewRecords(Path path, int rowLimit, MessageType schema = reader.getFooter().getFileMetaData().getSchema(); MessageColumnIO columnIO = new ColumnIOFactory().getColumnIO(schema); PageReadStore store = reader.readNextRowGroup(); - RecordReader recordReader = - columnIO.getRecordReader(store, new GroupRecordConverter(schema)); + RecordReader recordReader = columnIO.getRecordReader(store, new GroupRecordConverter(schema)); int fieldSize = schema.getFields().size(); for (int i = 0; i < rowLimit; i++) { SimpleGroup group = (SimpleGroup) recordReader.read(); @@ -45,16 +42,6 @@ public static List> previewRecords(Path path, int rowLimit, return result; } - public static List getColumns(Path path) throws IOException { - LocalInputFile file = new LocalInputFile(path); - try (ParquetFileReader reader = ParquetFileReader.open(file)) { - var schema = reader.getFooter().getFileMetaData().getSchema(); - return IntStream.range(0, schema.getFieldCount()).mapToObj(schema::getFieldName).toList(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - public static Map retrieveDimensions(Path path) throws FileNotFoundException { LocalInputFile file = new LocalInputFile(path); try (ParquetFileReader reader = ParquetFileReader.open(file)) { diff --git a/armadillo/src/main/java/org/molgenis/armadillo/storage/StorageService.java b/armadillo/src/main/java/org/molgenis/armadillo/storage/StorageService.java index 246799f44..8c471999d 100644 --- a/armadillo/src/main/java/org/molgenis/armadillo/storage/StorageService.java +++ b/armadillo/src/main/java/org/molgenis/armadillo/storage/StorageService.java @@ -1,6 +1,5 @@ package org.molgenis.armadillo.storage; -import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; import java.util.List; @@ -10,9 +9,6 @@ public interface StorageService { boolean objectExists(String bucket, String objectName); - List getUnavailableVariables(String bucketName, String objectName, String variables) - throws IOException; - void createBucketIfNotExists(String bucketName); void deleteBucket(String bucketName); diff --git a/armadillo/src/test/java/org/molgenis/armadillo/controller/DataControllerTest.java b/armadillo/src/test/java/org/molgenis/armadillo/controller/DataControllerTest.java index 1d3c5da66..0b181dd9d 100644 --- a/armadillo/src/test/java/org/molgenis/armadillo/controller/DataControllerTest.java +++ b/armadillo/src/test/java/org/molgenis/armadillo/controller/DataControllerTest.java @@ -11,8 +11,6 @@ import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.*; import static org.molgenis.armadillo.controller.DataController.TABLE_RESOURCE_REGEX; -import static org.molgenis.armadillo.storage.ArmadilloStorageService.LINK_FILE; -import static org.molgenis.armadillo.storage.ArmadilloStorageService.PARQUET; import static org.obiba.datashield.core.DSMethodType.AGGREGATE; import static org.obiba.datashield.core.DSMethodType.ASSIGN; import static org.springframework.http.MediaType.*; @@ -20,7 +18,6 @@ import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.*; import com.github.dockerjava.api.DockerClient; -import java.io.InputStream; import java.security.Principal; import java.util.*; import java.util.concurrent.CompletableFuture; @@ -37,11 +34,9 @@ import org.molgenis.armadillo.command.Commands.ArmadilloCommandStatus; import org.molgenis.armadillo.exceptions.ExpressionException; import org.molgenis.armadillo.exceptions.UnknownProfileException; -import org.molgenis.armadillo.exceptions.UnknownVariableException; import org.molgenis.armadillo.model.Workspace; import org.molgenis.armadillo.service.DSEnvironmentCache; import org.molgenis.armadillo.service.ExpressionRewriter; -import org.molgenis.armadillo.storage.ArmadilloLinkFile; import org.molgenis.armadillo.storage.ArmadilloStorageService; import org.molgenis.r.model.RPackage; import org.molgenis.r.rock.RockResult; @@ -675,12 +670,13 @@ void testAssignAsync() throws Exception { @Test @WithMockUser void testLoadTableDoesNotExist() throws Exception { - when(armadilloStorage.hasObject("gecko", "core/core-all.alf")).thenReturn(false); - when(armadilloStorage.hasObject("gecko", "core/core-all.parquet")).thenReturn(false); + when(armadilloStorage.tableExists("gecko", "core/core-all")).thenReturn(false); - mockMvc - .perform(post("/load-table?symbol=D&table=gecko/core/core-all").session(session)) - .andExpect(status().isNotFound()); + var result = + mockMvc + .perform(post("/load-table?symbol=D&table=gecko/core/core-all").session(session)) + .andReturn(); + mockMvc.perform(asyncDispatch(result)).andExpect(status().isNotFound()); auditEventValidator.validateAuditEvent( new AuditEvent( @@ -707,8 +703,7 @@ void testLoadTableDoesNotExist() throws Exception { @Test @WithMockUser void testLoadTable() throws Exception { - when(armadilloStorage.hasObject("project", "folder/table.alf")).thenReturn(false); - when(armadilloStorage.hasObject("project", "folder/table.parquet")).thenReturn(true); + when(armadilloStorage.tableExists("project", "folder/table")).thenReturn(true); when(commands.loadTable("D", "project/folder/table", emptyList())) .thenReturn(completedFuture(null)); @@ -737,149 +732,10 @@ void testLoadTable() throws Exception { "table"))); } - @Test - @WithMockUser - void testLoadTableLinkFile() throws Exception { - ArmadilloLinkFile alfMock = mock(ArmadilloLinkFile.class); - InputStream isMock = mock(InputStream.class); - String project = "project"; - String sourceProject = "source-project"; - String sourceObject = "source/object"; - String linkObject = "folder/table-view"; - String variables = "childId,rowId,age,weight"; - when(armadilloStorage.hasObject(project, linkObject + LINK_FILE)).thenReturn(true); - when(armadilloStorage.loadObject(project, linkObject + LINK_FILE)).thenReturn(isMock); - when(armadilloStorage.createArmadilloLinkFileFromStream(isMock, project, linkObject)) - .thenReturn(alfMock); - when(alfMock.getSourceObject()).thenReturn(sourceObject); - when(alfMock.getSourceProject()).thenReturn(sourceProject); - when(alfMock.getVariables()).thenReturn(variables); - when(armadilloStorage.hasObject(sourceProject, sourceObject + PARQUET)).thenReturn(true); - when(commands.loadTable( - "D", - sourceProject + "/" + sourceObject, - new ArrayList<>(Arrays.asList(variables.split(","))))) - .thenReturn(completedFuture(null)); - mockMvc - .perform( - post("/load-table?symbol=D&table=project/folder/table-view&async=false") - .session(session)) - .andExpect(status().isOk()); - - auditEventValidator.validateAuditEvent( - new AuditEvent( - instant, - "user", - "LOAD_TABLE", - Map.of( - "symbol", - "D", - "sessionId", - sessionId, - "roles", - List.of("ROLE_SU"), - "project", - project, - "folder", - "folder", - "table", - "table-view"))); - } - - @Test - @WithMockUser - void testLoadTableLinkFileWithVars() throws Exception { - ArmadilloLinkFile alfMock = mock(ArmadilloLinkFile.class); - InputStream isMock = mock(InputStream.class); - String project = "project"; - String sourceProject = "source-project"; - String sourceObject = "source/object"; - String linkObject = "folder/table-view"; - String variables = "childId,rowId,age,weight"; - ArrayList selectedVariables = new ArrayList<>(); - selectedVariables.add("age"); - selectedVariables.add("weight"); - when(armadilloStorage.hasObject(project, linkObject + LINK_FILE)).thenReturn(true); - when(armadilloStorage.loadObject(project, linkObject + LINK_FILE)).thenReturn(isMock); - when(armadilloStorage.createArmadilloLinkFileFromStream(isMock, project, linkObject)) - .thenReturn(alfMock); - when(alfMock.getSourceObject()).thenReturn(sourceObject); - when(alfMock.getSourceProject()).thenReturn(sourceProject); - when(alfMock.getVariables()).thenReturn(variables); - when(armadilloStorage.hasObject(sourceProject, sourceObject + PARQUET)).thenReturn(true); - when(commands.loadTable("D", sourceProject + "/" + sourceObject, selectedVariables)) - .thenReturn(completedFuture(null)); - mockMvc - .perform( - post("/load-table?symbol=D&table=project/folder/table-view&async=false&variables=age,weight") - .session(session)) - .andExpect(status().isOk()); - - auditEventValidator.validateAuditEvent( - new AuditEvent( - instant, - "user", - "LOAD_TABLE", - Map.of( - "symbol", - "D", - "sessionId", - sessionId, - "roles", - List.of("ROLE_SU"), - "project", - project, - "folder", - "folder", - "table", - "table-view"))); - } - - @Test - void testGetLinkedVariables() { - DataController dataController = - new DataController( - commands, armadilloStorage, auditEventPublisher, expressionRewriter, environments); - ArmadilloLinkFile alfMock = mock(ArmadilloLinkFile.class); - String selectedVariables = "my,variable,list"; - when(alfMock.getVariables()).thenReturn("my,full,complete,extended,variable,list"); - assertEquals( - Arrays.stream(selectedVariables.split(",")).toList(), - dataController.getLinkedVariables(alfMock, selectedVariables)); - } - - @Test - void testGetLinkedInvalidVariables() { - DataController dataController = - new DataController( - commands, armadilloStorage, auditEventPublisher, expressionRewriter, environments); - ArmadilloLinkFile alfMock = mock(ArmadilloLinkFile.class); - String selectedVariables = "not,my,variable,list"; - when(alfMock.getVariables()).thenReturn("my,full,complete,extended,variable,list"); - when(alfMock.getProject()).thenReturn("project"); - when(alfMock.getLinkObject()).thenReturn("object"); - try { - dataController.getLinkedVariables(alfMock, selectedVariables); - } catch (UnknownVariableException e) { - assertEquals("Variables '[not]' do not exist in object 'project/object'", e.getMessage()); - } - } - - @Test - void testGetVariableList() { - DataController dataController = - new DataController( - commands, armadilloStorage, auditEventPublisher, expressionRewriter, environments); - String variables = "1,b,c"; - List expected = Arrays.asList("1", "b", "c"); - assertEquals(expected, dataController.getVariableList(variables)); - } - @Test @WithMockUser void testLoadTableWithVariables() throws Exception { - when(armadilloStorage.hasObject("project", "folder/table.alf")).thenReturn(false); - when(armadilloStorage.hasObject("project", "folder/table.parquet")).thenReturn(true); + when(armadilloStorage.tableExists("project", "folder/table")).thenReturn(true); when(commands.loadTable("D", "project/folder/table", List.of("age", "weight"))) .thenReturn(completedFuture(null)); diff --git a/armadillo/src/test/java/org/molgenis/armadillo/controller/StorageControllerTest.java b/armadillo/src/test/java/org/molgenis/armadillo/controller/StorageControllerTest.java index 649a1750c..abc62dd98 100644 --- a/armadillo/src/test/java/org/molgenis/armadillo/controller/StorageControllerTest.java +++ b/armadillo/src/test/java/org/molgenis/armadillo/controller/StorageControllerTest.java @@ -428,37 +428,6 @@ void previewObject() throws Exception { mockSuAuditMap(Map.of(PROJECT, "lifecycle", OBJECT, "test.parquet")))); } - @Test - void createLinkedObject() throws Exception { - doNothing() - .when(storage) - .createLinkedObject("lifecycle", "test", "my-link", "lifecycle", "a,b,c"); - mockMvc - .perform( - post("/storage/projects/lifecycle/objects/link") - .content( - "{\"sourceProject\": \"lifecycle\", \"sourceObjectName\": \"test\", \"linkedObject\": \"my-link\", \"variables\": \"a,b,c\"}") - .contentType(APPLICATION_JSON) - .session(session)) - .andExpect(status().isNoContent()); - - auditEventValidator.validateAuditEvent( - new AuditEvent( - instant, - "user", - CREATE_LINKED_OBJECT, - mockSuAuditMap( - Map.of( - PROJECT, - "lifecycle", - "source", - "lifecycle/my-link", - "columns", - List.of("a", "b", "c"), - OBJECT, - "my-link.alf")))); - } - @Test void getObjectInfo() throws Exception { when(storage.getInfo("lifecycle", "test.parquet")) diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloLinkFileTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloLinkFileTest.java deleted file mode 100644 index b91a3dbab..000000000 --- a/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloLinkFileTest.java +++ /dev/null @@ -1,111 +0,0 @@ -package org.molgenis.armadillo.storage; - -import static java.lang.String.format; -import static org.junit.jupiter.api.Assertions.assertEquals; - -import com.google.gson.JsonObject; -import com.google.gson.JsonParseException; -import java.io.ByteArrayInputStream; -import java.io.InputStream; -import org.junit.jupiter.api.Test; - -public class ArmadilloLinkFileTest { - String srcObj = "folder/src-obj"; - String srcProject = "src-project"; - String vars = "var1,var2,var3"; - String testData = - "{\"sourceObject\":\"folder/src-obj\",\"sourceProject\":\"src-project\",\"variables\":\"var1,var2,var3\"}"; - String linkObj = "folder/link-obj"; - - String linkProject = "view-project"; - ArmadilloLinkFile alf = new ArmadilloLinkFile(srcProject, srcObj, vars, linkObj, linkProject); - - @Test - public void testBuildJson() { - JsonObject actual = alf.buildJson(); - assertEquals(srcObj, actual.get("sourceObject").getAsString()); - assertEquals(srcProject, actual.get("sourceProject").getAsString()); - assertEquals(vars, actual.get("variables").getAsString()); - } - - @Test - public void testToString() { - String actual = alf.toString(); - assertEquals(testData, actual); - } - - @Test - public void testGetFileName() { - String actual = alf.getFileName(); - assertEquals("folder/link-obj.alf", actual); - } - - @Test - public void testLoadFromStream() { - InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); - JsonObject actual = alf.loadFromStream(inputStream); - assertEquals(testData, actual.toString()); - } - - @Test - public void testLoadLinkFileFromStream() { - InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); - ArmadilloLinkFile alfFromStream = new ArmadilloLinkFile(inputStream, linkObj, linkProject); - JsonObject jsonFromStream = alfFromStream.buildJson(); - assertEquals(srcObj, jsonFromStream.get("sourceObject").getAsString()); - assertEquals(srcProject, jsonFromStream.get("sourceProject").getAsString()); - assertEquals(vars, jsonFromStream.get("variables").getAsString()); - } - - @Test - public void testLoadLinkFileFromStreamInvalidJson() { - String testData = - "\"sourceObject\":\"folder/src-obj\",\"sourceProject\":\"src-project\",\"variables\":\"var1,var2,var3\"}"; - InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); - try { - new ArmadilloLinkFile(inputStream, linkObj, linkProject); - } catch (JsonParseException e) { - String message = format("Cannot load [%s/%s] because JSON is invalid", linkProject, linkObj); - assertEquals(message, e.getMessage()); - } - } - - @Test - public void testLoadLinkFileFromStreamInvalidObj() { - String testData = - "{\"sourceObj\":\"folder/src-obj\",\"sourceProject\":\"src-project\",\"variables\":\"var1,var2,var3\"}"; - InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); - try { - new ArmadilloLinkFile(inputStream, linkObj, linkProject); - } catch (NullPointerException e) { - String message = format("Source object is missing from [%s/%s]", linkProject, linkObj); - assertEquals(message, e.getMessage()); - } - } - - @Test - public void testLoadLinkFileFromStreamInvalidProject() { - String testData = - "{\"sourceObject\":\"folder/src-obj\",\"sourceProj\":\"src-project\",\"variables\":\"var1,var2,var3\"}"; - InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); - try { - new ArmadilloLinkFile(inputStream, linkObj, linkProject); - } catch (NullPointerException e) { - String message = format("Source project is missing from [%s/%s]", linkProject, linkObj); - assertEquals(message, e.getMessage()); - } - } - - @Test - public void testLoadLinkFileFromStreamInvalidVariables() { - String testData = - "{\"sourceObject\":\"folder/src-obj\",\"sourceProject\":\"src-project\",\"variabls\":\"var1,var2,var3\"}"; - InputStream inputStream = new ByteArrayInputStream(testData.getBytes()); - try { - new ArmadilloLinkFile(inputStream, linkObj, linkProject); - } catch (NullPointerException e) { - String message = format("Variables are not defined on [%s/%s]", linkProject, linkObj); - assertEquals(message, e.getMessage()); - } - } -} diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloStorageServiceTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloStorageServiceTest.java index ad0810f42..b8997dc71 100644 --- a/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloStorageServiceTest.java +++ b/armadillo/src/test/java/org/molgenis/armadillo/storage/ArmadilloStorageServiceTest.java @@ -31,7 +31,10 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import org.molgenis.armadillo.exceptions.*; +import org.molgenis.armadillo.exceptions.DuplicateObjectException; +import org.molgenis.armadillo.exceptions.InvalidProjectNameException; +import org.molgenis.armadillo.exceptions.UnknownObjectException; +import org.molgenis.armadillo.exceptions.UnknownProjectException; import org.molgenis.armadillo.model.Workspace; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.mock.mockito.MockBean; @@ -565,81 +568,6 @@ void testSaveSystemFile() { assertEquals(testValue, new String(argument.getValue().readAllBytes())); } - @Test - @WithMockUser(roles = "SU") - void testCreateLinkedObject() { - mockExistingObject(SHARED_GECKO, "1_0_release_1_1/gecko.parquet"); - when(storageService.listBuckets()).thenReturn(List.of(SHARED_DIABETES, SHARED_GECKO)); - assertDoesNotThrow( - () -> - armadilloStorage.createLinkedObject( - "gecko", "1_0_release_1_1/gecko", "folder/my_link", "diabetes", "a,b,c")); - } - - @Test - @WithMockUser(roles = "SU") - void testCreateLinkedObjectUnknownSrcObj() { - when(storageService.listBuckets()).thenReturn(List.of(SHARED_DIABETES, SHARED_GECKO)); - assertThrows( - UnknownObjectException.class, - () -> - armadilloStorage.createLinkedObject( - "gecko", "1_0_release_1_1/gecko", "folder/my_link", "diabetes", "a,b,c")); - } - - @Test - @WithMockUser(roles = "SU") - void testCreateLinkedObjectUnknownSrcProject() { - when(storageService.listBuckets()).thenReturn(List.of(SHARED_DIABETES)); - assertThrows( - UnknownProjectException.class, - () -> - armadilloStorage.createLinkedObject( - "gecko", "1_0_release_1_1/gecko", "folder/my_link", "diabetes", "a,b,c")); - } - - @Test - @WithMockUser(roles = "SU") - void testCreateLinkedObjectUnknownLinkProject() { - mockExistingObject(SHARED_GECKO, "1_0_release_1_1/gecko.parquet"); - when(storageService.listBuckets()).thenReturn(List.of(SHARED_GECKO)); - assertThrows( - UnknownProjectException.class, - () -> - armadilloStorage.createLinkedObject( - "gecko", "1_0_release_1_1/gecko", "folder/my_link", "diabetes", "a,b,c")); - } - - @Test - @WithMockUser(roles = "SU") - void testCreateLinkedObjectDupLinkFile() { - String obj = "folder/my_link"; - String obj_file = obj + ".alf"; - mockExistingObject(SHARED_GECKO, "1_0_release_1_1/gecko.parquet"); - mockExistingObject(SHARED_DIABETES, obj_file); - when(storageService.listBuckets()).thenReturn(List.of(SHARED_GECKO, SHARED_DIABETES)); - when(storageService.objectExists(SHARED_DIABETES, obj_file)).thenReturn(true); - assertThrows( - DuplicateObjectException.class, - () -> - armadilloStorage.createLinkedObject( - "gecko", "1_0_release_1_1/gecko", obj, "diabetes", "a,b,c")); - } - - @Test - @WithMockUser(roles = "SU") - void testCreateLinkedObjectUnavailableVars() throws IOException { - String obj = "folder/my_link"; - String srcObj = "1_0_release_1_1/gecko"; - mockExistingObject(SHARED_GECKO, srcObj + ".parquet"); - when(storageService.listBuckets()).thenReturn(List.of(SHARED_GECKO, SHARED_DIABETES)); - when(storageService.getUnavailableVariables(SHARED_GECKO, srcObj, "a,b,c,x,y,z")) - .thenReturn(List.of("x,y,z")); - assertThrows( - UnknownVariableException.class, - () -> armadilloStorage.createLinkedObject("gecko", srcObj, obj, "diabetes", "a,b,c,x,y,z")); - } - @Test void testValidateProjectName() { assertDoesNotThrow(() -> validateProjectName("lifecycle")); diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/LocalStorageServiceTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/LocalStorageServiceTest.java index d16a958e0..7a821e5c0 100644 --- a/armadillo/src/test/java/org/molgenis/armadillo/storage/LocalStorageServiceTest.java +++ b/armadillo/src/test/java/org/molgenis/armadillo/storage/LocalStorageServiceTest.java @@ -18,7 +18,6 @@ import java.time.ZonedDateTime; import java.util.Collections; import java.util.HashMap; -import java.util.List; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.MockedStatic; @@ -192,36 +191,6 @@ void testGetPathIfObjectExists() { mockedFiles.close(); } - @Test - void testGetUnavailableVariables() throws IOException { - String bucket = "my-bucket"; - String object = "my-object"; - MockedStatic mockedPaths = Mockito.mockStatic(Paths.class, RETURNS_DEEP_STUBS); - MockedStatic mockedFiles = Mockito.mockStatic(Files.class); - MockedStatic mockedParquetUtils = Mockito.mockStatic(ParquetUtils.class); - Path bucketPathMock = mock(Path.class); - Path objectPathMock = mock(Path.class); - when(Paths.get(localStorageService.rootDir, bucket, object + ".parquet") - .toAbsolutePath() - .normalize()) - .thenReturn(objectPathMock); - when(Paths.get(localStorageService.rootDir, bucket)).thenReturn(bucketPathMock); - when(Paths.get(localStorageService.rootDir, bucket).toAbsolutePath()) - .thenReturn(bucketPathMock); - when(Paths.get(localStorageService.rootDir, bucket).toAbsolutePath().normalize()) - .thenReturn(bucketPathMock); - when(objectPathMock.startsWith(bucketPathMock)).thenReturn(Boolean.TRUE); - when(Files.exists(bucketPathMock)).thenReturn(Boolean.TRUE); - when(Files.exists(objectPathMock)).thenReturn(Boolean.TRUE); - when(ParquetUtils.getColumns(objectPathMock)).thenReturn(List.of("var1", "var2")); - List unavailableVariables = - localStorageService.getUnavailableVariables(bucket, object, "var1,var2,var3"); - assertEquals(unavailableVariables, List.of("var3")); - mockedPaths.close(); - mockedFiles.close(); - mockedParquetUtils.close(); - } - @Test void testGetPathIfObjectExistsThrowsErrorWhenNoBucket() { String object = "my-object.txt"; diff --git a/armadillo/src/test/java/org/molgenis/armadillo/storage/ParquetUtilsTest.java b/armadillo/src/test/java/org/molgenis/armadillo/storage/ParquetUtilsTest.java index 8f86bdfbd..83f7d2e08 100644 --- a/armadillo/src/test/java/org/molgenis/armadillo/storage/ParquetUtilsTest.java +++ b/armadillo/src/test/java/org/molgenis/armadillo/storage/ParquetUtilsTest.java @@ -25,10 +25,4 @@ void testRetrieveDimensions() throws URISyntaxException, FileNotFoundException { assertEquals("3", dimensions.get("columns")); assertEquals("11", dimensions.get("rows")); } - - @Test - void testGetColumns() throws URISyntaxException, IOException { - Path path = Path.of(this.getClass().getClassLoader().getResource("patient.parquet").toURI()); - assertEquals(List.of("id", "age", "name"), ParquetUtils.getColumns(path)); - } } diff --git a/scripts/release/release-test.R b/scripts/release/release-test.R index 4d0646fa8..29871c76e 100755 --- a/scripts/release/release-test.R +++ b/scripts/release/release-test.R @@ -156,8 +156,8 @@ get_from_api_with_header <- function(endpoint, key, auth_type){ } # add/edit user using armadillo api -set_user <- function(user, admin_pwd, isAdmin, project1, omics_project, link_project){ - args <- list(email = user, admin = isAdmin, projects= list(project1, omics_project, link_project)) +set_user <- function(user, admin_pwd, isAdmin, project1, omics_project){ + args <- list(email = user, admin = isAdmin, projects= list(project1, omics_project)) response <- put_to_api("access/users", admin_pwd, "basic", args) if(response$status_code != 204) { cli_alert_warning("Altering OIDC user failed, please do this manually") @@ -737,29 +737,6 @@ system.time({ }) cli_alert_info("Creating resource") -cli_h2("Creating linked view on table") -#TODO: replace with R code once that is created and released -auth_header <- get_auth_header(auth_type, token) -link_project <- generate_random_project_name(available_projects) -srcObj <- "core/nonrep" -srcProject <- "lifecycle" -linkObj <- "core-variables/nonrep" -json_body <- jsonlite::toJSON( - list(sourceObjectName = srcObj, - sourceProject = srcProject, - linkedObject = linkObj, - variables = "child_id,mother_id,row_id,ethn1_m"), auto_unbox=TRUE) -armadillo.create_project(link_project) -post_url <- sprintf("%sstorage/projects/%s/objects/link", armadillo_url, link_project) -response <- POST(post_url, - body=json_body, - encode="json", - config = c(httr::content_type_json(), httr::add_headers(auth_header))) -if (!response$status_code == 204) { - exit_test(sprintf("Unable to create linked object %s/%s from source: %s/%s, status code: %s", link_project, linkObj, srcProject, srcObj, response$status_code)) -} else { - cli_alert_success(sprintf("Successfully created linked object %s/%s from source: %s/%s", link_project, linkObj, srcProject, srcObj)) -} rds_url <- armadillo_url if(armadillo_url == "http://localhost:8080/") { @@ -782,7 +759,7 @@ if(!ADMIN_MODE){ update_auto <- readLines("stdin", n=1) } if(update_auto == "y"){ - set_user(user, admin_pwd, F, project1, omics_project, link_project) + set_user(user, admin_pwd, F, project1, omics_project) } if(update_auto != "y"){ cat("\nGo to the Users tab") @@ -817,25 +794,6 @@ if (identical(datatype, expected_type)){ cli_alert_info("Assigning expression for core_nonrep$coh_country") datashield.assign.expr(conns, "x", expr=quote(core_nonrep$coh_country)) -cli_alert_info("Testing linked table") -#TODO: replace this by following once implemented in R api: -#linked_data <- armadillo.load_table(link_project, "core-variables", "nonrep") -query <- list(table = paste0(link_project, "/core-variables/nonrep"), symbol = "core_nonrep", async = TRUE) -variables <- c("child_id", "mother_id") -query$variables <- paste(unlist(variables), collapse = ",") -response <- httr::POST( - handle = handle(armadillo_url), - path = "/load-table", - query = query, - config = httr::add_headers(get_auth_header(auth_type, token)) -) - -if (!response$status_code == 201) { - exit_test(sprintf("Unable to retrieve linked object %s/%s from source: %s/%s, status code: %s", link_project, linkObj, srcProject, srcObj, response$status_code)) -} else { - cli_alert_success(sprintf("Successfully retrieved linked object %s/%s from source: %s/%s with variables %s", link_project, linkObj, srcProject, srcObj, paste(variables, collapse = ", "))) -} - cli_alert_info("Verifying connecting to profile possible") con <- create_ds_connection(password = admin_pwd, token = token, url=armadillo_url, profile=profile) if (con@name == "armadillo") { @@ -927,7 +885,7 @@ dsDisconnect(con) cli_h2("Removing data as admin") cat("We're now continueing with the datamanager workflow as admin\n") if(update_auto == "y"){ - set_user(user, admin_pwd, T, project1, omics_project, link_project) + set_user(user, admin_pwd, T, project1, omics_project) } else{ cat("Make your account admin again") wait_for_input() @@ -938,7 +896,6 @@ armadillo.delete_table(project1, "2_1-core-1_0", "trimesterrep") armadillo.delete_table(project1, "2_1-core-1_0", "monthlyrep") armadillo.delete_table(project1, "1_1-outcome-1_0", "nonrep") armadillo.delete_table(project1, "1_1-outcome-1_0", "yearlyrep") - cat(sprintf("\nVerify in UI all data from [%s] is gone.", project1)) wait_for_input() armadillo.delete_project(project1) @@ -947,9 +904,6 @@ wait_for_input() armadillo.delete_project(omics_project) cat(sprintf("\nVerify in UI project [%s] is gone", omics_project)) wait_for_input() -armadillo.delete_project(link_project) -cat(sprintf("\nVerify in UI project [%s] is gone", link_project)) -wait_for_input() project2 <- generate_random_project_name(available_projects) available_projects <- c(available_projects, project2)