Merge branch 'hotfix-1.27.15'

Author: arteymix

gemma-core/pom.xml

@@ -3,7 +3,7 @@
     <parent>
         <artifactId>gemma</artifactId>
         <groupId>gemma</groupId>
-        <version>1.27.14</version>
+        <version>1.27.15</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>gemma-core</artifactId>

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileService.java

@@ -25,6 +25,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.io.Writer;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
@@ -120,6 +121,20 @@ List<DifferentialExpressionAnalysisResult> analysisResultSetToString( Expression
     File writeDataFile( ExpressionExperiment ee, boolean filtered, String fileName, boolean compress )
             throws IOException;
 
+    /**
+     * Write raw expression data to a given writer.
+     *
+     * Note: the preferred quantitations are used.
+     *
+     * Note: For compression, wrap a {@link java.util.zip.GZIPOutputStream} with a {@link java.io.OutputStreamWriter}.
+     * To write to a string, consider using {@link java.io.StringWriter}.
+     *
+     * @param ee the expression experiment
+     * @param writer the destination for the raw expression data
+     * @throws IOException if operations with the writer fails
+     */
+    void writeRawExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException;
+
     /**
      * Write or located the coexpression data file for a given experiment
      *

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataFileServiceImpl.java

@@ -23,8 +23,10 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.hibernate.Hibernate;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
 import ubic.basecode.util.FileTools;
 import ubic.basecode.util.StringUtil;
 import ubic.gemma.core.analysis.expression.diff.DifferentialExpressionAnalysisConfig;
@@ -42,6 +44,7 @@
 import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssayData.DesignElementDataVector;
+import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
 import ubic.gemma.model.expression.designElement.CompositeSequence;
 import ubic.gemma.model.expression.experiment.*;
 import ubic.gemma.model.genome.Taxon;
@@ -57,6 +60,7 @@
 import java.io.*;
 import java.util.*;
 import java.util.Map.Entry;
+import java.util.stream.Collectors;
 import java.util.zip.GZIPOutputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -347,6 +351,23 @@ public void writeDiffExArchiveFile( BioAssaySet experimentAnalyzed, Differential
         }
     }
 
+    @Override
+    @Transactional(readOnly = true)
+    public void writeRawExpressionData( ExpressionExperiment ee, Writer writer ) throws IOException {
+        ee = expressionExperimentService.find( ee );
+        if ( ee == null ) {
+            throw new IllegalArgumentException( "ExpressionExperiment has been removed." );
+        }
+        // pre-initialize it so that it get fetched in a single query without a jointure with the EE
+        Hibernate.initialize( ee.getRawExpressionDataVectors() );
+        ExpressionDataDoubleMatrix matrix = expressionDataMatrixService.getRawExpressionDataMatrix( ee );
+        Set<ArrayDesign> ads = ee.getRawExpressionDataVectors().stream()
+                .map( RawExpressionDataVector::getDesignElement )
+                .map( CompositeSequence::getArrayDesign )
+                .collect( Collectors.toSet() );
+        new MatrixWriter().writeWithStringifiedGeneAnnotations( writer, matrix, getGeneAnnotationsAsStringsByProbe( ads ), true );
+    }
+
     @Override
     public File writeOrLocateCoexpressionDataFile( ExpressionExperiment ee, boolean forceWrite ) {
 

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixService.java

@@ -14,6 +14,7 @@
  */
 package ubic.gemma.core.analysis.service;
 
+import org.springframework.transaction.annotation.Transactional;
 import ubic.basecode.dataStructure.matrix.DoubleMatrix;
 import ubic.gemma.core.analysis.preprocess.filter.FilterConfig;
 import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
@@ -61,6 +62,13 @@ ExpressionDataDoubleMatrix getFilteredMatrix( String arrayDesignName, FilterConf
      */
     ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionExperiment ee );
 
+    /**
+     * @throws IllegalArgumentException if the expression experiment has no preferred raw quantitation types
+     * @param ee
+     * @return
+     */
+    ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee );
+
     DoubleMatrix<Gene, ExpressionExperiment> getRankMatrix( Collection<Gene> genes,
             Collection<ExpressionExperiment> ees, ProcessedExpressionDataVectorDao.RankMethod method );
 

gemma-core/src/main/java/ubic/gemma/core/analysis/service/ExpressionDataMatrixServiceImpl.java

@@ -19,35 +19,39 @@
 package ubic.gemma.core.analysis.service;
 
 import cern.colt.list.DoubleArrayList;
+import lombok.extern.apachecommons.CommonsLog;
 import org.apache.commons.lang3.ArrayUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
 import ubic.basecode.dataStructure.matrix.DenseDoubleMatrix;
 import ubic.basecode.dataStructure.matrix.DoubleMatrix;
 import ubic.basecode.math.DescriptiveWithMissing;
 import ubic.gemma.core.analysis.preprocess.filter.ExpressionExperimentFilter;
 import ubic.gemma.core.analysis.preprocess.filter.FilterConfig;
 import ubic.gemma.core.datastructure.matrix.ExpressionDataDoubleMatrix;
+import ubic.gemma.model.common.quantitationtype.QuantitationType;
 import ubic.gemma.model.expression.arrayDesign.ArrayDesign;
 import ubic.gemma.model.expression.bioAssayData.ProcessedExpressionDataVector;
+import ubic.gemma.model.expression.bioAssayData.RawExpressionDataVector;
 import ubic.gemma.model.expression.experiment.ExpressionExperiment;
 import ubic.gemma.model.genome.Gene;
 import ubic.gemma.persistence.service.expression.arrayDesign.ArrayDesignService;
 import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorDao;
 import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService;
+import ubic.gemma.persistence.service.expression.bioAssayData.RawExpressionDataVectorService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashSet;
-import java.util.Map;
+import java.util.*;
+import java.util.stream.Collectors;
 
 /**
  * Tools for easily getting data matrices for analysis in a consistent way.
  *
  * @author keshav
  */
 @Component
+@CommonsLog
 public class ExpressionDataMatrixServiceImpl implements ExpressionDataMatrixService {
 
     @Autowired
@@ -56,6 +60,9 @@ public class ExpressionDataMatrixServiceImpl implements ExpressionDataMatrixServ
     @Autowired
     private ProcessedExpressionDataVectorService processedExpressionDataVectorService;
 
+    @Autowired
+    private RawExpressionDataVectorService rawExpressionDataVectorService;
+
     @Autowired
     private ArrayDesignService arrayDesignService;
 
@@ -97,6 +104,32 @@ public ExpressionDataDoubleMatrix getProcessedExpressionDataMatrix( ExpressionEx
         return new ExpressionDataDoubleMatrix( dataVectors );
     }
 
+    @Override
+    @Transactional(readOnly = true)
+    public ExpressionDataDoubleMatrix getRawExpressionDataMatrix( ExpressionExperiment ee ) {
+        Map<QuantitationType, List<RawExpressionDataVector>> rawVectorsByQt = ee.getRawExpressionDataVectors().stream()
+                .collect( Collectors.groupingBy( RawExpressionDataVector::getQuantitationType, Collectors.toList() ) );
+
+        Set<QuantitationType> preferredQuantitationTypes = rawVectorsByQt.keySet().stream()
+                .filter( QuantitationType::getIsPreferred )
+                .collect( Collectors.toSet() );
+
+        if ( preferredQuantitationTypes.isEmpty() ) {
+            throw new IllegalArgumentException( "There are no RawExpressionDataVectors for " + ee + ", they must be created first." );
+        }
+
+        if ( preferredQuantitationTypes.size() > 1 ) {
+            log.warn( "There are more than one preferred quantitation type for " + ee + " raw expression vectors." );
+        }
+
+        // pick the QT with the maximum ID, which should be the latest one created
+        QuantitationType pickedQuantitationType = preferredQuantitationTypes.stream()
+                .max( Comparator.comparing( QuantitationType::getId ) )
+                .orElse( null );
+
+        return new ExpressionDataDoubleMatrix( rawVectorsByQt.get( pickedQuantitationType ) );
+    }
+
     @Override
     public DoubleMatrix<Gene, ExpressionExperiment> getRankMatrix( Collection<Gene> genes,
             Collection<ExpressionExperiment> ees, ProcessedExpressionDataVectorDao.RankMethod method ) {

gemma-web/pom.xml

@@ -3,7 +3,7 @@
 	<parent>
 		<artifactId>gemma</artifactId>
 		<groupId>gemma</groupId>
-		<version>1.27.14</version>
+		<version>1.27.15</version>
 	</parent>
 	<modelVersion>4.0.0</modelVersion>
 	<artifactId>gemma-web</artifactId>

gemma-web/src/main/java/ubic/gemma/web/services/rest/DatasetsWebService.java

@@ -15,6 +15,9 @@
 package ubic.gemma.web.services.rest;
 
 import io.swagger.v3.oas.annotations.Operation;
+import io.swagger.v3.oas.annotations.media.Content;
+import io.swagger.v3.oas.annotations.media.Schema;
+import io.swagger.v3.oas.annotations.responses.ApiResponse;
 import lombok.extern.apachecommons.CommonsLog;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
@@ -40,19 +43,22 @@
 import ubic.gemma.persistence.service.expression.bioAssayData.ProcessedExpressionDataVectorService;
 import ubic.gemma.persistence.service.expression.experiment.ExpressionExperimentService;
 import ubic.gemma.persistence.util.Filters;
-import ubic.gemma.web.services.rest.util.ArgUtils;
-import ubic.gemma.web.services.rest.util.PaginatedResponseDataObject;
-import ubic.gemma.web.services.rest.util.Responder;
-import ubic.gemma.web.services.rest.util.ResponseDataObject;
+import ubic.gemma.web.services.rest.annotations.GZIP;
+import ubic.gemma.web.services.rest.util.*;
 import ubic.gemma.web.services.rest.util.args.*;
 
 import javax.servlet.http.HttpServletResponse;
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
 import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
 import java.io.File;
-import java.util.*;
+import java.io.OutputStreamWriter;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 
 /**
  * RESTful interface for datasets.
@@ -249,6 +255,29 @@ public Response datasetData( // Params:
         return this.outputDataFile( ee, filterData.getValue() );
     }
 
+    /**
+     * Retrieve raw expression data.
+     *
+     * The payload is transparently compressed via a <code>Content-Encoding</code> header and streamed to avoid dumping
+     * the whole payload in memory.
+     */
+    @GZIP
+    @GET
+    @Path("/{dataset}/data/raw")
+    @Produces("text/tab-separated-values; charset=UTF-8")
+    @Operation(summary = "Retrieve raw expression data of a dataset", responses = {
+            @ApiResponse(responseCode = "200", content = @Content(mediaType = "text/tab-separated-values; charset=UTF-8",
+                    schema = @Schema(type = "string", format = "binary"))),
+            @ApiResponse(responseCode = "404", description = "The dataset does not exist.",
+                    content = @Content(mediaType = MediaType.APPLICATION_JSON, schema = @Schema(implementation = ResponseErrorObject.class))) })
+    public Response getDatasetRawExpression( @PathParam("dataset") DatasetArg<?> datasetArg ) {
+        ExpressionExperiment ee = datasetArg.getEntity( expressionExperimentService );
+        StreamingOutput stream = ( output ) -> expressionDataFileService.writeRawExpressionData( ee, new OutputStreamWriter( output ) );
+        return Response.ok( stream )
+                .header( "Content-Disposition", String.format( "attachment; filename=%d_%s_expmat.unfilt.raw.data.txt", ee.getId(), ee.getShortName() ) )
+                .build();
+    }
+
     /**
      * Retrieves the design for the given dataset.
      *

gemma-web/src/test/java/ubic/gemma/web/services/rest/DatasetsRestTest.java

@@ -17,6 +17,10 @@
 import ubic.gemma.web.services.rest.util.args.*;
 import ubic.gemma.web.util.BaseSpringWebTest;
 
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.StreamingOutput;
+import java.io.*;
+import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -239,4 +243,21 @@ public void testFilterByGeeqPublicationScore() {
                 SortArg.valueOf( "+id" ),
                 new MockHttpServletResponse() );
     }
+
+    @Test
+    public void testGetDatasetRawExpression() throws IOException {
+        ExpressionExperiment ee = ees.get( 0 );
+        Response response = datasetsWebService.getDatasetRawExpression( DatasetArg.valueOf( String.valueOf( ee.getId() ) ) );
+        byte[] payload;
+        try ( ByteArrayOutputStream os = new ByteArrayOutputStream() ) {
+            ( ( StreamingOutput ) response.getEntity() ).write( os );
+            payload = os.toByteArray();
+        }
+        String decodedPayload = new String( payload, StandardCharsets.UTF_8 );
+        // there's 7 comment lines, 1 header and then one line per raw EV (there are two platforms the default collection size in the fixture)
+        assertThat( decodedPayload )
+                .isNotEmpty()
+                .contains( ee.getShortName() )
+                .hasLineCount( 8 + 2 * testHelper.getTestElementCollectionSize() );
+    }
 }

pom.xml

@@ -5,7 +5,7 @@
 	<name>Gemma</name>
 	<groupId>gemma</groupId>
 	<artifactId>gemma</artifactId>
-	<version>1.27.14</version>
+	<version>1.27.15</version>
 	<inceptionYear>2005</inceptionYear>
 	<description>The Gemma Project for meta-analysis of genomics data</description>
 	<url>https://gemma.msl.ubc.ca</url>