Merge branch 'develop' into km-doc-updates

broadinstitute · Feb 14, 2024 · 10e8bf2 · 10e8bf2
2 parents 493db08 + 69c6827
commit 10e8bf2
Show file tree

Hide file tree

Showing 20 changed files with 106 additions and 53 deletions.
diff --git a/pipelines/skylab/multiome/Multiome.changelog.md b/pipelines/skylab/multiome/Multiome.changelog.md
@@ -1,3 +1,8 @@
+# 3.1.3
+2024-02-07 (Date of Last Commit)
+
+* Updated the Metrics tasks to exclude mitochondrial genes from reads_mapped_uniquely, reads_mapped_multiple and reads_mapped_exonic, reads_mapped_exonic_as and reads_mapped_intergenic
+
 # 3.1.2
 2024-02-01 (Date of Last Commit)
 

diff --git a/pipelines/skylab/multiome/Multiome.wdl b/pipelines/skylab/multiome/Multiome.wdl
@@ -6,7 +6,7 @@ import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils
 import "https://raw.githubusercontent.com/broadinstitute/CellBender/v0.3.0/wdl/cellbender_remove_background.wdl" as CellBender
 
 workflow Multiome {
-    String pipeline_version = "3.1.2"
+    String pipeline_version = "3.1.3"
 
     input {
         String input_id

diff --git a/pipelines/skylab/multiome/atac.changelog.md b/pipelines/skylab/multiome/atac.changelog.md
@@ -1,3 +1,8 @@
+# 1.1.8
+2024-02-07 (Date of Last Commit)
+
+* Updated the Metrics tasks to exclude mitochondrial genes from reads_mapped_uniquely, reads_mapped_multiple and reads_mapped_exonic, reads_mapped_exonic_as and reads_mapped_intergenic
+
 # 1.1.7
 2024-02-01 (Date of Last Commit)
 

diff --git a/pipelines/skylab/multiome/atac.wdl b/pipelines/skylab/multiome/atac.wdl
@@ -41,7 +41,7 @@ workflow ATAC {
     String adapter_seq_read3 = "TCGTCGGCAGCGTCAGATGTGTATAAGAGACAG"
   }
 
-  String pipeline_version = "1.1.7"
+  String pipeline_version = "1.1.8"
 
   parameter_meta {
     read1_fastq_gzipped: "read 1 FASTQ file as input for the pipeline, contains read 1 of paired reads"

diff --git a/pipelines/skylab/optimus/Optimus.changelog.md b/pipelines/skylab/optimus/Optimus.changelog.md
@@ -1,3 +1,7 @@
+# 6.3.6
+2024-02-07 (Date of Last Commit)
+* Updated the Metrics tasks to exclude mitochondrial genes from reads_mapped_uniquely, reads_mapped_multiple and reads_mapped_exonic, reads_mapped_exonic_as and reads_mapped_intergenic
+
 # 6.3.5
 2024-01-30 (Date of Last Commit)
 * Added task GetNumSplits before FastqProcess ATAC task to determine the number of splits based on the bwa-mem2 machine specs

diff --git a/pipelines/skylab/optimus/Optimus.wdl b/pipelines/skylab/optimus/Optimus.wdl
@@ -65,7 +65,7 @@ workflow Optimus {
 
   # version of this pipeline
 
-  String pipeline_version = "6.3.5"
+  String pipeline_version = "6.3.6"
 
   # this is used to scatter matched [r1_fastq, r2_fastq, i1_fastq] arrays
   Array[Int] indices = range(length(r1_fastq))
@@ -146,6 +146,7 @@ workflow Optimus {
     input:
       bam_input = MergeBam.output_bam,
       mt_genes = mt_genes,
+      original_gtf = annotations_gtf,
       input_id = input_id
   }
 

diff --git a/pipelines/skylab/paired_tag/PairedTag.changelog.md b/pipelines/skylab/paired_tag/PairedTag.changelog.md
@@ -1,3 +1,8 @@
+# 0.0.7
+2024-02-07 (Date of Last Commit)
+
+* Updated the Metrics tasks to exclude mitochondrial genes from reads_mapped_uniquely, reads_mapped_multiple and reads_mapped_exonic, reads_mapped_exonic_as and reads_mapped_intergenic
+
 # 0.0.6
 2024-02-01 (Date of Last Commit)
 

diff --git a/pipelines/skylab/paired_tag/PairedTag.wdl b/pipelines/skylab/paired_tag/PairedTag.wdl
@@ -5,7 +5,7 @@ import "../../../pipelines/skylab/optimus/Optimus.wdl" as optimus
 import "../../../tasks/skylab/H5adUtils.wdl" as H5adUtils
 import "../../../tasks/skylab/PairedTagUtils.wdl" as Demultiplexing
 workflow PairedTag {
-    String pipeline_version = "0.0.6"
+    String pipeline_version = "0.0.7"
 
     input {
         String input_id

diff --git a/pipelines/skylab/slideseq/SlideSeq.changelog.md b/pipelines/skylab/slideseq/SlideSeq.changelog.md
@@ -1,8 +1,14 @@
+# 3.0.1
+2024-02-13 (Date of Last Commit)
+
+* Updated the Metrics tasks to exclude mitochondrial genes from reads_mapped_uniquely, reads_mapped_multiple and reads_mapped_exonic, reads_mapped_exonic_as and reads_mapped_intergenic; this does affect the SlideSeq workflow
+
 # 3.0.0
 2024-02-12 (Date of Last Commit)
 
 * Updated the SlideSeq WDL output to utilize the h5ad format in place of Loom
 
+
 # 2.1.6
 2024-01-30 (Date of Last Commit)
 

diff --git a/pipelines/skylab/slideseq/SlideSeq.wdl b/pipelines/skylab/slideseq/SlideSeq.wdl
@@ -23,7 +23,7 @@ import "../../../tasks/skylab/MergeSortBam.wdl" as Merge
 
 workflow SlideSeq {
 
-    String pipeline_version = "3.0.0"
+    String pipeline_version = "3.0.1"
 
     input {
         Array[File] r1_fastq
@@ -91,11 +91,13 @@ workflow SlideSeq {
     call Metrics.CalculateGeneMetrics as GeneMetrics {
         input:
             bam_input = MergeBam.output_bam,
+            original_gtf = annotations_gtf,
             input_id = input_id
     }
     call Metrics.CalculateUMIsMetrics as UMIsMetrics {
         input:
             bam_input = MergeBam.output_bam,
+            original_gtf = annotations_gtf,
             input_id = input_id
     }
 

diff --git a/tasks/skylab/FastqProcessing.wdl b/tasks/skylab/FastqProcessing.wdl
@@ -11,7 +11,7 @@ task FastqProcessing {
     String read_struct
 
     #using the latest build of warp-tools in GCR
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
     #runtime values
     Int machine_mem_mb = 40000
     Int cpu = 16   
@@ -246,7 +246,7 @@ task FastqProcessATAC {
 
         # [?] copied from corresponding optimus wdl for fastqprocessing
         # using the latest build of warp-tools in GCR
-        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
+        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
 
         # Runtime attributes [?]
         Int mem_size = 5

diff --git a/tasks/skylab/H5adUtils.wdl b/tasks/skylab/H5adUtils.wdl
@@ -6,8 +6,7 @@ task OptimusH5adGeneration {
 
   input {
     #runtime values
-    #String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.6-1692962087"
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
     # name of the sample
     String input_id
     # user provided id
@@ -106,8 +105,7 @@ task SingleNucleusOptimusH5adOutput {
 
     input {
         #runtime values
-        #String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.6-1692962087"
-        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
+        String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
         # name of the sample
         String input_id
         # user provided id

diff --git a/tasks/skylab/Metrics.wdl b/tasks/skylab/Metrics.wdl
@@ -8,12 +8,11 @@ task CalculateCellMetrics {
     String input_id
 
     # runtime values
-    #String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.9-1700252065"
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
     Int machine_mem_mb = 8000
     Int cpu = 4
     Int disk = ceil(size(bam_input, "Gi") * 4) + ceil((size(original_gtf, "Gi") * 3)) 
-    Int preemptible = 3
+    Int preemptible = 1
   }
 
   meta {
@@ -81,16 +80,16 @@ task CalculateCellMetrics {
 task CalculateGeneMetrics {
   input {
     File bam_input
+    File original_gtf
     File? mt_genes
     String input_id
     # runtime values
 
-    #String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.9-1700252065"
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
     Int machine_mem_mb = 32000
     Int cpu = 4
-    Int disk = ceil(size(bam_input, "Gi") * 4) 
-    Int preemptible = 3
+    Int disk = ceil(size(bam_input, "Gi") * 4) + ceil((size(original_gtf, "Gi") * 3)) 
+    Int preemptible = 1
   }
 
 
@@ -109,9 +108,21 @@ task CalculateGeneMetrics {
 
   command {
     set -e
+
+     # create the tmp folder
     mkdir temp
+
+    # if GTF file in compressed then uncompress
+    if [[ ~{original_gtf} =~ \.gz$ ]]
+    then
+        gunzip -c ~{original_gtf} > annotation.gtf
+    else
+        mv  ~{original_gtf}  annotation.gtf
+    fi
 
+    # call TagSort with gene as metric type
     TagSort --bam-input ~{bam_input} \
+    --gtf-file annotation.gtf \
     --metric-output "~{input_id}.gene-metrics.csv" \
     --compute-metric \
     --metric-type gene \
@@ -149,11 +160,13 @@ task CalculateGeneMetrics {
 task CalculateUMIsMetrics {
   input {
     File bam_input
+    File original_gtf
     File? mt_genes
     String input_id
+
     # runtime values
     # Did not update docker image as this task uses loom which does not play nice with the changes
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:1.0.9-1700252065"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
     Int machine_mem_mb = 16000
     Int cpu = 8
     Int disk = ceil(size(bam_input, "Gi") * 4)
@@ -179,7 +192,16 @@ task CalculateUMIsMetrics {
     set -e
     mkdir temp
 
+    # if GTF file in compressed then uncompress
+    if [[ ~{original_gtf} =~ \.gz$ ]]
+    then
+        gunzip -c ~{original_gtf} > annotation.gtf
+    else
+        mv  ~{original_gtf}  annotation.gtf
+    fi
+
     TagSort --bam-input ~{bam_input} \
+    --gtf-file annotation.gtf \
     --metric-output "~{input_id}.umi-metrics.csv" \
     --compute-metric \
     --metric-type umi \
@@ -219,8 +241,7 @@ task FastqMetricsSlideSeq {
 
 
     # Runtime attributes
-    #String docker =  "us.gcr.io/broad-gotc-prod/warp-tools:1.0.9-1700252065"
-    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.0"
+    String docker = "us.gcr.io/broad-gotc-prod/warp-tools:2.0.1"
     Int cpu = 16
     Int machine_mb = 40000
     Int disk = ceil(size(r1_fastq, "GiB")*3)  + 50

diff --git a/website/docs/Pipelines/ATAC/README.md b/website/docs/Pipelines/ATAC/README.md
@@ -8,7 +8,9 @@ slug: /Pipelines/ATAC/README
 
 | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback |
 | :----: | :---: | :----: | :--------------: |
-| [1.1.7](https://github.com/broadinstitute/warp/releases) | February, 2024 | Kaylee Mathews | Please file GitHub issues in warp or contact [the WARP team](mailto:warp-pipelines-help@broadinstitute.org) |
+
+| [1.1.8](https://github.com/broadinstitute/warp/releases) | January, 2024 | Kaylee Mathews | Please file GitHub issues in warp or contact [the WARP team](mailto:warp-pipelines-help@broadinstitute.org) |
+
 
 ## Introduction to the ATAC workflow
 ATAC is an open-source, cloud-optimized pipeline developed in collaboration with members of the [BRAIN Initiative](https://braininitiative.nih.gov/) (BICCN and [BICAN](https://brainblog.nih.gov/brain-blog/brain-issues-suite-funding-opportunities-advance-brain-cell-atlases-through-centers) Sequencing Working Group) and [SCORCH](https://nida.nih.gov/about-nida/organization/divisions/division-neuroscience-behavior-dnb/basic-research-hiv-substance-use-disorder/scorch-program) (see [Acknowledgements](#acknowledgements) below). It supports the processing of 10x single-nucleus data generated with 10x Multiome [ATAC-seq (Assay for Transposase-Accessible Chromatin)](https://www.10xgenomics.com/products/single-cell-multiome-atac-plus-gene-expression), a technique used in molecular biology to assess genome-wide chromatin accessibility. 

diff --git a/website/docs/Pipelines/Multiome_Pipeline/README.md b/website/docs/Pipelines/Multiome_Pipeline/README.md
@@ -8,8 +8,7 @@ slug: /Pipelines/Multiome_Pipeline/README
 | Pipeline Version | Date Updated | Documentation Author | Questions or Feedback |
 | :----: | :---: | :----: | :--------------: |
 
-| [Multiome v3.1.2](https://github.com/broadinstitute/warp/releases) | February, 2024 | Kaylee Mathews | Please file GitHub issues in warp or contact the [WARP Pipeline Development team](mailto:warp-pipelines-help@broadinstitute.org) |
-
+| [Multiome v3.1.3](https://github.com/broadinstitute/warp/releases) | February, 2024 | Kaylee Mathews | Please file GitHub issues in warp or contact the [WARP Pipeline Development team](mailto:warp-pipelines-help@broadinstitute.org) |
 
 ![Multiome_diagram](./multiome_diagram.png)