Merge branch 'develop' into np_address_github_terra_delay_1

pipelines/skylab/atac/atac.changelog.md

@@ -1,9 +1,10 @@
 # 2.7.1
-2025-02-12 (Date of Last Commit)
+2025-02-25 (Date of Last Commit)
 
 * Added a new warning for peak calling step if the probability_threshold is too low, resutling in a null matrix after doublet filtering
 * Updated the probability threshold default to 0.5
 * Updated the warp-tools docker image to include an update to the GroupQCs function in sctools; this does not affect the outputs of the pipeline
+* Added reference information to the BAM header
 
 # 2.7.0
 2025-02-03 (Date of Last Commit)

pipelines/skylab/atac/atac.wdl

@@ -353,6 +353,7 @@ task BWAPairedEndAlignment {
     Array[File] read1_fastq
     Array[File] read3_fastq
     File tar_bwa_reference
+    String reference_path = tar_bwa_reference
     String read_group_id = "RG1"
     String read_group_sample_name = "RGSN1"
     String suffix = "trimmed_adapters.fastq.gz"
@@ -477,7 +478,11 @@ task BWAPairedEndAlignment {
     ls
 
     # rename file to this
-    mv final.sorted.bam ~{bam_aligned_output_name}
+    echo "Reheading BAM with reference"
+    /usr/temp/Open-Omics-Acceleration-Framework/applications/samtools/samtools view -H final.sorted.bam > header.txt
+    echo -e "@CO\tReference genome used: ~{reference_path}" >> header.txt
+    /usr/temp/Open-Omics-Acceleration-Framework/applications/samtools/samtools reheader header.txt final.sorted.bam > final.sorted.reheader.bam
+    mv final.sorted.reheader.bam ~{bam_aligned_output_name}
 
     echo "the present working dir"
     pwd

pipelines/skylab/multiome/Multiome.changelog.md

@@ -1,8 +1,9 @@
 # 5.11.0
-2025-02-05 (Date of Last Commit)
+2025-02-25 (Date of Last Commit)
 
 * Refactored the Peak Calling step of Multiome to use the JoinBarcodes output h5ad as the input for peak calling, ensuring the h5ad files have both GEX and ATAC barcodes
 * Updated the warp-tools docker image to include an update to the GroupQCs function in sctools; this does not affect the outputs of the pipeline
+* Added reference information to the BAM headers
 
 # 5.10.0
 2025-02-03 (Date of Last Commit)

pipelines/skylab/optimus/Optimus.changelog.md

@@ -1,7 +1,8 @@
 # 7.9.2
-2025-02-12 (Date of Last Commit)
+2025-02-25 (Date of Last Commit)
 
 * Updated the warp-tools docker image to include an update to the GroupQCs function in sctools; this does not affect the outputs of the pipeline
+* Added reference information to the BAM header
 
 # 7.9.1
 2025-01-13 (Date of Last Commit)

pipelines/skylab/paired_tag/PairedTag.changelog.md

@@ -1,8 +1,9 @@
 # 1.10.2 
-2025-02-06 (Date of Last Commit)
+2025-02-25 (Date of Last Commit)
 
 * Updated the SnapATAC2 docker image to the latest SnapATAC2, allowing for future peak calling implementation
 * Updated the warp-tools docker image to include an update to the GroupQCs function in sctools; this does not affect the outputs of the pipeline
+* Added reference information to the BAM headers
 
 # 1.10.1
 2025-02-03 (Date of Last Commit)

pipelines/skylab/slideseq/SlideSeq.changelog.md

@@ -1,6 +1,7 @@
 # 3.4.9
-2025-02-12 (Date of Last Commit)
+2025-02-25 (Date of Last Commit)
 * Updated the warp-tools docker image to include an update to the GroupQCs function in sctools; this does not affect the outputs of the pipeline
+* Added reference information to the BAM header for Optimus and ATAC workflows; this does not impact Slideseq
 
 # 3.4.8
 2025-01-13 (Date of Last Commit)

pipelines/skylab/smartseq2_single_nucleus_multisample/MultiSampleSmartSeq2SingleNucleus.changelog.md

@@ -1,7 +1,8 @@
 # 2.0.8
-2025-02-12 (Date of Last Commit)
+2025-02-25 (Date of Last Commit)
 
 * Updated the warp-tools docker image to include an update to the GroupQCs function in sctools; this does not affect the outputs of the pipeline
+* Added reference information to the BAM header for Optimus and ATAC; does not impact snSS2
 
 # 2.0.7
 2025-01-13 (Date of Last Commit)

scripts/firecloud_api/firecloud_api.py

@@ -256,6 +256,7 @@ def attempt_creation():
             logging.error(f"Response body: {response.text}")
             raise Exception(f"Failed to create method configuration for {pipeline_name} on the branch {branch_name}")
 
+
     def upload_test_inputs(self, pipeline_name, test_inputs, branch_name, test_type):
         """
         Uploads test inputs to the workspace via Firecloud API.
@@ -337,19 +338,12 @@ def poll_job_status(self, submission_id):
         workflow_status_map = {}
 
         # Set up retry parameters
-        max_retry_duration = 15 * 60  # 15 minutes in seconds
         start_time = time.time()
         retry_delay = 5  # Start with a 5-second delay between retries
         max_retry_delay = 30  # Maximum retry delay in seconds
 
         # Continuously poll the status of the submission until completion
         while True:
-            # Check if we've exceeded the maximum retry duration
-            current_time = time.time()
-            if current_time - start_time > max_retry_duration:
-                logging.error(f"Exceeded maximum retry duration of {max_retry_duration/60} minutes.")
-                return workflow_status_map
-
             try:
                 # Get the token and headers
                 token = self.get_user_token(self.delegated_creds)
@@ -397,6 +391,7 @@ def poll_job_status(self, submission_id):
                 # Check if the submission is complete
                 submission_status = status_data.get("status", "")
                 if submission_status == "Done":
+                    logging.info("Submission is done.")
                     break
 
                 # Wait for 20 seconds before polling again

tasks/skylab/StarAlign.wdl

@@ -224,6 +224,7 @@ task STARsoloFastq {
     String output_bam_basename
     Boolean? count_exons
     String? soloMultiMappers
+    String reference_path = tar_star_reference
 
     # runtime values
     String samtools_star_docker_path
@@ -339,6 +340,10 @@ task STARsoloFastq {
 
     # validate the bam with samtools quickcheck
     samtools quickcheck -v Aligned.sortedByCoord.out.bam
+    # reheader the BAM
+    samtools view -H Aligned.sortedByCoord.out.bam > header.txt
+    echo -e "@CO\tReference genome used: ~{reference_path}" >> header.txt
+    samtools reheader header.txt Aligned.sortedByCoord.out.bam > Aligned.sortedByCoord.out.reheader.bam
 
 
     echo "UMI LEN " $UMILen
@@ -419,7 +424,7 @@ task STARsoloFastq {
     else
       echo Error: unknown counting mode: "$counting_mode". Should be either sn_rna or sc_rna.
     fi
-    mv Aligned.sortedByCoord.out.bam ~{output_bam_basename}.bam
+    mv Aligned.sortedByCoord.out.reheader.bam ~{output_bam_basename}.bam
 
   >>>