Merge branch 'develop' into aa-remove-task

broadinstitute · Mar 4, 2025 · 85d2ae8 · 85d2ae8
2 parents 51586e3 + feec84a
commit 85d2ae8
Show file tree

Hide file tree

Showing 21 changed files with 888 additions and 50 deletions.
diff --git a/.dockstore.yml b/.dockstore.yml
@@ -83,6 +83,14 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/arrays/imputation/Imputation.wdl
 
+  - name: ImputationBeagle
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/broad/arrays/imputation_beagle/ImputationBeagle.wdl
+
+  - name: ArrayImputationQuotaConsumed
+    subclass: WDL
+    primaryDescriptorPath: /pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.wdl
+
   - name: RNAWithUMIsPipeline
     subclass: WDL
     primaryDescriptorPath: /pipelines/broad/rna_seq/RNAWithUMIsPipeline.wdl
@@ -155,6 +163,10 @@ workflows:
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestImputation.wdl
 
+  - name: TestImputationBeagle
+    subclass: WDL
+    primaryDescriptorPath: /verification/test-wdls/TestImputationBeagle.wdl
+
   - name: TestJointGenotyping
     subclass: WDL
     primaryDescriptorPath: /verification/test-wdls/TestJointGenotyping.wdl

diff --git a/.github/workflows/test_imputation_beagle.yml b/.github/workflows/test_imputation_beagle.yml
@@ -0,0 +1,75 @@
+name: Test ImputationBeagle
+
+# Controls when the workflow will run
+on:
+  pull_request:
+    branches: [ "develop", "staging", "master" ]
+    # Only run if files in these paths changed:
+    ####################################
+    # SET PIPELINE SPECIFIC PATHS HERE #
+    ####################################
+    paths:
+      - 'pipelines/broad/arrays/imputation_beagle/**'
+      - 'structs/imputation/ImputationBeagleStructs.wdl'
+      - 'tasks/broad/ImputationTasks.wdl'
+      - 'tasks/broad/ImputationBeagleTasks.wdl'
+      - 'verification/VerifyImputationBeagle.wdl'
+      - 'verification/test-wdls/TestImputationBeagle.wdl'
+      - 'tasks/broad/Utilities.wdl'
+      - 'tasks/broad/TerraCopyFilesFromCloudToCloud.wdl'
+      - '.github/workflows/test_imputation_beagle.yml'
+      - '.github/workflows/warp_test_workflow.yml'
+
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+    inputs:
+      useCallCache:
+        description: 'Use call cache (default: true)'
+        required: false
+        default: "true"
+      updateTruth:
+        description: 'Update truth files (default: false)'
+        required: false
+        default: "false"
+      testType:
+        description: 'Specify the type of test (Plumbing or Scientific)'
+        required: false
+        type: choice
+        options:
+          - Plumbing
+          - Scientific
+      truthBranch:
+        description: 'Specify the branch for truth files (default: master)'
+        required: false
+        default: "master"
+
+env:
+  # pipeline configuration
+  PIPELINE_NAME: TestImputationBeagle
+  DOCKSTORE_PIPELINE_NAME: ImputationBeagle
+  PIPELINE_DIR: "pipelines/broad/arrays/imputation_beagle"
+
+  # workspace configuration
+  TESTING_WORKSPACE: WARP Tests
+  WORKSPACE_NAMESPACE: warp-pipelines
+
+  # service account configuration
+  SA_JSON_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+  USER: pdt-tester@warp-pipeline-dev.iam.gserviceaccount.com
+
+
+jobs:
+  TestImputationBeagle:
+    uses: ./.github/workflows/warp_test_workflow.yml
+    with:
+      pipeline_name: TestImputationBeagle
+      dockstore_pipeline_name: ImputationBeagle
+      pipeline_dir: pipelines/broad/arrays/imputation_beagle
+      use_call_cache: ${{ github.event.inputs.useCallCache || 'true' }}
+      update_truth: ${{ github.event.inputs.updateTruth || 'false' }}
+      test_type: ${{ github.event.inputs.testType }}
+      truth_branch: ${{ github.event.inputs.truthBranch || 'master' }}
+    secrets:
+      PDT_TESTER_SA_B64: ${{ secrets.PDT_TESTER_SA_B64 }}
+      DOCKSTORE_TOKEN: ${{ secrets.DOCKSTORE_TOKEN }}
diff --git a/pipeline_versions.txt b/pipeline_versions.txt
@@ -1,15 +1,17 @@
 Pipeline Name	Version	Date of Last Commit
 Arrays	 2.6.30	2024-11-04 
 ValidateChip	 1.16.7	2024-11-04 
-Imputation	 1.1.15	2024-11-04 
+ArrayImputationQuotaConsumed	 1.0.0	2025-02-24 
+ImputationBeagle	 1.0.0	2025-02-26 
+Imputation	 1.1.16	2025-02-24 
 MultiSampleArrays	 1.6.2	2024-08-02 
 WholeGenomeReprocessing	 3.3.3	2024-11-04 
 ExomeReprocessing	 3.3.3	2024-11-04 
 CramToUnmappedBams	 1.1.3	2024-08-02 
 ExternalWholeGenomeReprocessing	 2.3.3	2024-11-04 
 ExternalExomeReprocessing	 3.3.3	2024-11-04 
 BroadInternalArrays	 1.1.14	2024-11-04 
-BroadInternalImputation	 1.1.14	2024-11-04 
+BroadInternalImputation	 1.1.15	2025-02-24 
 BroadInternalRNAWithUMIs	 1.0.36	2024-11-04 
 BroadInternalUltimaGenomics	 1.1.3	2024-12-05 
 RNAWithUMIsPipeline	 1.0.18	2024-11-04 

diff --git a/pipelines/broad/arrays/imputation/Imputation.changelog.md b/pipelines/broad/arrays/imputation/Imputation.changelog.md
@@ -1,3 +1,8 @@
+# 1.1.16
+2025-02-24 (Date of Last Commit)
+
+* Updated runtime parameters in some ImputationTasks, and added an explicit definition of a vcf_index.
+
 # 1.1.15
 2024-11-04 (Date of Last Commit)
 

diff --git a/pipelines/broad/arrays/imputation/Imputation.wdl b/pipelines/broad/arrays/imputation/Imputation.wdl
@@ -6,7 +6,7 @@ import "../../../../tasks/broad/Utilities.wdl" as utils
 
 workflow Imputation {
 
-  String pipeline_version = "1.1.15"
+  String pipeline_version = "1.1.16"
 
   input {
     Int chunkLength = 25000000
@@ -242,6 +242,7 @@ workflow Imputation {
       call tasks.SelectVariantsByIds {
         input:
           vcf = SetIdsVcfToImpute.output_vcf,
+          vcf_index = SetIdsVcfToImpute.output_vcf_index,
           ids = FindSitesUniqueToFileTwoOnly.missing_sites,
           basename = "imputed_sites_to_recover"
       }

diff --git a/pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.changelog.md b/pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.changelog.md
@@ -0,0 +1,4 @@
+# 1.0.0
+2025-02-24 (Date of Last Commit)
+
+* Initial release of pipeline to calculate the number of samples, i.e. quota used by an imputation service that uses ImputationBeagle.wdl.
diff --git a/pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.wdl b/pipelines/broad/arrays/imputation_beagle/ArrayImputationQuotaConsumed.wdl
@@ -0,0 +1,29 @@
+version 1.0
+
+import "../../../../tasks/broad/ImputationTasks.wdl" as tasks
+
+workflow QuotaConsumed {
+    String pipeline_version = "1.0.0"
+
+    input {
+        Int chunkLength = 25000000
+        Int chunkOverlaps = 5000000
+
+        File multi_sample_vcf
+
+        File ref_dict
+        Array[String] contigs
+        String reference_panel_path_prefix
+        String genetic_maps_path
+        String output_basename
+    }
+
+    call tasks.CountSamples {
+        input:
+            vcf = multi_sample_vcf
+    }
+
+    output {
+        Int quota_consumed = CountSamples.nSamples
+    }
+}
diff --git a/pipelines/broad/arrays/imputation_beagle/ImputationBeagle.changelog.md b/pipelines/broad/arrays/imputation_beagle/ImputationBeagle.changelog.md
@@ -0,0 +1,5 @@
+# 1.0.0
+2025-02-26 (Date of Last Commit)
+
+* Initial public release of the ImputationBeagle pipeline.
+  * The ImputationBeagle pipeline imputes missing genotypes from a multi-sample VCF using a large genomic reference panel. It is based on the Michigan Imputation Server pipeline but uses the Beagle imputation tool instead of minimac. Overall, the pipeline filters, phases, and performs imputation on a multi-sample VCF. It outputs the imputed VCF.