feat: add terraform plan job

.github/workflows/terraform-core.yml

@@ -61,7 +61,7 @@ on:
       artefact_path:
         required: false
         type: string
-        default: ""
+        default: "nonexistentfile.txt"
         description: "If there are artefacts created from data sources, specify the path from the TF stack dir here so they are uploaded before applying"
 
     secrets:
@@ -327,7 +327,7 @@ jobs:
         id: download_plan
         if: ( inputs.download_existing_plan == true && steps.state_empty.outputs.state_empty == 'false' )
         with:
-          name: "${{ env.state_name }}-artefacts"
+          name: "${{ env.state_name }}-${{ inputs.environment_name }}-artefacts"
           path: ${{ matrix.stack.directory }}
 
       - name: Decrypt Terraform plan
@@ -443,14 +443,19 @@ jobs:
           pass_file=$(mktemp)
           printf "%s" "$ENCRYPTION_PASSPHRASE" > "$pass_file"
           gpg --batch --symmetric --passphrase-file "$pass_file" tfplan
+          gpg --batch --symmetric --passphrase-file "$pass_file" tfplan.json
+
+          # Delete unencrypted plan files incase of accidental upload.
+          rm tfplan tfplan.json
 
       - name: Upload Terraform Plan and matrix
         uses: actions/upload-artifact@v4
         if: ${{ inputs.upload_plan }}
         with:
-          name: "${{ env.state_name }}-artefacts"
+          name: "${{ env.state_name }}-${{ inputs.environment_name }}-artefacts"
           path: |
             ${{ matrix.stack.directory }}/tfplan.gpg
+            ${{ matrix.stack.directory }}/tfplan.json.gpg
             ${{ matrix.stack.directory }}/updated_matrix.json
             ${{ matrix.stack.directory }}/${{ inputs.artefact_path }}
           if-no-files-found: warn

.github/workflows/terraform-plan-apply.yml

@@ -40,8 +40,8 @@ on:
       artefact_path:
         required: false
         type: string
-        default: ""
-        description: "If there are artefacts created from data sources, specify the path here so they are downloaded before applying"
+        default: "nonexistentfile.txt"
+        description: "If there are artefacts created from data sources, specify the path here so they are downloaded before applying" 
 
     secrets:
       AWS_ROLE_NAME:
@@ -240,26 +240,6 @@ jobs:
       artefact_path: ${{ inputs.artefact_path }}
     secrets: inherit
 
-  # update-deployment:
-  #   name: Update Github deployment
-  #   if: always()
-  #   runs-on: ubuntu-latest
-  #   steps:
-  #     - run: |
-  #         deployid=$(gh api   -H "Accept: application/vnd.github+json"  \
-  #           --method GET \
-  #           -H "X-GitHub-Api-Version: 2022-11-28"  \
-  #           -f 'q=sha:${{ github.sha }};environment:${{ inputs.environment_name }}' \
-  #           /repos/ukhsa-collaboration/${{ github.event.repository.name }}/deployments)
-
-  #         gh api \
-  #           --method POST \
-  #           -H "Accept: application/vnd.github+json" \
-  #           -H "X-GitHub-Api-Version: 2022-11-28" \
-  #           /repos/OWNER/${{ github.event.repository.name }}/deployments/"$deployid"/statuses \
-  #           -f "state=success" \
-  #           -f "description=Deployment finished successfully."
-
   post-deployment-qa-checks:
     name: Run post deployment QA checks.
     uses: ./.github/workflows/terraform-post-deployment-qa.yml

.github/workflows/terraform-plan.yml

@@ -0,0 +1,233 @@
+name: "[CI] Plan Terraform Stacks"
+on:
+  workflow_call:
+    inputs:
+      environment_name:
+        required: false
+        default: dev
+        type: string
+      aws_region:
+        required: false
+        default: eu-west-2
+        type: string
+      repo:
+        required: false
+        type: string
+        default: ${{ github.repository }}
+      ref:
+        required: false
+        type: string
+        default: ${{ github.ref }}
+
+    secrets:
+      AWS_ROLE_NAME:
+        required: false
+      AWS_ACCOUNT_ID:
+        required: false
+      AWS_ACCESS_KEY_ID:
+        required: false
+      AWS_SECRET_ACCESS_KEY:
+        required: false
+      TF_MODULES_SSH_DEPLOY_KEY:
+        required: false
+      REPO_SSH_DEPLOY_KEY:
+        required: false
+      TF_PLAN_ENCRYPTION_PASSPHRASE:
+        required: true
+        description: "The passphrase used to encrypt Terraform Plans before uploading them as Github Artifacts"
+
+
+jobs:
+  define_matrix:
+    name: Define directory matrix for build
+    runs-on: ubuntu-latest
+    outputs:
+      stack_config: "${{ steps.stack_config.outputs.json_directory_list }}"
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.repo }}
+          ref: ${{ inputs.ref }}
+          ssh-key: ${{ secrets.REPO_SSH_DEPLOY_KEY }}
+      - name: Determine order to run Terraform stacks
+        uses: >-
+          ukhsa-collaboration/devops-github-actions/.github/actions/terraform-dependency-sort@v0.8.0
+        id: stack_config
+
+  filter_matrix:
+    name: Filter matrix for only planned changes
+    needs:
+      - define_matrix
+    runs-on: ubuntu-latest
+    outputs:
+      filtered_matrix: ${{ steps.filter_matrix.outputs.filtered_matrix }}
+    steps:
+      - name: Filter Stacks
+        env:
+          ENVIRONMENT_NAME: ${{ inputs.environment_name }}
+        id: filter_matrix
+        run: |
+          echo '${{ needs.define_matrix.outputs.stack_config }}' > initial_matrix.json
+
+          filtered_matrix=$(jq -c --arg env_name "${ENVIRONMENT_NAME}" '
+          [ .[]
+            | select(.planned_changes == true)
+            | if .runner_label == "self-hosted"
+              then .runner_label = ["self-hosted", $env_name]
+              else .
+              end
+          ]
+          ' initial_matrix.json)
+
+          echo "Filtered Matrix: $filtered_matrix"
+          echo "filtered_matrix=$filtered_matrix" >> $GITHUB_OUTPUT
+
+  plan:
+    name: Plan Terraform
+    uses: ./.github/workflows/terraform-core.yml
+    needs:
+      - filter_matrix
+    with:
+      environment_name: ${{ inputs.environment_name }}
+      aws_region: ${{ inputs.aws_region }}
+      repo: ${{ inputs.repo }}
+      ref:  ${{ inputs.ref }}
+      stack_config: "${{ needs.filter_matrix.outputs.filtered_matrix }}"
+      terraform_action: "apply"
+      execute_terraform_plan: false
+      upload_plan: true
+      download_existing_plan: false
+    secrets: inherit
+
+  comment:
+    name: Comment plan on Pull Request
+    if: github.event_name == 'pull_request'
+    needs:
+      - plan
+      - filter_matrix
+    runs-on: "${{ matrix.stack.runner_label }}"
+    strategy:
+      matrix:
+        stack: "${{ fromJSON(needs.filter_matrix.outputs.filtered_matrix) }}"
+    steps:
+      - name: Export variables
+        id: variables
+        run: | 
+          echo "state_name=$(basename ${{ matrix.stack.directory }})" >> $GITHUB_OUTPUT
+
+      - uses: actions/download-artifact@v4
+        id: download
+        with:
+          name: "${{ steps.variables.outputs.state_name }}-${{ inputs.environment_name }}-artefacts"
+          path: "${{ inputs.environment_name }}/${{ steps.variables.outputs.state_name }}"
+
+      - name: Decrypt Terraform Plan
+        working-directory: ${{ steps.download.outputs.download-path }}
+        env:
+          ENCRYPTION_PASSPHRASE: ${{ secrets.TF_PLAN_ENCRYPTION_PASSPHRASE }}
+        run: |
+          pass_file=$(mktemp)
+          printf "%s" "$ENCRYPTION_PASSPHRASE" > "$pass_file"
+          gpg --decrypt --batch --passphrase-file "$pass_file" --out tfplan.json tfplan.json.gpg
+
+      - name: Comment plan on PR
+        uses: actions/github-script@v7
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          script: |
+            const fs = require('fs');
+            const path = require('path');
+
+            const environment = "${{ inputs.environment_name }}";
+            const stack = "${{ steps.variables.outputs.state_name }}";
+
+            try {
+              const issue_number = context.payload.pull_request.number;
+              const owner = context.repo.owner;
+              const repo = context.repo.repo;
+              const commentIdentifier = `<!-- planComment-${environment}-${stack} -->`;
+              const filePath = path.join(process.env.GITHUB_WORKSPACE, environment, stack, 'tfplan.json');
+
+              if (!fs.existsSync(filePath)) {
+                console.log(`❌ Terraform plan file not found: ${filePath}`);
+                core.setFailed('Terraform plan file not found.');
+                return;
+              }
+
+              const fileContent = fs.readFileSync(filePath, 'utf8');
+
+              let planJson;
+              try {
+                planJson = JSON.parse(fileContent);
+              } catch (parseError) {
+                core.setFailed(`Failed to parse tfplan.json: ${parseError.message}`);
+                return;
+              }
+
+              const summary = { create: [], update: [], delete: [], replace: [] };
+
+              const changes = planJson.resource_changes || [];
+              changes.forEach(change => {
+                const address = change.address;
+                const actions = change.change.actions;
+                // If the actions include both delete and create, it's a replacement.
+                if (actions.includes("delete") && actions.includes("create")) {
+                  summary.replace.push(address);
+                } else if (actions.includes("create")) {
+                  summary.create.push(address);
+                } else if (actions.includes("update")) {
+                  summary.update.push(address);
+                } else if (actions.includes("delete")) {
+                  summary.delete.push(address);
+                }
+              });
+
+              let summaryText = `### Terraform Plan Summary (${environment.toUpperCase()}/${stack.toUpperCase()})\n\n`;
+              if (summary.delete.length > 0) {
+                summaryText += `🔴 **Resources to be destroyed:**\n${summary.delete.map(addr => `- ${addr}`).join("\n")}\n\n`;
+              }
+              if (summary.update.length > 0) {
+                summaryText += `🟡 **Resources to be updated:**\n${summary.update.map(addr => `- ${addr}`).join("\n")}\n\n`;
+              }
+              if (summary.create.length > 0) {
+                summaryText += `🟢 **Resources to be created:**\n${summary.create.map(addr => `- ${addr}`).join("\n")}\n\n`;
+              }
+              if (summary.replace.length > 0) {
+                summaryText += `🟣 **Resources to be replaced:**\n${summary.replace.map(addr => `- ${addr}`).join("\n")}\n\n`;
+              }
+              if (summary.update.length == 0 && summary.delete.length == 0 && summary.create.length == 0 && summary.replace.length == 0) {
+                summaryText += `👌 **No resources will be changed**\n\n`;
+              }
+
+              // Append the comment identifier so that subsequent runs can update the correct comment.
+              summaryText += commentIdentifier;
+
+              const comments = await github.rest.issues.listComments({
+                owner,
+                repo,
+                issue_number,
+              });
+
+              const botComment = comments.data.find(comment => comment.body.includes(commentIdentifier));
+
+              if (botComment) {
+                await github.rest.issues.updateComment({
+                  owner,
+                  repo,
+                  comment_id: botComment.id,
+                  body: summaryText,
+                });
+                console.log(`✅ Updated existing PR comment for environment: ${environment} / stack: ${stack}`);
+              } else {
+                await github.rest.issues.createComment({
+                  owner,
+                  repo,
+                  issue_number,
+                  body: summaryText,
+                });
+                console.log(`✅ Created a new PR comment for environment: ${environment} / stack: ${stack}`);
+              }
+            } catch (error) {
+              core.setFailed(`🚨 Failed to comment on PR for environment ${environment}: ${error.message}`);
+            }

.github/workflows/terraform-post-deployment-qa.yml

@@ -80,7 +80,7 @@ jobs:
       - uses: actions/download-artifact@v4
         id: download_plan
         with:
-          name: "${{ env.state_name }}-artefacts"
+          name: "${{ env.state_name }}-${{ inputs.environment_name}}-artefacts"
           path: ${{ matrix.stack.directory }}
 
       - name: Deep SAST Scan Terraform code

_test-terraform-plan.yml

@@ -0,0 +1,34 @@
+name: "Test Plan Terraform stacks"
+# Regression testing to try and catch and breaking changes
+
+on:
+  pull_request:
+    branches:
+      - main
+    paths:
+      - .github/workflows/terraform-plan.yml
+      - .github/workflows/_test-terraform-plan.yml
+      - .github/workflows/terraform-core.yml
+      - .github/workflows/terraform-code-check.yml
+
+jobs:
+  plan_tf_stacks_regression_test:
+    name: Test using devops-terraform-example-project@main
+    uses: ./.github/workflows/terraform-plan.yml
+    with:
+      repo: ukhsa-collaboration/devops-terraform-example-project
+      ref: "main"
+    permissions:
+      packages: read
+      actions: read
+      contents: read
+      security-events: write
+      statuses: write
+      checks: write
+      id-token: write
+    secrets:
+      REPO_SSH_DEPLOY_KEY: ${{ secrets.SSH_DEPLOY_KEY }}
+      AWS_ACCOUNT_ID: ${{ secrets.TEST_AWS_ACCOUNT_ID }}
+      AWS_ROLE_NAME: ${{ secrets.TEST_AWS_ROLE_NAME }}
+      TF_MODULES_SSH_DEPLOY_KEY: ${{ secrets.TF_MODULES_SSH_DEPLOY_KEY }}
+      TF_PLAN_ENCRYPTION_PASSPHRASE: ${{ secrets.TF_PLAN_ENCRYPTION_PASSPHRASE }}