restructure TF-managed services (#354)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

.github/workflows/deploy-daily-snapshot.yml

@@ -1,45 +1,48 @@
 name: Snapshot Service
-concurrency: ci-${{ github.ref }}
+concurrency: ci-${{ github.ref }}-snapshot-service
 
 on:
   pull_request:
     branches:
       - main
     paths:
-      - 'terraform/daily_snapshot/**'
-      - 'terraform/modules/daily_snapshot/**'
+      - 'tf-managed/modules/daily_snapshot/**'
+      - 'tf-managed/scripts/**'
+      - 'tf-managed/live/environments/prod/applications/snapshot-service/**'
     # This needs to be declared explicitly so that the job is actually
     # run when moved out of draft.
     types: [opened, synchronize, reopened, ready_for_review]
   push:
     branches:
       - main
     paths:
-      - 'terraform/daily_snapshot/**'
-      - 'terraform/modules/daily_snapshot/**'
+      - 'tf-managed/modules/daily_snapshot/**'
+      - 'tf-managed/scripts/**'
+      - 'tf-managed/live/environments/prod/applications/snapshot-service'
   workflow_dispatch:
 
 jobs:
-  deploy-daily-snapshot-calibnet:
-    name: Deploy
+  deploy-daily-snapshot:
+    env:
+      TF_VAR_monitoring: "{ \"enable\": true,\"slack_enable\":true,\"slack_destination_id\":\"${{ secrets.SLACK_DESTINATION_ID }}\",\"slack_channel_id\":\"${{ secrets.SLACK_CHANNEL_ID }}\"}"
     runs-on: ubuntu-latest
     permissions: write-all
     steps:
       - name: Checkout the code
         uses: actions/checkout@v4
 
-      # Using Custom Composite action in ./composite-action/terraform folder
-      - name: Composite Action for Deploying Terraform Resources
-        uses: ./composite-action/terraform
+      # Using Custom Composite action in ./composite-action/terragrunt folder
+      - name: Composite Action for Deploying Terragrunt Resources
+        uses: ./composite-action/terragrunt
         with:
           do_token: ${{ secrets.DO_TOKEN }}
           aws_access_key_id:  ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
           r2_access_key: ${{ secrets.R2_ACCESS_KEY }}
           r2_secret_key: ${{ secrets.R2_SECRET_KEY }}
           slack_token: ${{ secrets.SLACK_TOKEN }}
-          working_directory: terraform/daily_snapshot/prod
-          environment: Snapshot Service
+          working_directory: tf-managed/live/environments/prod/applications/snapshot-service
+          service_name: Snapshot Service
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
           new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
+          ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

.github/workflows/deploy-sync-check.yml

@@ -1,40 +1,46 @@
 name: Sync Check Service
-concurrency: ci-${{ github.ref }}
+concurrency: ci-${{ github.ref }}-sync-check
 
 on:
   pull_request:
     branches:
       - main
     paths:
-      - 'terraform/sync_check/**'
-      - 'terraform/modules/sync_check/**'
+      - 'tf-managed/modules/sync-check/**'
+      - 'tf-managed/scripts/**'
+      - 'tf-managed/live/environments/prod/applications/sync-check/**'
+    # This needs to be declared explicitly so that the job is actually
+    # run when moved out of draft.
+    types: [opened, synchronize, reopened, ready_for_review]
   push:
     branches:
       - main
     paths:
-      - 'terraform/sync_check/**'
-      - 'terraform/modules/sync_check/**'
+      - 'tf-managed/modules/sync-check/**'
+      - 'tf-managed/scripts/**'
+      - 'tf-managed/live/environments/prod/applications/sync-check'
   workflow_dispatch:
 
 jobs:
-  sync-check:
-    name: Deploy
+  deploy-sync-check:
     runs-on: ubuntu-latest
     permissions: write-all
     steps:
       - name: Checkout the code
         uses: actions/checkout@v4
 
-       # Using Custom Composite action in ./composite-action/terraform folder
-      - name: Composite Action for Deploying Terraform Resources
-        uses: ./composite-action/terraform
+      # Using Custom Composite action in ./composite-action/terragrunt folder
+      - name: Composite Action for Deploying Terragrunt Resources
+        uses: ./composite-action/terragrunt
         with:
           do_token: ${{ secrets.DO_TOKEN }}
           aws_access_key_id:  ${{ secrets.AWS_ACCESS_KEY_ID }}
           aws_secret_access_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}
+          r2_access_key: ${{ secrets.R2_ACCESS_KEY }}
+          r2_secret_key: ${{ secrets.R2_SECRET_KEY }}
           slack_token: ${{ secrets.SLACK_TOKEN }}
-          working_directory: terraform/sync_check
-          environment: Sync Check Service
+          working_directory: tf-managed/live/environments/prod/applications/sync-check
+          service_name: Sync Check Service
           new_relic_account_id: ${{ secrets.NEW_RELIC_ACCOUNT_ID }}
           new_relic_api_key: ${{ secrets.NEW_RELIC_API_KEY }}
+          ssh_private_key: ${{ secrets.SSH_PRIVATE_KEY }}

.github/workflows/scripts-lint.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Run rubocop
         run: |
           gem install rubocop --no-document
-          rubocop scripts/ # TODO: Apply rubocop to terraform/modules/
+          rubocop tf-managed/scripts/ # TODO: Apply rubocop to tf-managed/modules/
   run-js-linters:
     runs-on: ubuntu-latest
     steps:

.tflint.hcl

@@ -1,5 +1,5 @@
 plugin "terraform" {
-    enabled = true
-    version = "0.2.2"
-    source  = "github.com/terraform-linters/tflint-ruleset-terraform"
+  enabled = true
+  version = "0.5.0"
+  source  = "github.com/terraform-linters/tflint-ruleset-terraform"
 }

composite-action/terragrunt/action.yml

@@ -0,0 +1,190 @@
+name: Custom Composite action to deploy terragrunt resources
+
+description: |
+  This action deploys the Forest infrastructure with Terragrunt
+
+inputs:
+  service_name:
+    description: 'Human-readable name of the service'
+    required: true
+  do_token:
+    description: 'The DigitalOcean access token to use for deploying the infrastructure'
+    required: true
+  aws_access_key_id:
+    description: 'S3 access keys id used by terraform and service like sync check, Deploy Snapshot Service etc'
+    required: true
+  aws_secret_access_key:
+    description: 'S3 secret access keys used by terraform and service like sync check, Deploy Snapshot Service etc'
+    required: true
+  working_directory:
+    description: 'The working Directory'
+    required: true
+  ssh_private_key:
+    description: 'The SSH private key to use for connecting to Droplets via SSH'
+  slack_token:
+    description: 'The slack token secret used to connect the Infrastructure to Slack'
+  new_relic_api_key:
+    description: 'The New Relic API KEY'
+  nr_license_key:
+    description: 'The New Relic Access Token'
+  new_relic_account_id:
+    description: 'The New Relic Platform Region'
+  r2_access_key:
+    description: 'CloudFlare R2 access key id'
+  r2_secret_key:
+    description: 'CloudFlare R2 private access key'
+
+runs:
+  using: "composite"
+  steps:
+    # Workaround for https://github.com/orgs/community/discussions/51280
+    - name: Set TF/TG versions
+      shell: bash
+      run: |
+        echo "tf_version=1.6.6" >> $GITHUB_ENV
+        echo "tg_version=0.53.2" >> $GITHUB_ENV
+
+    - name: Setup Terraform
+      uses: hashicorp/setup-terraform@v2
+      with:
+        terraform_version: v${{ env.tf_version }}
+        # This is required for Terragrunt to parse Terraform outputs.
+        terraform_wrapper: false
+
+    - name: Setup Terragrunt
+      shell: bash
+      run: |
+        sudo wget -q -O /bin/terragrunt "https://github.com/gruntwork-io/terragrunt/releases/download/v${{ env.tg_version }}/terragrunt_linux_amd64"
+        sudo chmod +x /bin/terragrunt
+        terragrunt -v
+
+    - name: Check terragrunt HCL
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      run: |
+        terragrunt hclfmt --terragrunt-check --terragrunt-diff
+
+    - name: Validate
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      run: |
+        terragrunt validate
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+
+    - name: Plan
+      if: github.event_name == 'pull_request'
+      id: plan
+      shell: bash
+      working-directory: ${{ inputs.working_directory }}
+      continue-on-error: true
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+        TF_VAR_digitalocean_token: ${{ inputs.do_token }}
+        TF_VAR_AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        TF_VAR_AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+        TF_VAR_R2_ACCESS_KEY: ${{ inputs.r2_access_key }}
+        TF_VAR_R2_SECRET_KEY: ${{ inputs.r2_secret_key }}
+        TF_VAR_slack_token: ${{ inputs.slack_token }}
+        TF_VAR_new_relic_api_key: ${{ inputs.new_relic_api_key }}
+        TF_VAR_new_relic_account_id: ${{ inputs.new_relic_account_id }}
+      run: |
+        terragrunt plan -no-color --terragrunt-non-interactive -out ${{ github.workspace }}/tfplan | tee output
+        echo 'stdout<<EOF' >> $GITHUB_OUTPUT
+        cat output >> $GITHUB_OUTPUT
+        echo 'EOF' >> $GITHUB_OUTPUT
+
+    - name: Find Comment
+      if: github.event.pull_request.draft == false &&
+          github.event_name == 'pull_request'
+      uses: peter-evans/find-comment@v2
+      id: fc
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-regex: "^### Forest: ${{ inputs.service_name }} Infrastructure Plan"
+
+
+    - name: Create or Update Comment
+      if: github.event.pull_request.draft == false &&
+          github.event_name == 'pull_request' &&
+          !contains(steps.plan.outputs.stdout, 'No changes. Your infrastructure matches the configuration.')
+      uses: peter-evans/create-or-update-comment@v2
+      with:
+        comment-id: ${{ steps.fc.outputs.comment-id }}
+        issue-number: ${{ github.event.pull_request.number }}
+        body: |
+          ### Forest: ${{ inputs.service_name }} Infrastructure Plan: ${{ steps.plan.outcome }}
+
+          <details><summary>Show Plan</summary>
+
+          ```
+          ${{ steps.plan.outputs.stdout }}
+          ```
+
+          </details>
+        edit-mode: replace
+
+    - name: Delete Comment
+      uses: detomarco/delete-comments@v1.0.4
+      if: github.event.pull_request.draft == false &&
+          github.event_name == 'pull_request' &&
+          contains(steps.plan.outputs.stdout, 'No changes. Your infrastructure matches the configuration.')
+      with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+
+    - name: Terragrunt Plan Status
+      if: steps.plan.outcome == 'failure'
+      shell: bash
+      run: exit 1
+
+    - name: Configure ssh-agent
+      if: github.ref == 'refs/heads/main' && ( github.event_name == 'push' || github.event_name == 'workflow_dispatch' )
+      uses: webfactory/ssh-agent@v0.8.0
+      with:
+        ssh-private-key: ${{ inputs.ssh_private_key }}
+
+    - name: Terragrunt Apply
+      if: github.ref == 'refs/heads/main' && github.event_name == 'push'
+      shell: bash
+      run: |
+        if grep -q 'No changes.' ${{ github.workspace }}/tfplan; then
+          echo "No changes detected."
+        else
+          echo "Changes detected. Redeploying everything..."
+          terragrunt destroy -auto-approve --terragrunt-non-interactive
+          terragrunt apply -auto-approve --terragrunt-non-interactive
+        fi
+      working-directory: ${{ inputs.working_directory }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+        TF_VAR_digitalocean_token: ${{ inputs.do_token }}
+        TF_VAR_AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        TF_VAR_AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+        TF_VAR_R2_ACCESS_KEY: ${{ inputs.r2_access_key }}
+        TF_VAR_R2_SECRET_KEY: ${{ inputs.r2_secret_key }}
+        TF_VAR_slack_token: ${{ inputs.slack_token }}
+        TF_VAR_new_relic_api_key: ${{ inputs.new_relic_api_key }}
+        TF_VAR_new_relic_account_id: ${{ inputs.new_relic_account_id }}
+
+    - name: Terragrunt Force Apply
+      if: github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch'
+      working-directory: ${{ inputs.working_directory }}
+      env:
+        AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+        TF_VAR_digitalocean_token: ${{ inputs.do_token }}
+        TF_VAR_AWS_ACCESS_KEY_ID: ${{ inputs.aws_access_key_id }}
+        TF_VAR_AWS_SECRET_ACCESS_KEY: ${{ inputs.aws_secret_access_key }}
+        TF_VAR_R2_ACCESS_KEY: ${{ inputs.r2_access_key }}
+        TF_VAR_R2_SECRET_KEY: ${{ inputs.r2_secret_key }}
+        TF_VAR_slack_token: ${{ inputs.slack_token }}
+        TF_VAR_new_relic_api_key: ${{ inputs.new_relic_api_key }}
+        TF_VAR_new_relic_account_id: ${{ inputs.new_relic_account_id }}
+      shell: bash
+      run: |
+        terragrunt destroy -auto-approve --terragrunt-non-interactive
+        terragrunt apply -auto-approve --terragrunt-non-interactive