Implement automation for archival snapshots

.github/workflows/deploy-archival-snapshots.yml

@@ -0,0 +1,41 @@
+name: Run Ansible playbook for archival snapshots
+
+on:
+  schedule:
+    - cron: '0 0 */10 * *'
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Install Ansible
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y ansible
+
+      - name: Configure SSH
+        env:
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+        run: |
+          mkdir -p ~/.ssh
+          echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+          chmod 600 ~/.ssh/id_rsa
+          ssh-keyscan -t rsa archie.chainsafe.dev >> ~/.ssh/known_hosts
+
+      - name: Run Ansible playbook
+        env:
+          ANSIBLE_HOST_KEY_CHECKING: True
+          SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+          ARCHIVAL_SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }}
+          ENDPOINT: ${{ secrets.CLOUDFLARE_ENDPOINT }}
+        run: |
+          ansible-playbook playbook.yml -i inventory.ini

ansible/archival-snapshots/inventory.ini

@@ -0,0 +1,2 @@
+[remote_server]
+archie.chainsafe.dev ansible_user=archie ansible_ssh_private_key_file=~/.ssh/id_rsa

ansible/archival-snapshots/playbook.yml

@@ -0,0 +1,31 @@
+---
+- name: Automate packaging, transferring, and executing a script
+  hosts: remote_server
+  vars:
+    local_resources_path: "resources"
+    remote_resources_path: "/mnt/md0/exported/archival"
+    zip_file_name: "resources.zip"
+  tasks:
+    - name: Zip the resources folder
+      ansible.builtin.command:
+        cmd: "zip -r {{ zip_file_name }} ."
+        chdir: "{{ local_resources_path }}"
+      delegate_to: localhost
+
+    - name: Transfer the zip file to the remote server
+      ansible.builtin.copy:
+        src: "{{ local_resources_path }}/{{ zip_file_name }}"
+        dest: "{{ remote_resources_path }}/{{ zip_file_name }}"
+
+    - name: Unzip the resources folder on the remote server
+      ansible.builtin.command:
+        cmd: "unzip {{ zip_file_name }}"
+        chdir: "{{ remote_resources_path }}"
+
+    - name: Execute the init.sh script
+      ansible.builtin.shell:
+        cmd: "./init.sh"
+        chdir: "{{ remote_resources_path }}"
+      environment:
+        ARCHIVAL_SLACK_TOKEN: "{{ lookup('env', 'ARCHIVAL_SLACK_TOKEN') }}"
+        ENDPOINT: "{{ lookup('env', 'ENDPOINT') }}"

ansible/archival-snapshots/resources/diff_script.sh

@@ -0,0 +1,24 @@
+#!/bin/env bash
+
+set -euxo pipefail
+
+FOREST=~/forest-v0.14.0/forest-tool
+
+EPOCH_START="$1"
+shift
+DIFF_STEP=3000
+DIFF_COUNT=10
+GENESIS_TIMESTAMP=1598306400
+SECONDS_PER_EPOCH=30
+
+for i in $(seq 1 $DIFF_COUNT); do
+    EPOCH=$((EPOCH_START+DIFF_STEP*i))
+    EPOCH_TIMESTAMP=$((GENESIS_TIMESTAMP + EPOCH*SECONDS_PER_EPOCH))
+    DATE=$(date --date=@"$EPOCH_TIMESTAMP" -u -I)
+    FILE="diff_snapshots/forest_diff_mainnet_${DATE}_height_$((EPOCH-DIFF_STEP))+$DIFF_STEP.forest.car.zst"
+    if ! test -f "$FILE"; then
+        "$FOREST" archive export --depth "$DIFF_STEP" --epoch "$EPOCH" --diff $((EPOCH-DIFF_STEP)) --diff-depth 900 --output-path "$FILE" "$@"
+    else
+        echo "Skipping $FILE"
+    fi
+done

ansible/archival-snapshots/resources/init.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+## Enable strict error handling, command tracing, and pipefail
+set -euxo pipefail
+
+FOREST_CLI="/home/archie/forest-v0.16.3/forest-cli"
+FOREST_TOOL="/home/archie/forest-v0.16.3/forest-tool"
+
+# Setting DEBIAN_FRONTEND to ensure non-interactive operations for APT
+export DEBIAN_FRONTEND=noninteractive
+
+CURRENT_SNAPSHOT=$(aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/lite/" | sort | tail -n 1 | awk '{print $NF}')
+CURRENT_EPOCH=$(echo "$CURRENT_SNAPSHOT" | awk -F'_' '{gsub(/[^0-9]/, "", $6); print $6}')
+
+echo "$CURRENT_EPOCH"
+
+LATEST_EPOCH=$($FOREST_CLI sync status | grep "Height:" | awk '{print $2}')
+echo "$LATEST_EPOCH"
+
+while ((LATEST_EPOCH - CURRENT_EPOCH > 30000)); do
+   NEW_EPOCH=$((CURRENT_EPOCH + 30000))
+   NEW_SNAPSHOT=$($FOREST_CLI snapshot export --tipset "$NEW_EPOCH" --depth 30000 | grep forest |awk -F'[:]' '{print $1}')
+
+   # Generate and upload lite snapshot
+   $FOREST_TOOL archive export --epoch "$NEW_EPOCH" --output-path ./lite_snapshots/ "$NEW_SNAPSHOT"
+   aws --profile prod --endpoint "$ENDPOINT" s3 cp ./lite_snapshots/"$NEW_SNAPSHOT" "s3://forest-archive/mainnet/lite/"
+
+   # Generate and upload diff snapshots
+   ./diff_script.sh "$CURRENT_EPOCH" "$CURRENT_SNAPSHOT" "$NEW_SNAPSHOT"
+   ./upload_diff.sh "$ENDPOINT"
+
+   CURRENT_EPOCH=$NEW_EPOCH
+done
+
+# Send notification on slack channel
+ruby notify.rb "$CURRENT_EPOCH"

ansible/archival-snapshots/resources/notify.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'slack-ruby-client'
+
+CHANNEL = '#forest-notifications'
+SLACK_TOKEN = ENV.fetch('ARCHIVAL_SLACK_TOKEN')
+EPOCH = ARGV[0]
+
+client = Slack::Web::Client.new(token: SLACK_TOKEN)
+
+message = "✅ Lite and Diff snapshots updated till #{EPOCH}. 🌲🌳🌲🌳🌲"
+
+client.chat_postMessage(channel: CHANNEL, text: message, as_user: true)

ansible/archival-snapshots/resources/upload_diff.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+ENDPOINT="$1"
+
+# list files to upload
+ls diff_snapshots/ > upload_files.txt
+
+while read -r file; do
+    # Upload the file to the S3 bucket
+    aws --profile prod --endpoint "$ENDPOINT" s3 cp diff_snapshots/"$file" "s3://forest-archive/mainnet/diff/"
+done < upload_files.txt
+
+# Remove uploaded diff snapshots
+rm diff_snapshots/* -rf