diff --git a/.github/workflows/deploy-archival-snapshots.yml b/.github/workflows/deploy-archival-snapshots.yml new file mode 100644 index 000000000..214e06899 --- /dev/null +++ b/.github/workflows/deploy-archival-snapshots.yml @@ -0,0 +1,57 @@ +name: Export archival snapshots + +on: + schedule: + - cron: '0 0 * * *' + pull_request: + paths: + - 'ansible/archival-snapshots/**' + push: + paths: + - 'ansible/archival-snapshots/**' + workflow_dispatch: + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + + - name: Install Ansible + run: | + sudo apt-get update + sudo apt-get install -y ansible + + - name: Download and install Cloudflared + run: | + wget https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb + sudo dpkg -i cloudflared-linux-amd64.deb + cloudflared --version + + - name: Configure ssh-agent + uses: webfactory/ssh-agent@v0.8.0 + with: + ssh-private-key: ${{ secrets.ARCHIE_PRIVATE_KEY }} + + - name: Store SSH key + env: + SSH_PRIVATE_KEY: ${{ secrets.ARCHIE_PRIVATE_KEY }} + run: | + cat "$GITHUB_WORKSPACE/ansible/archival-snapshots/resources/ssh_config" >> ~/.ssh/config + echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa_archie + chmod 600 ~/.ssh/id_rsa_archie + + - name: Run Ansible playbook + env: + ANSIBLE_HOST_KEY_CHECKING: "False" + ARCHIVAL_SLACK_TOKEN: ${{ secrets.SLACK_TOKEN }} + ENDPOINT: https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com/ + run: | + ansible-playbook -i ansible/archival-snapshots/inventory.ini ansible/archival-snapshots/playbook.yml diff --git a/ansible/archival-snapshots/inventory.ini b/ansible/archival-snapshots/inventory.ini new file mode 100644 index 000000000..9249c40d8 --- /dev/null +++ b/ansible/archival-snapshots/inventory.ini @@ -0,0 +1,2 @@ +[remote_server] +archie.chainsafe.dev ansible_user=archie ansible_ssh_private_key_file=~/.ssh/id_rsa_archie diff --git a/ansible/archival-snapshots/playbook.yml b/ansible/archival-snapshots/playbook.yml new file mode 100644 index 000000000..2090403b2 --- /dev/null +++ b/ansible/archival-snapshots/playbook.yml @@ -0,0 +1,66 @@ +--- +- name: Automate packaging, transferring, and executing a script + hosts: remote_server + vars: + local_resources_path: "resources" + remote_resources_path: "/mnt/md0/exported/archival" + zip_file_name: "resources.zip" + forest_version: "v0.17.2" + forest_release_url: "https://github.com/ChainSafe/forest/releases/download/{{ forest_version }}/forest-{{ forest_version }}-linux-amd64.zip" + tasks: + - name: Check if AWS CLI is installed + ansible.builtin.command: + cmd: "which aws" + register: aws_installed + changed_when: false + ignore_errors: true + + - name: Install AWS CLI if not installed + ansible.builtin.command: + cmd: "sudo apt-get update && sudo apt-get install -y awscli" + when: aws_installed.rc != 0 + + - name: Check if Ruby is installed + ansible.builtin.command: + cmd: "which ruby" + register: ruby_installed + changed_when: false + ignore_errors: true + + - name: Install Ruby if not installed + ansible.builtin.command: + cmd: "sudo apt-get update && sudo apt-get install -y ruby-full" + when: ruby_installed.rc != 0 + + - name: Zip the resources folder + ansible.builtin.command: + cmd: "zip -r {{ zip_file_name }} ." + chdir: "{{ local_resources_path }}" + delegate_to: localhost + + - name: Transfer the zip file to the remote server + ansible.builtin.copy: + src: "{{ local_resources_path }}/{{ zip_file_name }}" + dest: "{{ remote_resources_path }}/{{ zip_file_name }}" + + - name: Unzip the resources folder on the remote server + ansible.builtin.command: + cmd: "unzip -o {{ zip_file_name }}" + chdir: "{{ remote_resources_path }}" + + - name: Download Forest release package + ansible.builtin.get_url: + url: "{{ forest_release_url }}" + dest: "{{ remote_resources_path }}/forest.zip" + + - name: Unzip the Forest release package on the remote server + ansible.builtin.command: + cmd: "unzip -jo {{ remote_resources_path }}/forest.zip -d {{ remote_resources_path }}/forest/" + + - name: Execute the init.sh script + ansible.builtin.shell: + cmd: "nohup ./init.sh > init.log 2>&1 &" + chdir: "{{ remote_resources_path }}" + environment: + ARCHIVAL_SLACK_TOKEN: "{{ lookup('env', 'ARCHIVAL_SLACK_TOKEN') }}" + ENDPOINT: "{{ lookup('env', 'ENDPOINT') }}" diff --git a/ansible/archival-snapshots/resources/config.toml b/ansible/archival-snapshots/resources/config.toml new file mode 100644 index 000000000..178345fb9 --- /dev/null +++ b/ansible/archival-snapshots/resources/config.toml @@ -0,0 +1,3 @@ +[client] +data_dir = "/mnt/md0/forest-archival-data" +encrypt_keystore = false diff --git a/ansible/archival-snapshots/resources/diff_script.sh b/ansible/archival-snapshots/resources/diff_script.sh new file mode 100755 index 000000000..9d436c487 --- /dev/null +++ b/ansible/archival-snapshots/resources/diff_script.sh @@ -0,0 +1,41 @@ +#!/bin/env bash + +set -euxo pipefail + +FOREST=/mnt/md0/exported/archival/forest/forest-tool +UPLOADED_DIFFS=/mnt/md0/exported/archival/uploaded-diff-snaps.txt +UPLOAD_QUEUE="/mnt/md0/exported/archival/upload_files.txt" + +EPOCH_START="$1" +shift +DIFF_STEP=3000 +DIFF_COUNT=10 +GENESIS_TIMESTAMP=1598306400 +SECONDS_PER_EPOCH=30 + +# Clear Upload List +if [ -f "$UPLOAD_QUEUE" ]; then + # Clear the contents of the file + true > "$UPLOAD_QUEUE" +fi + + +aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/diff/" > "$UPLOADED_DIFFS" + +for i in $(seq 1 $DIFF_COUNT); do + EPOCH=$((EPOCH_START+DIFF_STEP*i)) + EPOCH_TIMESTAMP=$((GENESIS_TIMESTAMP + EPOCH*SECONDS_PER_EPOCH)) + DATE=$(date --date=@"$EPOCH_TIMESTAMP" -u -I) + FILE_NAME="forest_diff_mainnet_${DATE}_height_$((EPOCH-DIFF_STEP))+$DIFF_STEP.forest.car.zst" + FILE="/mnt/md0/exported/archival/diff_snapshots/$FILE_NAME" + if ! grep -q "$FILE_NAME" "$UPLOADED_DIFFS"; then + if ! test -f "$FILE"; then + # Export diff snapshot + "$FOREST" archive export --depth "$DIFF_STEP" --epoch "$EPOCH" --diff $((EPOCH-DIFF_STEP)) --diff-depth 900 --output-path "$FILE" "$@" + fi + # Add exported diff snapshot to upload queue + echo "$FILE" >> "$UPLOAD_QUEUE" + else + echo "Skipping $FILE" + fi +done diff --git a/ansible/archival-snapshots/resources/init.sh b/ansible/archival-snapshots/resources/init.sh new file mode 100755 index 000000000..8a8cf492d --- /dev/null +++ b/ansible/archival-snapshots/resources/init.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +## Enable strict error handling, command tracing, and pipefail +set -eux + +# Initialize snapshots directory +LITE_SNAPSHOT_DIR="/mnt/md0/exported/archival/lite_snapshots" +DIFF_SNAPSHOT_DIR="/mnt/md0/exported/archival/diff_snapshots" +FULL_SNAPSHOTS_DIR=/mnt/md0/exported/archival/snapshots + +if [ ! -d "$LITE_SNAPSHOT_DIR" ]; then + mkdir -p "$LITE_SNAPSHOT_DIR" + echo "Created $LITE_SNAPSHOT_DIR" +else + echo "$LITE_SNAPSHOT_DIR exists" +fi + +if [ ! -d "$DIFF_SNAPSHOT_DIR" ]; then + mkdir -p "$DIFF_SNAPSHOT_DIR" + echo "Created $DIFF_SNAPSHOT_DIR" +else + echo "$DIFF_SNAPSHOT_DIR exists" +fi + +if [ ! -d "$FULL_SNAPSHOTS_DIR" ]; then + mkdir -p "$FULL_SNAPSHOTS_DIR" + echo "Created $FULL_SNAPSHOTS_DIR" +else + echo "$FULL_SNAPSHOTS_DIR exists" +fi + +# Trigger main script +./main.sh + +EXIT_STATUS=$? + +# Notify on slack channel +if [ "$EXIT_STATUS" -eq 0 ]; then + echo "Script executed successfully" + ruby notify.rb success +else + echo "Script execution failed" + ruby notify.rb failure +fi + +exit "$EXIT_STATUS" diff --git a/ansible/archival-snapshots/resources/main.sh b/ansible/archival-snapshots/resources/main.sh new file mode 100755 index 000000000..b23c84477 --- /dev/null +++ b/ansible/archival-snapshots/resources/main.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +## Enable strict error handling, command tracing, and pipefail +set -euxo pipefail + +## Set constants +GENESIS_TIMESTAMP=1598306400 +SECONDS_PER_EPOCH=30 + +## Set forest artifacts path +FOREST="/mnt/md0/exported/archival/forest/forest" +FOREST_CLI="/mnt/md0/exported/archival/forest/forest-cli" +FOREST_TOOL="/mnt/md0/exported/archival/forest/forest-tool" + +LITE_SNAPSHOT_DIR=/mnt/md0/exported/archival/lite_snapshots +FULL_SNAPSHOTS_DIR=/mnt/md0/exported/archival/snapshots + +## Fetch last snapshot details +LAST_SNAPSHOT=$(aws --profile prod --endpoint "$ENDPOINT" s3 ls "s3://forest-archive/mainnet/lite/" | sort | tail -n 1 | awk '{print $NF}') +LAST_EPOCH=$(echo "$LAST_SNAPSHOT" | awk -F'_' '{gsub(/[^0-9]/, "", $6); print $6}') +LAST_FULL_SNAPSHOT_PATH="$FULL_SNAPSHOTS_DIR/$LAST_SNAPSHOT" + +if [ ! -f "$LAST_FULL_SNAPSHOT_PATH" ]; then + echo "Downloading last snapshot: $LAST_FULL_SNAPSHOT_PATH" + aws --profile prod --endpoint "$ENDPOINT" s3 cp "s3://forest-archive/mainnet/lite/$LAST_SNAPSHOT" "$LAST_FULL_SNAPSHOT_PATH" + echo "Last snapshot download: $LAST_FULL_SNAPSHOT_PATH" +else + echo "$LAST_FULL_SNAPSHOT_PATH snapshot exists." +fi + +# Clean forest db +$FOREST_TOOL db destroy --force + +echo "Starting forest daemon" +nohup $FOREST --no-gc --config ./config.toml --save-token ./admin_token --rpc-address 127.0.0.1:3456 --metrics-address 127.0.0.1:5000 --import-snapshot "$LAST_FULL_SNAPSHOT_PATH" > forest.log 2>&1 & +FOREST_NODE_PID=$! + +sleep 30 +echo "Forest process started with PID: $FOREST_NODE_PID" + +# Function to kill Forest daemon +function kill_forest_daemon { + echo "Killing Forest daemon with PID: $FOREST_NODE_PID" + kill -KILL $FOREST_NODE_PID +} + +# Set trap to kill Forest daemon on script exit or error +trap kill_forest_daemon EXIT + +# Set required env variables +function set_fullnode_api_info { + ADMIN_TOKEN=$(cat admin_token) + export FULLNODE_API_INFO="$ADMIN_TOKEN:/ip4/127.0.0.1/tcp/3456/http" + echo "Using: $FULLNODE_API_INFO" +} +set_fullnode_api_info + +# Wait for network to sync +echo "Waiting for forest to sync to latest network head" +$FOREST_CLI sync wait + +# Get latest epoch using sync status +echo "Current Height: $LAST_EPOCH" +LATEST_EPOCH=$($FOREST_CLI sync status | grep "Height:" | awk '{print $2}') +echo "Latest Height: $LATEST_EPOCH" + +while ((LATEST_EPOCH - LAST_EPOCH > 30000)); do + set_fullnode_api_info + NEW_EPOCH=$((LAST_EPOCH + 30000)) + echo "Next Height: $NEW_EPOCH" + + # Export full snapshot to generate lite and diff snapshots + EPOCH_TIMESTAMP=$((GENESIS_TIMESTAMP + NEW_EPOCH*SECONDS_PER_EPOCH)) + DATE=$(date --date=@"$EPOCH_TIMESTAMP" -u -I) + NEW_SNAPSHOT="forest_snapshot_mainnet_${DATE}_height_${NEW_EPOCH}.forest.car.zst" + if [ ! -f "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" ]; then + echo "Exporting snapshot: $FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" + echo "USING FULLNODE API: $FULLNODE_API_INFO" + $FOREST_CLI snapshot export --tipset "$NEW_EPOCH" --depth 30000 -o "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" > export.txt + echo "Snapshot exported: $FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" + else + echo "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT already exists." + fi + + # Generate and upload lite snapshot + if [ ! -f "$LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" ]; then + echo "Generating Lite snapshot: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" + $FOREST_TOOL archive export --epoch "$NEW_EPOCH" --output-path "$LITE_SNAPSHOT_DIR" "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" + echo "Lite snapshot generated: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" + else + echo "$NEW_SNAPSHOT lite snapshot already exists." + fi + echo "Uploading Lite snapshot: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" + aws --profile prod --endpoint "$ENDPOINT" s3 cp "$LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" "s3://forest-archive/mainnet/lite/" + echo "Lite snapshot uploaded: $LITE_SNAPSHOT_DIR/$NEW_SNAPSHOT" + + # Generate and upload diff snapshots + if [ ! -f "$LAST_FULL_SNAPSHOT_PATH" ]; then + echo "File does not exist. Exporting..." + $FOREST_CLI snapshot export --tipset "$LAST_EPOCH" --depth 30000 -o "$LAST_FULL_SNAPSHOT_PATH" + else + echo "$LAST_FULL_SNAPSHOT_PATH file exists." + fi + echo "Generating Diff snapshots: $LAST_EPOCH - $NEW_EPOCH" + ./diff_script.sh "$LAST_EPOCH" "$LAST_FULL_SNAPSHOT_PATH" "$FULL_SNAPSHOTS_DIR/$NEW_SNAPSHOT" + echo "Diff snapshots generated successfully" + echo "Uploading Diff snapshots" + ./upload_diff.sh "$ENDPOINT" + echo "Diff snapshots uploaded successfully" + + LAST_EPOCH=$NEW_EPOCH +done diff --git a/ansible/archival-snapshots/resources/notify.rb b/ansible/archival-snapshots/resources/notify.rb new file mode 100644 index 000000000..07844b770 --- /dev/null +++ b/ansible/archival-snapshots/resources/notify.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'slack-ruby-client' + +CHANNEL = '#forest-dump' +SLACK_TOKEN = ENV.fetch('ARCHIVAL_SLACK_TOKEN') +STATUS = ARGV[0] + +client = Slack::Web::Client.new(token: SLACK_TOKEN) + +message = if STATUS == 'success' + '✅ Lite and Diff snapshots updated. 🌲🌳🌲🌳🌲' + else + '❌ Failed to update Lite and Diff snapshots. 🔥🌲🔥' + end + +client.chat_postMessage(channel: CHANNEL, text: message, as_user: true) diff --git a/ansible/archival-snapshots/resources/ssh_config b/ansible/archival-snapshots/resources/ssh_config new file mode 100644 index 000000000..a176dc0ed --- /dev/null +++ b/ansible/archival-snapshots/resources/ssh_config @@ -0,0 +1,3 @@ +Host archie.chainsafe.dev + ProxyCommand /usr/local/bin/cloudflared access ssh --hostname %h + User archie diff --git a/ansible/archival-snapshots/resources/upload_diff.sh b/ansible/archival-snapshots/resources/upload_diff.sh new file mode 100755 index 000000000..ba5258b6f --- /dev/null +++ b/ansible/archival-snapshots/resources/upload_diff.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +## Enable strict error handling, command tracing, and pipefail +set -euxo pipefail + +ENDPOINT="$1" + +while read -r file; do + # Upload the file to the S3 bucket + aws --profile prod --endpoint "$ENDPOINT" s3 cp "$file" "s3://forest-archive/mainnet/diff/" +done < /mnt/md0/exported/archival/upload_files.txt