feat: mirror filecoin built-in actors

.github/workflows/mirror-builtin-actors.yml

@@ -0,0 +1,86 @@
+name: Mirror Builtin Actors Releases
+on:
+  schedule:
+    - cron: '0 * * * *'  # Runs every hour
+  pull_request:
+    paths:
+      - 'scripts/mirror-actors/**'
+  push:
+    paths:
+      - 'scripts/mirror-actors/**'
+  workflow_dispatch:
+
+jobs:
+  mirror-releases-do:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install dependencies
+        working-directory: scripts/mirror-actors
+        run: poetry install --no-interaction --no-root
+
+      - name: Mirror Actors to DigitalOcean
+        working-directory: scripts/mirror-actors
+        if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+        run: |
+          source .venv/bin/activate
+          python3 mirror_actors/
+        env:
+            SLACK_API_TOKEN: ${{ secrets.SLACK_TOKEN }}
+            SLACK_CHANNEL: "#forest-notifications"
+            BUCKET_NAME: filecoin-builtin-actors
+            AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            REGION_NAME: fra1
+            ENDPOINT_URL: https://fra1.digitaloceanspaces.com
+
+  mirror-releases-cf:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Install dependencies
+        working-directory: scripts/mirror-actors
+        run: poetry install --no-interaction --no-root
+
+      - name: Mirror Actors to CloudFlare
+        working-directory: scripts/mirror-actors
+        if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+        run: |
+          source .venv/bin/activate
+          python3 mirror_actors/
+        env:
+            SLACK_API_TOKEN: ${{ secrets.SLACK_TOKEN }}
+            SLACK_CHANNEL: "#forest-notifications"
+            BUCKET_NAME: filecoin-builtin-actors
+            AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY }}
+            AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_KEY }}
+            REGION_NAME: "auto"
+            ENDPOINT_URL: "https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com"

.github/workflows/scripts-lint.yml

@@ -38,3 +38,24 @@ jobs:
         yarn install
         yarn lint
         yarn js-check
+  run-py-linters:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+
+    - name: Install dependencies
+      working-directory: scripts/mirror-actors
+      run: poetry install --no-interaction --no-root
+
+    - name: Lint Python Code
+      working-directory: scripts/mirror-actors
+      run: poetry run pylint mirror_actors/ -f actions

scripts/mirror-actors/README.md

@@ -0,0 +1,46 @@
+# Overview
+This project automates the process of mirroring Filecoin's built-in actors' releases from GitHub to cloud storage services (DigitalOcean Spaces and CloudFlare R2). The script checks for new releases on GitHub, downloads them, and uploads them to the specified cloud storage. It's designed to run periodically and ensures that the latest releases are always available in the cloud storage.
+
+
+# Workflow
+
+The project uses GitHub Actions for automated deployment:
+
+- **Frequency**: The script runs every hour (0 * * * *).
+- **Triggered By**: Changes in the scripts/mirror-actors/** path in the repository. This includes both pull requests and push events.
+- **Manual Trigger**: The workflow can also be triggered manually via the GitHub UI (workflow_dispatch event).
+
+# Manual deployments
+
+## Requirements
+
+### Software
+
+* [Python3](https://www.python.org/downloads/)
+* [Poetry](https://python-poetry.org/docs/)
+
+For manual deployments, particularly useful for testing and debugging, set the following environment variables:
+
+## Required environment variables
+
+```bash
+# DigitalOcean or CloudFlare Access Tokens depending which cloud you want to mirror to
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+
+# Slack Access Token and channel
+export SLACK_API_TOKEN=
+export SLACK_CHANNEL=
+
+# s3 Boto client Configurations
+export BUCKET_NAME=
+export REGION_NAME=
+export ENDPOINT_URL=
+```
+
+Playbook:
+
+```bash
+$ poetry install --no-interaction --no-root         # Install dependencies
+$ poetry run python3 mirror_actors/                 # Run the mirroring script
+```

scripts/mirror-actors/mirror_actors/__main__.py

@@ -0,0 +1,75 @@
+"""
+This script mirrors Filecoin Actor releases from GitHub to an S3 bucket
+and sends alerts to a Slack channel in case of failures.
+"""
+
+import os
+import re
+from datetime import datetime
+import requests
+from dateutil.relativedelta import relativedelta
+import boto3
+from slack_sdk.web import WebClient
+from github import Github
+
+# Define environment variables
+SLACK_API_TOKEN = os.environ["SLACK_API_TOKEN"]
+SLACK_CHANNEL = os.environ["SLACK_CHANNEL"]
+S3_BUCKET = os.environ["S3_BUCKET"]
+ENDPOINT_URL = os.environ["ENDPOINT_URL"]
+REGION_NAME = os.environ["REGION_NAME"]
+
+GITHUB_REPO = "filecoin-project/builtin-actors"
+RELEASE_PATTERN = r'^v\d+\.\d+\.\d+.*$'
+
+# Initialize clients
+slack = WebClient(token=SLACK_API_TOKEN)
+github = Github()
+s3 = boto3.client('s3',
+                       region_name=REGION_NAME,
+                       endpoint_url=ENDPOINT_URL)
+
+# Calculate the cutoff date (3 years ago from current date)
+three_years_ago = datetime.now() - relativedelta(years=3)
+
+
+def send_slack_alert(message):
+    """
+    Send an alert message to a predefined Slack channel.
+
+    Args:
+        message (str): The message to be sent to Slack.
+    """
+    slack.chat_postMessage(
+        channel=SLACK_CHANNEL,
+        text=message
+    ).validate()
+
+# Process GitHub releases
+try:
+    releases = github.get_repo(GITHUB_REPO).get_releases()
+    # Fetch already mirrored objects from S3
+    s3_response = s3.list_objects(Bucket=S3_BUCKET)
+    already_mirrored = set()
+    if 'Contents' in s3_response:
+        already_mirrored = set(obj["Key"] for obj in s3_response["Contents"])
+
+    for release in releases:
+        tag_name = release.tag_name
+        # Removing timezone info for comparison
+        published_at = release.published_at.replace(tzinfo=None)
+        # Skip the release if it's older than 3 years
+        if published_at < three_years_ago:
+            continue
+
+        if re.match(RELEASE_PATTERN, tag_name):
+            for asset in release.get_assets():
+                key = f"{tag_name}/{asset.name}"
+                if key not in already_mirrored:
+                    response = requests.get(asset.browser_download_url, timeout=30)
+                    response.raise_for_status()
+                    s3.put_object(Bucket=S3_BUCKET, Key=key, Body=response.content)
+
+except Exception as e:
+    send_slack_alert(f"⛔ Filecoin Actor mirroring failed: {e}")
+    raise