feat: mirror filecoin built-in actors

.github/workflows/mirror-builtin-actors.yml

@@ -0,0 +1,114 @@
+name: Mirror Builtin Actors Releases
+on:
+  schedule:
+    - cron: '0 * * * *'  # Runs every hour
+  pull_request:
+    paths:
+      - 'scripts/mirror-actors/**'
+  push:
+    paths:
+      - 'scripts/mirror-actors/**'
+  workflow_dispatch:
+
+jobs:
+  mirror-releases-do:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Load cached Poetry installation
+        id: cached-poetry
+        uses: actions/cache@v3
+        with:
+          path: ~/.local
+          key: poetry-0
+
+      - name: Install Poetry
+        if: steps.cached-poetry.outputs.cache-hit != 'true'
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached dependencies Install
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: scripts/mirror-actors/.venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        working-directory: scripts/mirror-actors
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Mirror Actors to DigitalOcean
+        working-directory: scripts/mirror-actors
+        if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+        run: poetry run python -m mirror_actors
+        env:
+            SLACK_API_TOKEN: ${{ secrets.SLACK_TOKEN }}
+            SLACK_CHANNEL: "#forest-notifications"
+            BUCKET_NAME: filecoin-builtin-actors
+            AWS_ACCESS_KEY_ID:  ${{ secrets.AWS_ACCESS_KEY_ID }}
+            AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+            REGION_NAME: fra1
+            ENDPOINT_URL: https://fra1.digitaloceanspaces.com
+
+  mirror-releases-cf:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Load cached Poetry installation
+        id: cached-poetry
+        uses: actions/cache@v3
+        with:
+          path: ~/.local
+          key: poetry-1
+
+      - name: Install Poetry
+        if: steps.cached-poetry.outputs.cache-hit != 'true'
+        uses: snok/install-poetry@v1
+        with:
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+          installer-parallel: true
+
+      - name: Load cached dependencies Install
+        id: cached-poetry-dependencies
+        uses: actions/cache@v3
+        with:
+          path: scripts/mirror-actors/.venv
+          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Install dependencies
+        working-directory: scripts/mirror-actors
+        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+        run: poetry install --no-interaction --no-root
+
+      - name: Mirror Actors to CloudFlare
+        working-directory: scripts/mirror-actors
+        if: github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')
+        run: poetry run python -m mirror_actors
+        env:
+            SLACK_API_TOKEN: ${{ secrets.SLACK_TOKEN }}
+            SLACK_CHANNEL: "#forest-notifications"
+            BUCKET_NAME: filecoin-builtin-actors
+            AWS_ACCESS_KEY_ID: ${{ secrets.R2_ACTORS_ACCESS_KEY }}
+            AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_ACTORS_SECRET_KEY }}
+            REGION_NAME: "auto"
+            ENDPOINT_URL: "https://2238a825c5aca59233eab1f221f7aefb.r2.cloudflarestorage.com"

.github/workflows/scripts-lint.yml

@@ -38,3 +38,24 @@ jobs:
         yarn install
         yarn lint
         yarn js-check
+  run-py-linters:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - name: Install Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.12'
+
+    - name: Install Poetry
+      uses: snok/install-poetry@v1
+      with:
+        virtualenvs-create: true
+
+    - name: Install dependencies
+      working-directory: scripts/mirror-actors
+      run: poetry install --no-interaction --no-root
+
+    - name: Lint Python Code
+      working-directory: scripts/mirror-actors
+      run: poetry run pylint mirror_actors/ -f actions

scripts/mirror-actors/README.md

@@ -0,0 +1,55 @@
+# Overview
+This project automates the process of mirroring Filecoin's built-in actors' releases from GitHub to cloud storage services (DigitalOcean Spaces and CloudFlare R2). The script checks for new releases on GitHub, downloads them, and uploads them to the specified cloud storage. It's designed to run periodically and ensures that the latest releases are always available in the cloud storage.
+
+
+# Workflow
+
+The project uses GitHub Actions for automated deployment:
+
+- **Frequency**: The script runs every hour `(0 * * * *)`.
+- **Triggered By**: Changes in the `scripts/mirror-actors/**` path in the repository. This includes both pull requests and push events.
+- **Manual Trigger**: The workflow can also be triggered manually via the GitHub UI `(workflow_dispatch event)`.
+
+# Manual deployments
+
+## Requirements
+
+### Software
+
+* [Python3](https://www.python.org/downloads/)
+* [Poetry](https://python-poetry.org/docs/)
+
+For manual deployments, particularly useful for testing and debugging, set the following environment variables:
+
+## Required environment variables
+
+```bash
+
+# Defines if mirroring actors are on local storage or s3. Set to 'True' for local, 'False' for s3.
+export USE_LOCAL=
+
+# Path to save mirrored data if local mirroring is enabled (USE_LOCAL=true)
+export LOCAL_SAVE_PATH=
+
+# Configuration for s3 mirroring (USE_LOCAL=false)
+## Slack credentials and target channel
+export SLACK_API_TOKEN=
+export SLACK_CHANNEL=
+
+## Settings for s3 Boto3 client
+### Credentials for DigitalOcean or CloudFlare, depending on chosen cloud service
+export AWS_ACCESS_KEY_ID=
+export AWS_SECRET_ACCESS_KEY=
+
+export BUCKET_NAME=
+export REGION_NAME=
+export ENDPOINT_URL=
+
+```
+
+Playbook:
+
+```bash
+$ poetry install --no-interaction --no-root         # Install dependencies
+$ poetry run python -m mirror_actors                 # Run the mirroring script
+```

scripts/mirror-actors/mirror_actors/__main__.py

@@ -0,0 +1,103 @@
+"""
+This script mirrors Filecoin Actor releases from GitHub to a specified storage sink.
+It supports uploading to an S3 bucket or saving locally, based on configuration.
+Alerts are sent to a Slack channel in case of failures.
+"""
+import os
+import re
+from datetime import datetime
+import requests
+from dateutil.relativedelta import relativedelta
+import boto3
+from slack_sdk.web import WebClient
+from github import Github
+
+GITHUB_REPO = "filecoin-project/builtin-actors"
+RELEASE_PATTERN = r'^v\d+\.\d+\.\d+.*$'
+
+# Initialize GitHub client
+github = Github()
+
+# Calculate the cutoff date (3 years ago from the current date)
+three_years_ago = datetime.now() - relativedelta(years=3)
+
+def send_slack_alert(message):
+    """
+    Send an alert message to a predefined Slack channel.
+    """
+    slack_api_token = os.environ["SLACK_API_TOKEN"]
+    slack_channel = os.environ["SLACK_CHANNEL"]
+    slack = WebClient(token=slack_api_token)
+    slack.chat_postMessage(channel=slack_channel, text=message).validate()
+
+def list_s3_objects(bucket_name, endpoint_url, region_name):
+    """
+    List all objects in an S3 bucket.
+    """
+    s3 = boto3.client('s3', endpoint_url=endpoint_url, region_name=region_name)
+    s3_response = s3.list_objects_v2(Bucket=bucket_name)
+
+    if 'Contents' in s3_response and s3_response['Contents']:
+        return {item['Key'] for item in s3_response['Contents']}
+    return set()
+
+def save_to_s3(bucket_name, key, content, endpoint_url, region_name):
+    """
+    Mirror Actors to S3 bucket.
+    """
+    s3 = boto3.client('s3', endpoint_url=endpoint_url, region_name=region_name)
+    s3.put_object(Bucket=bucket_name, Key=key, Body=content)
+
+def save_to_local(base_dir, key, content):
+    """
+    Mirror Actors to local filesystem.
+    """
+    full_path = os.path.join(base_dir, key)
+    os.makedirs(os.path.dirname(full_path), exist_ok=True)
+    with open(full_path, 'wb') as local_file:
+        local_file.write(content)
+
+# Configuration
+USE_LOCAL = os.environ.get("USE_LOCAL", "False") == "True"
+LOCAL_SAVE_PATH = os.environ.get("LOCAL_SAVE_PATH", ".")
+BUCKET_NAME = os.environ.get("BUCKET_NAME", "")
+ENDPOINT_URL = os.environ.get("ENDPOINT_URL", "")
+REGION_NAME = os.environ.get("REGION_NAME", "")
+
+# Process GitHub releases
+try:
+    releases = github.get_repo(GITHUB_REPO).get_releases()
+    already_mirrored = set()
+
+    if USE_LOCAL:
+        for root, _, files in os.walk(LOCAL_SAVE_PATH):
+            for file in files:
+                already_mirrored.add(os.path.join(root, file))
+    else:
+        already_mirrored = list_s3_objects(BUCKET_NAME, ENDPOINT_URL, REGION_NAME)
+
+    for release in releases:
+        tag_name = release.tag_name
+        published_at = release.published_at.replace(tzinfo=None)
+        if published_at < three_years_ago:
+            continue
+
+        if re.match(RELEASE_PATTERN, tag_name):
+            for asset in release.get_assets():
+                release = f"{tag_name}/{asset.name}"
+                if release in already_mirrored:
+                    continue  # Skip already mirrored assets
+
+                response = requests.get(asset.browser_download_url, timeout=30)
+                response.raise_for_status()
+
+                # Save using the appropriate sink
+                if USE_LOCAL:
+                    save_to_local(LOCAL_SAVE_PATH, release, response.content)
+                else:
+                    save_to_s3(BUCKET_NAME, release, response.content, ENDPOINT_URL, REGION_NAME)
+
+except Exception as e:
+    error_message = f"⛔ Filecoin Actor mirroring failed: {e}"
+    send_slack_alert(error_message)
+    raise