Tinybird Forward Deployment (#379)

* Initial commit

* Fix port

* Try to deploy the diff only when needed

* Fix grep return code

* Deploy to the cloud

* Change grep pattern

* Print CLI version

* Add debug to tb deploy

* Add copy for the initial populate

* Make populate pipe a copy

* Add MV

.github/workflows/forward_deployment.yml

@@ -0,0 +1,80 @@
+name: Tinybird - Forward Deployment demo
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 * * * *"
+  pull_request:
+    branches:
+      - main
+      - master
+    types: [opened, reopened, labeled, unlabeled, synchronize]
+
+concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+
+env:
+  TINYBIRD_FWD_HOST: ${{ secrets.TINYBIRD_FWD_HOST }}
+  TINYBIRD_FWD_TOKEN: ${{ secrets.TINYBIRD_FWD_TOKEN }}
+
+jobs:
+  forward_deployment:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: './forward_deployment'
+    services:
+      tinybird:
+        image: tinybirdco/tinybird-local:beta
+        ports:
+          - 7181:7181
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install Tinybird CLI
+        run: curl -LsSf https://tbrd.co/fwd | sh
+      - name: Tinybird CLI version
+        run: tb --version
+      - name: Get datasource definition
+        run: |
+          DATASOURCE_SORTING_KEY=$(curl \
+            -H "Authorization: Bearer $TINYBIRD_FWD_TOKEN" \
+            -X GET "$TINYBIRD_FWD_HOST/v0/datasources/users.datasource" | grep 'ENGINE_SORTING_KEY "tuple()"' || true)
+
+          echo "DATASOURCE_SORTING_KEY=$DATASOURCE_SORTING_KEY" >> $GITHUB_ENV
+      - name: Apply patch
+        run: |
+          if [ -n "$DATASOURCE_SORTING_KEY" ]; then
+            cp -r .diff/* . && git diff
+          else
+            echo "No changes to apply"
+          fi
+      - name: Build project
+        run: tb build
+      - name: Deploy project
+        run: tb --host $TINYBIRD_FWD_HOST --token $TINYBIRD_FWD_TOKEN --cloud deployment create --auto --wait
+      - name: Populate datasource if empty
+        run: |
+          COUNT=$(tb --host $TINYBIRD_FWD_HOST --token $TINYBIRD_FWD_TOKEN --cloud sql "SELECT count() AS c FROM users" --format json | grep '"c":' | awk '{print $2}')
+          if [ "$COUNT" -eq 0 ]; then
+            tb --host $TINYBIRD_FWD_HOST --token $TINYBIRD_FWD_TOKEN --cloud copy run populate --wait
+          else
+            echo "Datasource is not empty, skipping population"
+          fi
+      - name: Check users datasource
+        run: |
+          COUNT=$(tb --host $TINYBIRD_FWD_HOST --token $TINYBIRD_FWD_TOKEN --cloud sql "SELECT count() AS c FROM users" --format json | grep '"c":' | awk '{print $2}')
+          if [ "$COUNT" -eq 1000000000 ]; then
+            echo "Datasource data is correct"
+          else
+            echo "Datasource data is incorrect"
+            exit 1
+          fi
+      - name: Check user_stats datasource
+        run: |
+          COUNT=$(tb --host $TINYBIRD_FWD_HOST --token $TINYBIRD_FWD_TOKEN --cloud sql "SELECT count() AS c FROM user_stats" --format json | grep '"c":' | awk '{print $2}')
+          if [ "$COUNT" -gt 0 ]; then
+            echo "Datasource data is correct"
+          else
+            echo "Datasource data is incorrect"
+            exit 1
+          fi
+

forward_deployment/.diff/datasources/users.datasource

@@ -0,0 +1,9 @@
+SCHEMA >
+
+    id UInt64 `json:$.user_id`,
+    name String `json:$.name`,
+    category LowCardinality(String) `json:$.category`,
+    created_at DateTime `json:$.created_at` DEFAULT now()
+
+ENGINE MergeTree
+ENGINE_SORTING_KEY "id"

forward_deployment/copies/populate.pipe

@@ -0,0 +1,11 @@
+NODE populate_users
+SQL >
+  SELECT id % 100000 AS id, name, arrayElement(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j'], (category_id % 10) + 1) AS category, created_at
+  FROM generateRandom('id UInt64, name String, category_id UInt8, created_at DateTime')
+  LIMIT 1000000000
+
+TYPE copy
+
+TARGET_DATASOURCE users
+COPY_SCHEDULE @on-demand
+COPY_MODE replace

forward_deployment/datasources/user_stats.datasource

@@ -0,0 +1,8 @@
+SCHEMA >
+
+    id UInt64,
+    categories AggregateFunction(uniq, String),
+    count AggregateFunction(count, UInt64)
+
+ENGINE AggregatingMergeTree
+ENGINE_SORTING_KEY id

forward_deployment/datasources/users.datasource

@@ -0,0 +1,9 @@
+SCHEMA >
+
+    id UInt64 `json:$.user_id`,
+    name String `json:$.name`,
+    category LowCardinality(String) `json:$.category`,
+    created_at DateTime `json:$.created_at` DEFAULT now()
+
+ENGINE MergeTree
+ENGINE_SORTING_KEY ""

forward_deployment/materializations/mv_1.pipe

@@ -0,0 +1,12 @@
+NODE mv_user_stats
+SQL >
+
+  SELECT 
+    id, 
+    uniqState(category) AS categories,
+    countState() AS count
+  FROM users
+  GROUP BY id
+
+TYPE materialized
+DATASOURCE user_stats