diff --git a/.github/workflows/delete_simple_resource_cd.yml b/.github/workflows/delete_simple_resource_cd.yml index 95cb59a6..4343e262 100644 --- a/.github/workflows/delete_simple_resource_cd.yml +++ b/.github/workflows/delete_simple_resource_cd.yml @@ -17,7 +17,7 @@ cd: uses: tinybirdco/ci/.github/workflows/cd.yml@main with: - tb_deploy: false + tb_deploy: true data_project_dir: ./delete_simple_resource secrets: tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_SIMPLE_RESOURCE }} diff --git a/.github/workflows/delete_simple_resource_ci.yml b/.github/workflows/delete_simple_resource_ci.yml index 79aa7774..3e445005 100644 --- a/.github/workflows/delete_simple_resource_ci.yml +++ b/.github/workflows/delete_simple_resource_ci.yml @@ -21,6 +21,7 @@ ci: uses: tinybirdco/ci/.github/workflows/ci.yml@main with: + tb_deploy: true data_project_dir: ./delete_simple_resource secrets: tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_SIMPLE_RESOURCE }} diff --git a/.github/workflows/v3_delete_simple_resource_cd.yml b/.github/workflows/v3_delete_simple_resource_cd.yml deleted file mode 100644 index b2335d05..00000000 --- a/.github/workflows/v3_delete_simple_resource_cd.yml +++ /dev/null @@ -1,25 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CD Workflow - - on: - workflow_dispatch: - push: - branches: - - main - - master - paths: - - 'v3/delete_simple_resource/**' - jobs: - cd: # deploy changes to workspace 'v3_delete_simple_resource' - uses: tinybirdco/ci/.github/workflows/cd.yml@main - with: - tb_deploy: true - data_project_dir: v3/delete_simple_resource - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_SIMPLE_RESOURCE_V3 }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/.github/workflows/v3_delete_simple_resource_ci.yml b/.github/workflows/v3_delete_simple_resource_ci.yml deleted file mode 100644 index fcdb7d32..00000000 --- a/.github/workflows/v3_delete_simple_resource_ci.yml +++ /dev/null @@ -1,29 +0,0 @@ - - ################################################## - ### Visit https://github.com/tinybirdco/ci ### - ### for more details or custom CI/CD ### - ################################################## - - name: Tinybird - CI Workflow - - on: - workflow_dispatch: - pull_request: - branches: - - main - - master - types: [opened, reopened, labeled, unlabeled, synchronize, closed] - paths: - - 'v3/delete_simple_resource/**' - - concurrency: ${{ github.workflow }}-${{ github.event.pull_request.number }} - - jobs: - ci: # ci using environments from workspace 'v3_delete_simple_resource' - uses: tinybirdco/ci/.github/workflows/ci.yml@main - with: - tb_deploy: true - data_project_dir: v3/delete_simple_resource - secrets: - tb_admin_token: ${{ secrets.TB_ADMIN_TOKEN_DELETE_SIMPLE_RESOURCE_V3 }} # set the Workspace admin token in GitHub secrets - tb_host: https://api.tinybird.co diff --git a/README.md b/README.md index 9254c43a..327421c4 100644 --- a/README.md +++ b/README.md @@ -11,27 +11,7 @@ Either you want to reproduce a use case or create a new one, please follow the [ This repository contains all the use cases you can iterate with Versions: - [Delete simple resource](delete_simple_resource) -- [Delete simple resource using Releases](v3/delete_simple_resource) -- [Change Data Source sorting key](change_sorting_key) -- [Change S3 Data Source sorting key](change_sorting_key_to_s3_data_source) -- [Change S3 Data Source sorting key with reingestion](change_sorting_key_to_s3_data_source_with_reingestion) -- [Change Kafka Data Source sorting key](change_sorting_key_to_kafka_data_source) -- [Recover data from quarantine](v3/Quarentine_V3) using a copy Pipe -- [Add a new column to a Landing Data Source](add_column_landing_ds) -- [Add a new column to a Landing Data Source using V3](v3/add_column_landing_ds) -- [Add a new column to a Materialized View](add_column_materialized_view) -- [Add a new column to a Materialized View using Releases](v3/add_nullable_column_to_landing_datas_source) -- [Add a new column in a BigQuery Data Source](add_column_BQ_ds) -- [Add a new column to a Snowflake Data Source](add_column_snowflake_ds) -- [Add a new column to an S3 Data Source](add_column_to_s3_ds) -- [Add a new column to a Kafka Data Source](add_column_kafka_ds) -- [Change a Data Source's TTL](change_ttl) -- [Change column type from String to LowCardinality](change_column_type_to_lowcardinality) -- [Remove Column from a Landing Data Source using Releases](v3/delete_column_from_landing_data_source_releases) -- [Remove Column from a Materialized View](delete_column_materialized_view) -- [Remove Column from a Snowflake Data Source](delete_column_snowflake_ds) -- [Change Copy pipe and its target Data Source](change_copy_pipe) -- [Use shared Data Source](use_new_columns_from_shared_datasource) + ## Caveats diff --git a/delete_simple_resource/.gitignore b/delete_simple_resource/.gitignore deleted file mode 100644 index 0fec8669..00000000 --- a/delete_simple_resource/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -.DS_Store -.venv -.tinyb -.vercel -node_modules -dist -.e \ No newline at end of file diff --git a/delete_simple_resource/.tinyenv b/delete_simple_resource/.tinyenv index 8303cedc..ffb6da70 100644 --- a/delete_simple_resource/.tinyenv +++ b/delete_simple_resource/.tinyenv @@ -1 +1,2 @@ +TB_VERSION_WARNING=0 VERSION=0.0.1 diff --git a/delete_simple_resource/README.md b/delete_simple_resource/README.md index cbfac146..637735f5 100644 --- a/delete_simple_resource/README.md +++ b/delete_simple_resource/README.md @@ -1,21 +1,11 @@ # Delete simple resource -> Important! Resources Deletion has been improved and it's much easier using Releases: [Delete simple resource using Releases](v3/delete_simple_resource) -Delete simple Data Sources or Pipes easily. Create a [Pull Request](https://github.com/tinybirdco/use-case-examples/pull/5) following these steps: +Delete simple Data Sources or Pipes easily. Create a [Pull Request](https://github.com/tinybirdco/use-case-examples/pull/143) following these steps: > Remember to follow the [instructions](../README.md) to setup your Tinybird Data Project before jumping into the use-case steps -- Create a new branch -- Remove the Pipes or Data Sources -- Generate a new CI/CD version `tb release generate --semver 0.0.1` -- Include your delete operations in both CI/CD files -- Push your changes +- Remove the Pipes or Data Sources datafiles you want to delete +- Bump .tinyenv +- CI/CD will protect you from removing resources with depedencies and doing in proper order -Go to your Workspace and check the new Environment created. These changes will be displayed: - -![Changes in environment](./images/delete-resources.png) - -> For the moment, you need to specify the commands for the CI and CD, but in future versions it won't be needed - - -[Internal workspace](https://ui.tinybird.co/128be410-8de1-4b1c-805c-145fdcf2566a/dashboard) \ No newline at end of file +[workspace](https://ui.tinybird.co/128be410-8de1-4b1c-805c-145fdcf2566a/dashboard) diff --git a/delete_simple_resource/datasources/analytics_pages_mv.datasource b/delete_simple_resource/datasources/analytics_pages_mv.datasource index 509061a4..68a76061 100644 --- a/delete_simple_resource/datasources/analytics_pages_mv.datasource +++ b/delete_simple_resource/datasources/analytics_pages_mv.datasource @@ -1,4 +1,3 @@ -# Data Source created from Pipe 'analytics_pages' SCHEMA > `date` Date, diff --git a/delete_simple_resource/datasources/analytics_sessions_mv.datasource b/delete_simple_resource/datasources/analytics_sessions_mv.datasource deleted file mode 100644 index 1ca003fc..00000000 --- a/delete_simple_resource/datasources/analytics_sessions_mv.datasource +++ /dev/null @@ -1,15 +0,0 @@ -# Data Source created from Pipe 'analytics_sessions' - -SCHEMA > - `date` Date, - `session_id` String, - `device` SimpleAggregateFunction(any, String), - `browser` SimpleAggregateFunction(any, String), - `location` SimpleAggregateFunction(any, String), - `first_hit` SimpleAggregateFunction(min, DateTime), - `latest_hit` SimpleAggregateFunction(max, DateTime), - `hits` AggregateFunction(count) - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(date)" -ENGINE_SORTING_KEY "date, session_id" diff --git a/delete_simple_resource/datasources/analytics_sources_mv.datasource b/delete_simple_resource/datasources/analytics_sources_mv.datasource index 904469fd..7235d468 100644 --- a/delete_simple_resource/datasources/analytics_sources_mv.datasource +++ b/delete_simple_resource/datasources/analytics_sources_mv.datasource @@ -1,4 +1,3 @@ -# Data Source created from Pipe 'analytics_sources' SCHEMA > `date` Date, diff --git a/v3/delete_simple_resource/datasources/fixtures/analytics_events.ndjson b/delete_simple_resource/datasources/fixtures/analytics_events.ndjson similarity index 100% rename from v3/delete_simple_resource/datasources/fixtures/analytics_events.ndjson rename to delete_simple_resource/datasources/fixtures/analytics_events.ndjson diff --git a/delete_simple_resource/deploy/0.0.1/cd-deploy.sh b/delete_simple_resource/deploy/0.0.1/cd-deploy.sh deleted file mode 100755 index 8d029237..00000000 --- a/delete_simple_resource/deploy/0.0.1/cd-deploy.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e - -tb pipe rm trend --yes -tb pipe rm kpis --yes \ No newline at end of file diff --git a/delete_simple_resource/deploy/0.0.1/ci-deploy.sh b/delete_simple_resource/deploy/0.0.1/ci-deploy.sh deleted file mode 100755 index 8d029237..00000000 --- a/delete_simple_resource/deploy/0.0.1/ci-deploy.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -set -e - -tb pipe rm trend --yes -tb pipe rm kpis --yes \ No newline at end of file diff --git a/delete_simple_resource/images/delete-resources.png b/delete_simple_resource/images/delete-resources.png deleted file mode 100644 index c9269f1c..00000000 Binary files a/delete_simple_resource/images/delete-resources.png and /dev/null differ diff --git a/delete_simple_resource/pipes/analytics_pages.pipe b/delete_simple_resource/pipes/analytics_pages.pipe index 099cd16c..51e8fb4e 100644 --- a/delete_simple_resource/pipes/analytics_pages.pipe +++ b/delete_simple_resource/pipes/analytics_pages.pipe @@ -22,7 +22,5 @@ SQL > TYPE materialized DATASOURCE analytics_pages_mv -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(date)" -ENGINE_SORTING_KEY "date, device, browser, location, pathname" + diff --git a/delete_simple_resource/pipes/analytics_sessions.pipe b/delete_simple_resource/pipes/analytics_sessions.pipe deleted file mode 100644 index d7f804de..00000000 --- a/delete_simple_resource/pipes/analytics_sessions.pipe +++ /dev/null @@ -1,26 +0,0 @@ -NODE analytics_sessions_1 -DESCRIPTION > - Aggregate by session_id and calculate session metrics - -SQL > - - SELECT - toDate(timestamp) AS date, - session_id, - anySimpleState(device) AS device, - anySimpleState(browser) AS browser, - anySimpleState(location) AS location, - minSimpleState(timestamp) AS first_hit, - maxSimpleState(timestamp) AS latest_hit, - countState() AS hits - FROM analytics_hits - GROUP BY - date, - session_id - -TYPE materialized -DATASOURCE analytics_sessions_mv -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(date)" -ENGINE_SORTING_KEY "date, session_id" - diff --git a/delete_simple_resource/pipes/analytics_sources.pipe b/delete_simple_resource/pipes/analytics_sources.pipe index 3660edb9..21972934 100644 --- a/delete_simple_resource/pipes/analytics_sources.pipe +++ b/delete_simple_resource/pipes/analytics_sources.pipe @@ -7,6 +7,7 @@ SQL > WITH ( SELECT domainWithoutWWW(href) FROM analytics_hits + WHERE href is not null and href != '' LIMIT 1 ) AS currenct_domain SELECT @@ -28,7 +29,5 @@ SQL > TYPE materialized DATASOURCE analytics_sources_mv -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(date)" -ENGINE_SORTING_KEY "date, device, browser, location, referrer" + diff --git a/delete_simple_resource/pipes/top_browsers.pipe b/delete_simple_resource/pipes/top_browsers.pipe index e0aeac7f..4a335814 100644 --- a/delete_simple_resource/pipes/top_browsers.pipe +++ b/delete_simple_resource/pipes/top_browsers.pipe @@ -13,27 +13,27 @@ DESCRIPTION > SQL > % - select - browser, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_sources_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - browser - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + select + browser, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + browser + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} diff --git a/delete_simple_resource/pipes/top_devices.pipe b/delete_simple_resource/pipes/top_devices.pipe index bc4be7c3..2592b5c8 100644 --- a/delete_simple_resource/pipes/top_devices.pipe +++ b/delete_simple_resource/pipes/top_devices.pipe @@ -13,27 +13,27 @@ DESCRIPTION > SQL > % - select - device, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_sources_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - device - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + select + device, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + device + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} diff --git a/delete_simple_resource/pipes/top_locations.pipe b/delete_simple_resource/pipes/top_locations.pipe index 92e05806..79018a32 100644 --- a/delete_simple_resource/pipes/top_locations.pipe +++ b/delete_simple_resource/pipes/top_locations.pipe @@ -13,27 +13,27 @@ DESCRIPTION > SQL > % - select - location, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_pages_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - location - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + select + location, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_pages_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + location + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} diff --git a/delete_simple_resource/pipes/top_pages.pipe b/delete_simple_resource/pipes/top_pages.pipe index bbb232bf..394f362d 100644 --- a/delete_simple_resource/pipes/top_pages.pipe +++ b/delete_simple_resource/pipes/top_pages.pipe @@ -13,27 +13,27 @@ DESCRIPTION > SQL > % - select - pathname, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_pages_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - pathname - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + select + pathname, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_pages_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + pathname + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} diff --git a/delete_simple_resource/pipes/top_sources.pipe b/delete_simple_resource/pipes/top_sources.pipe index 0ae2709d..eb7756ff 100644 --- a/delete_simple_resource/pipes/top_sources.pipe +++ b/delete_simple_resource/pipes/top_sources.pipe @@ -13,27 +13,27 @@ DESCRIPTION > SQL > % - select - domainWithoutWWW(referrer) as referrer, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_sources_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - referrer - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} + select + domainWithoutWWW(referrer) as referrer, + uniqMerge(visits) as visits, + countMerge(hits) as hits + from + analytics_sources_mv + where + {% if defined(date_from) %} + date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} + {% else %} + date >= timestampAdd(today(), interval -7 day) + {% end %} + {% if defined(date_to) %} + and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} + {% else %} + and date <= today() + {% end %} + group by + referrer + order by + visits desc + limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} diff --git a/v3/delete_simple_resource/pipes/trend.pipe b/delete_simple_resource/pipes/trend.pipe similarity index 100% rename from v3/delete_simple_resource/pipes/trend.pipe rename to delete_simple_resource/pipes/trend.pipe diff --git a/v3/delete_simple_resource/requirements.txt b/delete_simple_resource/requirements.txt similarity index 100% rename from v3/delete_simple_resource/requirements.txt rename to delete_simple_resource/requirements.txt diff --git a/v3/delete_simple_resource/.tinyenv b/v3/delete_simple_resource/.tinyenv deleted file mode 100644 index ffb6da70..00000000 --- a/v3/delete_simple_resource/.tinyenv +++ /dev/null @@ -1,2 +0,0 @@ -TB_VERSION_WARNING=0 -VERSION=0.0.1 diff --git a/v3/delete_simple_resource/README.md b/v3/delete_simple_resource/README.md deleted file mode 100644 index 637735f5..00000000 --- a/v3/delete_simple_resource/README.md +++ /dev/null @@ -1,11 +0,0 @@ -# Delete simple resource - -Delete simple Data Sources or Pipes easily. Create a [Pull Request](https://github.com/tinybirdco/use-case-examples/pull/143) following these steps: - -> Remember to follow the [instructions](../README.md) to setup your Tinybird Data Project before jumping into the use-case steps - -- Remove the Pipes or Data Sources datafiles you want to delete -- Bump .tinyenv -- CI/CD will protect you from removing resources with depedencies and doing in proper order - -[workspace](https://ui.tinybird.co/128be410-8de1-4b1c-805c-145fdcf2566a/dashboard) diff --git a/v3/delete_simple_resource/datasources/analytics_events.datasource b/v3/delete_simple_resource/datasources/analytics_events.datasource deleted file mode 100644 index 241f6984..00000000 --- a/v3/delete_simple_resource/datasources/analytics_events.datasource +++ /dev/null @@ -1,16 +0,0 @@ -TOKEN "tracker" APPEND - -DESCRIPTION > - Analytics events landing data source - -SCHEMA > - `timestamp` DateTime `json:$.timestamp`, - `session_id` String `json:$.session_id`, - `action` LowCardinality(String) `json:$.action`, - `version` LowCardinality(String) `json:$.version`, - `payload` String `json:$.payload` - -ENGINE "MergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(timestamp)" -ENGINE_SORTING_KEY "timestamp" -ENGINE_TTL "timestamp + toIntervalDay(60)" diff --git a/v3/delete_simple_resource/datasources/analytics_pages_mv.datasource b/v3/delete_simple_resource/datasources/analytics_pages_mv.datasource deleted file mode 100644 index 68a76061..00000000 --- a/v3/delete_simple_resource/datasources/analytics_pages_mv.datasource +++ /dev/null @@ -1,13 +0,0 @@ - -SCHEMA > - `date` Date, - `device` String, - `browser` String, - `location` String, - `pathname` String, - `visits` AggregateFunction(uniq, String), - `hits` AggregateFunction(count) - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(date)" -ENGINE_SORTING_KEY "date, device, browser, location, pathname" diff --git a/v3/delete_simple_resource/datasources/analytics_sources_mv.datasource b/v3/delete_simple_resource/datasources/analytics_sources_mv.datasource deleted file mode 100644 index 7235d468..00000000 --- a/v3/delete_simple_resource/datasources/analytics_sources_mv.datasource +++ /dev/null @@ -1,13 +0,0 @@ - -SCHEMA > - `date` Date, - `device` String, - `browser` String, - `location` String, - `referrer` String, - `visits` AggregateFunction(uniq, String), - `hits` AggregateFunction(count) - -ENGINE "AggregatingMergeTree" -ENGINE_PARTITION_KEY "toYYYYMM(date)" -ENGINE_SORTING_KEY "date, device, browser, location, referrer" diff --git a/v3/delete_simple_resource/pipes/analytics_hits.pipe b/v3/delete_simple_resource/pipes/analytics_hits.pipe deleted file mode 100644 index 1a226c80..00000000 --- a/v3/delete_simple_resource/pipes/analytics_hits.pipe +++ /dev/null @@ -1,59 +0,0 @@ -DESCRIPTION > - Parsed `page_hit` events, implementing `browser` and `device` detection logic. - - -TOKEN "dashboard" READ - -NODE parsed_hits -DESCRIPTION > - Parse raw page_hit events - -SQL > - - SELECT - timestamp, - action, - version, - coalesce(session_id,'0') as session_id, - JSONExtractString(payload, 'locale') as locale, - JSONExtractString(payload, 'location') as location, - JSONExtractString(payload, 'referrer') as referrer, - JSONExtractString(payload, 'pathname') as pathname, - JSONExtractString(payload, 'href') as href, - lower(JSONExtractString(payload, 'user-agent')) as user_agent - FROM - analytics_events - where action = 'page_hit' - - - -NODE endpoint -SQL > - - SELECT - timestamp, - action, - version, - session_id, - location, - referrer, - pathname, - href, - case - when match(user_agent, 'wget|ahrefsbot|curl|urllib|bitdiscovery|\+https://|googlebot') then 'bot' - when match(user_agent, 'android') then 'mobile-android' - when match(user_agent, 'ipad|iphone|ipod') then 'mobile-ios' - else 'desktop' - END as device, - case - when match(user_agent, 'firefox') then 'firefox' - when match(user_agent, 'chrome|crios') then 'chrome' - when match(user_agent, 'opera') then 'opera' - when match(user_agent, 'msie|trident') then 'ie' - when match(user_agent, 'iphone|ipad|safari') then 'safari' - else 'Unknown' - END as browser - FROM - parsed_hits - - diff --git a/v3/delete_simple_resource/pipes/analytics_pages.pipe b/v3/delete_simple_resource/pipes/analytics_pages.pipe deleted file mode 100644 index 51e8fb4e..00000000 --- a/v3/delete_simple_resource/pipes/analytics_pages.pipe +++ /dev/null @@ -1,26 +0,0 @@ -NODE analytics_pages_1 -DESCRIPTION > - Aggregate by pathname and calculate session and hits - -SQL > - - SELECT - toDate(timestamp) AS date, - device, - browser, - location, - pathname, - uniqState(session_id) AS visits, - countState() AS hits - FROM analytics_hits - GROUP BY - date, - device, - browser, - location, - pathname - -TYPE materialized -DATASOURCE analytics_pages_mv - - diff --git a/v3/delete_simple_resource/pipes/analytics_sources.pipe b/v3/delete_simple_resource/pipes/analytics_sources.pipe deleted file mode 100644 index 21972934..00000000 --- a/v3/delete_simple_resource/pipes/analytics_sources.pipe +++ /dev/null @@ -1,33 +0,0 @@ -NODE analytics_sources_1 -DESCRIPTION > - Aggregate by referral and calculate session and hits - -SQL > - - WITH ( - SELECT domainWithoutWWW(href) - FROM analytics_hits - WHERE href is not null and href != '' - LIMIT 1 - ) AS currenct_domain - SELECT - toDate(timestamp) AS date, - device, - browser, - location, - referrer, - uniqState(session_id) AS visits, - countState() AS hits - FROM analytics_hits - WHERE domainWithoutWWW(referrer) != currenct_domain - GROUP BY - date, - device, - browser, - location, - referrer - -TYPE materialized -DATASOURCE analytics_sources_mv - - diff --git a/v3/delete_simple_resource/pipes/top_browsers.pipe b/v3/delete_simple_resource/pipes/top_browsers.pipe deleted file mode 100644 index 4a335814..00000000 --- a/v3/delete_simple_resource/pipes/top_browsers.pipe +++ /dev/null @@ -1,39 +0,0 @@ -DESCRIPTION > - Top Browsers ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by browser and calcualte hits and visits - -SQL > - - % - select - browser, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_sources_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - browser - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} - - diff --git a/v3/delete_simple_resource/pipes/top_devices.pipe b/v3/delete_simple_resource/pipes/top_devices.pipe deleted file mode 100644 index 2592b5c8..00000000 --- a/v3/delete_simple_resource/pipes/top_devices.pipe +++ /dev/null @@ -1,39 +0,0 @@ -DESCRIPTION > - Top Device Types ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by device and calcualte hits and visits - -SQL > - - % - select - device, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_sources_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - device - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} - - diff --git a/v3/delete_simple_resource/pipes/top_locations.pipe b/v3/delete_simple_resource/pipes/top_locations.pipe deleted file mode 100644 index 79018a32..00000000 --- a/v3/delete_simple_resource/pipes/top_locations.pipe +++ /dev/null @@ -1,39 +0,0 @@ -DESCRIPTION > - Top visting Countries ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by pagepath and calcualte hits and visits - -SQL > - - % - select - location, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_pages_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - location - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} - - diff --git a/v3/delete_simple_resource/pipes/top_pages.pipe b/v3/delete_simple_resource/pipes/top_pages.pipe deleted file mode 100644 index 394f362d..00000000 --- a/v3/delete_simple_resource/pipes/top_pages.pipe +++ /dev/null @@ -1,39 +0,0 @@ -DESCRIPTION > - Most visited pages for a given period. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by pagepath and calcualte hits and visits - -SQL > - - % - select - pathname, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_pages_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - pathname - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} - - diff --git a/v3/delete_simple_resource/pipes/top_sources.pipe b/v3/delete_simple_resource/pipes/top_sources.pipe deleted file mode 100644 index eb7756ff..00000000 --- a/v3/delete_simple_resource/pipes/top_sources.pipe +++ /dev/null @@ -1,39 +0,0 @@ -DESCRIPTION > - Top traffic sources (domains), ordered by most visits. - Accepts `date_from` and `date_to` date filter. Defaults to last 7 days. - Also `skip` and `limit` parameters for pagination. - - -TOKEN "dashboard" READ - -NODE endpoint -DESCRIPTION > - Group by referral and calcualte hits and visits - -SQL > - - % - select - domainWithoutWWW(referrer) as referrer, - uniqMerge(visits) as visits, - countMerge(hits) as hits - from - analytics_sources_mv - where - {% if defined(date_from) %} - date >= {{Date(date_from, description="Starting day for filtering a date range", required=False)}} - {% else %} - date >= timestampAdd(today(), interval -7 day) - {% end %} - {% if defined(date_to) %} - and date <= {{Date(date_to, description="Finishing day for filtering a date range", required=False)}} - {% else %} - and date <= today() - {% end %} - group by - referrer - order by - visits desc - limit {{Int32(skip, 0)}},{{Int32(limit, 50)}} - - diff --git a/v3/delete_simple_resource/scripts/exec_test.sh b/v3/delete_simple_resource/scripts/exec_test.sh deleted file mode 100755 index 09d6d2b5..00000000 --- a/v3/delete_simple_resource/scripts/exec_test.sh +++ /dev/null @@ -1,21 +0,0 @@ - -#!/usr/bin/env bash - -fail=0; - -for t in `find ./tests -name "*.test"`; do - echo "** Running $t **" - echo "** $(cat $t)" - if res=$(bash $t $1 | diff -B ${t}.result -); then - echo 'OK'; - else - echo "failed, diff:"; - echo "$res"; - fail=1 - fi - echo "" -done; - -if [ $fail == 1 ]; then - exit -1; -fi