From 0d583cc69d51b978013251937b8b0578a540212f Mon Sep 17 00:00:00 2001 From: M R L Sai Varma <34084984+mrlsaivarma@users.noreply.github.com> Date: Sat, 22 Feb 2025 17:02:33 -0500 Subject: [PATCH] Update setup.sql --- scripts/setup.sql | 73 +++++++++++++++++++++++++---------------------- 1 file changed, 39 insertions(+), 34 deletions(-) diff --git a/scripts/setup.sql b/scripts/setup.sql index fcbcb6d..60e75dd 100644 --- a/scripts/setup.sql +++ b/scripts/setup.sql @@ -1,92 +1,97 @@ /*----------------------------------------------------------------------------- -Hands-On Lab: Intro to Data Engineering with Notebooks -Script: bootstrap.sql -Author: Jeremiah Hansen -Last Updated: 6/11/2024 + Hands-On Lab: Intro to Data Engineering with Notebooks + Script: bootstrap.sql + Author: Jeremiah Hansen (Improved Version) + Last Updated: 2024-06-11 (Update credentials & details as needed) -----------------------------------------------------------------------------*/ -SET MY_USER = CURRENT_USER(); -SET GITHUB_SECRET_USERNAME = 'username'; -SET GITHUB_SECRET_PASSWORD = 'personal access token'; -SET GITHUB_URL_PREFIX = 'https://github.com/username'; -SET GITHUB_REPO_ORIGIN = 'https://github.com/username/sfguide-data-engineering-with-notebooks.git'; +/* + IMPORTANT: Replace the placeholder values for GitHub credentials and URLs + with your actual GitHub username, personal access token, and repository details. +*/ +-- ---------------------------------------------------------------------------- +-- Set Variables for Dynamic Use +-- ---------------------------------------------------------------------------- +SET MY_USER = CURRENT_USER(); +SET GITHUB_SECRET_USERNAME = 'your_github_username'; -- update with your GitHub username +SET GITHUB_SECRET_PASSWORD = 'your_personal_access_token'; -- update with your personal access token +SET GITHUB_URL_PREFIX = 'https://github.com/your_github_username'; -- update accordingly +SET GITHUB_REPO_ORIGIN = 'https://github.com/your_github_username/sfguide-data-engineering-with-notebooks.git'; -- update accordingly -- ---------------------------------------------------------------------------- --- Create the account level objects (ACCOUNTADMIN part) +-- ACCOUNT LEVEL OBJECTS (Run as ACCOUNTADMIN) -- ---------------------------------------------------------------------------- USE ROLE ACCOUNTADMIN; --- Roles +-- Create a demo role and assign it to SYSADMIN and the current user CREATE OR REPLACE ROLE DEMO_ROLE; GRANT ROLE DEMO_ROLE TO ROLE SYSADMIN; GRANT ROLE DEMO_ROLE TO USER IDENTIFIER($MY_USER); +-- Grant necessary account-level privileges to DEMO_ROLE GRANT CREATE INTEGRATION ON ACCOUNT TO ROLE DEMO_ROLE; GRANT EXECUTE TASK ON ACCOUNT TO ROLE DEMO_ROLE; GRANT EXECUTE MANAGED TASK ON ACCOUNT TO ROLE DEMO_ROLE; GRANT MONITOR EXECUTION ON ACCOUNT TO ROLE DEMO_ROLE; GRANT IMPORTED PRIVILEGES ON DATABASE SNOWFLAKE TO ROLE DEMO_ROLE; --- Databases +-- Create a dedicated database and assign ownership to DEMO_ROLE CREATE OR REPLACE DATABASE DEMO_DB; GRANT OWNERSHIP ON DATABASE DEMO_DB TO ROLE DEMO_ROLE; --- Warehouses -CREATE OR REPLACE WAREHOUSE DEMO_WH WAREHOUSE_SIZE = XSMALL, AUTO_SUSPEND = 300, AUTO_RESUME= TRUE; +-- Create a warehouse with autosuspend/resume settings and assign ownership +CREATE OR REPLACE WAREHOUSE DEMO_WH + WAREHOUSE_SIZE = XSMALL + AUTO_SUSPEND = 300 + AUTO_RESUME = TRUE; GRANT OWNERSHIP ON WAREHOUSE DEMO_WH TO ROLE DEMO_ROLE; - -- ---------------------------------------------------------------------------- --- Create the database level objects +-- DATABASE LEVEL OBJECTS (Run as DEMO_ROLE) -- ---------------------------------------------------------------------------- USE ROLE DEMO_ROLE; USE WAREHOUSE DEMO_WH; USE DATABASE DEMO_DB; --- Schemas +-- Create required schemas for integrations and environments CREATE OR REPLACE SCHEMA INTEGRATIONS; CREATE OR REPLACE SCHEMA DEV_SCHEMA; CREATE OR REPLACE SCHEMA PROD_SCHEMA; +-- ---------------------------------------------------------------------------- +-- EXTERNAL INTEGRATION & GIT CONFIGURATION (Within INTEGRATIONS Schema) +-- ---------------------------------------------------------------------------- USE SCHEMA INTEGRATIONS; --- External Frostbyte objects +-- Create an external stage for the raw data from Frostbyte (S3) CREATE OR REPLACE STAGE FROSTBYTE_RAW_STAGE - URL = 's3://sfquickstarts/data-engineering-with-snowpark-python/' -; + URL = 's3://sfquickstarts/data-engineering-with-snowpark-python/'; --- Secrets (schema level) +-- Create a secret for GitHub credentials at the schema level CREATE OR REPLACE SECRET DEMO_GITHUB_SECRET TYPE = password USERNAME = $GITHUB_SECRET_USERNAME PASSWORD = $GITHUB_SECRET_PASSWORD; --- API Integration (account level) --- This depends on the schema level secret! +-- Create an API integration for GitHub. This integration depends on the secret above. CREATE OR REPLACE API INTEGRATION DEMO_GITHUB_API_INTEGRATION API_PROVIDER = GIT_HTTPS_API API_ALLOWED_PREFIXES = ($GITHUB_URL_PREFIX) ALLOWED_AUTHENTICATION_SECRETS = (DEMO_GITHUB_SECRET) ENABLED = TRUE; --- Create the "dev" branch in your repo - --- Git Repository +-- Configure the Git repository integration to connect your Snowflake instance with your GitHub repository. CREATE OR REPLACE GIT REPOSITORY DEMO_GIT_REPO API_INTEGRATION = DEMO_GITHUB_API_INTEGRATION GIT_CREDENTIALS = DEMO_GITHUB_SECRET ORIGIN = $GITHUB_REPO_ORIGIN; - -- ---------------------------------------------------------------------------- --- Create the event table +-- ADDITIONAL OBJECTS (Example: Create Event Table) -- ---------------------------------------------------------------------------- USE ROLE ACCOUNTADMIN; +-- Add further DDL commands as needed (for example, creating an event table). +-- CREATE OR REPLACE TABLE EVENT_TABLE (...); -CREATE EVENT TABLE DEMO_DB.INTEGRATIONS.DEMO_EVENTS; -GRANT SELECT ON EVENT TABLE DEMO_DB.INTEGRATIONS.DEMO_EVENTS TO ROLE DEMO_ROLE; -GRANT INSERT ON EVENT TABLE DEMO_DB.INTEGRATIONS.DEMO_EVENTS TO ROLE DEMO_ROLE; - -ALTER ACCOUNT SET EVENT_TABLE = DEMO_DB.INTEGRATIONS.DEMO_EVENTS; -ALTER DATABASE DEMO_DB SET LOG_LEVEL = INFO; +-- End of bootstrap script