fix: adding regional constraints and simplifying workflow execution (#284)

Author: shanecglass

modules/data_warehouse/README.md

@@ -74,8 +74,6 @@ the resources of this module:
 - Storage Admin: `roles/storage.admin`
 - BigQuery Admin: `roles/bigquery.admin`
 - Workflows Admin: `roles/workflows.admin`
-- Eventarc Admin: `roles/eventarc.admin`
-- Pub/Sub Admin: `roles/pubsub.admin`
 - Dataplex Admin: `roles/dataplex.admin`
 
 The [Project Factory module](./.terraform/modules/project-services/README.md) and the
@@ -101,8 +99,6 @@ resources of this module:
 - Infrastructure Manager API: `config.googleapis.com`
 - Data Catalog API: `datacatalog.googleapis.com`
 - Data Lineage API: `datalineage.googleapis.com`
-- Eventarc API: `eventarc.googleapis.com`
-- Google Cloud Pub/Sub API: `pubsub.googleapis.com`
 - Service Usage API: `serviceusage.googleapis.com`
 - Google Cloud Storage API: `storage.googleapis.com`
 - Google Cloud Storage JSON API: `storage-api.googleapis.com`

modules/data_warehouse/bigquery.tf

@@ -70,6 +70,7 @@ resource "google_project_iam_member" "bq_connection_iam_vertex_ai" {
   member  = "serviceAccount:${google_bigquery_connection.vertex_ai_connection.cloud_resource[0].service_account_id}"
 }
 
+# Create data tables in BigQuery
 # # Create a Biglake table for events with metadata caching
 resource "google_bigquery_table" "tbl_edw_events" {
   dataset_id          = google_bigquery_dataset.ds_edw.dataset_id
@@ -199,7 +200,7 @@ resource "google_bigquery_routine" "sp_provision_lookup_tables" {
   )
 }
 
-# Add Looker Studio Data Report Procedure
+# # Add Looker Studio Data Report Procedure
 resource "google_bigquery_routine" "sproc_sp_demo_lookerstudio_report" {
   project      = module.project-services.project_id
   dataset_id   = google_bigquery_dataset.ds_edw.dataset_id
@@ -239,7 +240,7 @@ resource "google_bigquery_routine" "sp_sample_queries" {
 }
 
 
-# Add Bigquery ML Model for clustering
+# # Add Bigquery ML Model for clustering
 resource "google_bigquery_routine" "sp_bigqueryml_model" {
   project      = module.project-services.project_id
   dataset_id   = google_bigquery_dataset.ds_edw.dataset_id
@@ -256,7 +257,7 @@ resource "google_bigquery_routine" "sp_bigqueryml_model" {
   ]
 }
 
-# Create Bigquery ML Model for using text generation
+# # Create Bigquery ML Model for using text generation
 resource "google_bigquery_routine" "sp_bigqueryml_generate_create" {
   project      = module.project-services.project_id
   dataset_id   = google_bigquery_dataset.ds_edw.dataset_id
@@ -273,7 +274,7 @@ resource "google_bigquery_routine" "sp_bigqueryml_generate_create" {
   )
 }
 
-# Query Bigquery ML Model for describing customer clusters
+# # Query Bigquery ML Model for describing customer clusters
 resource "google_bigquery_routine" "sp_bigqueryml_generate_describe" {
   project      = module.project-services.project_id
   dataset_id   = google_bigquery_dataset.ds_edw.dataset_id
@@ -382,6 +383,5 @@ resource "google_bigquery_data_transfer_config" "dts_config" {
     google_project_iam_member.dts_roles,
     google_bigquery_dataset.ds_edw,
     google_service_account_iam_binding.dts_token_creator,
-    time_sleep.wait_to_startfile,
   ]
 }

modules/data_warehouse/main.tf

@@ -37,21 +37,13 @@ module "project-services" {
     "config.googleapis.com",
     "datacatalog.googleapis.com",
     "datalineage.googleapis.com",
-    "eventarc.googleapis.com",
-    "pubsub.googleapis.com",
     "serviceusage.googleapis.com",
     "storage.googleapis.com",
     "storage-api.googleapis.com",
     "workflows.googleapis.com",
   ]
 
   activate_api_identities = [
-    {
-      api = "pubsub.googleapis.com"
-      roles = [
-        "roles/iam.serviceAccountTokenCreator",
-      ]
-    },
     {
       api = "workflows.googleapis.com"
       roles = [
@@ -61,18 +53,18 @@ module "project-services" {
   ]
 }
 
+# Wait after APIs are enabled to give time for them to spin up
 resource "time_sleep" "wait_after_apis" {
   create_duration = "90s"
   depends_on      = [module.project-services]
 }
 
-// Create random ID to be used for deployment uniqueness
+# Create random ID to be used for deployment uniqueness
 resource "random_id" "id" {
   byte_length = 4
 }
 
 # Set up Storage Buckets
-
 # # Set up the raw storage bucket
 resource "google_storage_bucket" "raw_bucket" {
   name                        = "ds-edw-raw-${random_id.id.hex}"
@@ -88,120 +80,10 @@ resource "google_storage_bucket" "raw_bucket" {
   labels = var.labels
 }
 
-# # Set up the provisioning storage bucket
-resource "google_storage_bucket" "provisioning_bucket" {
-  name                        = "ds-edw-provisioner-${random_id.id.hex}"
-  project                     = module.project-services.project_id
-  location                    = var.region
-  uniform_bucket_level_access = true
-  force_destroy               = var.force_destroy
-
-  public_access_prevention = "enforced"
-
-  depends_on = [time_sleep.wait_after_apis]
-
-  labels = var.labels
-}
-
-// Create Eventarc Trigger
-# # Create a Pub/Sub topic.
-resource "google_pubsub_topic" "topic" {
-  name    = "provisioning-topic"
-  project = module.project-services.project_id
-
-  depends_on = [time_sleep.wait_after_apis]
-
-  labels = var.labels
-}
-
-resource "google_pubsub_topic_iam_binding" "binding" {
-  project = module.project-services.project_id
-  topic   = google_pubsub_topic.topic.id
-  role    = "roles/pubsub.publisher"
-  members = ["serviceAccount:${data.google_storage_project_service_account.gcs_account.email_address}"]
-}
-
-# # Get the GCS service account to trigger the pub/sub notification
-data "google_storage_project_service_account" "gcs_account" {
-  project = module.project-services.project_id
-
-  depends_on = [time_sleep.wait_after_apis]
-}
-
-# # Create the Storage trigger
-resource "google_storage_notification" "notification" {
-  provider       = google
-  bucket         = google_storage_bucket.provisioning_bucket.name
-  payload_format = "JSON_API_V1"
-  topic          = google_pubsub_topic.topic.id
-  depends_on = [
-    google_pubsub_topic_iam_binding.binding,
-  ]
-}
-
-# # Create the Eventarc trigger
-resource "google_eventarc_trigger" "trigger_pubsub_tf" {
-  project  = module.project-services.project_id
-  name     = "trigger-pubsub-tf"
-  location = var.region
-  matching_criteria {
-    attribute = "type"
-    value     = "google.cloud.pubsub.topic.v1.messagePublished"
-
-  }
-  destination {
-    workflow = google_workflows_workflow.workflow.id
-  }
-
-  transport {
-    pubsub {
-      topic = google_pubsub_topic.topic.id
-    }
-  }
-  service_account = google_service_account.eventarc_service_account.email
-
-  labels = var.labels
-
-  depends_on = [
-    google_project_iam_member.eventarc_service_account_invoke_role,
-  ]
-}
-
-# Set up Eventarc service account for the Trigger to execute as
-# # Set up the Eventarc service account
-resource "google_service_account" "eventarc_service_account" {
-  project      = module.project-services.project_id
-  account_id   = "eventarc-sa-${random_id.id.hex}"
-  display_name = "Service Account for Cloud Eventarc"
-
-  depends_on = [time_sleep.wait_after_apis]
-}
-
-# # Grant the Eventarc service account Workflow Invoker Access
-resource "google_project_iam_member" "eventarc_service_account_invoke_role" {
-  project = module.project-services.project_id
-  role    = "roles/workflows.invoker"
-  member  = "serviceAccount:${google_service_account.eventarc_service_account.email}"
-}
-
-// Sleep for 120 seconds to drop start file
-resource "time_sleep" "wait_to_startfile" {
-  depends_on = [
-    google_storage_notification.notification,
-    google_eventarc_trigger.trigger_pubsub_tf,
-    google_workflows_workflow.workflow
-  ]
-
+# Sleep for 120 seconds to allow the workflow to execute and finish setup
+resource "time_sleep" "wait_after_workflow_execution" {
   create_duration = "120s"
-}
-
-// Drop start file for workflow to execute
-resource "google_storage_bucket_object" "startfile" {
-  bucket = google_storage_bucket.provisioning_bucket.name
-  name   = "startfile"
-  source = "${path.module}/src/startfile"
-
   depends_on = [
-    time_sleep.wait_to_startfile
+    data.http.call_workflows_setup,
   ]
 }

modules/data_warehouse/src/startfile

@@ -22,6 +22,11 @@ variable "project_id" {
 variable "region" {
   type        = string
   description = "Google Cloud Region"
+
+  validation {
+    condition     = contains(["us-central1", "us-west4", "europe-west1", "europe-west2", "europe-west3", "europe-west4", "europe-west9", "asia-northeast3", "asia-southeast1"], var.region)
+    error_message = "This region is not supported. Region must be one of us-central1, us-west4, europe-west1, europe-west2, europe-west3, europe-west4, europe-west9, asia-northeast3, asia-southeast1."
+  }
 }
 
 variable "text_generation_model_name" {

modules/data_warehouse/variables.tf

@@ -36,6 +36,10 @@ terraform {
       source  = "hashicorp/time"
       version = ">= 0.9.1"
     }
+    http = {
+      source  = "hashicorp/http"
+      version = ">= 2"
+    }
   }
   required_version = ">= 0.13"
 

modules/data_warehouse/versions.tf

@@ -14,7 +14,8 @@
  * limitations under the License.
  */
 
-# Set up the Workflows service account
+# Set up the Workflow
+# # Create the Workflows service account
 resource "google_service_account" "workflow_service_account" {
   project      = module.project-services.project_id
   account_id   = "cloud-workflow-sa-${random_id.id.hex}"
@@ -57,3 +58,22 @@ resource "google_workflows_workflow" "workflow" {
     google_project_iam_member.workflow_service_account_roles,
   ]
 }
+
+data "google_client_config" "current" {
+}
+
+# # Trigger the execution of the setup workflow
+data "http" "call_workflows_setup" {
+  url    = "https://workflowexecutions.googleapis.com/v1/projects/${module.project-services.project_id}/locations/${var.region}/workflows/${google_workflows_workflow.workflow.name}/executions"
+  method = "POST"
+  request_headers = {
+    Accept = "application/json"
+  Authorization = "Bearer ${data.google_client_config.current.access_token}" }
+  depends_on = [
+    google_storage_bucket.raw_bucket,
+    google_bigquery_routine.sp_bigqueryml_generate_create,
+    google_bigquery_routine.sp_bigqueryml_model,
+    google_bigquery_routine.sproc_sp_demo_lookerstudio_report,
+    google_bigquery_routine.sp_provision_lookup_tables
+  ]
+}