fix: adding workaround for cloud workflows issue (#310)

Author: shanecglass

modules/data_warehouse/bigquery.tf

@@ -44,7 +44,7 @@ resource "google_project_iam_member" "bq_connection_iam_object_viewer" {
   role    = "roles/storage.objectViewer"
   member  = "serviceAccount:${google_bigquery_connection.ds_connection.cloud_resource[0].service_account_id}"
 
-  depends_on = [google_storage_bucket.raw_bucket, google_bigquery_connection.ds_connection]
+  depends_on = [google_project_iam_member.workflow_manage_sa_roles, google_bigquery_connection.ds_connection]
 }
 
 ## Create a BigQuery connection for Vertex AI to support GenerativeAI use cases
@@ -372,6 +372,6 @@ resource "google_bigquery_data_transfer_config" "dts_config" {
   depends_on = [
     google_project_iam_member.dts_roles,
     google_bigquery_dataset.ds_edw,
-    time_sleep.complete_workflow
+    module.workflow_polling_4
   ]
 }

modules/data_warehouse/metadata.yaml

@@ -32,29 +32,22 @@ spec:
     description: {}
     icon: assets/data_warehouse_icon_v1.png
     deploymentDuration:
-      configurationSecs: 120
-      deploymentSecs: 420
+      configurationSecs: "120"
+      deploymentSecs: "420"
     costEstimate:
       description: cost of this solution is $0.65
       url: https://cloud.google.com/products/calculator/#id=857776c6-49e8-4c6a-adc5-42a15b8fb67d
     cloudProducts:
       - productId: search_BIGQUERY_SECTION
-        pageUrl: ""
       - productId: WORKFLOWS_SECTION
-        pageUrl: ""
       - productId: STORAGE_SECTION
-        pageUrl: ""
       - productId: ai-platform
-        pageUrl: ""
       - productId: LOOKER_STUDIO_SECTION
         pageUrl: lookerstudio.google.com
         isExternal: true
       - productId: CLOUD_DMS_SECTION
-        pageUrl: ""
       - productId: FUNCTIONS_SECTION
-        pageUrl: ""
       - productId: DATAFORM_SECTION
-        pageUrl: ""
   content:
     architecture:
       diagramUrl: www.gstatic.com/pantheon/images/solutions/data-warehouse-architecture_v6.svg

modules/data_warehouse/notebook_deployment.tf

@@ -155,7 +155,12 @@ resource "google_dataform_repository_iam_member" "workflow_manage_repo" {
   count      = length(local.notebook_names)
   repository = local.notebook_names[count.index]
 
-  depends_on = [google_service_account_iam_member.workflow_auth_function, google_dataform_repository_iam_member.function_manage_repo, google_dataform_repository.notebook_repo]
+  depends_on = [
+    google_project_iam_member.workflow_manage_sa_roles,
+    google_service_account_iam_member.workflow_auth_function,
+    google_dataform_repository_iam_member.function_manage_repo,
+    google_dataform_repository.notebook_repo
+  ]
 }
 
 # Create and deploy a Cloud Function to deploy notebooks

modules/data_warehouse/src/sql/sp_sample_queries.sql

@@ -47,7 +47,7 @@ SELECT
   GROUP BY
     1, 2
   ORDER BY
-    WeekdayNumber, 3
+    WeekdayNumber
 ;
 
 -- Query: Items with less than 30 days of inventory remaining
@@ -58,7 +58,7 @@ WITH Orders AS (
   FROM
     `${project_id}.${dataset_id}.order_items` AS order_items
   WHERE
-    order_items.created_at > TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 30 DAY)
+    order_items.created_at > TIMESTAMP_SUB(TIMESTAMP("2023-09-26 00:00:00"), INTERVAL 30 DAY)
   GROUP BY
     product_id
 ),
@@ -99,7 +99,7 @@ SELECT
 FROM
   End30dInventory
 ORDER BY
-  rank ASC, current_stock DESC
+  rank ASC, count_sold_30d DESC, current_stock DESC
 ;
 
 -- Query: data summed by month, then pivoted by department

modules/data_warehouse/versions.tf

@@ -24,13 +24,9 @@ terraform {
       source  = "hashicorp/google-beta"
       version = ">= 4.52, < 6"
     }
-    random = {
-      source  = "hashicorp/random"
-      version = ">= 2"
-    }
     archive = {
       source  = "hashicorp/archive"
-      version = ">= 2"
+      version = "2.4.2"
     }
     time = {
       source  = "hashicorp/time"
@@ -44,6 +40,10 @@ terraform {
       source  = "hashicorp/local"
       version = ">=2.4"
     }
+    random = {
+      source  = "hashicorp/random"
+      version = "3.6.0"
+    }
   }
   required_version = ">= 0.13"
 

modules/data_warehouse/workflow_polling/main.tf

@@ -0,0 +1,98 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+data "google_client_config" "current" {
+}
+
+locals {
+  is_active = var.input_workflow_state == null || var.input_workflow_state == "FAILED" ? "no" : "yes"
+}
+
+## Trigger the execution of the setup workflow with an API call
+data "http" "call_workflows_setup" {
+  url    = "https://workflowexecutions.googleapis.com/v1/${var.workflow_id}/executions"
+  method = local.is_active == "no" ? "POST" : "GET"
+  # method = "POST"
+  request_headers = {
+    Accept = "application/json"
+  Authorization = "Bearer ${data.google_client_config.current.access_token}" }
+}
+
+resource "time_sleep" "workflow_execution_wait" {
+  create_duration = "120s"
+  depends_on = [
+    data.http.call_workflows_setup
+  ]
+}
+
+# locals {
+#   is_active = var.input_workflow_state == null || var.input_workflow_state == "FAILED" ? "no" : "yes"
+# }
+
+# ## Trigger the execution of the setup workflow with an API call
+# data "http" "call_workflows_setup" {
+#   url    = "https://workflowexecutions.googleapis.com/v1/${var.workflow_id}/executions"
+#   method = local.is_active == "no" ? "POST" : "GET"
+#   request_headers = {
+#     Accept = "application/json"
+#   Authorization = "Bearer ${data.google_client_config.current.access_token}" }
+# }
+
+# ## If the workflow last failed, sleep for 30 seconds before checking the workflow execution status.
+# ## If last execution did not fail, exit as quickly as possible (1 second)
+# resource "time_sleep" "workflow_execution_wait" {
+#   create_duration = local.is_active == "no" ? "30s" : "1s"
+#   depends_on = [
+#     data.http.call_workflows_setup,
+#   ]
+# }
+
+## Check the state of the setup workflow execution with an API call
+data "http" "call_workflows_state" {
+  url    = "https://workflowexecutions.googleapis.com/v1/${var.workflow_id}/executions"
+  method = "GET"
+  request_headers = {
+    Accept = "application/json"
+  Authorization = "Bearer ${data.google_client_config.current.access_token}" }
+  depends_on = [
+    data.http.call_workflows_setup,
+    time_sleep.workflow_execution_wait
+  ]
+}
+
+## Parse out the workflow execution state from the API call response
+locals {
+  response_body  = jsondecode(data.http.call_workflows_state.response_body)
+  workflow_state = local.response_body.executions[0].state
+}
+
+## Output the workflow state to use as input for subsequent invocations
+output "workflow_state" {
+  description = "State of the most recent workflow execution. Used to determine how to proceed with next polling run."
+  value       = local.workflow_state
+}
+
+# ## If workflow execution is actively running, sleep for 90 seconds to allow it to finish
+# ## If not, exit as quickly as possible (1 second)
+# resource "time_sleep" "complete_workflow" {
+#   create_duration = local.workflow_state == "ACTIVE" ? "90s" : "1s"
+#   depends_on = [
+#     data.http.call_workflows_setup,
+#     time_sleep.workflow_execution_wait,
+#     data.http.call_workflows_state,
+#     local.workflow_state
+#   ]
+# }

modules/data_warehouse/workflow_polling/variables.tf

@@ -0,0 +1,27 @@
+/**
+ * Copyright 2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "input_workflow_state" {
+  type        = string
+  description = "Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation"
+}
+
+variable "workflow_id" {
+  type        = string
+  description = "The identifer of a workflow created by Terraform. Format is projects/{project ID}/locations/{region}/workflows/{workflow name}"
+}
+
+

modules/data_warehouse/workflow_polling/versions.tf

@@ -0,0 +1,53 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 4.52, < 6"
+    }
+    google-beta = {
+      source  = "hashicorp/google-beta"
+      version = ">= 4.52, < 6"
+    }
+    archive = {
+      source  = "hashicorp/archive"
+      version = ">= 2"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = ">= 2"
+    }
+    time = {
+      source  = "hashicorp/time"
+      version = ">= 0.9.1"
+    }
+    http = {
+      source  = "hashicorp/http"
+      version = ">= 2"
+    }
+    local = {
+      source  = "hashicorp/local"
+      version = ">=2.4"
+    }
+  }
+  required_version = ">= 0.13"
+
+  provider_meta "google" {
+    module_name = "blueprints/terraform/terraform-google-bigquery:data_warehouse/v7.0.0"
+  }
+}