feat: added optional description parameter (#213)

Co-authored-by: Iban Tanovic <ibantanovic>
Co-authored-by: Awais Malik <malik.awais@gmail.com>
Co-authored-by: Awais Malik <awmalik@google.com>

Author: ibantanovic

README.md

@@ -193,14 +193,14 @@ This module provisions a dataset and a list of tables with associated JSON schem
 | deletion\_protection | Whether or not to allow Terraform to destroy the instance. Unless this field is set to false in Terraform state, a terraform destroy or terraform apply that would delete the instance will fail | `bool` | `false` | no |
 | description | Dataset description. | `string` | `null` | no |
 | encryption\_key | Default encryption key to apply to the dataset. Defaults to null (Google-managed). | `string` | `null` | no |
-| external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | <pre>list(object({<br>    table_id              = string,<br>    autodetect            = bool,<br>    compression           = string,<br>    ignore_unknown_values = bool,<br>    max_bad_records       = number,<br>    schema                = string,<br>    source_format         = string,<br>    source_uris           = list(string),<br>    csv_options = object({<br>      quote                 = string,<br>      allow_jagged_rows     = bool,<br>      allow_quoted_newlines = bool,<br>      encoding              = string,<br>      field_delimiter       = string,<br>      skip_leading_rows     = number,<br>    }),<br>    google_sheets_options = object({<br>      range             = string,<br>      skip_leading_rows = number,<br>    }),<br>    hive_partitioning_options = object({<br>      mode              = string,<br>      source_uri_prefix = string,<br>    }),<br>    expiration_time = string,<br>    labels          = map(string),<br>  }))</pre> | `[]` | no |
+| external\_tables | A list of objects which include table\_id, expiration\_time, external\_data\_configuration, and labels. | <pre>list(object({<br>    table_id              = string,<br>    description           = optional(string),<br>    autodetect            = bool,<br>    compression           = string,<br>    ignore_unknown_values = bool,<br>    max_bad_records       = number,<br>    schema                = string,<br>    source_format         = string,<br>    source_uris           = list(string),<br>    csv_options = object({<br>      quote                 = string,<br>      allow_jagged_rows     = bool,<br>      allow_quoted_newlines = bool,<br>      encoding              = string,<br>      field_delimiter       = string,<br>      skip_leading_rows     = number,<br>    }),<br>    google_sheets_options = object({<br>      range             = string,<br>      skip_leading_rows = number,<br>    }),<br>    hive_partitioning_options = object({<br>      mode              = string,<br>      source_uri_prefix = string,<br>    }),<br>    expiration_time = string,<br>    labels          = map(string),<br>  }))</pre> | `[]` | no |
 | location | The regional location for the dataset only US and EU are allowed in module | `string` | `"US"` | no |
-| materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | <pre>list(object({<br>    view_id             = string,<br>    query               = string,<br>    enable_refresh      = bool,<br>    refresh_interval_ms = string,<br>    clustering          = list(string),<br>    time_partitioning = object({<br>      expiration_ms            = string,<br>      field                    = string,<br>      type                     = string,<br>      require_partition_filter = bool,<br>    }),<br>    range_partitioning = object({<br>      field = string,<br>      range = object({<br>        start    = string,<br>        end      = string,<br>        interval = string,<br>      }),<br>    }),<br>    expiration_time = string,<br>    labels          = map(string),<br>  }))</pre> | `[]` | no |
+| materialized\_views | A list of objects which includes view\_id, view\_query, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels | <pre>list(object({<br>    view_id             = string,<br>    description         = optional(string),<br>    query               = string,<br>    enable_refresh      = bool,<br>    refresh_interval_ms = string,<br>    clustering          = list(string),<br>    time_partitioning = object({<br>      expiration_ms            = string,<br>      field                    = string,<br>      type                     = string,<br>      require_partition_filter = bool,<br>    }),<br>    range_partitioning = object({<br>      field = string,<br>      range = object({<br>        start    = string,<br>        end      = string,<br>        interval = string,<br>      }),<br>    }),<br>    expiration_time = string,<br>    labels          = map(string),<br>  }))</pre> | `[]` | no |
 | max\_time\_travel\_hours | Defines the time travel window in hours | `number` | `null` | no |
 | project\_id | Project where the dataset and table are created | `string` | n/a | yes |
 | routines | A list of objects which include routine\_id, routine\_type, routine\_language, definition\_body, return\_type, routine\_description and arguments. | <pre>list(object({<br>    routine_id      = string,<br>    routine_type    = string,<br>    language        = string,<br>    definition_body = string,<br>    return_type     = string,<br>    description     = string,<br>    arguments = list(object({<br>      name          = string,<br>      data_type     = string,<br>      argument_kind = string,<br>      mode          = string,<br>    })),<br>  }))</pre> | `[]` | no |
-| tables | A list of objects which include table\_id, table\_name, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | <pre>list(object({<br>    table_id   = string,<br>    table_name = optional(string),<br>    schema     = string,<br>    clustering = list(string),<br>    time_partitioning = object({<br>      expiration_ms            = string,<br>      field                    = string,<br>      type                     = string,<br>      require_partition_filter = bool,<br>    }),<br>    range_partitioning = object({<br>      field = string,<br>      range = object({<br>        start    = string,<br>        end      = string,<br>        interval = string,<br>      }),<br>    }),<br>    expiration_time = string,<br>    labels          = map(string),<br>  }))</pre> | `[]` | no |
-| views | A list of objects which include view\_id and view query | <pre>list(object({<br>    view_id        = string,<br>    query          = string,<br>    use_legacy_sql = bool,<br>    labels         = map(string),<br>  }))</pre> | `[]` | no |
+| tables | A list of objects which include table\_id, table\_name, schema, clustering, time\_partitioning, range\_partitioning, expiration\_time and labels. | <pre>list(object({<br>    table_id    = string,<br>    description = optional(string),<br>    table_name  = optional(string),<br>    schema      = string,<br>    clustering  = list(string),<br>    time_partitioning = object({<br>      expiration_ms            = string,<br>      field                    = string,<br>      type                     = string,<br>      require_partition_filter = bool,<br>    }),<br>    range_partitioning = object({<br>      field = string,<br>      range = object({<br>        start    = string,<br>        end      = string,<br>        interval = string,<br>      }),<br>    }),<br>    expiration_time = string,<br>    labels          = map(string),<br>  }))</pre> | `[]` | no |
+| views | A list of objects which include view\_id and view query | <pre>list(object({<br>    view_id        = string,<br>    description    = optional(string),<br>    query          = string,<br>    use_legacy_sql = bool,<br>    labels         = map(string),<br>  }))</pre> | `[]` | no |
 
 ## Outputs
 

main.tf

@@ -69,6 +69,7 @@ resource "google_bigquery_table" "main" {
   dataset_id          = google_bigquery_dataset.main.dataset_id
   friendly_name       = each.value["table_name"] != null ? each.value["table_name"] : each.key
   table_id            = each.key
+  description         = each.value["description"]
   labels              = each.value["labels"]
   schema              = each.value["schema"]
   clustering          = each.value["clustering"]
@@ -110,6 +111,7 @@ resource "google_bigquery_table" "view" {
   dataset_id          = google_bigquery_dataset.main.dataset_id
   friendly_name       = each.key
   table_id            = each.key
+  description         = each.value["description"]
   labels              = each.value["labels"]
   project             = var.project_id
   deletion_protection = false
@@ -131,6 +133,7 @@ resource "google_bigquery_table" "materialized_view" {
   dataset_id          = google_bigquery_dataset.main.dataset_id
   friendly_name       = each.key
   table_id            = each.key
+  description         = each.value["description"]
   labels              = each.value["labels"]
   clustering          = each.value["clustering"]
   expiration_time     = each.value["expiration_time"]
@@ -177,6 +180,7 @@ resource "google_bigquery_table" "external_table" {
   dataset_id          = google_bigquery_dataset.main.dataset_id
   friendly_name       = each.key
   table_id            = each.key
+  description         = each.value["description"]
   labels              = each.value["labels"]
   expiration_time     = each.value["expiration_time"]
   project             = var.project_id

metadata.yaml

@@ -90,6 +90,7 @@ spec:
     type: |-
       list(object({
           table_id              = string,
+          description           = optional(string),
           autodetect            = bool,
           compression           = string,
           ignore_unknown_values = bool,
@@ -128,6 +129,7 @@ spec:
     type: |-
       list(object({
           view_id             = string,
+          description         = optional(string),
           query               = string,
           enable_refresh      = bool,
           refresh_interval_ms = string,
@@ -178,9 +180,10 @@ spec:
     description: A list of objects which include table_id, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels.
     type: |-
       list(object({
-          table_id   = string,
-          schema     = string,
-          clustering = list(string),
+          table_id    = string,
+          description = optional(string),
+          schema      = string,
+          clustering  = list(string),
           time_partitioning = object({
             expiration_ms            = string,
             field                    = string,
@@ -205,6 +208,7 @@ spec:
     type: |-
       list(object({
           view_id        = string,
+          description    = optional(string),
           query          = string,
           use_legacy_sql = bool,
           labels         = map(string),

modules/data_warehouse/.gitignore

@@ -1,2 +1,2 @@
 *.zip
-terraform.tfvars
+terraform.tfvars

modules/data_warehouse/README.md

@@ -47,7 +47,9 @@ Functional examples are included in the
 | ds\_friendly\_name | Dataset name |
 | function\_uri | Function URI |
 | lookerstudio\_report\_url | The URL to create a new Looker Studio report displays a sample dashboard for the taxi data analysis |
+| neos\_tutorial\_url | The URL to launch the in-console tutorial for the EDW solution |
 | raw\_bucket | Raw bucket name |
+| solution\_guide\_url | The URL to Solution Guide for the EDW solution |
 
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
 

modules/data_warehouse/assets/workflows/workflow.yaml

@@ -1,11 +1,25 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 # Run the Cloud Function from Workflows
 
 main:
     params: [event]
     steps:
     - assign_values:
         assign:
-            - cloud_function_url: ${cloud_function_url} 
+            - cloud_function_url: ${cloud_function_url}
     - call_cloud_function:
         call: http.get
         args:

variables.tf

@@ -98,10 +98,11 @@ variable "tables" {
   description = "A list of objects which include table_id, table_name, schema, clustering, time_partitioning, range_partitioning, expiration_time and labels."
   default     = []
   type = list(object({
-    table_id   = string,
-    table_name = optional(string),
-    schema     = string,
-    clustering = list(string),
+    table_id    = string,
+    description = optional(string),
+    table_name  = optional(string),
+    schema      = string,
+    clustering  = list(string),
     time_partitioning = object({
       expiration_ms            = string,
       field                    = string,
@@ -126,6 +127,7 @@ variable "views" {
   default     = []
   type = list(object({
     view_id        = string,
+    description    = optional(string),
     query          = string,
     use_legacy_sql = bool,
     labels         = map(string),
@@ -137,6 +139,7 @@ variable "materialized_views" {
   default     = []
   type = list(object({
     view_id             = string,
+    description         = optional(string),
     query               = string,
     enable_refresh      = bool,
     refresh_interval_ms = string,
@@ -165,6 +168,7 @@ variable "external_tables" {
   default     = []
   type = list(object({
     table_id              = string,
+    description           = optional(string),
     autodetect            = bool,
     compression           = string,
     ignore_unknown_values = bool,