From c17fc8bd84fe37a9fd8cf48709e04b1d36ac3099 Mon Sep 17 00:00:00 2001 From: Andrea Zucchini Date: Fri, 30 Aug 2024 14:57:20 +0100 Subject: [PATCH] feat!: Retirement GCP Dataproc offering (#1325) --- .github/dependabot.yml | 7 - acceptance-tests/apps/dataprocapp/app/app.go | 51 ----- .../apps/dataprocapp/app/delete_job.go | 31 --- .../apps/dataprocapp/app/get_job.go | 34 --- .../apps/dataprocapp/app/run_job.go | 86 -------- .../dataprocapp/credentials/credentials.go | 46 ----- acceptance-tests/apps/dataprocapp/go.mod | 46 ----- acceptance-tests/apps/dataprocapp/go.sum | 195 ------------------ acceptance-tests/apps/dataprocapp/main.go | 32 --- acceptance-tests/dataproc_test.go | 43 ---- acceptance-tests/helpers/apps/testapps.go | 1 - .../update_and_upgrade_dataproc_test.go | 76 ------- docs/installation.md | 1 - google-dataproc.yml | 138 ------------- integration-tests/dataproc_test.go | 82 -------- manifest.yml | 1 - terraform/dataproc/bind/main.tf | 20 -- terraform/dataproc/bind/outputs.tf | 7 - terraform/dataproc/bind/provider.tf | 4 - terraform/dataproc/bind/variables.tf | 7 - terraform/dataproc/bind/versions.tf | 8 - terraform/dataproc/provision/main.tf | 25 --- terraform/dataproc/provision/outputs.tf | 3 - terraform/dataproc/provision/provider.tf | 4 - terraform/dataproc/provision/variables.tf | 15 -- terraform/dataproc/provision/versions.tf | 8 - 26 files changed, 971 deletions(-) delete mode 100644 acceptance-tests/apps/dataprocapp/app/app.go delete mode 100644 acceptance-tests/apps/dataprocapp/app/delete_job.go delete mode 100644 acceptance-tests/apps/dataprocapp/app/get_job.go delete mode 100644 acceptance-tests/apps/dataprocapp/app/run_job.go delete mode 100644 acceptance-tests/apps/dataprocapp/credentials/credentials.go delete mode 100644 acceptance-tests/apps/dataprocapp/go.mod delete mode 100644 acceptance-tests/apps/dataprocapp/go.sum delete mode 100644 acceptance-tests/apps/dataprocapp/main.go delete mode 100644 acceptance-tests/dataproc_test.go delete mode 100644 acceptance-tests/upgrade/update_and_upgrade_dataproc_test.go delete mode 100644 google-dataproc.yml delete mode 100644 integration-tests/dataproc_test.go delete mode 100644 terraform/dataproc/bind/main.tf delete mode 100644 terraform/dataproc/bind/outputs.tf delete mode 100644 terraform/dataproc/bind/provider.tf delete mode 100644 terraform/dataproc/bind/variables.tf delete mode 100644 terraform/dataproc/bind/versions.tf delete mode 100644 terraform/dataproc/provision/main.tf delete mode 100644 terraform/dataproc/provision/outputs.tf delete mode 100644 terraform/dataproc/provision/provider.tf delete mode 100644 terraform/dataproc/provision/variables.tf delete mode 100644 terraform/dataproc/provision/versions.tf diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 8c508894..6a11bec1 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -14,13 +14,6 @@ updates: schedule: interval: "weekly" day: "saturday" -- package-ecosystem: gomod - directory: "/acceptance-tests/apps/dataprocapp" - schedule: - interval: "weekly" - day: "saturday" - labels: - - "test-dependencies" - package-ecosystem: gomod directory: "/acceptance-tests/apps/mysqlapp" schedule: diff --git a/acceptance-tests/apps/dataprocapp/app/app.go b/acceptance-tests/apps/dataprocapp/app/app.go deleted file mode 100644 index 45ec66ff..00000000 --- a/acceptance-tests/apps/dataprocapp/app/app.go +++ /dev/null @@ -1,51 +0,0 @@ -package app - -import ( - "context" - "dataprocapp/credentials" - "fmt" - "log" - "net/http" - "strings" - - dataproc "cloud.google.com/go/dataproc/v2/apiv1" - "google.golang.org/api/option" -) - -func App(creds credentials.DataprocCredentials) http.HandlerFunc { - endpoint := fmt.Sprintf("%s-dataproc.googleapis.com:443", creds.Region) - client, err := dataproc.NewJobControllerClient( - context.Background(), - option.WithEndpoint(endpoint), - option.WithCredentialsJSON(creds.Credentials)) - if err != nil { - log.Fatalf("error creating the cluster client: %s\n", err) - } - - return func(w http.ResponseWriter, r *http.Request) { - job := strings.Trim(r.URL.Path, "/") - switch r.Method { - case http.MethodHead: - aliveness(w, r) - case http.MethodGet: - handleGetJob(w, job, client, creds) - case http.MethodPut: - handleRunJob(w, job, client, creds) - case http.MethodDelete: - handleDeleteJob(w, job, client, creds) - default: - fail(w, http.StatusMethodNotAllowed, http.StatusText(http.StatusMethodNotAllowed)) - } - } -} - -func aliveness(w http.ResponseWriter, r *http.Request) { - log.Printf("Handled aliveness test.") - w.WriteHeader(http.StatusNoContent) -} - -func fail(w http.ResponseWriter, code int, format string, a ...any) { - msg := fmt.Sprintf(format, a...) - log.Println(msg) - http.Error(w, msg, code) -} diff --git a/acceptance-tests/apps/dataprocapp/app/delete_job.go b/acceptance-tests/apps/dataprocapp/app/delete_job.go deleted file mode 100644 index 01aaf3dc..00000000 --- a/acceptance-tests/apps/dataprocapp/app/delete_job.go +++ /dev/null @@ -1,31 +0,0 @@ -package app - -import ( - "context" - "dataprocapp/credentials" - "log" - "net/http" - - dataproc "cloud.google.com/go/dataproc/v2/apiv1" - "cloud.google.com/go/dataproc/v2/apiv1/dataprocpb" -) - -func handleDeleteJob(w http.ResponseWriter, jobName string, jobClient *dataproc.JobControllerClient, creds credentials.DataprocCredentials) { - log.Printf("Handling deletion of job") - - jobReq := &dataprocpb.DeleteJobRequest{ - ProjectId: creds.ProjectID, - Region: creds.Region, - JobId: jobName, - } - - ctx := context.Background() - err := jobClient.DeleteJob(ctx, jobReq) - if err != nil { - fail(w, http.StatusFailedDependency, "error with request to delete job: %v", err) - return - } - - w.WriteHeader(http.StatusGone) - log.Printf("Job deleted") -} diff --git a/acceptance-tests/apps/dataprocapp/app/get_job.go b/acceptance-tests/apps/dataprocapp/app/get_job.go deleted file mode 100644 index 981f340f..00000000 --- a/acceptance-tests/apps/dataprocapp/app/get_job.go +++ /dev/null @@ -1,34 +0,0 @@ -package app - -import ( - "context" - "dataprocapp/credentials" - "log" - "net/http" - - dataproc "cloud.google.com/go/dataproc/v2/apiv1" - "cloud.google.com/go/dataproc/v2/apiv1/dataprocpb" -) - -func handleGetJob(w http.ResponseWriter, jobName string, jobClient *dataproc.JobControllerClient, creds credentials.DataprocCredentials) { - log.Printf("Handling getting job") - - jobReq := &dataprocpb.GetJobRequest{ - ProjectId: creds.ProjectID, - Region: creds.Region, - JobId: jobName, - } - - ctx := context.Background() - getJobOp, err := jobClient.GetJob(ctx, jobReq) - if err != nil { - fail(w, http.StatusFailedDependency, "error with request to getting job: %v", err) - return - } - - w.WriteHeader(http.StatusOK) - w.Header().Set("Content-Type", "text/html") - _, err = w.Write([]byte((getJobOp.Status.State.String()))) - - log.Printf("Job finished with status: %s", getJobOp.Status.State.String()) -} diff --git a/acceptance-tests/apps/dataprocapp/app/run_job.go b/acceptance-tests/apps/dataprocapp/app/run_job.go deleted file mode 100644 index 7b9082f3..00000000 --- a/acceptance-tests/apps/dataprocapp/app/run_job.go +++ /dev/null @@ -1,86 +0,0 @@ -package app - -import ( - "context" - "dataprocapp/credentials" - "fmt" - "io" - "log" - "net/http" - "regexp" - - dataproc "cloud.google.com/go/dataproc/v2/apiv1" - "cloud.google.com/go/dataproc/v2/apiv1/dataprocpb" - "cloud.google.com/go/storage" - "google.golang.org/api/option" -) - -func handleRunJob(w http.ResponseWriter, jobName string, jobClient *dataproc.JobControllerClient, creds credentials.DataprocCredentials) { - log.Printf("Handling job run") - - submitJobReq := &dataprocpb.SubmitJobRequest{ - ProjectId: creds.ProjectID, - Region: creds.Region, - Job: &dataprocpb.Job{ - Reference: &dataprocpb.JobReference{ - JobId: jobName, - }, - Placement: &dataprocpb.JobPlacement{ - ClusterName: creds.ClusterName, - }, - TypeJob: &dataprocpb.Job_PysparkJob{ - PysparkJob: &dataprocpb.PySparkJob{ - MainPythonFileUri: fmt.Sprintf("gs://dataproc_input_for_test_do_not_delete/script.py"), - }, - }, - }, - } - - ctx := context.Background() - submitJobOp, err := jobClient.SubmitJobAsOperation(ctx, submitJobReq) - if err != nil { - fail(w, http.StatusFailedDependency, "error with request to submitting job: %v", err) - return - } - - submitJobResp, err := submitJobOp.Wait(ctx) - if err != nil { - fail(w, http.StatusFailedDependency, "error submitting job: %v", err) - return - } - re := regexp.MustCompile("gs://(.+?)/(.+)") - matches := re.FindStringSubmatch(submitJobResp.DriverOutputResourceUri) - log.Printf("Job Response: %v", submitJobResp) - log.Printf("Job Response DriverOutputResourceUri: %s", submitJobResp.DriverOutputResourceUri) - - if len(matches) < 3 { - fail(w, http.StatusInternalServerError, "regex error: %s", submitJobResp.DriverOutputResourceUri) - return - } - - // Dataproc job outget gets saved to a GCS bucket allocated to it. - storageClient, err := storage.NewClient(context.Background(), option.WithCredentialsJSON(creds.Credentials)) - if err != nil { - fail(w, http.StatusFailedDependency, "error creating storage jobClient: %v", err) - return - } - - obj := fmt.Sprintf("%s.000000000", matches[2]) - reader, err := storageClient.Bucket(creds.BucketName).Object(obj).NewReader(ctx) - if err != nil { - fail(w, http.StatusFailedDependency, "error reading job output: %v", err) - return - } - - defer reader.Close() - - body, err := io.ReadAll(reader) - if err != nil { - fail(w, http.StatusFailedDependency, "could not read output from Dataproc Job: %v", err) - return - } - - log.Printf("Job finished successfully: %s", body) - - w.WriteHeader(http.StatusOK) -} diff --git a/acceptance-tests/apps/dataprocapp/credentials/credentials.go b/acceptance-tests/apps/dataprocapp/credentials/credentials.go deleted file mode 100644 index 6e5ac158..00000000 --- a/acceptance-tests/apps/dataprocapp/credentials/credentials.go +++ /dev/null @@ -1,46 +0,0 @@ -package credentials - -import ( - b64 "encoding/base64" - "fmt" - - "github.com/cloudfoundry-community/go-cfenv" - "github.com/mitchellh/mapstructure" -) - -type DataprocCredentials struct { - PrivateKey string `mapstructure:"private_key"` - BucketName string `mapstructure:"bucket_name"` - ClusterName string `mapstructure:"cluster_name"` - ProjectID string `mapstructure:"project_id"` - Region string `mapstructure:"region"` - Credentials []byte -} - -func Read() (DataprocCredentials, error) { - app, err := cfenv.Current() - if err != nil { - return DataprocCredentials{}, fmt.Errorf("error reading app env: %w", err) - } - svs, err := app.Services.WithTag("dataproc") - if err != nil { - return DataprocCredentials{}, fmt.Errorf("error reading Dataproc service details") - } - - var r DataprocCredentials - if err := mapstructure.Decode(svs[0].Credentials, &r); err != nil { - return DataprocCredentials{}, fmt.Errorf("failed to decode binding credentials: %w", err) - } - - if r.PrivateKey == "" || r.BucketName == "" || r.ClusterName == "" || r.ProjectID == "" || r.Region == "" { - return DataprocCredentials{}, fmt.Errorf("parsed credentials are not valid: %v", r) - } - - credential, err := b64.StdEncoding.DecodeString(r.PrivateKey) - if err != nil { - return DataprocCredentials{}, fmt.Errorf("failed to decode PrivateKey: %w", err) - } - - r.Credentials = credential - return r, nil -} diff --git a/acceptance-tests/apps/dataprocapp/go.mod b/acceptance-tests/apps/dataprocapp/go.mod deleted file mode 100644 index 53ffe6d6..00000000 --- a/acceptance-tests/apps/dataprocapp/go.mod +++ /dev/null @@ -1,46 +0,0 @@ -module dataprocapp - -go 1.22.6 - -require ( - cloud.google.com/go/dataproc/v2 v2.6.0 - cloud.google.com/go/storage v1.43.0 - github.com/cloudfoundry-community/go-cfenv v1.18.0 - github.com/mitchellh/mapstructure v1.5.0 - google.golang.org/api v0.194.0 -) - -require ( - cloud.google.com/go v0.115.1 // indirect - cloud.google.com/go/auth v0.9.1 // indirect - cloud.google.com/go/auth/oauth2adapt v0.2.4 // indirect - cloud.google.com/go/compute/metadata v0.5.0 // indirect - cloud.google.com/go/iam v1.1.13 // indirect - cloud.google.com/go/longrunning v0.5.12 // indirect - github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/google/s2a-go v0.1.8 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.13.0 // indirect - go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect - go.opentelemetry.io/otel/trace v1.28.0 // indirect - golang.org/x/crypto v0.26.0 // indirect - golang.org/x/net v0.28.0 // indirect - golang.org/x/oauth2 v0.22.0 // indirect - golang.org/x/sync v0.8.0 // indirect - golang.org/x/sys v0.24.0 // indirect - golang.org/x/text v0.17.0 // indirect - golang.org/x/time v0.6.0 // indirect - google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 // indirect - google.golang.org/grpc v1.65.0 // indirect - google.golang.org/protobuf v1.34.2 // indirect -) diff --git a/acceptance-tests/apps/dataprocapp/go.sum b/acceptance-tests/apps/dataprocapp/go.sum deleted file mode 100644 index 040bccfe..00000000 --- a/acceptance-tests/apps/dataprocapp/go.sum +++ /dev/null @@ -1,195 +0,0 @@ -cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.115.1 h1:Jo0SM9cQnSkYfp44+v+NQXHpcHqlnRJk2qxh6yvxxxQ= -cloud.google.com/go v0.115.1/go.mod h1:DuujITeaufu3gL68/lOFIirVNJwQeyf5UXyi+Wbgknc= -cloud.google.com/go/auth v0.9.1 h1:+pMtLEV2k0AXKvs/tGZojuj6QaioxfUjOpMsG5Gtx+w= -cloud.google.com/go/auth v0.9.1/go.mod h1:Sw8ocT5mhhXxFklyhT12Eiy0ed6tTrPMCJjSI8KhYLk= -cloud.google.com/go/auth/oauth2adapt v0.2.4 h1:0GWE/FUsXhf6C+jAkWgYm7X9tK8cuEIfy19DBn6B6bY= -cloud.google.com/go/auth/oauth2adapt v0.2.4/go.mod h1:jC/jOpwFP6JBxhB3P5Rr0a9HLMC/Pe3eaL4NmdvqPtc= -cloud.google.com/go/compute/metadata v0.5.0 h1:Zr0eK8JbFv6+Wi4ilXAR8FJ3wyNdpxHKJNPos6LTZOY= -cloud.google.com/go/compute/metadata v0.5.0/go.mod h1:aHnloV2TPI38yx4s9+wAZhHykWvVCfu7hQbF+9CWoiY= -cloud.google.com/go/dataproc/v2 v2.6.0 h1:YjgcpuzUYX+Q/xCbh/+5+Nwx0DGzsO9nss/O7Usy79c= -cloud.google.com/go/dataproc/v2 v2.6.0/go.mod h1:amsKInI+TU4GcXnz+gmmApYbiYM4Fw051SIMDoWCWeE= -cloud.google.com/go/iam v1.1.13 h1:7zWBXG9ERbMLrzQBRhFliAV+kjcRToDTgQT3CTwYyv4= -cloud.google.com/go/iam v1.1.13/go.mod h1:K8mY0uSXwEXS30KrnVb+j54LB/ntfZu1dr+4zFMNbus= -cloud.google.com/go/longrunning v0.5.12 h1:5LqSIdERr71CqfUsFlJdBpOkBH8FBCFD7P1nTWy3TYE= -cloud.google.com/go/longrunning v0.5.12/go.mod h1:S5hMV8CDJ6r50t2ubVJSKQVv5u0rmik5//KgLO3k4lU= -cloud.google.com/go/storage v1.43.0 h1:CcxnSohZwizt4LCzQHWvBf1/kvtHUn7gk9QERXPyXFs= -cloud.google.com/go/storage v1.43.0/go.mod h1:ajvxEa7WmZS1PxvKRq4bq0tFT3vMd502JwstCcYv0Q0= -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudfoundry-community/go-cfenv v1.18.0 h1:dOIRSHUSaj4r6Q9Cx+nzz2OytHt+QNKqtOuKTQsa+zw= -github.com/cloudfoundry-community/go-cfenv v1.18.0/go.mod h1:qGMSI6lygPzqugFs9M1NFjJBtEPgl0MgT6drMFZGUoU= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= -github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= -github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= -github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= -github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= -github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= -github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= -github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= -github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.13.0 h1:yitjD5f7jQHhyDsnhKEBU52NdvvdSeGzlAnDPT0hH1s= -github.com/googleapis/gax-go/v2 v2.13.0/go.mod h1:Z/fvTZXF8/uw7Xu5GuslPw+bplx6SS338j1Is2S+B7A= -github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo= -github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= -github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= -github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= -github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= -github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA= -github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= -go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0 h1:vS1Ao/R55RNV4O7TA2Qopok8yN+X0LIP6RVWLFkprck= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.52.0/go.mod h1:BMsdeOxN04K0L5FNUBfjFdvwWGNe/rkmSwH4Aelu/X0= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0 h1:4K4tsIXefpVJtvA/8srF4V4y0akAoPHkIslgAkjixJA= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.53.0/go.mod h1:jjdQuTGVsXV4vSs+CJ2qYDeDPf9yIJV23qlIzBm73Vg= -go.opentelemetry.io/otel v1.28.0 h1:/SqNcYk+idO0CxKEUOtKQClMK/MimZihKYMruSMViUo= -go.opentelemetry.io/otel v1.28.0/go.mod h1:q68ijF8Fc8CnMHKyzqL6akLO46ePnjkgfIMIjUIX9z4= -go.opentelemetry.io/otel/metric v1.28.0 h1:f0HGvSl1KRAU1DLgLGFjrwVyismPlnuU6JD6bOeuA5Q= -go.opentelemetry.io/otel/metric v1.28.0/go.mod h1:Fb1eVBFZmLVTMb6PPohq3TO9IIhUisDsbJoL/+uQW4s= -go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= -go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= -go.opentelemetry.io/otel/trace v1.28.0 h1:GhQ9cUuQGmNDd5BTCP2dAvv75RdMxEfTmYejp+lkx9g= -go.opentelemetry.io/otel/trace v1.28.0/go.mod h1:jPyXzNPg6da9+38HEwElrQiHlVMTnVfM3/yv2OlIHaI= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= -golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= -golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= -golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= -golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= -golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA= -golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= -golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= -golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= -golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= -golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= -golang.org/x/time v0.6.0 h1:eTDhh4ZXt5Qf0augr54TN6suAUudPcawVZeIAPU7D4U= -golang.org/x/time v0.6.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.194.0 h1:dztZKG9HgtIpbI35FhfuSNR/zmaMVdxNlntHj1sIS4s= -google.golang.org/api v0.194.0/go.mod h1:AgvUFdojGANh3vI+P7EVnxj3AISHllxGCJSFmggmnd0= -google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= -google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142 h1:oLiyxGgE+rt22duwci1+TG7bg2/L1LQsXwfjPlmuJA0= -google.golang.org/genproto v0.0.0-20240814211410-ddb44dafa142/go.mod h1:G11eXq53iI5Q+kyNOmCvnzBaxEA2Q/Ik5Tj7nqBE8j4= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142 h1:wKguEg1hsxI2/L3hUYrpo1RVi48K+uTyzKqprwLXsb8= -google.golang.org/genproto/googleapis/api v0.0.0-20240814211410-ddb44dafa142/go.mod h1:d6be+8HhtEtucleCbxpPW9PA9XwISACu8nvpPqF0BVo= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142 h1:e7S5W7MGGLaSu8j3YjdezkZ+m1/Nm0uRVRMEMGk26Xs= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240814211410-ddb44dafa142/go.mod h1:UqMtugtsSgubUsoxbuAoiCXvqvErP7Gf0so0mK9tHxU= -google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.65.0 h1:bs/cUb4lp1G5iImFFd3u5ixQzweKizoZJAwBNLR42lc= -google.golang.org/grpc v1.65.0/go.mod h1:WgYC2ypjlB0EiQi6wdKixMqukr6lBc0Vo+oOgjrM5ZQ= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= -google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= -google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1 h1:mUhvW9EsL+naU5Q3cakzfE91YhliOondGd6ZrsDBHQE= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/acceptance-tests/apps/dataprocapp/main.go b/acceptance-tests/apps/dataprocapp/main.go deleted file mode 100644 index 418254da..00000000 --- a/acceptance-tests/apps/dataprocapp/main.go +++ /dev/null @@ -1,32 +0,0 @@ -package main - -import ( - "dataprocapp/app" - "dataprocapp/credentials" - "fmt" - "log" - "net/http" - "os" -) - -func main() { - log.Println("Starting.") - - log.Println("Reading credentials.") - creds, err := credentials.Read() - if err != nil { - panic(err) - } - - port := port() - log.Printf("Listening on port: %s", port) - http.Handle("/", app.App(creds)) - http.ListenAndServe(port, nil) -} - -func port() string { - if port := os.Getenv("PORT"); port != "" { - return fmt.Sprintf(":%s", port) - } - return ":8080" -} diff --git a/acceptance-tests/dataproc_test.go b/acceptance-tests/dataproc_test.go deleted file mode 100644 index 7cef17d7..00000000 --- a/acceptance-tests/dataproc_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package acceptance_test - -import ( - "csbbrokerpakgcp/acceptance-tests/helpers/apps" - "csbbrokerpakgcp/acceptance-tests/helpers/matchers" - "csbbrokerpakgcp/acceptance-tests/helpers/random" - "csbbrokerpakgcp/acceptance-tests/helpers/services" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("Dataproc", Label("dataproc"), func() { - It("can be accessed by an app", func() { - By("creating a service instance") - serviceInstance := services.CreateInstance("csb-google-dataproc", "standard") - defer serviceInstance.Delete() - - By("pushing the unstarted app") - appOne := apps.Push(apps.WithApp(apps.Dataproc)) - defer apps.Delete(appOne) - - By("binding the app to the service instance") - binding := serviceInstance.Bind(appOne) - - By("starting the apps") - apps.Start(appOne) - - By("checking that the app environment has a credhub reference for credentials") - Expect(binding.Credential()).To(matchers.HaveCredHubRef) - - By("running a job") - jobName := random.Hexadecimal() - appOne.PUT("", jobName) - - By("getting the job status") - status := appOne.GET(jobName).String() - Expect(status).To(Equal("DONE")) - - By("deleting the job") - appOne.DELETE(jobName) - }) -}) diff --git a/acceptance-tests/helpers/apps/testapps.go b/acceptance-tests/helpers/apps/testapps.go index 99318771..08c143f6 100644 --- a/acceptance-tests/helpers/apps/testapps.go +++ b/acceptance-tests/helpers/apps/testapps.go @@ -7,7 +7,6 @@ import ( type AppCode string const ( - Dataproc AppCode = "dataprocapp" Spanner AppCode = "spannerapp" Storage AppCode = "storageapp" MySQL AppCode = "mysqlapp" diff --git a/acceptance-tests/upgrade/update_and_upgrade_dataproc_test.go b/acceptance-tests/upgrade/update_and_upgrade_dataproc_test.go deleted file mode 100644 index 937e9eca..00000000 --- a/acceptance-tests/upgrade/update_and_upgrade_dataproc_test.go +++ /dev/null @@ -1,76 +0,0 @@ -package upgrade_test - -import ( - "fmt" - - "csbbrokerpakgcp/acceptance-tests/helpers/apps" - "csbbrokerpakgcp/acceptance-tests/helpers/brokers" - "csbbrokerpakgcp/acceptance-tests/helpers/matchers" - "csbbrokerpakgcp/acceptance-tests/helpers/random" - "csbbrokerpakgcp/acceptance-tests/helpers/services" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("UpgradeDataprocTest", Label("dataproc"), func() { - When("upgrading broker version", func() { - It("should continue to work", func() { - By("pushing latest released broker version") - serviceBroker := brokers.Create( - brokers.WithPrefix("csb-dataproc"), - brokers.WithSourceDir(releasedBuildDir), - brokers.WithReleasedEnv(releasedBuildDir), - ) - defer serviceBroker.Delete() - - By("creating a service instance") - serviceInstance := services.CreateInstance( - "csb-google-dataproc", - "standard", - services.WithBroker(serviceBroker), - services.WithParameters(map[string]any{"name": fmt.Sprintf("csb-dataproc-%s", serviceBroker.Name)}), - ) - defer serviceInstance.Delete() - - By("pushing the unstarted app") - appOne := apps.Push(apps.WithApp(apps.Dataproc)) - defer apps.Delete(appOne) - - By("binding the app to the service instance") - binding := serviceInstance.Bind(appOne) - - By("starting the apps") - apps.Start(appOne) - - By("checking that the app environment has a credhub reference for credentials") - Expect(binding.Credential()).To(matchers.HaveCredHubRef) - - By("running a job") - jobName := random.Hexadecimal() - appOne.PUT("", jobName) - - By("getting the job status") - status := appOne.GET(jobName).String() - Expect(status).To(Equal("DONE")) - - By("pushing the development version of the broker") - serviceBroker.UpdateBroker(developmentBuildDir) - - By("upgrading service instance") - serviceInstance.Upgrade() - - By("checking the job status is still accessible") - Expect(appOne.GET(jobName).String()).To(Equal("DONE")) - - By("updating the instance config params") - serviceInstance.Update(services.WithParameters(`{}`)) - - By("checking the job status is still accessible") - Expect(appOne.GET(jobName).String()).To(Equal("DONE")) - - By("deleting the job") - appOne.DELETE(jobName) - }) - }) -}) diff --git a/docs/installation.md b/docs/installation.md index bab2e208..d59c5fad 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -31,7 +31,6 @@ Enable the following services in **[APIs and services > Library](https://console 1. If you want to enable Bigtable as a service, enable the [Bigtable Admin API](https://console.cloud.google.com/apis/library/bigtable.googleapis.com) 1. If you want to enable Datastore as a service, enable the [Datastore API](https://console.cloud.google.com/apis/api/datastore.googleapis.com/overview) 1. If you want to enable Redis as a service, enable the [Redis API](https://console.cloud.google.com/apis/library/redis.googleapis.com) -1. If you want to enable Dataproc as a service, enable the [Dataproc API](https://console.developers.google.com/apis/api/dataproc.googleapis.com/overview) 1. If you want to enable Cloud Spanner as a service, enable the [Cloud Spanner API](https://console.developers.google.com/apis/api/spanner.googleapis.com/overview) #### [Create a root service account](#service-account) diff --git a/google-dataproc.yml b/google-dataproc.yml deleted file mode 100644 index 92532fe0..00000000 --- a/google-dataproc.yml +++ /dev/null @@ -1,138 +0,0 @@ -version: 1 -name: csb-google-dataproc -id: ebb35d15-8c7a-4c4e-8aa8-d8d751a9d8d3 -description: Beta - Dataproc is a fully-managed service for running Apache Spark and Apache Hadoop clusters in a simpler, more cost-efficient way. -display_name: Google Cloud Dataproc (Beta) -image_url: file://service-images/csb.png -documentation_url: https://docs.vmware.com/en/Cloud-Service-Broker-for-VMware-Tanzu/index.html -provider_display_name: VMware -support_url: https://cloud.google.com/dataproc/docs/support/getting-support -tags: [gcp, dataproc, beta] -plans: -- id: ed8c2ad0-edc7-4f36-a332-fd63d81ec276 - name: standard - display_name: Standard (Beta - 1 master, N workers) - description: Beta - Basic plan with 1 master and N workers. - properties: -- id: 71cc321b-3ba3-4f0f-b058-90cfc978e743 - name: ha - display_name: High Availability (Beta - 3 masters, N workers) - description: Beta - High availability plan with 3 masters and N workers. - properties: - master_count: 3 -provision: - user_inputs: - - default: n1-standard-2 - field_name: worker_machine_type - type: string - details: 'The name of a Google Compute Engine machine type to create for the worker(s).' - - default: 2 - field_name: worker_count - type: integer - details: 'Specifies the number of worker nodes to create.' - - default: n1-standard-2 - field_name: master_machine_type - type: string - details: 'The name of a Google Compute Engine machine type to create for the master(s).' - - default: 1 - field_name: master_count - type: integer - details: 'Specifies the number of master nodes to create.' - - default: 0 - field_name: preemptible_count - type: integer - details: 'Specifies the number of preemptible nodes to create. This is independent of the worker count and will be the type specified by `worker_machine_type`.' - - field_name: name - type: string - details: The name of the cluster. - default: csb-dataproc-${request.instance_id} - constraints: - maxLength: 222 - minLength: 3 - pattern: ^[a-z0-9-\.]+$ - - field_name: region - type: string - details: 'The region in which the cluster and associated nodes will be created in.' - default: 'us-central1' - constraints: - examples: - - us-central1 - - asia-northeast1 - pattern: ^[a-z][a-z0-9-]+$ - - field_name: credentials - type: string - details: GCP credentials - default: ${config("gcp.credentials")} - - field_name: project - type: string - details: GCP project - default: ${config("gcp.project")} - computed_inputs: - - name: labels - default: ${json.marshal(request.default_labels)} - overwrite: true - type: object - template_refs: - provider: terraform/dataproc/provision/provider.tf - versions: terraform/dataproc/provision/versions.tf - main: terraform/dataproc/provision/main.tf - variables: terraform/dataproc/provision/variables.tf - outputs: terraform/dataproc/provision/outputs.tf - outputs: - - field_name: bucket_name - type: string - details: Name of the bucket used for staging jobs. - - field_name: cluster_name - type: string - details: The name of the Dataproc cluster. - - field_name: region - type: string - details: The GCP region of the Dataproc cluster. -bind: - plan_inputs: [] - user_inputs: - - field_name: credentials - type: string - details: GCP credentials - default: ${config("gcp.credentials")} - - field_name: project - type: string - details: GCP project - default: ${config("gcp.project")} - computed_inputs: - - name: service_account_name - default: ${str.truncate(20, "pcf-binding-${request.binding_id}")} - overwrite: true - - name: bucket - default: ${instance.details["bucket_name"]} - overwrite: true - template_refs: - provider: terraform/dataproc/bind/provider.tf - main: terraform/dataproc/bind/main.tf - variables: terraform/dataproc/bind/variables.tf - outputs: terraform/dataproc/bind/outputs.tf - versions: terraform/dataproc/bind/versions.tf - outputs: - - field_name: email - type: string - details: Email address of the service account. - - field_name: private_key - type: string - details: Service account private key data. Base64 encoded JSON. - - field_name: project_id - type: string - details: ID of the project that owns the service account. - - field_name: name - type: string - details: Name of service account -examples: -- name: basic - description: Create a standard Dataproc cluster with a service account that can kick off jobs (roles/dataproc.editor) and has objectAdmin access to the bucket that's created. - plan_id: ed8c2ad0-edc7-4f36-a332-fd63d81ec276 - provision_params: {} - bind_params: {} -- name: ha - description: Create a HA Dataproc cluster with a service account that can kick off jobs (roles/dataproc.editor) and has objectAdmin access to the bucket that's created. - plan_id: 71cc321b-3ba3-4f0f-b058-90cfc978e743 - provision_params: {} - bind_params: {} diff --git a/integration-tests/dataproc_test.go b/integration-tests/dataproc_test.go deleted file mode 100644 index 86ab184c..00000000 --- a/integration-tests/dataproc_test.go +++ /dev/null @@ -1,82 +0,0 @@ -package integration_test - -import ( - testframework "github.com/cloudfoundry/cloud-service-broker/v2/brokerpaktestframework" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - . "github.com/onsi/gomega/gstruct" -) - -const ( - dataprocServiceName = "csb-google-dataproc" - dataprocServiceID = "ebb35d15-8c7a-4c4e-8aa8-d8d751a9d8d3" - dataprocServiceDisplayName = "Google Cloud Dataproc (Beta)" - dataprocServiceDescription = "Beta - Dataproc is a fully-managed service for running Apache Spark and Apache Hadoop clusters in a simpler, more cost-efficient way." - dataprocServiceSupportURL = "https://cloud.google.com/dataproc/docs/support/getting-support" - dataprocStandardPlanName = "standard" - dataprocStandardPlanID = "ed8c2ad0-edc7-4f36-a332-fd63d81ec276" - dataprocHAPlanName = "ha" - dataprocHAPlanID = "71cc321b-3ba3-4f0f-b058-90cfc978e743" -) - -var _ = Describe("Dataproc", func() { - BeforeEach(func() { - Expect(mockTerraform.SetTFState([]testframework.TFStateValue{})).To(Succeed()) - }) - - AfterEach(func() { - Expect(mockTerraform.Reset()).To(Succeed()) - }) - - It("publishes in the catalog", func() { - catalog, err := broker.Catalog() - Expect(err).NotTo(HaveOccurred()) - - service := testframework.FindService(catalog, dataprocServiceName) - Expect(service.ID).To(Equal(dataprocServiceID)) - Expect(service.Description).To(Equal(dataprocServiceDescription)) - Expect(service.Tags).To(ConsistOf("gcp", "dataproc", "beta")) - Expect(service.Metadata.ImageUrl).To(ContainSubstring("data:image/png;base64,")) - Expect(service.Metadata.DisplayName).To(Equal(dataprocServiceDisplayName)) - Expect(service.Metadata.DocumentationUrl).To(Equal(cloudServiceBrokerDocumentationURL)) - Expect(service.Metadata.ProviderDisplayName).To(Equal(providerDisplayName)) - Expect(service.Metadata.SupportUrl).To(Equal(dataprocServiceSupportURL)) - Expect(service.Plans).To( - ConsistOf( - MatchFields(IgnoreExtras, Fields{ - ID: Equal(dataprocStandardPlanID), - Name: Equal(dataprocStandardPlanName), - }), - MatchFields(IgnoreExtras, Fields{ - ID: Equal(dataprocHAPlanID), - Name: Equal(dataprocHAPlanName), - }), - ), - ) - }) - - Describe("provisioning", func() { - It("should check region constraints", func() { - _, err := broker.Provision(dataprocServiceName, "standard", map[string]any{"region": "-Asia-northeast1"}) - Expect(err).To(MatchError(ContainSubstring("region: Does not match pattern '^[a-z][a-z0-9-]+$'"))) - }) - }) - - Describe("updating instance", func() { - var instanceID string - - BeforeEach(func() { - var err error - instanceID, err = broker.Provision(dataprocServiceName, "standard", nil) - - Expect(err).NotTo(HaveOccurred()) - Expect(mockTerraform.Reset()).To(Succeed()) - }) - - It("should allow updating region because it is not flagged as `prohibit_update` and not specified in the plan", func() { - err := broker.Update(instanceID, dataprocServiceName, "standard", map[string]any{"region": "asia-southeast1"}) - - Expect(err).NotTo(HaveOccurred()) - }) - }) -}) diff --git a/manifest.yml b/manifest.yml index 3691d212..846d9299 100644 --- a/manifest.yml +++ b/manifest.yml @@ -44,5 +44,4 @@ service_definitions: - google-postgresql.yml - google-spanner.yml - google-bigquery.yml -- google-dataproc.yml - google-stackdriver-trace.yml diff --git a/terraform/dataproc/bind/main.tf b/terraform/dataproc/bind/main.tf deleted file mode 100644 index f53bca1f..00000000 --- a/terraform/dataproc/bind/main.tf +++ /dev/null @@ -1,20 +0,0 @@ -resource "google_service_account" "account" { - account_id = var.service_account_name - display_name = var.service_account_name -} - -resource "google_service_account_key" "key" { - service_account_id = google_service_account.account.name -} - -resource "google_storage_bucket_iam_member" "member" { - bucket = var.bucket - role = "roles/storage.objectAdmin" - member = "serviceAccount:${google_service_account.account.email}" -} - -resource "google_project_iam_member" "member" { - project = var.project - role = "roles/dataproc.editor" - member = "serviceAccount:${google_service_account.account.email}" -} diff --git a/terraform/dataproc/bind/outputs.tf b/terraform/dataproc/bind/outputs.tf deleted file mode 100644 index 148a7c16..00000000 --- a/terraform/dataproc/bind/outputs.tf +++ /dev/null @@ -1,7 +0,0 @@ -output "email" { value = google_service_account.account.email } -output "private_key" { - sensitive = true - value = google_service_account_key.key.private_key -} -output "project_id" { value = google_service_account.account.project } -output "name" { value = google_service_account.account.account_id } diff --git a/terraform/dataproc/bind/provider.tf b/terraform/dataproc/bind/provider.tf deleted file mode 100644 index e34e2155..00000000 --- a/terraform/dataproc/bind/provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "google" { - credentials = var.credentials - project = var.project -} diff --git a/terraform/dataproc/bind/variables.tf b/terraform/dataproc/bind/variables.tf deleted file mode 100644 index fcdc3a86..00000000 --- a/terraform/dataproc/bind/variables.tf +++ /dev/null @@ -1,7 +0,0 @@ -variable "credentials" { - type = string - sensitive = true -} -variable "project" { type = string } -variable "service_account_name" { type = string } -variable "bucket" { type = string } diff --git a/terraform/dataproc/bind/versions.tf b/terraform/dataproc/bind/versions.tf deleted file mode 100644 index 29e387d6..00000000 --- a/terraform/dataproc/bind/versions.tf +++ /dev/null @@ -1,8 +0,0 @@ -terraform { - required_providers { - google = { - source = "registry.terraform.io/hashicorp/google" - version = "~> 5" - } - } -} diff --git a/terraform/dataproc/provision/main.tf b/terraform/dataproc/provision/main.tf deleted file mode 100644 index 234a7c0d..00000000 --- a/terraform/dataproc/provision/main.tf +++ /dev/null @@ -1,25 +0,0 @@ -resource "google_dataproc_cluster" "cluster" { - name = var.name - region = var.region - labels = var.labels - - cluster_config { - master_config { - num_instances = var.master_count - machine_type = var.master_machine_type - } - - worker_config { - num_instances = var.worker_count - machine_type = var.worker_machine_type - } - - preemptible_worker_config { - num_instances = var.preemptible_count - } - } - - lifecycle { - prevent_destroy = true - } -} diff --git a/terraform/dataproc/provision/outputs.tf b/terraform/dataproc/provision/outputs.tf deleted file mode 100644 index 2fcf69fc..00000000 --- a/terraform/dataproc/provision/outputs.tf +++ /dev/null @@ -1,3 +0,0 @@ -output "bucket_name" { value = google_dataproc_cluster.cluster.cluster_config.0.bucket } -output "cluster_name" { value = google_dataproc_cluster.cluster.name } -output "region" { value = google_dataproc_cluster.cluster.region } diff --git a/terraform/dataproc/provision/provider.tf b/terraform/dataproc/provision/provider.tf deleted file mode 100644 index e34e2155..00000000 --- a/terraform/dataproc/provision/provider.tf +++ /dev/null @@ -1,4 +0,0 @@ -provider "google" { - credentials = var.credentials - project = var.project -} diff --git a/terraform/dataproc/provision/variables.tf b/terraform/dataproc/provision/variables.tf deleted file mode 100644 index 3ecf34c0..00000000 --- a/terraform/dataproc/provision/variables.tf +++ /dev/null @@ -1,15 +0,0 @@ -variable "worker_machine_type" { type = string } -variable "master_machine_type" { type = string } -variable "worker_count" { type = number } -variable "master_count" { type = number } -variable "preemptible_count" { type = number } - -variable "name" { type = string } -variable "region" { type = string } -variable "labels" { type = map(any) } - -variable "credentials" { - type = string - sensitive = true -} -variable "project" { type = string } diff --git a/terraform/dataproc/provision/versions.tf b/terraform/dataproc/provision/versions.tf deleted file mode 100644 index 29e387d6..00000000 --- a/terraform/dataproc/provision/versions.tf +++ /dev/null @@ -1,8 +0,0 @@ -terraform { - required_providers { - google = { - source = "registry.terraform.io/hashicorp/google" - version = "~> 5" - } - } -}