From 63934e7a0354f3c47c46ffcb92a203ba11ddcd61 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 15:33:32 +0100 Subject: [PATCH 1/9] acc: replace LocalOnly option with Local & Cloud --- acceptance/acceptance_test.go | 8 ++++++-- acceptance/auth/bundle_and_profile/test.toml | 2 -- acceptance/auth/credentials/test.toml | 2 -- acceptance/bundle/debug/test.toml | 2 +- acceptance/bundle/generate/git_job/test.toml | 2 +- acceptance/bundle/libraries/maven/test.toml | 2 +- acceptance/bundle/libraries/pypi/test.toml | 2 +- .../bundle/templates-machinery/helpers-error/test.toml | 7 ------- acceptance/bundle/templates-machinery/helpers/test.toml | 1 - acceptance/bundle/templates-machinery/test.toml | 2 -- acceptance/bundle/templates/test.toml | 2 -- .../trampoline/warning_message_with_new_spark/test.toml | 2 +- .../trampoline/warning_message_with_old_spark/test.toml | 2 +- acceptance/bundle/variables/test.toml | 2 +- acceptance/cmd/workspace/apps/test.toml | 1 - acceptance/config_test.go | 7 +++++-- acceptance/selftest/server/test.toml | 1 - acceptance/selftest/test.toml | 1 - acceptance/terraform/test.toml | 3 +++ acceptance/workspace/jobs/create-error/test.toml | 1 - acceptance/workspace/jobs/create/test.toml | 1 - 21 files changed, 21 insertions(+), 32 deletions(-) delete mode 100644 acceptance/bundle/templates-machinery/helpers-error/test.toml delete mode 100644 acceptance/bundle/templates-machinery/helpers/test.toml delete mode 100644 acceptance/bundle/templates-machinery/test.toml delete mode 100644 acceptance/bundle/templates/test.toml delete mode 100644 acceptance/selftest/test.toml diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index 465df5ec74..066a84299e 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -217,8 +217,12 @@ func runTest(t *testing.T, dir, coverDir string, repls testdiff.ReplacementsCont } cloudEnv := os.Getenv("CLOUD_ENV") - if isTruePtr(config.LocalOnly) && cloudEnv != "" { - t.Skipf("Disabled via LocalOnly setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv) + if !isTruePtr(config.Local) && cloudEnv == "" { + t.Skipf("Disabled via Local setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv) + } + + if !isTruePtr(config.Cloud) && cloudEnv != "" { + t.Skipf("Disabled via Cloud setting in %s (CLOUD_ENV=%s)", configPath, cloudEnv) } var tmpDir string diff --git a/acceptance/auth/bundle_and_profile/test.toml b/acceptance/auth/bundle_and_profile/test.toml index 58ef79eaed..697281ee56 100644 --- a/acceptance/auth/bundle_and_profile/test.toml +++ b/acceptance/auth/bundle_and_profile/test.toml @@ -1,5 +1,3 @@ -LocalOnly=true - # Some of the clouds have DATABRICKS_HOST variable setup without https:// prefix # In the result, output is replaced with DATABRICKS_URL variable instead of DATABRICKS_HOST # This is a workaround to replace DATABRICKS_URL with DATABRICKS_HOST diff --git a/acceptance/auth/credentials/test.toml b/acceptance/auth/credentials/test.toml index 89438f43af..dc775ea627 100644 --- a/acceptance/auth/credentials/test.toml +++ b/acceptance/auth/credentials/test.toml @@ -1,5 +1,3 @@ -LocalOnly = true - RecordRequests = true IncludeRequestHeaders = ["Authorization", "User-Agent"] diff --git a/acceptance/bundle/debug/test.toml b/acceptance/bundle/debug/test.toml index bb0fcb395f..79d1b9ee6b 100644 --- a/acceptance/bundle/debug/test.toml +++ b/acceptance/bundle/debug/test.toml @@ -1,4 +1,4 @@ -LocalOnly = true +Cloud = false [[Repls]] # The keys are unsorted and also vary per OS diff --git a/acceptance/bundle/generate/git_job/test.toml b/acceptance/bundle/generate/git_job/test.toml index 28b473245b..fce46071af 100644 --- a/acceptance/bundle/generate/git_job/test.toml +++ b/acceptance/bundle/generate/git_job/test.toml @@ -1,4 +1,4 @@ -LocalOnly = true # This test needs to run against stubbed Databricks API +Cloud = false # This test needs to run against stubbed Databricks API [[Server]] Pattern = "GET /api/2.1/jobs/get" diff --git a/acceptance/bundle/libraries/maven/test.toml b/acceptance/bundle/libraries/maven/test.toml index 911438c24e..62ba36982b 100644 --- a/acceptance/bundle/libraries/maven/test.toml +++ b/acceptance/bundle/libraries/maven/test.toml @@ -1,5 +1,5 @@ # We run this test only locally for now because we need to figure out how to do # bundle destroy on script.cleanup first. -LocalOnly = true +Cloud = false RecordRequests = true diff --git a/acceptance/bundle/libraries/pypi/test.toml b/acceptance/bundle/libraries/pypi/test.toml index 911438c24e..62ba36982b 100644 --- a/acceptance/bundle/libraries/pypi/test.toml +++ b/acceptance/bundle/libraries/pypi/test.toml @@ -1,5 +1,5 @@ # We run this test only locally for now because we need to figure out how to do # bundle destroy on script.cleanup first. -LocalOnly = true +Cloud = false RecordRequests = true diff --git a/acceptance/bundle/templates-machinery/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml deleted file mode 100644 index 77f4ed94b7..0000000000 --- a/acceptance/bundle/templates-machinery/helpers-error/test.toml +++ /dev/null @@ -1,7 +0,0 @@ -Badness = '''(minor) error message is not great: executing "" at : error calling user_name:''' -LocalOnly = true - -[[Server]] -Pattern = "GET /api/2.0/preview/scim/v2/Me" -Response.Body = '{}' -Response.StatusCode = 500 diff --git a/acceptance/bundle/templates-machinery/helpers/test.toml b/acceptance/bundle/templates-machinery/helpers/test.toml deleted file mode 100644 index b76e712fbc..0000000000 --- a/acceptance/bundle/templates-machinery/helpers/test.toml +++ /dev/null @@ -1 +0,0 @@ -LocalOnly = true diff --git a/acceptance/bundle/templates-machinery/test.toml b/acceptance/bundle/templates-machinery/test.toml deleted file mode 100644 index 9083ecd1be..0000000000 --- a/acceptance/bundle/templates-machinery/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# Testing template machinery, by default there is no need to check against cloud. -LocalOnly = true diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml deleted file mode 100644 index 90539263de..0000000000 --- a/acceptance/bundle/templates/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -# At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. -LocalOnly = true diff --git a/acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml b/acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml index 811364d3bf..4e52dbb5e4 100644 --- a/acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_new_spark/test.toml @@ -1,6 +1,6 @@ # Since we use existing cluster id value which is not available in cloud envs, we need to stub the request # and run this test only locally -LocalOnly = true +Cloud = false [[Server]] Pattern = "GET /api/2.1/clusters/get" diff --git a/acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml b/acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml index b1963548e6..09021bfc00 100644 --- a/acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml +++ b/acceptance/bundle/trampoline/warning_message_with_old_spark/test.toml @@ -1,6 +1,6 @@ # Since we use existing cluster id value which is not available in cloud envs, we need to stub the request # and run this test only locally -LocalOnly = true +Cloud = false [[Server]] Pattern = "GET /api/2.1/clusters/get" diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml index 32398e8288..4ee938cce7 100644 --- a/acceptance/bundle/variables/test.toml +++ b/acceptance/bundle/variables/test.toml @@ -1,3 +1,3 @@ # The tests here intend to test variable interpolation via "bundle validate". # Even though "bundle validate" does a few API calls, that's not the focus there. -LocalOnly = true +Local = false diff --git a/acceptance/cmd/workspace/apps/test.toml b/acceptance/cmd/workspace/apps/test.toml index 4f919b315c..972ae1c505 100644 --- a/acceptance/cmd/workspace/apps/test.toml +++ b/acceptance/cmd/workspace/apps/test.toml @@ -1,4 +1,3 @@ -LocalOnly = true RecordRequests = true [[Server]] diff --git a/acceptance/config_test.go b/acceptance/config_test.go index a3589b2e5c..4edfee69d0 100644 --- a/acceptance/config_test.go +++ b/acceptance/config_test.go @@ -24,8 +24,11 @@ type TestConfig struct { // If absent, default to true. GOOS map[string]bool - // If true, do not run this test against cloud environment - LocalOnly *bool + // If true, run this test when running locally with a testserver + Local *bool + + // If true, run this test when running with cloud env configured + Cloud *bool // List of additional replacements to apply on this test. // Old is a regexp, New is a replacement expression. diff --git a/acceptance/selftest/server/test.toml b/acceptance/selftest/server/test.toml index 43ad1e85b0..8fc7b3cacd 100644 --- a/acceptance/selftest/server/test.toml +++ b/acceptance/selftest/server/test.toml @@ -1,4 +1,3 @@ -LocalOnly = true RecordRequests = true [[Server]] diff --git a/acceptance/selftest/test.toml b/acceptance/selftest/test.toml deleted file mode 100644 index b76e712fbc..0000000000 --- a/acceptance/selftest/test.toml +++ /dev/null @@ -1 +0,0 @@ -LocalOnly = true diff --git a/acceptance/terraform/test.toml b/acceptance/terraform/test.toml index a6849e30f2..9fbd709431 100644 --- a/acceptance/terraform/test.toml +++ b/acceptance/terraform/test.toml @@ -1,3 +1,6 @@ +Local = true +Cloud = true + [[Repls]] Old = 'Read complete after [^\s]+' New = 'Read complete after (redacted)' diff --git a/acceptance/workspace/jobs/create-error/test.toml b/acceptance/workspace/jobs/create-error/test.toml index b45bf77e58..a7b86accb4 100644 --- a/acceptance/workspace/jobs/create-error/test.toml +++ b/acceptance/workspace/jobs/create-error/test.toml @@ -1,4 +1,3 @@ -LocalOnly = true # request recording currently does not work with cloud environment RecordRequests = true [[Server]] diff --git a/acceptance/workspace/jobs/create/test.toml b/acceptance/workspace/jobs/create/test.toml index 1fd9b3cec3..f08bc0e63a 100644 --- a/acceptance/workspace/jobs/create/test.toml +++ b/acceptance/workspace/jobs/create/test.toml @@ -1,4 +1,3 @@ -LocalOnly = true # request recording currently does not work with cloud environment RecordRequests = true IncludeRequestHeaders = ["Authorization", "User-Agent"] From 495f1c4579dfd9bbf9b290ece085e394def102c9 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 15:39:40 +0100 Subject: [PATCH 2/9] restore --- .../bundle/templates-machinery/helpers-error/test.toml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 acceptance/bundle/templates-machinery/helpers-error/test.toml diff --git a/acceptance/bundle/templates-machinery/helpers-error/test.toml b/acceptance/bundle/templates-machinery/helpers-error/test.toml new file mode 100644 index 0000000000..3839635db0 --- /dev/null +++ b/acceptance/bundle/templates-machinery/helpers-error/test.toml @@ -0,0 +1,6 @@ +Badness = '''(minor) error message is not great: executing "" at : error calling user_name:''' + +[[Server]] +Pattern = "GET /api/2.0/preview/scim/v2/Me" +Response.Body = '{}' +Response.StatusCode = 500 From 0c2a96691bb57a69c6004038d0ad0b8230d47971 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 15:40:36 +0100 Subject: [PATCH 3/9] add root test.toml --- acceptance/test.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 acceptance/test.toml diff --git a/acceptance/test.toml b/acceptance/test.toml new file mode 100644 index 0000000000..7d36fb9dc1 --- /dev/null +++ b/acceptance/test.toml @@ -0,0 +1,2 @@ +Local = true +Cloud = false From b04df5d4436c05b458478c3b5391fa88fed04aa2 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 15:42:35 +0100 Subject: [PATCH 4/9] disable bundle/templates-machinery on cloiud --- acceptance/bundle/templates-machinery/test.toml | 1 + 1 file changed, 1 insertion(+) create mode 100644 acceptance/bundle/templates-machinery/test.toml diff --git a/acceptance/bundle/templates-machinery/test.toml b/acceptance/bundle/templates-machinery/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/templates-machinery/test.toml @@ -0,0 +1 @@ +Cloud = false From 56fe08ee9274ca6118b4b5fa4709a54a927d3a2e Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 16:08:37 +0100 Subject: [PATCH 5/9] add missing test.toml in acceptance/bundle --- acceptance/bundle/test.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 acceptance/bundle/test.toml diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml new file mode 100644 index 0000000000..0e8c8a3840 --- /dev/null +++ b/acceptance/bundle/test.toml @@ -0,0 +1,2 @@ +Local = true +Cloud = true From 78c39579c8ec13e1a7181ffccb1be485afa73687 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 16:09:20 +0100 Subject: [PATCH 6/9] disable cloud in bundle/help --- acceptance/bundle/help/test.toml | 1 + 1 file changed, 1 insertion(+) create mode 100644 acceptance/bundle/help/test.toml diff --git a/acceptance/bundle/help/test.toml b/acceptance/bundle/help/test.toml new file mode 100644 index 0000000000..18b1a88417 --- /dev/null +++ b/acceptance/bundle/help/test.toml @@ -0,0 +1 @@ +Cloud = false From e5eea9b175b1d85104b995343ca4f63be1db1467 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 16:12:35 +0100 Subject: [PATCH 7/9] restore --- acceptance/bundle/templates/test.toml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 acceptance/bundle/templates/test.toml diff --git a/acceptance/bundle/templates/test.toml b/acceptance/bundle/templates/test.toml new file mode 100644 index 0000000000..d0d289b5cc --- /dev/null +++ b/acceptance/bundle/templates/test.toml @@ -0,0 +1,2 @@ +# At the moment, there are many differences across different envs w.r.t to catalog use, node type and so on. +Cloud = false From 8c51b8929f02b658f666ceafe4adc266c90bf968 Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 16:17:45 +0100 Subject: [PATCH 8/9] add a comment --- acceptance/test.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/acceptance/test.toml b/acceptance/test.toml index 7d36fb9dc1..0a009f397e 100644 --- a/acceptance/test.toml +++ b/acceptance/test.toml @@ -1,2 +1,3 @@ +# Default settings that apply to all tests unless overriden by test.toml files in inner directories. Local = true Cloud = false From 74337e98b1d44005e2b00fd1f70c105f6cd037ae Mon Sep 17 00:00:00 2001 From: Denis Bilenko Date: Wed, 26 Feb 2025 16:38:09 +0100 Subject: [PATCH 9/9] fix variables/test.toml --- acceptance/bundle/variables/test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/variables/test.toml b/acceptance/bundle/variables/test.toml index 4ee938cce7..8ed716ad0c 100644 --- a/acceptance/bundle/variables/test.toml +++ b/acceptance/bundle/variables/test.toml @@ -1,3 +1,3 @@ # The tests here intend to test variable interpolation via "bundle validate". # Even though "bundle validate" does a few API calls, that's not the focus there. -Local = false +Cloud = false