{Synapse} Update azure synapse artifact version to 0.20.0 and spark to 0.7.0 (Azure#30875)

* update spark version
update artifact version

* update test record

* fix error

* enable spark client credential_scopes

Author: kevinzz6

src/azure-cli/azure/cli/command_modules/synapse/manual/_client_factory.py

@@ -105,8 +105,7 @@ def cf_kusto_pool(cli_ctx, *_):
 def synapse_spark_factory(cli_ctx, workspace_name, sparkpool_name):
     from azure.synapse.spark import SparkClient
     from azure.cli.core._profile import Profile
-    # Uncomment this line after SparkClient is fixed. See https://github.com/Azure/azure-cli/pull/30788
-    # from azure.cli.core.auth.util import resource_to_scopes
+    from azure.cli.core.auth.util import resource_to_scopes
     from azure.cli.core.commands.client_factory import get_subscription_id
     subscription_id = get_subscription_id(cli_ctx)
     profile = Profile(cli_ctx=cli_ctx)
@@ -115,8 +114,7 @@ def synapse_spark_factory(cli_ctx, workspace_name, sparkpool_name):
         credential=cred,
         endpoint='{}{}{}'.format("https://", workspace_name, cli_ctx.cloud.suffixes.synapse_analytics_endpoint),
         spark_pool_name=sparkpool_name,
-        # Uncomment this line after SparkClient is fixed
-        # credential_scopes=resource_to_scopes(cli_ctx.cloud.endpoints.synapse_analytics_resource_id)
+        credential_scopes=resource_to_scopes(cli_ctx.cloud.endpoints.synapse_analytics_resource_id)
     )
 
 

src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_job.yaml

@@ -2038,15 +2038,15 @@ def test_ip_firewall_rules(self, resource_group, storage_account):
     @ResourceGroupPreparer(name_prefix='synapse-cli', random_name_length=16)
     def test_spark_job(self, resource_group):
         self.kwargs.update({
-            'spark-pool': 'testsparkpool',
-            'workspace': 'testsynapseworkspace',
+            'spark-pool': 'sparkpooltest',
+            'workspace': 'yanjuntestws001',
             'job': 'WordCount_Java',
-            'main-definition-file': 'abfss://testfilesystem@adlsgen2account.dfs.core.windows.net/samples/java/wordcount/wordcount.jar',
+            'main-definition-file': 'abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/wordcount_2.11-0.1.jar',
             'main-class-name': 'WordCount',
             'arguments': [
-                'abfss://testfilesystem@adlsgen2account.dfs.core.windows.net/samples/java/wordcount/shakespeare.txt',
-                'abfss://testfilesystem@adlsgen2account.dfs.core.windows.net/samples/java/wordcount/result/'],
-            'executors': 2,
+                'abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/input/shakespeare.txt',
+                'abfss://filesystem01@ywtestaccount.dfs.core.windows.net/wordcount/output/'],
+            'executors': 4,
             'executor-size': 'Medium',
             'configuration': '{\\"spark.dynamicAllocation.maxExecutors\\":\\"18\\"}'
         })
@@ -2091,8 +2091,8 @@ def test_spark_job(self, resource_group):
     @ResourceGroupPreparer(name_prefix='synapse-cli', random_name_length=16)
     def test_spark_session_and_statements(self, resource_group):
         self.kwargs.update({
-            'spark-pool': 'testsparkpool',
-            'workspace': 'testsynapseworkspace',
+            'spark-pool': 'sparkpooltest',
+            'workspace': 'yanjuntestws001',
             'job': self.create_random_name(prefix='clisession', length=14),
             'executor-size': 'Small',
             'executors': 2,
@@ -2161,10 +2161,7 @@ def test_spark_session_and_statements(self, resource_group):
 
         # cancel a spark session statement
         self.cmd('az synapse spark statement cancel --livy-id {statement-id} --session-id {session-id} '
-                 '--workspace-name {workspace} --spark-pool-name {spark-pool}  --yes',
-                 checks=[
-                     self.check('msg', 'canceled')
-                 ])
+                 '--workspace-name {workspace} --spark-pool-name {spark-pool}  --yes')
 
         # delete/cancel a spark session
         self.cmd('az synapse spark session cancel --livy-id {session-id} --workspace-name {workspace} '

src/azure-cli/azure/cli/command_modules/synapse/tests/latest/recordings/test_spark_session_and_statements.yaml

@@ -84,9 +84,9 @@ azure-multiapi-storage==1.3.0
 azure-nspkg==3.0.2
 azure-storage-common==1.4.2
 azure-synapse-accesscontrol==0.5.0
-azure-synapse-artifacts==0.19.0
+azure-synapse-artifacts==0.20.0
 azure-synapse-managedprivateendpoints==0.4.0
-azure-synapse-spark==0.2.0
+azure-synapse-spark==0.7.0
 bcrypt==3.2.0
 certifi==2024.7.4
 cffi==1.16.0

src/azure-cli/azure/cli/command_modules/synapse/tests/latest/test_synapse_scenario.py

@@ -84,9 +84,9 @@ azure-multiapi-storage==1.3.0
 azure-nspkg==3.0.2
 azure-storage-common==1.4.2
 azure-synapse-accesscontrol==0.5.0
-azure-synapse-artifacts==0.19.0
+azure-synapse-artifacts==0.20.0
 azure-synapse-managedprivateendpoints==0.4.0
-azure-synapse-spark==0.2.0
+azure-synapse-spark==0.7.0
 bcrypt==3.2.0
 certifi==2024.7.4
 cffi==1.16.0

src/azure-cli/requirements.py3.Darwin.txt

@@ -84,9 +84,9 @@ azure-multiapi-storage==1.3.0
 azure-nspkg==3.0.2
 azure-storage-common==1.4.2
 azure-synapse-accesscontrol==0.5.0
-azure-synapse-artifacts==0.19.0
+azure-synapse-artifacts==0.20.0
 azure-synapse-managedprivateendpoints==0.4.0
-azure-synapse-spark==0.2.0
+azure-synapse-spark==0.7.0
 bcrypt==3.2.0
 certifi==2024.7.4
 cffi==1.16.0

src/azure-cli/requirements.py3.Linux.txt

@@ -126,9 +126,9 @@
     'azure-multiapi-storage~=1.3.0',
     'azure-storage-common~=1.4',
     'azure-synapse-accesscontrol~=0.5.0',
-    'azure-synapse-artifacts~=0.19.0',
+    'azure-synapse-artifacts~=0.20.0',
     'azure-synapse-managedprivateendpoints~=0.4.0',
-    'azure-synapse-spark~=0.2.0',
+    'azure-synapse-spark~=0.7.0',
     'chardet~=5.2.0',
     'colorama~=0.4.4',
     # On Linux, the distribution (Ubuntu, Debian, etc) and version are checked for `az feedback`