Merge branch 'Azure:dev' into dev

Author: ganga1980

scripts/regression_test/extension_regression_test.yml

@@ -24,7 +24,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'checkout branch'
       inputs:
-        azureSubscription: 'Azure CLI release'
+        azureSubscription: $(AZURE_SDK_INFRA_SUB_CONNECTED_SERVICE)
         scriptLocation: inlineScript
         inlineScript: |
           set -ev
@@ -81,7 +81,7 @@ jobs:
   - task: AzureCLI@1
     displayName: 'checkout cli and extension repo'
     inputs:
-      azureSubscription: 'Azure CLI release'
+      azureSubscription: $(AZURE_SDK_INFRA_SUB_CONNECTED_SERVICE)
       scriptLocation: inlineScript
       inlineScript: |
         set -ev
@@ -106,17 +106,20 @@ jobs:
   - template: ../../.azure-pipelines/templates/azdev_setup.yml
     parameters:
       CLIExtensionRepoPath: ./azure-cli-extensions
-  - bash: |
-      set -ev
-
-      source env/bin/activate
-      cd azure-cli-extensions
-
-      az login -u $(CLI_LIVE_TEST_ACCOUNT) -p "$(CLI_LIVE_TEST_PASSWORD)"
-      az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590
-
-      python ../scripts/ci/automation_full_test.py "12" "$(Instance_idx)" "latest" "" "True" "extension"
-    displayName: "Rerun tests"
+  - task: AzureCLI@1
+    displayName: 'Rerun tests'
+    inputs:
+      azureSubscription: $(AZURE_SDK_TEST_SUB_CONNECTED_SERVICE)
+      scriptLocation: inlineScript
+      inlineScript: |
+        set -ev
+  
+        source env/bin/activate
+        cd azure-cli-extensions
+  
+        az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590
+  
+        python ../scripts/ci/automation_full_test.py "12" "$(Instance_idx)" "latest" "" "True" "extension"
   - bash: |
       publishErrorModules='false'
       if [[ -f '/$(HOME)/.azdev/env_config/mnt/vss/_work/1/s/env/test_results_error_modules_$(Instance_idx).txt' ]]; then
@@ -159,7 +162,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'Result Summary'
       inputs:
-        azureSubscription: 'Azure CLI release'
+        azureSubscription: $(AZURE_SDK_INFRA_SUB_CONNECTED_SERVICE)
         scriptLocation: inlineScript
         inlineScript: |
           set -ev

scripts/regression_test/regression_test.yml

@@ -25,7 +25,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'update version'
       inputs:
-        azureSubscription: 'Azure CLI release'
+        azureSubscription: $(AZURE_SDK_INFRA_SUB_CONNECTED_SERVICE)
         scriptLocation: inlineScript
         inlineScript: |
           set -ev
@@ -93,7 +93,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'Checkout Target Branch'
       inputs:
-        azureSubscription: 'Azure CLI release'
+        azureSubscription: $(AZURE_SDK_INFRA_SUB_CONNECTED_SERVICE)
         scriptLocation: inlineScript
         inlineScript: |
           set -ev
@@ -114,21 +114,24 @@ jobs:
 
           git checkout -b ${GITHUB_BRANCH} ${GITHUB_REPO}/${GITHUB_BRANCH}
     - template: ../../.azure-pipelines/templates/azdev_setup.yml
-    - bash: |
-        set -ev
-
-        source env/bin/activate
-
-        if [[ -n "$(CUSTOM_WHL_URL)" ]]; then
-          pip install $(CUSTOM_WHL_URL) --force-reinstall
-        fi
-
-        az login -u $(CLI_LIVE_TEST_ACCOUNT) -p "$(CLI_LIVE_TEST_PASSWORD)"
-        az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590
-
-        serial_modules="appservice botservice cloud network azure-cli-core azure-cli-telemetry"
-        python scripts/ci/automation_full_test.py "8" "$(Instance_idx)" "latest" "$serial_modules" "True"
-      displayName: "Rerun tests"
+    - task: AzureCLI@1
+      displayName: 'Rerun tests'
+      inputs:
+        azureSubscription: $(AZURE_SDK_TEST_SUB_CONNECTED_SERVICE)
+        scriptLocation: inlineScript
+        inlineScript: |
+          set -ev
+  
+          source env/bin/activate
+  
+          if [[ -n "$(CUSTOM_WHL_URL)" ]]; then
+            pip install $(CUSTOM_WHL_URL) --force-reinstall
+          fi
+  
+          az account set -s 0b1f6471-1bf0-4dda-aec3-cb9272f09590
+  
+          serial_modules="appservice botservice cloud network azure-cli-core azure-cli-telemetry"
+          python scripts/ci/automation_full_test.py "8" "$(Instance_idx)" "latest" "$serial_modules" "True"
     - task: PublishTestResults@2
       inputs:
         testResultsFiles: '/$(HOME)/.azdev/env_config/mnt/vss/_work/1/s/env/test_results_*.xml'
@@ -144,7 +147,7 @@ jobs:
     - task: AzureCLI@1
       displayName: 'Create PR'
       inputs:
-        azureSubscription: 'Azure CLI release'
+        azureSubscription: $(AZURE_SDK_INFRA_SUB_CONNECTED_SERVICE)
         scriptLocation: inlineScript
         inlineScript: |
           set -ev

src/azure-cli/azure/cli/command_modules/appconfig/_kv_helpers.py

@@ -18,7 +18,7 @@
 from knack.log import get_logger
 from knack.util import CLIError
 
-from azure.cli.command_modules.keyvault.vendored_sdks.azure_keyvault_t1.key_vault_id import KeyVaultIdentifier
+from azure.keyvault.secrets._shared import parse_key_vault_id
 from azure.appconfiguration import ResourceReadOnlyError, ConfigurationSetting
 from azure.core.exceptions import HttpResponseError
 from azure.cli.core.util import user_confirmation
@@ -314,12 +314,6 @@ def __read_kv_from_config_store(azconfig_client,
     elif top is None:
         top = 100
 
-    if cli_ctx:
-        from azure.cli.command_modules.keyvault._client_factory import keyvault_data_plane_factory
-        keyvault_client = keyvault_data_plane_factory(cli_ctx)
-    else:
-        keyvault_client = None
-
     for setting in configsetting_iterable:
         kv = convert_configurationsetting_to_keyvalue(setting)
 
@@ -333,8 +327,8 @@ def __read_kv_from_config_store(azconfig_client,
 
             if kv.content_type and kv.value:
                 # resolve key vault reference
-                if keyvault_client and __is_key_vault_ref(kv):
-                    __resolve_secret(keyvault_client, kv)
+                if cli_ctx and __is_key_vault_ref(kv):
+                    __resolve_secret(cli_ctx, kv)
 
         # trim unwanted fields from kv object instead of leaving them as null.
         if fields:
@@ -437,7 +431,7 @@ def __read_kv_from_app_service(cmd, appservice_account, prefix_to_add="", conten
                             secret_identifier = "https://{0}.vault.azure.net/secrets/{1}/{2}".format(vault_name, secret_name, secret_version)
                         try:
                             # this throws an exception for invalid format of secret identifier
-                            KeyVaultIdentifier(uri=secret_identifier)
+                            parse_key_vault_id(source_id=secret_identifier)
                             kv = KeyValue(key=key,
                                           value=json.dumps({"uri": secret_identifier}, ensure_ascii=False, separators=(',', ':')),
                                           tags=tags,
@@ -816,15 +810,15 @@ def __compact_key_values(key_values):
     return compacted
 
 
-def __resolve_secret(keyvault_client, keyvault_reference):
-    from azure.cli.command_modules.keyvault.vendored_sdks.azure_keyvault_t1.key_vault_id import SecretId
+def __resolve_secret(cli_ctx, keyvault_reference):
     try:
         secret_id = json.loads(keyvault_reference.value)["uri"]
-        kv_identifier = SecretId(uri=secret_id)
+        kv_identifier = parse_key_vault_id(source_id=secret_id)
+        from azure.cli.command_modules.keyvault._client_factory import data_plane_azure_keyvault_secret_client
+        keyvault_client = data_plane_azure_keyvault_secret_client(cli_ctx, kv_identifier.vault_url)
 
-        secret = keyvault_client.get_secret(vault_base_url=kv_identifier.vault,
-                                            secret_name=kv_identifier.name,
-                                            secret_version=kv_identifier.version)
+        secret = keyvault_client.get_secret(name=kv_identifier.name,
+                                            version=kv_identifier.version)
         keyvault_reference.value = secret.value
         return keyvault_reference
     except (TypeError, ValueError):
@@ -890,7 +884,7 @@ def __validate_import_keyvault_ref(kv):
             # URL with a valid scheme and netloc is a valid url, but keyvault ref has path as well, so validate it
             if parsed_url.scheme and parsed_url.netloc and parsed_url.path:
                 try:
-                    KeyVaultIdentifier(uri=value['uri'])
+                    parse_key_vault_id(source_id=value['uri'])
                     return True
                 except Exception:  # pylint: disable=broad-except
                     pass

src/azure-cli/azure/cli/command_modules/appconfig/_validators.py

@@ -227,12 +227,12 @@ def validate_identity(namespace):
 
 def validate_secret_identifier(namespace):
     """ Validate the format of keyvault reference secret identifier """
-    from azure.cli.command_modules.keyvault.vendored_sdks.azure_keyvault_t1.key_vault_id import KeyVaultIdentifier
+    from azure.keyvault.secrets._shared import parse_key_vault_id
 
     identifier = getattr(namespace, 'secret_identifier', None)
     try:
         # this throws an exception for invalid format of secret identifier
-        KeyVaultIdentifier(uri=identifier)
+        parse_key_vault_id(source_id=identifier)
     except Exception as e:
         raise CLIError("Received an exception while validating the format of secret identifier.\n{0}".format(str(e)))
 

src/azure-cli/azure/cli/command_modules/keyvault/_client_factory.py

@@ -109,9 +109,7 @@ def get_client_factory(resource_type, client_name=''):
     if is_mgmt_plane(resource_type):
         return keyvault_mgmt_client_factory(resource_type, client_name)
     if resource_type == ResourceType.DATA_KEYVAULT:
-        if client_name == Clients.private_7_2:
-            return keyvault_private_data_plane_factory_v7_2_preview
-        return keyvault_data_plane_factory
+        return keyvault_private_data_plane_factory_v7_2_preview
     if resource_type == ResourceType.DATA_KEYVAULT_ADMINISTRATION_BACKUP:
         return data_plane_azure_keyvault_administration_backup_client
     if resource_type == ResourceType.DATA_KEYVAULT_ADMINISTRATION_ACCESS_CONTROL:
@@ -167,31 +165,6 @@ def _keyvault_mgmt_client_factory(cli_ctx, _):
     return _keyvault_mgmt_client_factory
 
 
-def keyvault_data_plane_factory(cli_ctx, *_):
-    from azure.cli.command_modules.keyvault.vendored_sdks.azure_keyvault_t1 import (
-        KeyVaultAuthentication, KeyVaultClient)
-    from azure.cli.core.util import should_disable_connection_verify
-
-    version = str(get_api_version(cli_ctx, ResourceType.DATA_KEYVAULT))
-
-    def get_token(server, resource, scope):  # pylint: disable=unused-argument
-        return Profile(cli_ctx=cli_ctx).get_raw_token(resource=resource,
-                                                      subscription=cli_ctx.data.get('subscription_id'))[0]
-
-    client = KeyVaultClient(KeyVaultAuthentication(get_token), api_version=version)
-
-    # HACK, work around the fact that KeyVault library does't take confiuration object on constructor
-    # which could be used to turn off the verifiaction. Remove this once we migrate to new data plane library
-    # pylint: disable=protected-access
-    if hasattr(client, '_client') and hasattr(client._client, 'config'):
-        verify = not should_disable_connection_verify()
-        client._client.config.connection.verify = verify
-    else:
-        logger.info('Could not find the configuration object to turn off the verification if needed')
-
-    return client
-
-
 def keyvault_private_data_plane_factory_v7_2_preview(cli_ctx, _):
     from azure.cli.command_modules.keyvault.vendored_sdks.azure_keyvault_t1 import (
         KeyVaultAuthentication, KeyVaultClient)

src/azure-cli/azure/cli/command_modules/keyvault/_command_type.py

@@ -29,11 +29,9 @@ def _encode_hex(item):
     return item
 
 
-def keyvault_exception_handler(cmd, ex):
+def keyvault_exception_handler(ex):
     from msrest.exceptions import ValidationError, ClientRequestError
-    from azure.cli.core.profiles import ResourceType
-    KeyVaultErrorException = cmd.get_models('KeyVaultErrorException', resource_type=ResourceType.DATA_KEYVAULT)
-    if isinstance(ex, (ValidationError, KeyVaultErrorException)):
+    if isinstance(ex, ValidationError):
         try:
             raise CLIError(ex.inner_exception.error.message)
         except AttributeError:
@@ -54,10 +52,9 @@ def keyvault_exception_handler(cmd, ex):
 class KeyVaultCommandGroup(AzCommandGroup):
 
     def __init__(self, command_loader, group_name, **kwargs):
-        from azure.cli.command_modules.keyvault._client_factory import keyvault_data_plane_factory
-        # all regular and custom commands should use the keyvault data plane client
+        from azure.cli.command_modules.keyvault._client_factory import keyvault_mgmt_client_factory
         merged_kwargs = self._merge_kwargs(kwargs, base_kwargs=command_loader.module_kwargs)
-        merged_kwargs['custom_command_type'].settings['client_factory'] = keyvault_data_plane_factory
+        merged_kwargs['custom_command_type'].settings['client_factory'] = keyvault_mgmt_client_factory
         super(KeyVaultCommandGroup, self).__init__(command_loader, group_name, **kwargs)
 
     def _create_keyvault_command(self, name, method_name=None, command_type_name=None, **kwargs):
@@ -135,7 +132,7 @@ def keyvault_command_handler(command_args):
                         show_exception_handler(ex)
                     except Exception:  # pylint: disable=broad-except
                         pass
-                return keyvault_exception_handler(self.command_loader, ex)
+                return keyvault_exception_handler(ex)
 
         self.command_loader._cli_command(command_name, handler=keyvault_command_handler,  # pylint: disable=protected-access
                                          argument_loader=keyvault_arguments_loader,

src/azure-cli/azure/cli/command_modules/keyvault/_completers.py

@@ -7,24 +7,35 @@
 from azure.cli.core._profile import Profile
 
 
-def _get_token(cli_ctx, server, resource, scope):  # pylint: disable=unused-argument
-    return Profile(cli_ctx=cli_ctx).get_raw_token(resource)[0]
-
-
 def get_keyvault_name_completion_list(resource_name):
 
     @Completer
     def completer(cmd, prefix, namespace, **kwargs):  # pylint: disable=unused-argument
-        from .vendored_sdks.azure_keyvault_t1 import KeyVaultAuthentication, KeyVaultClient
-        from azure.cli.core.profiles import ResourceType, get_api_version
-        version = str(get_api_version(cmd.cli_ctx, ResourceType.DATA_KEYVAULT))
-        client = KeyVaultClient(KeyVaultAuthentication(_get_token), api_version=version)
-        func_name = 'get_{}s'.format(resource_name)
+        func_name = 'list_properties_of_{}s'.format(resource_name)
         vault = namespace.vault_base_url
+        profile = Profile(cli_ctx=cmd.cli_ctx)
+        credential, _, _ = profile.get_login_credentials(subscription_id=cmd.cli_ctx.data.get('subscription_id'))
+        if resource_name == 'key':
+            from azure.keyvault.keys import KeyClient
+            from azure.cli.command_modules.keyvault._client_factory import is_azure_stack_profile
+            version = '7.5-preview.1' if not is_azure_stack_profile(cmd=cmd) else '2016-10-01'
+            client = KeyClient(vault_url=vault, credential=credential, api_version=version,
+                               verify_challenge_resource=False)
+        elif resource_name == 'secret':
+            from azure.keyvault.secrets import SecretClient
+            from azure.cli.command_modules.keyvault._client_factory import is_azure_stack_profile
+            version = '7.4' if not is_azure_stack_profile(cmd=cmd) else '2016-10-01'
+            client = SecretClient(vault_url=vault, credential=credential, api_version=version,
+                                  verify_challenge_resource=False)
+        else:
+            from azure.keyvault.certificates import CertificateClient
+            from azure.cli.command_modules.keyvault._client_factory import is_azure_stack_profile
+            version = '7.4' if not is_azure_stack_profile(cmd=cmd) else '2016-10-01'
+            client = CertificateClient(vault_url=vault, credential=credential, api_version=version,
+                                       verify_challenge_resource=False)
         items = []
-        for y in list(getattr(client, func_name)(vault)):
-            id_val = getattr(y, 'id', None) or getattr(y, 'kid', None)
-            items.append(id_val.rsplit('/', 1)[1])
+        for y in list(getattr(client, func_name)()):
+            items.append(y.name)
         return items
 
     return completer
@@ -34,17 +45,31 @@ def get_keyvault_version_completion_list(resource_name):
 
     @Completer
     def completer(cmd, prefix, namespace, **kwargs):  # pylint: disable=unused-argument
-        from .vendored_sdks.azure_keyvault_t1 import KeyVaultAuthentication, KeyVaultClient
-        from azure.cli.core.profiles import ResourceType, get_api_version
-        version = str(get_api_version(cmd.cli_ctx, ResourceType.DATA_KEYVAULT))
-        client = KeyVaultClient(KeyVaultAuthentication(_get_token), api_version=version)
-        func_name = 'get_{}_versions'.format(resource_name)
+        func_name = 'list_properties_of_{}_versions'.format(resource_name)
         vault = namespace.vault_base_url
-        name = getattr(namespace, '{}_name'.format(resource_name))
+        profile = Profile(cli_ctx=cmd.cli_ctx)
+        credential, _, _ = profile.get_login_credentials(subscription_id=cmd.cli_ctx.data.get('subscription_id'))
+        if resource_name == 'key':
+            from azure.keyvault.keys import KeyClient
+            from azure.cli.command_modules.keyvault._client_factory import is_azure_stack_profile
+            version = '7.5-preview.1' if not is_azure_stack_profile(cmd=cmd) else '2016-10-01'
+            client = KeyClient(vault_url=vault, credential=credential, api_version=version,
+                               verify_challenge_resource=False)
+        elif resource_name == 'secret':
+            from azure.keyvault.secrets import SecretClient
+            from azure.cli.command_modules.keyvault._client_factory import is_azure_stack_profile
+            version = '7.4' if not is_azure_stack_profile(cmd=cmd) else '2016-10-01'
+            client = SecretClient(vault_url=vault, credential=credential, api_version=version,
+                                  verify_challenge_resource=False)
+        else:
+            from azure.keyvault.certificates import CertificateClient
+            from azure.cli.command_modules.keyvault._client_factory import is_azure_stack_profile
+            version = '7.4' if not is_azure_stack_profile(cmd=cmd) else '2016-10-01'
+            client = CertificateClient(vault_url=vault, credential=credential, api_version=version,
+                                       verify_challenge_resource=False)
         items = []
-        for y in list(getattr(client, func_name)(vault, name)):
-            id_val = getattr(y, 'id', None) or getattr(y, 'kid', None)
-            items.append(id_val.rsplit('/', 1)[1])
+        for y in list(getattr(client, func_name)()):
+            items.append(y.version)
         return items
 
     return completer