From d19ca45f9948559562174f191603766e7069e390 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 10 Jul 2024 15:09:03 -0700 Subject: [PATCH 1/8] Update qiime info to show parallel config info --- q2cli/builtin/info.py | 37 +++++++++++++++++++++++++++++++++++++ q2cli/commands.py | 4 ++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/q2cli/builtin/info.py b/q2cli/builtin/info.py index f8c004cf..6a2530af 100644 --- a/q2cli/builtin/info.py +++ b/q2cli/builtin/info.py @@ -42,6 +42,10 @@ def info(): import q2cli.util # This import improves performance for repeated _echo_plugins import q2cli.core.cache + from qiime2.sdk.parallel_config import \ + (PARALLEL_CONFIG, get_vendored_config) + from tomlkit import dumps + from parsl import Config click.secho('System versions', fg='green') _echo_version() @@ -51,5 +55,38 @@ def info(): click.secho('\nApplication config directory', fg='green') click.secho(q2cli.util.get_app_dir()) + click.secho('\nParallel Config', fg='green') + parallel_config = PARALLEL_CONFIG.parallel_config + config_source = 'Memory' + + mapping = PARALLEL_CONFIG.action_executor_mapping + mapping_source = 'Memory' + + if not parallel_config or not mapping: + vendored_config, vendored_mapping, vendored_source = \ + get_vendored_config() + + if not parallel_config: + config_source = vendored_source + parallel_config = vendored_config + + if not mapping: + mapping_source = vendored_source + mapping = vendored_mapping + + click.secho(f'Config Source: {config_source}') + if isinstance(parallel_config, Config): + click.secho(parallel_config.__dict__) + elif parallel_config: + click.secho(dumps(parallel_config)) + else: + click.secho('{}') + + click.secho(f'\nMapping Source: {mapping_source}') + if not mapping: + click.secho('{}') + else: + click.secho(dumps(mapping)) + click.secho('\nGetting help', fg='green') click.secho('To get help with QIIME 2, visit https://qiime2.org') diff --git a/q2cli/commands.py b/q2cli/commands.py index e912b4ab..1ec71475 100644 --- a/q2cli/commands.py +++ b/q2cli/commands.py @@ -509,14 +509,14 @@ def __call__(self, **kwargs): with qiime2.util.redirected_stdio(stdout=log, stderr=log): if parallel: from qiime2.sdk.parallel_config import \ - (get_config_from_file, ParallelConfig) + (load_config_from_file, ParallelConfig) action = action.parallel if parallel_config_fp is None: parallel_config = ParallelConfig() else: config, mapping = \ - get_config_from_file(parallel_config_fp) + load_config_from_file(parallel_config_fp) parallel_config = ParallelConfig(config, mapping) with parallel_config: From 5fa379f31a23177be664738820347bf63a230189 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 10 Jul 2024 15:26:16 -0700 Subject: [PATCH 2/8] update test --- q2cli/tests/test_cli.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/q2cli/tests/test_cli.py b/q2cli/tests/test_cli.py index 76a3c6d9..036073a1 100644 --- a/q2cli/tests/test_cli.py +++ b/q2cli/tests/test_cli.py @@ -54,6 +54,9 @@ def test_info(self): self.assertIn('Installed plugins', result.output) self.assertIn('dummy-plugin', result.output) self.assertIn('other-plugin', result.output) + self.assertIn('Parallel Config', result.output) + self.assertIn('Config Source: test config dict', result.output) + self.assertIn('Mapping Source: test config dict', result.output) def test_list_commands(self): # top level commands, including a plugin, are present From 39586a7a44d602c1d1e02c1a89e6861fad850252 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 16 Jul 2024 15:52:55 -0700 Subject: [PATCH 3/8] Add levels to config display in qiime info --- q2cli/builtin/info.py | 52 ++++++++++++++++++------------------------- 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/q2cli/builtin/info.py b/q2cli/builtin/info.py index 6a2530af..7b17571a 100644 --- a/q2cli/builtin/info.py +++ b/q2cli/builtin/info.py @@ -38,14 +38,20 @@ def _echo_plugins(): @click.command(help='Display information about current deployment.', cls=ToolCommand) -def info(): +@click.option('--config-level', + required=False, + default=1, + show_default=True, + type=click.IntRange(0, 3), + help='Specify the level of detail you want the config to be ' + 'displayed in') +def info(config_level): import q2cli.util # This import improves performance for repeated _echo_plugins import q2cli.core.cache - from qiime2.sdk.parallel_config import \ - (PARALLEL_CONFIG, get_vendored_config) + from qiime2.sdk.parallel_config import (get_vendored_config, + load_config_from_dict) from tomlkit import dumps - from parsl import Config click.secho('System versions', fg='green') _echo_version() @@ -55,38 +61,24 @@ def info(): click.secho('\nApplication config directory', fg='green') click.secho(q2cli.util.get_app_dir()) - click.secho('\nParallel Config', fg='green') - parallel_config = PARALLEL_CONFIG.parallel_config - config_source = 'Memory' + if config_level: + click.secho('\nConfig', fg='green') - mapping = PARALLEL_CONFIG.action_executor_mapping - mapping_source = 'Memory' - - if not parallel_config or not mapping: - vendored_config, vendored_mapping, vendored_source = \ + config, action_executor_mapping, vendored_source = \ get_vendored_config() - if not parallel_config: - config_source = vendored_source - parallel_config = vendored_config + click.secho(f'Config Source: {vendored_source}') - if not mapping: - mapping_source = vendored_source - mapping = vendored_mapping + if action_executor_mapping: + config['parsl.executor_mapping'] = action_executor_mapping - click.secho(f'Config Source: {config_source}') - if isinstance(parallel_config, Config): - click.secho(parallel_config.__dict__) - elif parallel_config: - click.secho(dumps(parallel_config)) - else: - click.secho('{}') + if config_level > 1: + if config_level == 2: + config = dumps(config) + elif config_level == 3: + config['parsl'], _ = load_config_from_dict(config) - click.secho(f'\nMapping Source: {mapping_source}') - if not mapping: - click.secho('{}') - else: - click.secho(dumps(mapping)) + click.secho(f'\n{config}') click.secho('\nGetting help', fg='green') click.secho('To get help with QIIME 2, visit https://qiime2.org') From 28ea22b3534d91c9c1d41eb9d06f2e6b6ce388ed Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Tue, 16 Jul 2024 15:53:01 -0700 Subject: [PATCH 4/8] Tests --- q2cli/tests/test_cli.py | 94 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 92 insertions(+), 2 deletions(-) diff --git a/q2cli/tests/test_cli.py b/q2cli/tests/test_cli.py index 036073a1..9c629875 100644 --- a/q2cli/tests/test_cli.py +++ b/q2cli/tests/test_cli.py @@ -27,6 +27,47 @@ from q2cli.click.type import QIIME2Type +CONFIG_LEVEL_2 = """[parsl] +strategy = "None" + +[[parsl.executors]] +class = "ThreadPoolExecutor" +label = "default" +max_threads = 1 + +[[parsl.executors]] +class = "_TEST_EXECUTOR_" +label = "test" +max_threads = 1""" + +# This is formatted this way to ensure the trailing spaces tomlkit.dumps throws +# at the end of a bunch of lines for no apparent reason are preserved here +CONFIG_LEVEL_3 = "{'parsl': Config(" +\ + "\n app_cache=True, " +\ + "\n checkpoint_files=None, " +\ + "\n checkpoint_mode=None, " +\ + "\n checkpoint_period=None, " +\ + "\n executors=(ThreadPoolExecutor(" +\ + "\n label='default', " +\ + "\n max_threads=1, " +\ + "\n storage_access=None, " +\ + "\n thread_name_prefix='', " +\ + "\n working_dir=None" +\ + "\n ), _TEST_EXECUTOR_()), " +\ + "\n garbage_collect=True, " +\ + "\n initialize_logging=True, " +\ + "\n internal_tasks_max_threads=10, " +\ + "\n max_idletime=120.0, " +\ + "\n monitoring=None, " +\ + "\n retries=0, " +\ + "\n retry_handler=None, " +\ + "\n run_dir='runinfo', " +\ + "\n strategy='none', " +\ + "\n strategy_period=5, " +\ + "\n usage_tracking=False" +\ + "\n)}" + + class CliTests(unittest.TestCase): def setUp(self): get_dummy_plugin() @@ -54,9 +95,58 @@ def test_info(self): self.assertIn('Installed plugins', result.output) self.assertIn('dummy-plugin', result.output) self.assertIn('other-plugin', result.output) - self.assertIn('Parallel Config', result.output) + self.assertIn('Config', result.output) + self.assertIn('Config Source: test config dict', result.output) + + def test_info_level_0(self): + result = self.runner.invoke(info, '--config-level 0') + self.assertEqual(result.exit_code, 0) + # May not always start with "System versions" if cache updating message + # is printed. + self.assertIn('System versions', result.output) + self.assertIn('Installed plugins', result.output) + self.assertIn('dummy-plugin', result.output) + self.assertIn('other-plugin', result.output) + self.assertNotIn('Config', result.output) + self.assertNotIn('Config Source: test config dict', result.output) + + def test_info_level_1(self): + result = self.runner.invoke(info, '--config-level 1') + self.assertEqual(result.exit_code, 0) + # May not always start with "System versions" if cache updating message + # is printed. + self.assertIn('System versions', result.output) + self.assertIn('Installed plugins', result.output) + self.assertIn('dummy-plugin', result.output) + self.assertIn('other-plugin', result.output) + self.assertIn('Config', result.output) + self.assertIn('Config Source: test config dict', result.output) + + def test_info_level_2(self): + result = self.runner.invoke(info, '--config-level 2') + self.assertEqual(result.exit_code, 0) + # May not always start with "System versions" if cache updating message + # is printed. + self.assertIn('System versions', result.output) + self.assertIn('Installed plugins', result.output) + self.assertIn('dummy-plugin', result.output) + self.assertIn('other-plugin', result.output) + self.assertIn('Config', result.output) + self.assertIn('Config Source: test config dict', result.output) + self.assertIn(CONFIG_LEVEL_2, result.output) + + def test_info_level_3(self): + result = self.runner.invoke(info, '--config-level 3') + self.assertEqual(result.exit_code, 0) + # May not always start with "System versions" if cache updating message + # is printed. + self.assertIn('System versions', result.output) + self.assertIn('Installed plugins', result.output) + self.assertIn('dummy-plugin', result.output) + self.assertIn('other-plugin', result.output) + self.assertIn('Config', result.output) self.assertIn('Config Source: test config dict', result.output) - self.assertIn('Mapping Source: test config dict', result.output) + self.assertIn(CONFIG_LEVEL_3, result.output) def test_list_commands(self): # top level commands, including a plugin, are present From 79adae33493dbd42b4b2c4615ca84188082593b6 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Wed, 17 Jul 2024 13:13:29 -0700 Subject: [PATCH 5/8] Add parsl docs to info --- q2cli/builtin/info.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/q2cli/builtin/info.py b/q2cli/builtin/info.py index 7b17571a..354a96c1 100644 --- a/q2cli/builtin/info.py +++ b/q2cli/builtin/info.py @@ -82,3 +82,10 @@ def info(config_level): click.secho('\nGetting help', fg='green') click.secho('To get help with QIIME 2, visit https://qiime2.org') + + # When we have the final docs for this stuff up we should probably replace + # this with a reference to our docs which wil reference the parsl docs + if config_level: + click.secho('To get help with configuring/understanding QIIME 2 ' + 'parallelization visit ' + 'https://parsl.readthedocs.io/en/stable/') From b8991e3424e7e20f7f98ae0cf47a2db33d3172a8 Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Mon, 22 Jul 2024 15:11:26 -0700 Subject: [PATCH 6/8] Address some of @gregcaporaso's comments --- q2cli/builtin/info.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/q2cli/builtin/info.py b/q2cli/builtin/info.py index 354a96c1..2c9d5488 100644 --- a/q2cli/builtin/info.py +++ b/q2cli/builtin/info.py @@ -43,8 +43,8 @@ def _echo_plugins(): default=1, show_default=True, type=click.IntRange(0, 3), - help='Specify the level of detail you want the config to be ' - 'displayed in') + help='The level of detail to be used for displaying the ' + 'configuration summary.') def info(config_level): import q2cli.util # This import improves performance for repeated _echo_plugins @@ -61,7 +61,7 @@ def info(config_level): click.secho('\nApplication config directory', fg='green') click.secho(q2cli.util.get_app_dir()) - if config_level: + if config_level > 0: click.secho('\nConfig', fg='green') config, action_executor_mapping, vendored_source = \ @@ -83,8 +83,9 @@ def info(config_level): click.secho('\nGetting help', fg='green') click.secho('To get help with QIIME 2, visit https://qiime2.org') - # When we have the final docs for this stuff up we should probably replace - # this with a reference to our docs which wil reference the parsl docs + # TODO: When we have the final docs for this stuff up we should probably + # replace this with a reference to our docs which will reference the parsl + # docs if config_level: click.secho('To get help with configuring/understanding QIIME 2 ' 'parallelization visit ' From 844eef6c182c491a5d95e2761481de02a27a02af Mon Sep 17 00:00:00 2001 From: Oddant1 Date: Thu, 1 Aug 2024 15:25:40 -0700 Subject: [PATCH 7/8] Update parsl link to our docs --- q2cli/builtin/info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/q2cli/builtin/info.py b/q2cli/builtin/info.py index 2c9d5488..5d64aa7e 100644 --- a/q2cli/builtin/info.py +++ b/q2cli/builtin/info.py @@ -89,4 +89,5 @@ def info(config_level): if config_level: click.secho('To get help with configuring/understanding QIIME 2 ' 'parallelization visit ' - 'https://parsl.readthedocs.io/en/stable/') + 'https://develop.qiime2.org/en/latest/framework/' + 'how-to-guides/parallel-configuration.html') From 2b0b7cbef3a29880c39c03630d01fa1c6fd21921 Mon Sep 17 00:00:00 2001 From: Greg Caporaso Date: Wed, 14 Aug 2024 13:09:37 -0700 Subject: [PATCH 8/8] MAINT: a few formatting edits --- q2cli/builtin/info.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/q2cli/builtin/info.py b/q2cli/builtin/info.py index 5d64aa7e..c4ea04af 100644 --- a/q2cli/builtin/info.py +++ b/q2cli/builtin/info.py @@ -83,11 +83,13 @@ def info(config_level): click.secho('\nGetting help', fg='green') click.secho('To get help with QIIME 2, visit https://qiime2.org') - # TODO: When we have the final docs for this stuff up we should probably + # TODO: When we have user documentation on parallel QIIME 2 live, # replace this with a reference to our docs which will reference the parsl - # docs + # docs. if config_level: - click.secho('To get help with configuring/understanding QIIME 2 ' - 'parallelization visit ' + click.secho('To get help with configuring and/or understanding ' + 'QIIME 2 parallelization, visit ' 'https://develop.qiime2.org/en/latest/framework/' 'how-to-guides/parallel-configuration.html') + + click.secho('\n')