From f8269d5470d6bd41e03bbec4e0b2107a462d9535 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 4 Mar 2024 00:36:13 -0500 Subject: [PATCH 01/19] feat: add support for stdin to cli --- seqerakit/cli.py | 14 +++++++------ seqerakit/helper.py | 50 ++++++++++++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index f3e1b80..b35d6d1 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -21,7 +21,6 @@ import logging import sys -from pathlib import Path from seqerakit import seqeraplatform, helper, overwrite from seqerakit.seqeraplatform import ResourceExistsError, ResourceCreationError from seqerakit import __version__ @@ -66,7 +65,6 @@ def parse_args(args=None): yaml_processing = parser.add_argument_group("YAML Processing Options") yaml_processing.add_argument( "yaml", - type=Path, nargs="*", help="One or more YAML files with Seqera Platform resource definitions.", ) @@ -154,10 +152,14 @@ def main(args=None): return if not options.yaml: - logging.error( - " No YAML(s) provided. Please provide atleast one YAML configuration file." - ) - sys.exit(1) + if sys.stdin.isatty(): + logging.error( + " No YAML(s) provided and no input from stdin. Please provide " + "at least one YAML configuration file or pipe input from stdin." + ) + sys.exit(1) + else: + options.yaml = [sys.stdin] # Parse CLI arguments into a list cli_args_list = options.cli_args.split() if options.cli_args else [] diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 0b71311..eb32e0d 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -19,6 +19,7 @@ """ import yaml from seqerakit import utils +import sys def parse_yaml_block(yaml_data, block_name): @@ -56,24 +57,41 @@ def parse_all_yaml(file_paths, destroy=False): # If multiple yamls, merge them into one dictionary merged_data = {} - for file_path in file_paths: - with open(file_path, "r") as f: - data = yaml.safe_load(f) + # Special handling for stdin represented by "-" + if not file_paths or "-" in file_paths: + # Read YAML directly from stdin + data = yaml.safe_load(sys.stdin) + if data is None or not data: + raise ValueError( + " The input from stdin is empty or does not contain valid YAML data." + ) - # Check if the YAML file is empty or contains no valid data - if data is None or not data: - raise ValueError( - f" The file '{file_path}' is empty or does not contain valid data." - ) + merged_data.update(data) - for key, value in data.items(): - if key in merged_data: - try: - merged_data[key].extend(value) - except AttributeError: - merged_data[key] = [merged_data[key], value] - else: - merged_data[key] = value + for file_path in file_paths: + if file_path == "-": + continue + try: + with open(file_path, "r") as f: + data = yaml.safe_load(f) + if data is None or not data: + raise ValueError( + f" The file '{file_path}' is empty or " + "does not contain valid data." + ) + merged_data.update(data) + except FileNotFoundError: + print(f"Error: The file '{file_path}' was not found.") + sys.exit(1) + + for key, value in data.items(): + if key in merged_data: + try: + merged_data[key].extend(value) + except AttributeError: + merged_data[key] = [merged_data[key], value] + else: + merged_data[key] = value block_names = list(merged_data.keys()) From 72e2838165c97386f0dc9e68ba395051bf26732f Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Mar 2024 01:13:38 -0400 Subject: [PATCH 02/19] fix: fix duplicate name bug Re #123 --- seqerakit/cli.py | 10 ++++++---- seqerakit/helper.py | 21 +++++++++++++++------ 2 files changed, 21 insertions(+), 10 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index f3e1b80..9f360df 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -105,7 +105,7 @@ def __init__(self, sp, list_for_add_method): # Create an instance of Overwrite class self.overwrite_method = overwrite.Overwrite(self.sp) - def handle_block(self, block, args, destroy=False): + def handle_block(self, block, args, destroy=False, dryrun=False): # Check if delete is set to True, and call delete handler if destroy: logging.debug(" The '--delete' flag has been specified.\n") @@ -127,12 +127,12 @@ def handle_block(self, block, args, destroy=False): # Check if overwrite is set to True, and call overwrite handler overwrite_option = args.get("overwrite", False) - if overwrite_option: + if overwrite_option and dryrun is False: logging.debug(f" Overwrite is set to 'True' for {block}\n") self.overwrite_method.handle_overwrite( block, args["cmd_args"], overwrite_option ) - else: + elif dryrun is False: self.overwrite_method.handle_overwrite(block, args["cmd_args"]) if block in self.list_for_add_method: @@ -184,7 +184,9 @@ def main(args=None): for args in args_list: try: # Run the 'tw' methods for each block - block_manager.handle_block(block, args, destroy=options.delete) + block_manager.handle_block( + block, args, destroy=options.delete, dryrun=options.dryrun + ) except (ResourceExistsError, ResourceCreationError) as e: logging.error(e) sys.exit(1) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 0b71311..cc71770 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -340,10 +340,19 @@ def handle_pipelines(sp, args): def find_name(cmd_args): """ - Find and return the value associated with --name in the cmd_args list. + Find and return the value associated with --name in cmd_args, where cmd_args + can be a list, a tuple of lists, or nested lists/tuples. """ - args_list = cmd_args.get("cmd_args", []) - for i in range(len(args_list) - 1): - if args_list[i] == "--name": - return args_list[i + 1] - return None + + def search(args): + it = iter(args) + for arg in it: + if arg == "--name": + return next(it, None) + elif isinstance(arg, (list, tuple)): + result = search(arg) + if result is not None: + return result + return None + + return search(cmd_args.get("cmd_args", [])) From 1c7d8723bd8b9911745185281c7aba9720fd7fbd Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Mar 2024 19:51:16 -0400 Subject: [PATCH 03/19] fix: overwrite handling for teams cmd args --- seqerakit/overwrite.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index 7a18562..24fe817 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -202,7 +202,10 @@ def _get_json_data(self, block, args, keys_to_get): # Check if block data already exists if block in self.block_jsondata: self.cached_jsondata = self.block_jsondata[block] - sp_args = self._get_values_from_cmd_args(args, keys_to_get) + if block == "teams": + sp_args = self._get_values_from_cmd_args(args[0], keys_to_get) + else: + sp_args = self._get_values_from_cmd_args(args, keys_to_get) else: # Fetch the data if it does not exist if block == "teams": From e434f20d5657417644315201adf1f2cc32f82a23 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Mar 2024 20:28:49 -0400 Subject: [PATCH 04/19] feat: add support for creating labels --- seqerakit/cli.py | 1 + seqerakit/helper.py | 1 + seqerakit/overwrite.py | 31 ++++++++++++++++++++++++++++++- templates/labels.yml | 6 ++++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 templates/labels.yml diff --git a/seqerakit/cli.py b/seqerakit/cli.py index f3e1b80..4366114 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -169,6 +169,7 @@ def main(args=None): [ "organizations", # all use method.add "workspaces", + "labels", "credentials", "secrets", "actions", diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 0b71311..274638d 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -82,6 +82,7 @@ def parse_all_yaml(file_paths, destroy=False): "organizations", "teams", "workspaces", + "labels", "participants", "credentials", "compute-envs", diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index 7a18562..44fcd62 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -60,6 +60,11 @@ def __init__(self, sp): "method_args": self._get_organization_args, "name_key": "orgName", }, + "labels": { + "keys": ["name", "value", "workspace"], + "method_args": self._get_label_args, + "name_key": "name", + }, "teams": { "keys": ["name", "organization"], "method_args": self._get_team_args, @@ -171,6 +176,15 @@ def _get_workspace_args(self, args): workspace_id = self._find_workspace_id(args["organization"], args["name"]) return ("delete", "--id", str(workspace_id)) + def _get_label_args(self, args): + """ + Returns a list of arguments for the delete() method for labels. The + label_id used to delete will be retrieved using the _find_label_id() + method. + """ + label_id = self._find_label_id(args["name"], args["value"]) + return ("delete", "--id", str(label_id), "-w", args["workspace"]) + def _get_generic_deletion_args(self, args): """ Returns a list of arguments for the delete() method for resources that @@ -210,7 +224,10 @@ def _get_json_data(self, block, args, keys_to_get): self.cached_jsondata = json_method( block, "list", "-o", sp_args["organization"] ) - elif block in Overwrite.generic_deletion or block == "participants": + elif block in Overwrite.generic_deletion or block in { + "participants", + "labels", + }: sp_args = self._get_values_from_cmd_args(args, keys_to_get) self.cached_jsondata = json_method( block, "list", "-w", sp_args["workspace"] @@ -271,3 +288,15 @@ def _find_workspace_id(self, organization, workspace_name): ): return workspace.get("workspaceId") return None + + def _find_label_id(self, label_name, label_value): + """ + Custom method to find a label ID in a nested dictionary with a given + workspace name. This ID will be used to delete the label. + """ + jsondata = json.loads(self.cached_jsondata) + labels = jsondata["labels"] + for label in labels: + if label.get("name") == label_name and label.get("value") == label_value: + return label.get("id") + return None diff --git a/templates/labels.yml b/templates/labels.yml new file mode 100644 index 0000000..4b61b61 --- /dev/null +++ b/templates/labels.yml @@ -0,0 +1,6 @@ +## To see the full list of options available run: "tw labels add" +labels: + - name: 'label_name' # required + value: 'label_value' # required + workspace: 'my_organization/my_workspace' # required + overwrite: True # optional From bef8ad359e9ff24d6c23c7edd3be7b609099d07d Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Mar 2024 20:32:11 -0400 Subject: [PATCH 05/19] fix: revert prettier fix on template --- templates/labels.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/templates/labels.yml b/templates/labels.yml index 4b61b61..6a75adb 100644 --- a/templates/labels.yml +++ b/templates/labels.yml @@ -1,6 +1,6 @@ ## To see the full list of options available run: "tw labels add" labels: - - name: 'label_name' # required - value: 'label_value' # required - workspace: 'my_organization/my_workspace' # required - overwrite: True # optional + - name: 'label_name' # required + value: 'label_value' # required + workspace: 'my_organization/my_workspace' # required + overwrite: True # optional From 27bc0c940204659fffc09d79df179b5571620304 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Mar 2024 21:03:22 -0400 Subject: [PATCH 06/19] test: add tests for parsing teams args in yaml --- tests/unit/test_helper.py | 46 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 66b1ec7..6b2c57c 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -266,6 +266,52 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): assert result["pipelines"] == expected_block_output +def test_mock_teams_yaml(mock_yaml_file): + test_data = { + "teams": [ + { + "name": "test_team1", + "organization": "my_organization", + "description": "My test team 1", + "members": ["user1@org.io"], + "overwrite": True, + }, + ] + } + expected_block_output = [ + { + "cmd_args": ( + [ + "--description", + "My test team 1", + "--name", + "test_team1", + "--organization", + "my_organization", + ], + [ + [ + "--team", + "test_team1", + "--organization", + "my_organization", + "add", + "--member", + "user1@org.io", + ] + ], + ), + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "teams" in result + assert result["teams"] == expected_block_output + + def test_empty_yaml_file(mock_yaml_file): test_data = {} file_path = mock_yaml_file(test_data) From fd8f8edee6e539a7ea0ed5ff6f1eee570d97d730 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Wed, 20 Mar 2024 16:39:30 -0400 Subject: [PATCH 07/19] feat: add support for members --- seqerakit/cli.py | 1 + seqerakit/helper.py | 1 + seqerakit/overwrite.py | 25 +++++++++++++++++++++++++ templates/members.yml | 5 +++++ tests/unit/test_helper.py | 22 +++++++++++++++++++++- 5 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 templates/members.yml diff --git a/seqerakit/cli.py b/seqerakit/cli.py index e001cfa..0beb7b6 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -170,6 +170,7 @@ def main(args=None): "organizations", # all use method.add "workspaces", "labels", + "members", "credentials", "secrets", "actions", diff --git a/seqerakit/helper.py b/seqerakit/helper.py index d78b4a9..0eec9b3 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -83,6 +83,7 @@ def parse_all_yaml(file_paths, destroy=False): "teams", "workspaces", "labels", + "members", "participants", "credentials", "compute-envs", diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index 5739856..d624ddf 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -65,6 +65,11 @@ def __init__(self, sp): "method_args": self._get_label_args, "name_key": "name", }, + "members": { + "keys": ["user", "organization"], + "method_args": self._get_members_args, + "name_key": "email", + }, "teams": { "keys": ["name", "organization"], "method_args": self._get_team_args, @@ -106,6 +111,9 @@ def handle_overwrite(self, block, args, overwrite=False, destroy=False): self.block_operations["participants"]["name_key"] = "teamName" else: self.block_operations["participants"]["name_key"] = "email" + elif block == "members": + # Rename the user key to name to correctly index JSON data + sp_args["name"] = sp_args.pop("user") if self.check_resource_exists(operation["name_key"], sp_args): # if resource exists and overwrite is true, delete if overwrite: @@ -150,6 +158,18 @@ def _get_team_args(self, args): ) return ("delete", "--id", str(team_id), "--organization", args["organization"]) + def _get_members_args(self, args): + """ + Returns a list of arguments for the delete() method for members. + """ + return ( + "delete", + "--user", + args["name"], + "--organization", + args["organization"], + ) + def _get_participant_args(self, args): """ Returns a list of arguments for the delete() method for participants. @@ -235,6 +255,11 @@ def _get_json_data(self, block, args, keys_to_get): self.cached_jsondata = json_method( block, "list", "-w", sp_args["workspace"] ) + elif block == "members": # TODO + sp_args = self._get_values_from_cmd_args(args, keys_to_get) + self.cached_jsondata = json_method( + block, "list", "-o", sp_args["organization"] + ) else: sp_args = self._get_values_from_cmd_args(args, keys_to_get) self.cached_jsondata = json_method(block, "list") diff --git a/templates/members.yml b/templates/members.yml new file mode 100644 index 0000000..d3f2d82 --- /dev/null +++ b/templates/members.yml @@ -0,0 +1,5 @@ +## To see the full list of options available run: "tw members add" +participants: + - user: 'bob@myorg.io' # required + organization: 'myorg' # required + overwrite: True # optional \ No newline at end of file diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 6b2c57c..42a5fb8 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -266,7 +266,7 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): assert result["pipelines"] == expected_block_output -def test_mock_teams_yaml(mock_yaml_file): +def test_create_mock_teams_yaml(mock_yaml_file): test_data = { "teams": [ { @@ -312,6 +312,26 @@ def test_mock_teams_yaml(mock_yaml_file): assert result["teams"] == expected_block_output +def test_create_mock_members_yaml(mock_yaml_file): + test_data = {"members": [{"user": "bob@myorg.io", "organization": "myorg"}]} + expected_block_output = [ + { + "cmd_args": [ + "--organization", + "myorg", + "--user", + "bob@myorg.io", + ], + "overwrite": False, + } + ] + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "members" in result + assert result["members"] == expected_block_output + + def test_empty_yaml_file(mock_yaml_file): test_data = {} file_path = mock_yaml_file(test_data) From 40aa26f4b7c4c49b7177c1b45e78c94c28c9e792 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 21 Mar 2024 15:28:35 -0400 Subject: [PATCH 08/19] fix: uniqueness on name for members with user key --- seqerakit/helper.py | 17 ++++++++++++++--- tests/unit/test_helper.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 0eec9b3..3dec340 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -36,13 +36,14 @@ def parse_yaml_block(yaml_data, block_name): name_values = set() # Iterate over each item in the block. + # TODO: fix for resources that can be duplicate named in an org for item in block: cmd_args = parse_block(block_name, item) name = find_name(cmd_args) if name in name_values: raise ValueError( - f" Duplicate 'name' specified in config file" - f" for {block_name}: {name}. Please specify a unique name." + f" Duplicate name key specified in config file" + f" for {block_name}: {name}. Please specify a unique value." ) name_values.add(name) @@ -344,12 +345,22 @@ def find_name(cmd_args): """ Find and return the value associated with --name in cmd_args, where cmd_args can be a list, a tuple of lists, or nested lists/tuples. + + The function searches for the following keys: --name, --user, --email. + + Parameters: + - cmd_args: The command arguments (list, tuple, or nested structures). + + Returns: + - The value associated with the first key found, or None if none are found. """ + # Predefined list of keys to search for + keys = {"--name", "--user", "--email"} def search(args): it = iter(args) for arg in it: - if arg == "--name": + if isinstance(arg, str) and arg in keys: return next(it, None) elif isinstance(arg, (list, tuple)): result = search(arg) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 42a5fb8..0092f37 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -362,3 +362,35 @@ def test_error_type_yaml_file(mock_yaml_file): "Please specify at least 'type' or 'file-path' for creating the resource." in str(e.value) ) + + +def test_error_duplicate_name_yaml_file(mock_yaml_file): + test_data = { + "compute-envs": [ + { + "name": "test_computeenv", + "workspace": "my_organization/my_workspace", + "credentials": "my_credentials", + "type": "aws-batch", + "config-mode": "forge", + "wait": "AVAILABLE", + }, + { + "name": "test_computeenv", + "workspace": "my_organization/my_workspace", + "credentials": "my_credentials", + "type": "aws-batch", + "config-mode": "forge", + "wait": "AVAILABLE", + }, + ], + } + file_path = mock_yaml_file(test_data) + + with pytest.raises(ValueError) as e: + helper.parse_all_yaml([file_path]) + assert ( + "Duplicate name key specified in config file for " + "compute-envs: test_computeenv. Please specify " + "a unique value." in str(e.value) + ) From ccd77227c309994d8f62fac142d6cd405753a1f4 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 21 Mar 2024 15:32:41 -0400 Subject: [PATCH 09/19] chore: linting failures --- tests/unit/test_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 0092f37..fcdccfc 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -390,7 +390,7 @@ def test_error_duplicate_name_yaml_file(mock_yaml_file): with pytest.raises(ValueError) as e: helper.parse_all_yaml([file_path]) assert ( - "Duplicate name key specified in config file for " + "Duplicate name key specified in config file for " "compute-envs: test_computeenv. Please specify " "a unique value." in str(e.value) ) From d1629f14850c2ef220cb998a1a00827426f6b130 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 2 Apr 2024 17:52:06 +0200 Subject: [PATCH 10/19] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3e6e3de..ac45ab4 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The key features are: ## Prerequisites -You will need to have an account on Seqera Platform (see [Plans and pricing](https://cloud.tower.nf/pricing/)). +You will need to have an account on Seqera Platform (see [Plans and pricing](https://seqera.io/pricing/)). ## Installation @@ -77,7 +77,7 @@ pip show seqerakit ## Configuration -Create a Seqera Platform access token using the [Seqera Platform](https://tower.nf/) web interface via the **Your Tokens** page in your profile. +Create a Seqera Platform access token using the [Seqera Platform](https://seqera.io/) web interface via the **Your Tokens** page in your profile. `seqerakit` reads this token from the environment variable `TOWER_ACCESS_TOKEN`. Please export it into your terminal as shown below: @@ -89,7 +89,7 @@ For Enterprise installations of Seqera Platform, you will also need to configure ```bash export TOWER_API_ENDPOINT= ``` -By default, this is set to `https://api.tower.nf` to connect to Seqera Platform Cloud. +By default, this is set to `https://api.cloud.seqera.io` to connect to Seqera Platform Cloud. ## Usage @@ -225,7 +225,7 @@ compute-envs: You must provide a YAML file that defines the options for each of the entities you would like to create in Seqera Platform. -You will need to have an account on Seqera Platform (see [Plans and pricing](https://cloud.tower.nf/pricing/)). You will also need access to a Workspace and a pre-defined Compute Environment where you can launch a pipeline. +You will need to have an account on Seqera Platform (see [Plans and pricing](https://seqera.io/pricing/)). You will also need access to a Workspace and a pre-defined Compute Environment where you can launch a pipeline. ### Launch via YAML From 982a46146b9756441dd283c4b8cf35387bced1ae Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 25 Apr 2024 16:16:34 +0100 Subject: [PATCH 11/19] fix: catch errors more reliably seqerakit missed some errors raised by the cli and continued on. This PR fixes that bug and raises the error in the console. --- seqerakit/seqeraplatform.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index 6381e10..0308d27 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -96,7 +96,7 @@ def _execute_command(self, full_cmd, to_json=False): stdout, _ = process.communicate() stdout = stdout.decode("utf-8").strip() - if "ERROR: " in stdout: + if "ERROR: " in stdout or process.returncode != 0: self._handle_command_errors(stdout) return json.loads(stdout) if to_json else stdout @@ -114,11 +114,11 @@ def _handle_command_errors(self, stdout): r"ERROR: .*already (exists|a participant)", stdout, flags=re.IGNORECASE ): raise ResourceExistsError( - " Resource already exists. Please delete first or set 'overwrite: true'" + "Resource already exists. Please delete first or set 'overwrite: true'" ) else: raise ResourceCreationError( - f" Resource creation failed: '{stdout}'. " + f"Resource creation failed: '{stdout}'. " "Check your config and try again." ) From 382454fbcb0394f2e7dccda920edbb9eb70aa759 Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Thu, 25 Apr 2024 17:03:08 +0100 Subject: [PATCH 12/19] Mock the exit status 0 for all tests --- seqerakit/seqeraplatform.py | 2 +- tests/unit/test_seqeraplatform.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index 0308d27..e6a50d6 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -97,7 +97,7 @@ def _execute_command(self, full_cmd, to_json=False): stdout = stdout.decode("utf-8").strip() if "ERROR: " in stdout or process.returncode != 0: - self._handle_command_errors(stdout) + self._handle_command_errors(str(stdout)) return json.loads(stdout) if to_json else stdout diff --git a/tests/unit/test_seqeraplatform.py b/tests/unit/test_seqeraplatform.py index 10239f0..c95a25f 100644 --- a/tests/unit/test_seqeraplatform.py +++ b/tests/unit/test_seqeraplatform.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import patch +from unittest.mock import MagicMock, patch from seqerakit import seqeraplatform import json import subprocess @@ -26,6 +26,7 @@ def test_run_with_jsonout_command(self, mock_subprocess): "dateCreated": "2023-02-15T13:14:30Z", } # Mock the stdout of the Popen process + mock_subprocess.return_value = MagicMock(returncode=0) mock_subprocess.return_value.communicate.return_value = ( json.dumps(mock_pipelines_json).encode(), b"", @@ -93,6 +94,7 @@ def test_resource_creation_error(self): def test_json_parsing(self): with patch("subprocess.Popen") as mock_subprocess: # Mock the stdout of the Popen process to return JSON + mock_subprocess.return_value = MagicMock(returncode=0) mock_subprocess.return_value.communicate.return_value = ( b'{"key": "value"}', b"", @@ -112,6 +114,7 @@ def setUp(self): @patch("subprocess.Popen") def test_cli_args_inclusion(self, mock_subprocess): # Mock the stdout of the Popen process + mock_subprocess.return_value = MagicMock(returncode=0) mock_subprocess.return_value.communicate.return_value = ( json.dumps({"key": "value"}).encode(), b"", @@ -136,6 +139,7 @@ def test_cli_args_inclusion_ssl_certs(self, mock_subprocess): seqeraplatform.SeqeraPlatform(cli_args=self.cli_args) # Mock the stdout of the Popen process + mock_subprocess.return_value = MagicMock(returncode=0) mock_subprocess.return_value.communicate.return_value = ( json.dumps({"key": "value"}).encode(), b"", From fb977cfb136b2b464ad413b57837297c3b467da9 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 19:31:44 -0400 Subject: [PATCH 13/19] fix: logic for merging list of dicts, tests and add docs --- README.md | 12 +++++++++ seqerakit/helper.py | 36 ++++++++++++++++++++++---- tests/unit/test_helper.py | 53 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3e6e3de..59f8737 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,18 @@ Use `--version` or `-v` to retrieve the current version of your seqerakit instal ```bash seqerakit --version ``` +### Input +`seqerakit` supports input through either file paths to YAMLs or directly from standard input (stdin). + +#### Using File Path +```bash +seqerakit /path/to/file.yaml +``` +#### Using stdin +```console +$ cat file.yaml | seqerakit - +``` +See the [Defining your YAML file using CLI options](#defining-your-yaml-file-using-cli-options) section for guidance on formatting your input YAML file(s). ### Dryrun diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 5cc1064..3d02c85 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -62,11 +62,10 @@ def parse_all_yaml(file_paths, destroy=False): if not file_paths or "-" in file_paths: # Read YAML directly from stdin data = yaml.safe_load(sys.stdin) - if data is None or not data: + if not data: raise ValueError( " The input from stdin is empty or does not contain valid YAML data." ) - merged_data.update(data) for file_path in file_paths: @@ -75,12 +74,30 @@ def parse_all_yaml(file_paths, destroy=False): try: with open(file_path, "r") as f: data = yaml.safe_load(f) - if data is None or not data: + if not data: raise ValueError( f" The file '{file_path}' is empty or " "does not contain valid data." ) - merged_data.update(data) + + for key, new_value in data.items(): + if key in merged_data: + if isinstance(new_value, list) and all( + isinstance(i, dict) for i in new_value + ): + # Handle list of dictionaries & merge without duplication + existing_items = { + tuple(sorted(d.items())) for d in merged_data[key] + } + for item in new_value: + if tuple(sorted(item.items())) not in existing_items: + merged_data[key].append(item) + else: + # override if not list of dictionaries + merged_data[key] = new_value + else: + merged_data[key] = new_value + except FileNotFoundError: print(f"Error: The file '{file_path}' was not found.") sys.exit(1) @@ -94,6 +111,15 @@ def parse_all_yaml(file_paths, destroy=False): else: merged_data[key] = value + for key, value in data.items(): + if key in merged_data: + try: + merged_data[key].extend(value) + except AttributeError: + merged_data[key] = [merged_data[key], value] + else: + merged_data[key] = value + block_names = list(merged_data.keys()) # Define the order in which the resources should be created. @@ -113,7 +139,7 @@ def parse_all_yaml(file_paths, destroy=False): "launch", ] - # Reverse the order of resources if destroy is True + # Reverse the order of resources to delete if destroy is True if destroy: resource_order = resource_order[:-1][::-1] diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index fcdccfc..8038d1a 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -1,8 +1,8 @@ -from unittest.mock import mock_open +from unittest.mock import patch, mock_open from seqerakit import helper import yaml import pytest - +from io import StringIO # Fixture to mock a YAML file @pytest.fixture @@ -48,8 +48,8 @@ def test_create_mock_organization_yaml(mock_yaml_file): "overwrite": True, } ] - file_path = mock_yaml_file(test_data) + print(f"debug - file_path: {file_path}") result = helper.parse_all_yaml([file_path]) assert "organizations" in result @@ -343,6 +343,53 @@ def test_empty_yaml_file(mock_yaml_file): ) +def test_empty_stdin_file(): + # Prepare the mock to simulate empty stdin + with patch("sys.stdin", StringIO("")): + # Use '-' to indicate that stdin should be read + with pytest.raises(ValueError) as e: + helper.parse_all_yaml(["-"]) + assert ( + "The input from stdin is empty or does not contain valid YAML data." + in str(e.value) + ) + + +def test_stdin_yaml_file(): + # Prepare the mock to simulate stdin + yaml_data = """ +compute-envs: + - name: test_computeenv + config-mode: forge + workspace: my_organization/my_workspace + credentials: my_credentials + type: aws-batch + wait: AVAILABLE + """ + with patch("sys.stdin", StringIO(yaml_data)): + result = helper.parse_all_yaml(["-"]) + + expected_block_output = [ + { + "cmd_args": [ + "aws-batch", + "forge", + "--name", + "test_computeenv", + "--workspace", + "my_organization/my_workspace", + "--credentials", + "my_credentials", + "--wait", + "AVAILABLE", + ], + "overwrite": False, + } + ] + assert "compute-envs" in result + assert result["compute-envs"] == expected_block_output + + def test_error_type_yaml_file(mock_yaml_file): test_data = { "compute-envs": [ From b41fe2a9b727705497929e73b17aa448e7d514b4 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 19:35:31 -0400 Subject: [PATCH 14/19] fix: black linting --- tests/unit/test_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 8038d1a..76f5b55 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -4,6 +4,7 @@ import pytest from io import StringIO + # Fixture to mock a YAML file @pytest.fixture def mock_yaml_file(mocker): From 7b60445887fd1e68542195dae2bd081bf3ace58c Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 19:42:20 -0400 Subject: [PATCH 15/19] fix: remove extra logic that remained for some reason --- seqerakit/helper.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 3d02c85..b041230 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -97,29 +97,10 @@ def parse_all_yaml(file_paths, destroy=False): merged_data[key] = new_value else: merged_data[key] = new_value - except FileNotFoundError: print(f"Error: The file '{file_path}' was not found.") sys.exit(1) - for key, value in data.items(): - if key in merged_data: - try: - merged_data[key].extend(value) - except AttributeError: - merged_data[key] = [merged_data[key], value] - else: - merged_data[key] = value - - for key, value in data.items(): - if key in merged_data: - try: - merged_data[key].extend(value) - except AttributeError: - merged_data[key] = [merged_data[key], value] - else: - merged_data[key] = value - block_names = list(merged_data.keys()) # Define the order in which the resources should be created. From f94e57667ce6dd10951c3e46e45b9b83534d8e8a Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 20:15:04 -0400 Subject: [PATCH 16/19] fix: generate json data for workspaces with org argument for correct deletion --- seqerakit/overwrite.py | 2 +- seqerakit/utils.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index d624ddf..4be03a5 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -255,7 +255,7 @@ def _get_json_data(self, block, args, keys_to_get): self.cached_jsondata = json_method( block, "list", "-w", sp_args["workspace"] ) - elif block == "members": # TODO + elif block == "members" or block == "workspaces": # TODO sp_args = self._get_values_from_cmd_args(args, keys_to_get) self.cached_jsondata = json_method( block, "list", "-o", sp_args["organization"] diff --git a/seqerakit/utils.py b/seqerakit/utils.py index b58c3d1..bfa0203 100644 --- a/seqerakit/utils.py +++ b/seqerakit/utils.py @@ -66,7 +66,7 @@ def check_if_exists(json_data, namekey, namevalue): """ if not json_data: return False - logging.info(f"Checking if {namekey} {namevalue} exists in Seqera Platform...") + logging.info(f" Checking if {namekey} {namevalue} exists in Seqera Platform...") # Regex pattern to match environment variables in the string env_var_pattern = re.compile(r"\$\{?[\w]+\}?") From 5030df2a5e9ca3201615dd49f9bcf18b11623e1f Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 20:54:27 -0400 Subject: [PATCH 17/19] ci: add labels and members ymls to test e2e --- examples/yaml/e2e/labels.yml | 5 +++++ examples/yaml/e2e/members.yml | 4 ++++ 2 files changed, 9 insertions(+) create mode 100644 examples/yaml/e2e/labels.yml create mode 100644 examples/yaml/e2e/members.yml diff --git a/examples/yaml/e2e/labels.yml b/examples/yaml/e2e/labels.yml new file mode 100644 index 0000000..0c33b38 --- /dev/null +++ b/examples/yaml/e2e/labels.yml @@ -0,0 +1,5 @@ +labels: + - name: 'seqerakit' + value: 'test' + workspace: 'seqerakit-e2e/showcase' + overwrite: True diff --git a/examples/yaml/e2e/members.yml b/examples/yaml/e2e/members.yml new file mode 100644 index 0000000..54423f8 --- /dev/null +++ b/examples/yaml/e2e/members.yml @@ -0,0 +1,4 @@ +members: + - user: 'laramie.lindsey+owner@seqera.io' + organization: 'seqerakit-e2e' + overwrite: False From 306bcb535cc5feed030c6c29fbfd0d1045e59481 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 21:14:04 -0400 Subject: [PATCH 18/19] docs: update README links, version for release in setup --- README.md | 37 +++++++++++++++++++++---------------- setup.py | 2 +- 2 files changed, 22 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index d8fb2e3..64f4816 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,8 @@ -# seqerakit logo seqerakit +
+ seqerakit logo + seqerakit +
+
`seqerakit` is a Python wrapper for the [Seqera Platform CLI](https://github.com/seqeralabs/tower-cli). It can be leveraged to automate the creation of all of the entities in Seqera Platform via a simple configuration file in YAML format. @@ -220,7 +224,7 @@ DEBUG:root: Running command: tw organizations add --name $SEQERA_ORGANIZATION_NA ### 3. Specifying JSON configuration files with `file-path` The Seqera Platform CLI allows export and import of entities through JSON configuration files for pipelines and compute environments. To use these files to add a pipeline or compute environment to a workspace, use the `file-path` key to specify a path to a JSON configuration file. -An example of the `file-path` option is provided in the [compute-envs.yml](templates/compute-envs.yml) template: +An example of the `file-path` option is provided in the [compute-envs.yml](./templates/compute-envs.yml) template: ```yaml compute-envs: @@ -264,7 +268,7 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt You can also launch the same pipeline via a Python script. This will essentially allow you to extend the functionality on offer within the Seqera Platform CLI by leveraging the flexibility and customisation options available in Python. -1. Download the [`launch_hello_world.py`](https://github.com/seqeralabs/seqera-kit/blob/main/examples/python/launch_hello_world.py) Python script and customise the `` and `` entries as required. +1. Download the [`launch_hello_world.py`](./examples/python/launch_hello_world.py) Python script and customise the `` and `` entries as required. 2. Launch the pipeline with `seqerakit`: @@ -331,24 +335,25 @@ In this example: - Ensure that the indentation and structure of the YAML file are correct - YAML is sensitive to formatting. - Use quotes around strings that contain special characters or spaces. - When listing multiple values (`labels`, `instance-types`, `allow-buckets`, etc), separate them with commas as shown above. -- For complex configurations, refer to the [Templates](/templates/) provided in this repository. +- For complex configurations, refer to the [Templates](./templates/) provided in this repository. ## Templates We have provided template YAML files for each of the entities that can be created on Seqera Platform. These can be found in the [`templates/`](https://github.com/seqeralabs/blob/main/seqera-kit/templates) directory and should form a good starting point for you to add your own customization: -- [organizations.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/organizations.yml) -- [teams.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/teams.yml) -- [workspaces.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/workspaces.yml) -- [participants.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/participants.yml) -- [credentials.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/credentials.yml) -- [secrets.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/secrets.yml) -- [compute-envs.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/compute-envs.yml) -- [actions.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/actions.yml) -- [datasets.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/datasets.yml) -- [pipelines.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/pipelines.yml) -- [launch.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/launch.yml) +- [organizations.yml](./templates/organizations.yml) +- [teams.yml](./templates/teams.yml) +- [workspaces.yml](./templates/workspaces.yml) +- [participants.yml](./templates/participants.yml) +- [credentials.yml](./templates/credentials.yml) +- [secrets.yml](./templates/secrets.yml) +- [compute-envs.yml](./templates/compute-envs.yml) +- [actions.yml](./templates/actions.yml) +- [datasets.yml](./templates/datasets.yml) +- [labels.yml](./templates/labels.yml) +- [pipelines.yml](./templates/pipelines.yml) +- [launch.yml](./templates/launch.yml) ## Real world example @@ -370,7 +375,7 @@ $SENTIEON_LICENSE_BASE64 ## Contributions and Support -If you would like to contribute to `seqerakit`, please see the [contributing guidelines](https://github.com/seqeralabs/seqera-kit/blob/main/.github/CONTRIBUTING.md). +If you would like to contribute to `seqerakit`, please see the [contributing guidelines](./.github/CONTRIBUTING.md). For further information or help, please don't hesitate to create an [issue](https://github.com/seqeralabs/seqera-kit/issues) in this repository. diff --git a/setup.py b/setup.py index 268e0f1..0d94a03 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.6" +VERSION = "0.4.7" with open("README.md") as f: readme = f.read() From 810aa35807c6b4e594aa9bd8c08c0fa79865999b Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 21:30:09 -0400 Subject: [PATCH 19/19] docs: revert header --- README.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 64f4816..62f0c6a 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,5 @@ -
- seqerakit logo - seqerakit -
-
+# seqerakit logo seqerakit + `seqerakit` is a Python wrapper for the [Seqera Platform CLI](https://github.com/seqeralabs/tower-cli). It can be leveraged to automate the creation of all of the entities in Seqera Platform via a simple configuration file in YAML format.