From 0b107622d5d63d5a5c8dcec453bc5bd4c2205218 Mon Sep 17 00:00:00 2001 From: Purnendu Chakraborty Date: Wed, 18 Dec 2024 12:19:29 -0600 Subject: [PATCH 1/8] clone.py - (1) streamlined (2) branches are no longer in a detached state (3) updated tests (4) new tests --- src/mepo/cmdline/parser.py | 18 ++- src/mepo/command/clone.py | 215 ++++++++++++++++-------------------- src/mepo/command/reset.py | 2 +- src/mepo/git.py | 41 +++---- tests/test_component.py | 5 +- tests/test_mepo_clone.py | 92 +++++++++++++++ tests/test_mepo_commands.py | 4 +- 7 files changed, 224 insertions(+), 153 deletions(-) create mode 100644 tests/test_mepo_clone.py diff --git a/src/mepo/cmdline/parser.py b/src/mepo/cmdline/parser.py index 48811a46..416528ce 100644 --- a/src/mepo/cmdline/parser.py +++ b/src/mepo/cmdline/parser.py @@ -101,7 +101,7 @@ def __clone(self): aliases=mepoconfig.get_command_alias("clone"), ) clone.add_argument( - "repo_url", metavar="URL", nargs="?", default=None, help="URL to clone" + "url", metavar="URL", nargs="?", default=None, help="URL to clone" ) clone.add_argument( "directory", @@ -122,7 +122,7 @@ def __clone(self): metavar="registry", nargs="?", default=None, - help="Registry (default: components.yaml)", + help="Registry (default: %(default)s)", ) clone.add_argument( "--style", @@ -142,8 +142,18 @@ def __clone(self): metavar="partial-type", nargs="?", default=None, - choices=["off", "blobless", "treeless"], - help='Style of partial clone, default: None, allowed options: %(choices)s. Off means a "normal" full git clone, blobless means cloning with "--filter=blob:none" and treeless means cloning with "--filter=tree:0". NOTE: We do *not* recommend using "treeless" as it is very aggressive and will cause problems with many git commands.', + choices=[None, "blobless", "treeless"], + help=( + """ + Style of partial clone, default: %(default)s. + Allowed options: %(choices)s. + None: normal full git clone, + blobless: cloning with "--filter=blob:none", + treeless: cloning with "--filter=tree:0". + NOTE: We do *not* recommend using "treeless" as it is very + aggressive and will cause problems with many git commands. + """ + ), ) def __list(self): diff --git a/src/mepo/command/clone.py b/src/mepo/command/clone.py index 66718a25..53a69a99 100644 --- a/src/mepo/command/clone.py +++ b/src/mepo/command/clone.py @@ -1,145 +1,120 @@ import os import pathlib -import shutil -import shlex from urllib.parse import urlparse +from types import SimpleNamespace -from .init import run as mepo_init_run +from .init import run as mepo_init from ..state import MepoState from ..state import StateDoesNotExistError from ..git import GitRepository -from ..utilities import shellcmd from ..utilities import colors from ..utilities import mepoconfig - def run(args): - - # This protects against someone using branch without a URL - if args.branch and not args.repo_url: + """ + Entry point of clone. + Steps - + 1. Clone fixture - if url is passed + 2. Read state - if state does not exist, initialize mepo (write state) first + 3. Clone components + 4. Checkout all repos to the specified branch + """ + CWD = os.getcwd() + + if args.branch is not None and args.url is None: raise RuntimeError("The branch argument can only be used with a URL") - if args.allrepos and not args.branch: - raise RuntimeError("The allrepos option must be used with a branch/tag.") - - # We can get the blobless and treeless options from the config or the args - if args.partial: - # We need to set partial to None if it's off, otherwise we use the - # string. This is safe because argparse only allows for 'off', - # 'blobless', or 'treeless' - partial = None if args.partial == "off" else args.partial - elif mepoconfig.has_option("clone", "partial"): - allowed = ["blobless", "treeless"] - partial = mepoconfig.get("clone", "partial") - if partial not in allowed: - raise Exception( - f"Detected partial clone type [{partial}] from .mepoconfig is not an allowed partial clone type: {allowed}" - ) - else: - print(f"Found partial clone type [{partial}] in .mepoconfig") - else: - partial = None - - # If you pass in a registry, with clone, it could be outside the repo. - # So use the full path - passed_in_registry = False - if args.registry: - passed_in_registry = True - args.registry = os.path.abspath(args.registry) - else: - # If we don't pass in a registry, we need to "reset" the arg to the - # default name because we pass args to mepo_init - args.registry = "components.yaml" - - if args.repo_url: - p = urlparse(args.repo_url) - last_url_node = p.path.rsplit("/")[-1] - url_suffix = pathlib.Path(last_url_node).suffix - if args.directory: - local_clone(args.repo_url, args.branch, args.directory, partial) - os.chdir(args.directory) - else: - if url_suffix == ".git": - git_url_directory = pathlib.Path(last_url_node).stem - else: - git_url_directory = last_url_node - - local_clone(args.repo_url, args.branch, git_url_directory, partial) - os.chdir(git_url_directory) - - # Copy the new file into the repo only if we pass it in - if passed_in_registry: - try: - shutil.copy(args.registry, os.getcwd()) - except shutil.SameFileError: - pass + arg_partial = handle_partial(args.partial) - # This tries to read the state and if not, calls init, - # loops back, and reads the state + # Step 1 - clone fixture + if args.url is not None: + fixture_dir = clone_fixture(args.url, args.branch, args.directory, arg_partial) + os.chdir(fixture_dir) + + # Step 2 - Read state - if state does not exist, initialize mepo while True: try: allcomps = MepoState.read_state() except StateDoesNotExistError: - mepo_init_run(args) + mepo_init(SimpleNamespace(style=args.style, registry=get_registry(args.registry))) continue break - max_namelen = len(max([comp.name for comp in allcomps], key=len)) - for comp in allcomps: - if not comp.fixture: - git = GitRepository(comp.remote, comp.local) - version = comp.version.name - version = version.replace("origin/", "") - recurse = comp.recurse_submodules + # Step 3 - Clone componets + clone_components(allcomps, arg_partial) + + # Step 4 - Checkout all repos to the specified branch + # TODO - pchakrab: DO WE REALLY NEED THIS??? + if args.allrepos: + if args.branch is None: + raise RuntimeError("`allrepos` option must be used with a branch/tag.") + checkout_components(allcomps, args.branch) + + os.chdir(CWD) + + +def handle_partial(partial): + """ + The `partial` argument to clone can be set either via command line or + via .mepoconfig. Non-default value set via command line takes precedence. + The default value of `partial` is None, and possible choices are None/blobless/treeless + """ + ALLOWED_NON_DEFAULT = ["blobless", "treeless"] + if partial is None: # default value from command line + if mepoconfig.has_option("clone", "partial"): + partial = mepoconfig.get("clone", "partial") + if partial not in ALLOWED_NON_DEFAULT: + raise ValueError(f"Invalid partial type [{partial}] in .mepoconfig") + print(f"Found partial clone type [{partial}] in .mepoconfig") + return partial - # According to Git, treeless clones do not interact well with - # submodules. So we need to see if any comp has the recurse - # option set to True. If so, we need to clone that comp "normally" - _partial = None if partial == "treeless" and recurse else partial +def clone_fixture(url, branch=None, directory=None, partial=None): + if directory is None: + p = urlparse(url) + last_url_node = p.path.rsplit("/")[-1] + directory = pathlib.Path(last_url_node).stem + git = GitRepository(url, directory) + git.clone(branch, partial) + return directory - # We need the type to handle hashes in components.yaml - _type = comp.version.type - git.clone(version, recurse, _type, comp.name, _partial) - if comp.sparse: - git.sparsify(comp.sparse) - print_clone_info(comp, max_namelen) - if args.allrepos: - for comp in allcomps: - if not comp.fixture: - git = GitRepository(comp.remote, comp.local) - print( - "Checking out %s in %s" - % ( - colors.YELLOW + args.branch + colors.RESET, - colors.RESET + comp.name + colors.RESET, - ) - ) - git.checkout(args.branch, detach=True) - - -def print_clone_info(comp, name_width): - ver_name_type = "({}) {}".format(comp.version.type, comp.version.name) - print("{:<{width}} | {: Date: Wed, 18 Dec 2024 17:21:01 -0600 Subject: [PATCH 2/8] Black formatting changes --- src/mepo/command/clone.py | 11 +++++++---- tests/test_component.py | 4 +++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/mepo/command/clone.py b/src/mepo/command/clone.py index 53a69a99..c52d317b 100644 --- a/src/mepo/command/clone.py +++ b/src/mepo/command/clone.py @@ -10,6 +10,7 @@ from ..utilities import colors from ..utilities import mepoconfig + def run(args): """ Entry point of clone. @@ -36,7 +37,8 @@ def run(args): try: allcomps = MepoState.read_state() except StateDoesNotExistError: - mepo_init(SimpleNamespace(style=args.style, registry=get_registry(args.registry))) + registry = get_registry(args.registry) + mepo_init(SimpleNamespace(style=args.style, registry=registry)) continue break @@ -91,13 +93,14 @@ def clone_components(allcomps, partial): max_namelen = max([len(comp.name) for comp in allcomps]) for comp in allcomps: if comp.fixture: - continue # not cloning fixture + continue # not cloning fixture recurse_submodules = comp.recurse_submodules # According to Git, treeless clones do not interact well with # submodules. So if any comp has the recurse option set to True, # we do a non-partial clone partial = None if partial == "treeless" and recurse_submodules else partial - version = comp.version.name; version = version.replace("origin/", "") + version = comp.version.name + version = version.replace("origin/", "") git = GitRepository(comp.remote, comp.local) git.clone(version, recurse_submodules, partial) if comp.sparse: @@ -108,7 +111,7 @@ def clone_components(allcomps, partial): def checkout_components(allcomps, branch): for comp in allcomps: if comp.fixture: - continue # fixture is already on the right branch + continue # fixture is already on the right branch branch_y = colors.YELLOW + args.branch + colors.RESET print(f"Checking out {branch_y} in {comp.name}") git = GitRepository(comp.remote, comp.local) diff --git a/tests/test_component.py b/tests/test_component.py index 282531ec..28f74f25 100644 --- a/tests/test_component.py +++ b/tests/test_component.py @@ -67,5 +67,7 @@ def test_MepoComponent(): assert fvdycore == get_fvdycore_component() fvdycore_serialized = fvdycore.serialize() remote = fvdycore_serialized["remote"] - fvdycore_serialized["remote"] = remote.replace("git@github.com:", "https://github.com/") + fvdycore_serialized["remote"] = remote.replace( + "git@github.com:", "https://github.com/" + ) assert fvdycore_serialized == get_fvdycore_serialized() From a0e0207898b50f62ce6ff6f6145045436520256a Mon Sep 17 00:00:00 2001 From: Purnendu Chakraborty Date: Thu, 19 Dec 2024 08:13:23 -0600 Subject: [PATCH 3/8] Updated test_mepo_clone.py to use https://github.com/pchakraborty/GEOSfvdycore-mepo-testing.git instead --- .pre-commit-config.yaml | 2 +- tests/test_mepo_clone.py | 44 +++++++++++++++++++--------------------- 2 files changed, 22 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 983fce47..9edf3f08 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,4 +8,4 @@ repos: # supported by your project here, or alternatively use # pre-commit's default_language_version, see # https://pre-commit.com/#top_level-default_language_version - language_version: python3.11 + language_version: python3.9 diff --git a/tests/test_mepo_clone.py b/tests/test_mepo_clone.py index 0e5b172e..6c62a20a 100644 --- a/tests/test_mepo_clone.py +++ b/tests/test_mepo_clone.py @@ -13,7 +13,8 @@ import mepo.command.status as mepo_status -FIXTURE_URL = "https://github.com/GEOS-ESM/GEOSfvdycore.git" +FIXTURE_NAME = "GEOSfvdycore-mepo-testing" +FIXTURE_URL = f"https://github.com/pchakraborty/{FIXTURE_NAME}.git" def get_mepo_status(): @@ -29,9 +30,8 @@ def get_mepo_status(): def test_mepo_clone_url(): - DIRECTORY = "GEOSfvdycore" - if os.path.isdir(DIRECTORY): - shutil.rmtree(DIRECTORY) + if os.path.isdir(FIXTURE_NAME): + shutil.rmtree(FIXTURE_NAME) args = SimpleNamespace( style="prefix", registry=None, @@ -44,20 +44,19 @@ def test_mepo_clone_url(): mepo_clone.run(args) saved_output = """Checking status... GEOSfvdycore | (b) main -env | (t) v4.29.0 (DH) -cmake | (t) v3.51.0 (DH) -ecbuild | (t) geos/v1.3.0 (DH) -GMAO_Shared | (t) v1.9.8 (DH) -GEOS_Util | (t) v2.1.2 (DH) -MAPL | (t) v2.48.0 (DH) +env | (t) v4.29.1 (DH) +cmake | (t) v3.55.0 (DH) +ecbuild | (t) geos/v1.4.0 (DH) +GMAO_Shared | (t) v1.9.9 (DH) +GEOS_Util | (t) v2.1.3 (DH) FMS | (t) geos/2019.01.02+noaff.10 (DH) FVdycoreCubed_GridComp | (t) v2.12.0 (DH) -fvdycore | (t) geos/v2.9.0 (DH) +fvdycore | (t) geos/v2.9.1 (DH) """ - with contextlib_chdir(DIRECTORY): + with contextlib_chdir(FIXTURE_NAME): status_output = get_mepo_status() assert status_output == saved_output - shutil.rmtree(DIRECTORY) + shutil.rmtree(FIXTURE_NAME) def test_mepo_clone_url_branch_directory(): @@ -68,23 +67,22 @@ def test_mepo_clone_url_branch_directory(): style="postfix", registry=None, url=FIXTURE_URL, - branch="release/MAPL-v3", + branch="fvdycore-develop", directory=DIRECTORY, partial="blobless", allrepos=False, ) mepo_clone.run(args) saved_output = """Checking status... -GEOSfvdycore | (b) release/MAPL-v3 -env | (t) v4.29.0 (DH) -cmake | (t) v3.51.0 (DH) -ecbuild | (t) geos/v1.3.0 (DH) -GMAO_Shared | (t) v1.9.8 (DH) -GEOS_Util | (b) release/MAPL-v3 -MAPL | (b) release/MAPL-v3 +GEOSfvdycore | (b) fvdycore-develop +env | (t) v4.29.1 (DH) +cmake | (t) v3.55.0 (DH) +ecbuild | (t) geos/v1.4.0 (DH) +GMAO_Shared | (t) v1.9.9 (DH) +GEOS_Util | (t) v2.1.3 (DH) FMS | (t) geos/2019.01.02+noaff.10 (DH) -FVdycoreCubed_GridComp | (b) release/MAPL-v3 -fvdycore | (b) geos/release/MAPL-v3 +FVdycoreCubed_GridComp | (b) develop +fvdycore | (b) geos/develop """ with contextlib_chdir(DIRECTORY): status_output = get_mepo_status() From 2a8b76e641db8bd6afb35401d1df1cf01d974228 Mon Sep 17 00:00:00 2001 From: Purnendu Chakraborty Date: Thu, 19 Dec 2024 08:23:55 -0600 Subject: [PATCH 4/8] Updated CHANGELOG --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c76e8b9..c3bd6853 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,10 +13,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Hidden option `--location` for mepo that returns the path to the mepo directory - Added ability to print number of stashes in `mepo status` +- Added new tests for `mepo clone` ### Changed - Removed legacy `bin` directory +- Checked out branches are no longer in 'detached head' state ## [2.1.0] - 2024-10-02 From bb13a773a1e5991afa5ebd8e532af751c573a1d5 Mon Sep 17 00:00:00 2001 From: Purnendu Chakraborty Date: Thu, 19 Dec 2024 18:26:36 -0600 Subject: [PATCH 5/8] Some more refactoring --- src/mepo/command/clone.py | 54 +++++++++++++++++---------------------- 1 file changed, 23 insertions(+), 31 deletions(-) diff --git a/src/mepo/command/clone.py b/src/mepo/command/clone.py index c52d317b..5579b372 100644 --- a/src/mepo/command/clone.py +++ b/src/mepo/command/clone.py @@ -15,42 +15,22 @@ def run(args): """ Entry point of clone. Steps - - 1. Clone fixture - if url is passed + 1. Clone fixture - if url is provided 2. Read state - if state does not exist, initialize mepo (write state) first 3. Clone components 4. Checkout all repos to the specified branch """ CWD = os.getcwd() - if args.branch is not None and args.url is None: - raise RuntimeError("The branch argument can only be used with a URL") - arg_partial = handle_partial(args.partial) - # Step 1 - clone fixture if args.url is not None: fixture_dir = clone_fixture(args.url, args.branch, args.directory, arg_partial) os.chdir(fixture_dir) - - # Step 2 - Read state - if state does not exist, initialize mepo - while True: - try: - allcomps = MepoState.read_state() - except StateDoesNotExistError: - registry = get_registry(args.registry) - mepo_init(SimpleNamespace(style=args.style, registry=registry)) - continue - break - - # Step 3 - Clone componets + allcomps = read_state(args.style, args.registry) clone_components(allcomps, arg_partial) - - # Step 4 - Checkout all repos to the specified branch - # TODO - pchakrab: DO WE REALLY NEED THIS??? if args.allrepos: - if args.branch is None: - raise RuntimeError("`allrepos` option must be used with a branch/tag.") - checkout_components(allcomps, args.branch) + checkout_all_repos(allcomps, args.branch) os.chdir(CWD) @@ -81,6 +61,18 @@ def clone_fixture(url, branch=None, directory=None, partial=None): return directory +def read_state(arg_style, arg_registry): + while True: + try: + allcomps = MepoState.read_state() + except StateDoesNotExistError: + registry = get_registry(arg_registry) + mepo_init(SimpleNamespace(style=arg_style, registry=registry)) + continue + break + return allcomps + + def get_registry(arg_registry): registry = "components.yaml" if arg_registry is not None: @@ -108,16 +100,16 @@ def clone_components(allcomps, partial): print_clone_info(comp.name, comp.version, max_namelen) -def checkout_components(allcomps, branch): +def print_clone_info(comp_name, comp_version, name_width): + ver_name_type = f"({comp_version.type}) {comp_version.name}" + print(f"{comp_name:<{name_width}} | {ver_name_type: Date: Fri, 20 Dec 2024 07:24:09 -0600 Subject: [PATCH 6/8] clone.py - updated comments --- src/mepo/command/clone.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/mepo/command/clone.py b/src/mepo/command/clone.py index 5579b372..afb08b25 100644 --- a/src/mepo/command/clone.py +++ b/src/mepo/command/clone.py @@ -14,9 +14,19 @@ def run(args): """ Entry point of clone. + + Multiple ways to run clone + 1. After fixture has been cloned (via git clone) + a. mepo init + mepo clone + b. mepo clone (initializes mepo) + 2. Clone fixture as well + a. mepo clone [] + b. mepo clone -b [] + Steps - 1. Clone fixture - if url is provided - 2. Read state - if state does not exist, initialize mepo (write state) first + 2. Read state - initialize mepo (write state) first, if needed 3. Clone components 4. Checkout all repos to the specified branch """ From 8e8849376f7c48067bcb11dcd90c48c04817afe6 Mon Sep 17 00:00:00 2001 From: Purnendu Chakraborty Date: Fri, 20 Dec 2024 08:48:35 -0600 Subject: [PATCH 7/8] Added deprecation warning for 'mepo init' --- src/mepo/cmdline/parser.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/mepo/cmdline/parser.py b/src/mepo/cmdline/parser.py index 416528ce..3966c119 100644 --- a/src/mepo/cmdline/parser.py +++ b/src/mepo/cmdline/parser.py @@ -1,4 +1,5 @@ import argparse +import warnings from .branch_parser import MepoBranchArgParser from .stash_parser import MepoStashArgParser @@ -74,6 +75,7 @@ def parse(self): return self.parser.parse_args() def __init(self): + warnings.warn("init will be removed in version 3, use clone instead") init = self.subparsers.add_parser( "init", description="Initialize mepo based on `config-file`", From 5b092f51420e9c98d8f6b296185abb1ff0508036 Mon Sep 17 00:00:00 2001 From: Purnendu Chakraborty Date: Mon, 23 Dec 2024 07:04:10 -0600 Subject: [PATCH 8/8] init removal is a deprecation warning --- src/mepo/cmdline/parser.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/mepo/cmdline/parser.py b/src/mepo/cmdline/parser.py index 3966c119..4f70a4c8 100644 --- a/src/mepo/cmdline/parser.py +++ b/src/mepo/cmdline/parser.py @@ -75,7 +75,9 @@ def parse(self): return self.parser.parse_args() def __init(self): - warnings.warn("init will be removed in version 3, use clone instead") + warnings.warn( + "init will be removed in version 3, use clone instead", DeprecationWarning + ) init = self.subparsers.add_parser( "init", description="Initialize mepo based on `config-file`",