diff --git a/setup.py b/setup.py index 1d68190d..f5276a2b 100644 --- a/setup.py +++ b/setup.py @@ -46,6 +46,7 @@ 'pykwalify', 'setuptools', 'packaging', + 'joblib' ], python_requires='>=3.8', entry_points={'console_scripts': ('west = west.app.main:main',)}, diff --git a/src/west/app/project.py b/src/west/app/project.py index a046b65c..56c148cf 100644 --- a/src/west/app/project.py +++ b/src/west/app/project.py @@ -6,6 +6,7 @@ '''West project commands''' import argparse + from functools import partial import logging import os @@ -30,6 +31,12 @@ from west.manifest import QUAL_MANIFEST_REV_BRANCH as QUAL_MANIFEST_REV from west.manifest import QUAL_REFS_WEST as QUAL_REFS +JOBLIB_PRESENT = True +try: + from joblib import Parallel, delayed +except ModuleNotFoundError: + JOBLIB_PRESENT = False + # # Project-related or multi-repo commands, like "init", "update", # "diff", etc. @@ -883,6 +890,11 @@ def do_add_parser(self, parser_adder): parser.add_argument('--stats', action='store_true', help='''print performance statistics for update operations''') + if JOBLIB_PRESENT: + parser.add_argument('-j','--jobs', nargs='?', const=-1, default=1, type=int, action='store', + help='''Use multiple jobs to paralelize update process. + Pass -1 to use all avaliable jobs. + ''') group = parser.add_argument_group( title='local project clone caches', @@ -1018,18 +1030,26 @@ def update_all(self): import_flags=ImportFlag.FORCE_PROJECTS) failed = [] - for project in self.manifest.projects: + + def project_update(project): if (isinstance(project, ManifestProject) or project.name in self.updated): - continue + return try: if not self.project_is_active(project): self.dbg(f'{project.name}: skipping inactive project') - continue - self.update(project) + return self.updated.add(project.name) + self.update(project) except subprocess.CalledProcessError: failed.append(project) + + if not JOBLIB_PRESENT or self.args.jobs == 1: + for project in self.manifest.projects: + project_update(project) + else: + Parallel(n_jobs=self.args.jobs, require='sharedmem')(delayed(project_update)(project) + for project in self.manifest.projects) self._handle_failed(self.args, failed) def update_importer(self, project, path): @@ -1090,13 +1110,23 @@ def update_some(self): projects = self._projects(self.args.projects) failed = [] - for project in projects: + + def project_update_some(project): if isinstance(project, ManifestProject): - continue + return try: self.update(project) except subprocess.CalledProcessError: failed.append(project) + + + if not JOBLIB_PRESENT or self.args.jobs == 1: + for project in projects: + project_update_some(project) + else: + Parallel(n_jobs=self.args.jobs, require='sharedmem')(delayed(project_update_some)(project) + for project in projects) + self._handle_failed(self.args, failed) def toplevel_projects(self): diff --git a/tests/test_project.py b/tests/test_project.py index 985669f0..672e053d 100644 --- a/tests/test_project.py +++ b/tests/test_project.py @@ -416,14 +416,14 @@ def test_grep(west_init_tmpdir): assert re.search('west-commands', cmd('grep -- -- -commands')) - -def test_update_projects(west_init_tmpdir): +@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j"]) +def test_update_projects(options, west_init_tmpdir): # Test the 'west update' command. It calls through to the same backend # functions that are used for automatic updates and 'west init' # reinitialization. # create local repositories - cmd('update') + cmd('update ' + options) # Add commits to the local repos. ur = update_helper(west_init_tmpdir) @@ -613,7 +613,8 @@ def test_update_head_0(west_init_tmpdir): assert modified_files.strip() == "M CODEOWNERS", \ 'local zephyr change not preserved' -def test_update_some_with_imports(repos_tmpdir): +@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j -1"]) +def test_update_some_with_imports(options, repos_tmpdir): # 'west update project1 project2' should work fine even when # imports are used, as long as the relevant projects are all # defined in the manifest repository. @@ -654,19 +655,19 @@ def test_update_some_with_imports(repos_tmpdir): # Updating unknown projects should fail as always. with pytest.raises(subprocess.CalledProcessError): - cmd('update unknown-project', cwd=ws) + cmd(f'update {options} unknown-project', cwd=ws) # Updating a list of projects when some are resolved via project # imports must fail. with pytest.raises(subprocess.CalledProcessError): - cmd('update Kconfiglib net-tools', cwd=ws) + cmd(f'update {options} Kconfiglib net-tools', cwd=ws) # Updates of projects defined in the manifest repository or all # projects must succeed, and behave the same as if no imports # existed. - cmd('update net-tools', cwd=ws) + cmd(f'update {options} net-tools', cwd=ws) with pytest.raises(ManifestImportFailed): Manifest.from_topdir(topdir=ws) manifest = Manifest.from_topdir(topdir=ws, @@ -677,10 +678,10 @@ def test_update_some_with_imports(repos_tmpdir): assert net_tools_project.is_cloned() assert not zephyr_project.is_cloned() - cmd('update zephyr', cwd=ws) + cmd(f'update {options} zephyr', cwd=ws) assert zephyr_project.is_cloned() - cmd('update', cwd=ws) + cmd(f'update {options}', cwd=ws) manifest = Manifest.from_topdir(topdir=ws) assert manifest.get_projects(['Kconfiglib'])[0].is_cloned()