Merge branch 'linker-lib-flags' into new_additional_compilers

Documentation/source/site-specific-config.rst

@@ -148,6 +148,20 @@ rsync, ar, ...).
     tool_box = ToolBox()
     default_c_compiler = tool_box.get_tool(Category.C_COMPILER)
 
+There is special handling for compilers and linkers: the build
+configuration stores the information if an MPI and/or OpenMP build
+is requested. So when a default tool is requested by the ToolBox
+from the ToolRepository (i.e. when the user has not added specific
+compilers or linkers), this information is taken into account, and
+only a compiler that will fulfil the requirements is returned. For
+example, if you have `gfortran` and `mpif90-gfortran` defined in this
+order in the ToolRepository, and request the default compiler for an
+MPI build, the `mpif90-gfortran` instance is returned, not `gfortran`.
+On the other hand, if no MPI is requested, an MPI-enabled compiler
+might be returned, which does not affect the final result, since
+an MPI compiler just adds include- and library-paths.
+
+
 Compiler Wrapper
 ================
 Fab supports the concept of a compiler wrapper, which is typically
@@ -205,7 +219,6 @@ applied to the wrapper as well:
     assert my_mpicc.flags == ["-a", "-b"]
 
 
-
 TODO
 ====
 At this stage compiler flags are still set in the corresponding Fab

run_configs/gcom/build_gcom_ar.py

@@ -15,7 +15,7 @@
 if __name__ == '__main__':
 
     with BuildConfig(project_label='gcom object archive $compiler',
-                     mpi=False, openmp=False, tool_box=ToolBox()) as state:
+                     mpi=True, openmp=False, tool_box=ToolBox()) as state:
         common_build_steps(state)
         archive_objects(state, output_fpath='$output/libgcom.a')
         cleanup_prebuilds(state, all_unused=True)

run_configs/gcom/build_gcom_so.py

@@ -20,7 +20,7 @@
     parsed_args = arg_parser.parse_args()
 
     with BuildConfig(project_label='gcom shared library $compiler',
-                     mpi=False, openmp=False, tool_box=ToolBox()) as state:
+                     mpi=True, openmp=False, tool_box=ToolBox()) as state:
         common_build_steps(state, fpic=True)
         link_shared_object(state, output_fpath='$output/libgcom.so')
         cleanup_prebuilds(state, all_unused=True)

run_configs/gcom/grab_gcom.py

@@ -14,7 +14,7 @@
 
 # we put this here so the two build configs can read its source_root
 grab_config = BuildConfig(project_label=f'gcom_source {revision}',
-                          mpi=False, openmp=False, tool_box=ToolBox())
+                          tool_box=ToolBox())
 
 
 if __name__ == '__main__':

run_configs/lfric/grab_lfric.py

@@ -18,10 +18,10 @@
 tool_box = ToolBox()
 lfric_source_config = BuildConfig(
     project_label=f'lfric source {LFRIC_REVISION}',
-    mpi=False, openmp=False, tool_box=tool_box)
+    tool_box=tool_box)
 gpl_utils_source_config = BuildConfig(
     project_label=f'lfric source {LFRIC_REVISION}',
-    mpi=False, openmp=False, tool_box=tool_box)
+    tool_box=tool_box)
 
 
 if __name__ == '__main__':

run_configs/lfric/gungho.py

@@ -33,7 +33,7 @@
     gpl_utils_source = gpl_utils_source_config.source_root / 'gpl_utils'
 
     with BuildConfig(project_label='gungho $compiler $two_stage',
-                     mpi=False, openmp=False, tool_box=ToolBox()) as state:
+                     mpi=True, openmp=True, tool_box=ToolBox()) as state:
         grab_folder(state, src=lfric_source / 'infrastructure/source/', dst_label='')
         grab_folder(state, src=lfric_source / 'components/driver/source/', dst_label='')
         grab_folder(state, src=lfric_source / 'components' / 'inventory' / 'source', dst_label='')
@@ -87,7 +87,7 @@
             state,
             common_flags=[
                 '-c',
-                '-ffree-line-length-none', '-fopenmp',
+                '-ffree-line-length-none',
                 '-g',
                 '-std=f2008',
 
@@ -104,8 +104,6 @@
         link_exe(
             state,
             flags=[
-                '-fopenmp',
-
                 '-lyaxt', '-lyaxt_c', '-lnetcdff', '-lnetcdf', '-lhdf5',  # EXTERNAL_DYNAMIC_LIBRARIES
                 '-lxios',  # EXTERNAL_STATIC_LIBRARIES
                 '-lstdc++',

run_configs/lfric/mesh_tools.py

@@ -25,7 +25,7 @@
     psyclone_overrides = Path(__file__).parent / 'mesh_tools_overrides'
 
     with BuildConfig(project_label='mesh tools $compiler $two_stage',
-                     mpi=False, openmp=False, tool_box=ToolBox()) as state:
+                     mpi=True, openmp=False, tool_box=ToolBox()) as state:
         grab_folder(state, src=lfric_source / 'infrastructure/source/', dst_label='')
         grab_folder(state, src=lfric_source / 'mesh_tools/source/', dst_label='')
         grab_folder(state, src=lfric_source / 'components/science/source/', dst_label='')

run_configs/tiny_fortran/build_tiny_fortran.py

@@ -31,7 +31,7 @@ def __init__(self):
     tool_box.add_tool(Linker(compiler=fc))
 
     with BuildConfig(project_label='tiny_fortran $compiler',
-                     mpi=False, openmp=False, tool_box=tool_box) as state:
+                     tool_box=tool_box) as state:
         git_checkout(state, src='https://github.com/metomi/fab-test-data.git',
                      revision='main', dst_label='src')
 

run_configs/um/build_um.py

@@ -124,9 +124,11 @@ def replace_in_file(inpath, outpath, find, replace):
     revision = 'vn12.1'
     um_revision = revision.replace('vn', 'um')
 
+    # The original build script disabled openmp, so for now
+    # we keep this disabled.
     state = BuildConfig(
         project_label=f'um atmos safe {revision} $compiler $two_stage',
-        mpi=False, openmp=False, tool_box=ToolBox())
+        mpi=True, openmp=False, tool_box=ToolBox())
 
     # compiler-specific flags
     compiler = state.tool_box[Category.FORTRAN_COMPILER]

source/fab/build_config.py

@@ -42,8 +42,8 @@ class BuildConfig():
     """
     def __init__(self, project_label: str,
                  tool_box: ToolBox,
-                 mpi: bool,
-                 openmp: bool,
+                 mpi: bool = False,
+                 openmp: bool = False,
                  multiprocessing: bool = True,
                  n_procs: Optional[int] = None,
                  reuse_artefacts: bool = False,
@@ -56,9 +56,13 @@ def __init__(self, project_label: str,
             created from this name, with spaces replaced by underscores.
         :param tool_box: The ToolBox with all tools to use in the build.
         :param mpi: whether the project uses MPI or not. This is used to
-            pick a default compiler (if not explicitly set in the ToolBox),
-            and controls PSyclone parameters.
-        :param openmp: whether the project should use OpenMP or not.
+            pick a default compiler (if none is explicitly set in the
+            ToolBox), and controls PSyclone parameters.
+        :param openmp: as with `mpi`, this controls whether the project is
+            using OpenMP or not. This is used to pick a default compiler
+            (if none is explicitly set in the ToolBox). The compiler-specific
+            flag to enable OpenMP will automatically be added when compiling
+            and linking.
         :param multiprocessing:
             An option to disable multiprocessing to aid debugging.
         :param n_procs:
@@ -85,7 +89,8 @@ def __init__(self, project_label: str,
         self._openmp = openmp
         self.two_stage = two_stage
         self.verbose = verbose
-        compiler = tool_box.get_tool(Category.FORTRAN_COMPILER, mpi=mpi)
+        compiler = tool_box.get_tool(Category.FORTRAN_COMPILER, mpi=mpi,
+                                     openmp=openmp)
         project_label = Template(project_label).safe_substitute(
             compiler=compiler.name,
             two_stage=f'{int(two_stage)+1}stage')

source/fab/cli.py

@@ -34,7 +34,7 @@ def _generic_build_config(folder: Path, kwargs=None) -> BuildConfig:
     # Set the default Fortran compiler as linker (otherwise e.g. the
     # C compiler might be used in linking, requiring additional flags)
     tr = ToolRepository()
-    fc = tr.get_default(Category.FORTRAN_COMPILER, mpi=False)
+    fc = tr.get_default(Category.FORTRAN_COMPILER, mpi=False, openmp=False)
     # TODO: This assumes a mapping of compiler name to the corresponding
     # linker name (i.e. `linker-gfortran` or `linker-ifort`). Still, that's
     # better than hard-coding gnu here.

source/fab/steps/compile_c.py

@@ -81,7 +81,8 @@ def compile_c(config, common_flags: Optional[List[str]] = None,
         # No need to look for compiler etc if there is nothing to do
         return
 
-    compiler = config.tool_box.get_tool(Category.C_COMPILER, config.mpi)
+    compiler = config.tool_box.get_tool(Category.C_COMPILER, mpi=config.mpi,
+                                        openmp=config.openmp)
     logger.info(f'C compiler is {compiler}')
 
     mp_payload = MpCommonArgs(config=config, flags=flags)
@@ -147,7 +148,7 @@ def _compile_file(arg: Tuple[AnalysedC, MpCommonArgs]):
                 compiler.compile_file(analysed_file.fpath, obj_file_prebuild,
                                       openmp=config.openmp,
                                       add_flags=flags)
-            except Exception as err:
+            except RuntimeError as err:
                 return FabException(f"error compiling "
                                     f"{analysed_file.fpath}:\n{err}")
 

source/fab/steps/link.py

@@ -61,7 +61,8 @@ def link_exe(config,
         output from compiler steps, which typically is the expected behaviour.
 
     """
-    linker = config.tool_box.get_tool(Category.LINKER, config.mpi)
+    linker = config.tool_box.get_tool(Category.LINKER, mpi=config.mpi,
+                                      openmp=config.openmp)
     logger.info(f'Linker is {linker.name}')
 
     libs = libs or []

source/fab/tools/compiler.py

@@ -34,12 +34,13 @@ class Compiler(CompilerSuiteTool):
         the version number from the version output of the compiler. The
         version is taken from the first group of a match.
     :param category: the Category (C_COMPILER or FORTRAN_COMPILER).
+    :param mpi: whether the compiler or linker support MPI.
     :param compile_flag: the compilation flag to use when only requesting
         compilation (not linking).
-    :param mpi: whether MPI is supported by this compiler or not.
     :param output_flag: the compilation flag to use to indicate the name
         of the output file
-    :param openmp_flag: the flag to use to enable OpenMP
+    :param openmp_flag: the flag to use to enable OpenMP. If no flag is
+        specified, it is assumed that the compiler does not support OpenMP.
     :param availability_option: a command line option for the tool to test
         if the tool is available on the current system. Defaults to
         `--version`.
@@ -71,6 +72,12 @@ def mpi(self) -> bool:
         '''Returns whether this compiler supports MPI or not.'''
         return self._mpi
 
+    @property
+    def openmp(self) -> bool:
+        ''':returns: if the compiler supports openmp or not
+        '''
+        return self._openmp_flag != ""
+
     @property
     def openmp_flag(self) -> str:
         '''Returns the flag to enable OpenMP.'''
@@ -344,6 +351,7 @@ class Gcc(CCompiler):
 
     :param name: name of this compiler.
     :param exec_name: name of the executable.
+    :param mpi: whether the compiler supports MPI.
     '''
     def __init__(self,
                  name: str = "gcc",
@@ -365,6 +373,7 @@ class Gfortran(FortranCompiler):
 
     :param name: name of this compiler.
     :param exec_name: name of the executable.
+    :param mpi: whether the compiler supports MPI.
     '''
 
     def __init__(self, name: str = "gfortran",
@@ -385,6 +394,7 @@ class Icc(CCompiler):
 
     :param name: name of this compiler.
     :param exec_name: name of the executable.
+    :param mpi: whether the compiler supports MPI.
     '''
 
     def __init__(self, name: str = "icc", exec_name: str = "icc"):
@@ -399,6 +409,7 @@ class Ifort(FortranCompiler):
 
     :param name: name of this compiler.
     :param exec_name: name of the executable.
+    :param mpi: whether the compiler supports MPI.
     '''
 
     def __init__(self, name: str = "ifort", exec_name: str = "ifort"):

source/fab/tools/tool_box.py

@@ -46,13 +46,18 @@ def add_tool(self, tool: Tool,
                           f"'{tool}'.")
         self._all_tools[tool.category] = tool
 
-    def get_tool(self, category: Category, mpi: Optional[bool] = None) -> Tool:
+    def get_tool(self, category: Category, mpi: Optional[bool] = None,
+                 openmp: Optional[bool] = None) -> Tool:
         '''Returns the tool for the specified category.
 
         :param category: the name of the category in which to look
             for the tool.
-        :param mpi: if no compiler or linker is specified when requesting one,
-            use the MPI setting to find an appropriate default.
+        :param mpi: if no compiler or linker is explicitly specified in this
+            tool box, use the MPI and OpenMP setting to find an appropriate
+            default from the tool repository.
+        :param mpi: if no compiler or linker is explicitly specified in this
+            tool box, use the MPI and OpenMP setting to find an appropriate
+            default from the tool repository.
 
         :raises KeyError: if the category is not known.
         '''
@@ -69,6 +74,6 @@ def get_tool(self, category: Category, mpi: Optional[bool] = None) -> Tool:
         # from the ToolRepository, and add it, so we don't need to look
         # it up again later.
         tr = ToolRepository()
-        tool = tr.get_default(category, mpi=mpi)
+        tool = tr.get_default(category, mpi=mpi, openmp=openmp)
         self._all_tools[category] = tool
         return tool

source/fab/tools/tool_repository.py

@@ -153,7 +153,8 @@ def set_default_compiler_suite(self, suite: str):
                                    f"in the suite '{suite}'.")
 
     def get_default(self, category: Category,
-                    mpi: Optional[bool] = None):
+                    mpi: Optional[bool] = None,
+                    openmp: Optional[bool] = None):
         '''Returns the default tool for a given category. For most tools
         that will be the first entry in the list of tools. The exception
         are compilers and linker: in this case it must be specified if
@@ -162,6 +163,7 @@ def get_default(self, category: Category,
 
         :param category: the category for which to return the default tool.
         :param mpi: if a compiler or linker is required that supports MPI.
+        :param open: if a compiler or linker is required that supports OpenMP.
 
         :raises KeyError: if the category does not exist.
         :raises RuntimeError: if no compiler/linker is found with the
@@ -180,11 +182,29 @@ def get_default(self, category: Category,
             raise RuntimeError(f"Invalid or missing mpi specification "
                                f"for '{category}'.")
 
+        if not isinstance(openmp, bool):
+            raise RuntimeError(f"Invalid or missing openmp specification "
+                               f"for '{category}'.")
+
         for tool in self[category]:
-            # If the tool supports/does not support MPI, return the first one
+            # If OpenMP is request, but the tool does not support openmp,
+            # ignore it.
+            if openmp and not tool.openmp:
+                continue
+            # If the tool supports/does not support MPI, return it.
             if mpi == tool.mpi:
                 return tool
 
         # Don't bother returning an MPI enabled tool if no-MPI is requested -
         # that seems to be an unlikely scenario.
-        raise RuntimeError(f"Could not find '{category}' that supports MPI.")
+        if mpi:
+            if openmp:
+                raise RuntimeError(f"Could not find '{category}' that "
+                                   f"supports MPI and OpenMP.")
+            raise RuntimeError(f"Could not find '{category}' that "
+                               f"supports MPI.")
+
+        if openmp:
+            raise RuntimeError(f"Could not find '{category}' that "
+                               f"supports OpenMP.")
+        raise RuntimeError(f"Could not find any '{category}'.")

tests/system_tests/CFortranInterop/test_CFortranInterop.py

@@ -27,8 +27,7 @@ def test_CFortranInterop(tmp_path):
 
     # build
     with BuildConfig(fab_workspace=tmp_path, project_label='foo',
-                     mpi=False, openmp=False, tool_box=ToolBox(),
-                     multiprocessing=False) as config:
+                     tool_box=ToolBox(), multiprocessing=False) as config:
         grab_folder(config, src=PROJECT_SOURCE)
         find_source_files(config)
         c_pragma_injector(config)

tests/system_tests/CUserHeader/test_CUserHeader.py

@@ -24,8 +24,7 @@ def test_CUseHeader(tmp_path):
 
     # build
     with BuildConfig(fab_workspace=tmp_path, tool_box=ToolBox(),
-                     mpi=False, openmp=False, project_label='foo',
-                     multiprocessing=False) as config:
+                     project_label='foo', multiprocessing=False) as config:
 
         grab_folder(config, PROJECT_SOURCE)
         find_source_files(config)

tests/system_tests/FortranDependencies/test_FortranDependencies.py

@@ -25,7 +25,6 @@ def test_fortran_dependencies(tmp_path):
 
     # build
     with BuildConfig(fab_workspace=tmp_path, tool_box=ToolBox(),
-                     mpi=False, openmp=False,
                      project_label='foo', multiprocessing=False) as config:
         grab_folder(config, src=Path(__file__).parent / 'project-source')
         find_source_files(config)

tests/system_tests/FortranPreProcess/test_FortranPreProcess.py

@@ -22,8 +22,7 @@
 
 def build(fab_workspace, fpp_flags=None):
     with BuildConfig(fab_workspace=fab_workspace, tool_box=ToolBox(),
-                     mpi=False, openmp=False, project_label='foo',
-                     multiprocessing=False) as config:
+                     project_label='foo', multiprocessing=False) as config:
         grab_folder(config, Path(__file__).parent / 'project-source')
         find_source_files(config)
         preprocess_fortran(config, common_flags=fpp_flags)

tests/system_tests/MinimalC/test_MinimalC.py

@@ -24,8 +24,7 @@ def test_minimal_c(tmp_path):
 
     # build
     with BuildConfig(fab_workspace=tmp_path, tool_box=ToolBox(),
-                     mpi=False, openmp=False, project_label='foo',
-                     multiprocessing=False) as config:
+                     project_label='foo', multiprocessing=False) as config:
 
         grab_folder(config, PROJECT_SOURCE)
         find_source_files(config)