Merge branch 'main' into how_to_algo_selection

optimagic-dev · Oct 31, 2024 · a49433f · a49433f
2 parents 8d6a7a1 + cc049d1
commit a49433f
Show file tree

Hide file tree

Showing 26 changed files with 217 additions and 57 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -70,9 +70,10 @@ jobs:
           micromamba activate optimagic
           pytest -m "not slow and not jax"
   run-tests-with-old-pandas:
-    # This job is only for testing if optimagic works with older pandas versions, as
-    # many pandas functions we use will be deprecated in pandas 3. optimagic's behavior
-    # for older verions is handled in src/optimagic/compat.py.
+    # This job is only for testing if optimagic works with pandas<2, as many pandas
+    # functions we use will be deprecated in pandas 3. optimagic's behavior for older
+    # verions is handled in src/optimagic/compat.py. For compatibility with we have to
+    # restrict numpy<2.
     name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with pandas 1
     runs-on: ${{ matrix.os }}
     strategy:
@@ -96,6 +97,32 @@ jobs:
         run: |
           micromamba activate optimagic
           pytest -m "not slow and not jax"
+  run-tests-with-old-numpy:
+    # This job is only for testing if optimagic works with numpy<2. Because we already
+    # test pandas<2 with numpy<2, in this environment we restrict pandas>=2.
+    name: Run tests for ${{ matrix.os}} on ${{ matrix.python-version }} with numpy 1
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+        python-version:
+          - '3.10'
+    steps:
+      - uses: actions/checkout@v4
+      - name: create build environment
+        uses: mamba-org/setup-micromamba@v1
+        with:
+          environment-file: ./.tools/envs/testenv-numpy.yml
+          cache-environment: true
+          create-args: |
+            python=${{ matrix.python-version }}
+      - name: run pytest
+        shell: bash -l {0}
+        run: |
+          micromamba activate optimagic
+          pytest -m "not slow and not jax"
   code-in-docs:
     name: Run code snippets in documentation
     runs-on: ubuntu-latest

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -19,7 +19,7 @@ repos:
         always_run: true
         require_serial: true
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-added-large-files
         args:
@@ -56,7 +56,7 @@ repos:
       - id: yamllint
         exclude: tests/optimagic/optimizers/_pounders/fixtures
   - repo: https://github.com/PyCQA/docformatter
-    rev: v1.7.5
+    rev: eb1df34
     hooks:
       - id: docformatter
         args:
@@ -68,7 +68,7 @@ repos:
           - --blank
         exclude: src/optimagic/optimization/algo_options.py
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.6.3
+    rev: v0.7.1
     hooks:
       # Run the linter.
       - id: ruff
@@ -85,7 +85,7 @@ repos:
           - pyi
           - jupyter
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.18
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -97,7 +97,7 @@ repos:
           - '88'
         files: (README\.md)
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.18
     hooks:
       - id: mdformat
         additional_dependencies:
@@ -119,12 +119,12 @@ repos:
         args:
           - --drop-empty-cells
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.11.2
+    rev: v1.13.0
     hooks:
       - id: mypy
         files: src|tests
         additional_dependencies:
-          - numpy<2.0
+          - numpy
           - packaging
           - pandas-stubs
           - sqlalchemy-stubs

diff --git a/.tools/envs/testenv-linux.yml b/.tools/envs/testenv-linux.yml
@@ -16,19 +16,19 @@ dependencies:
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum >= 0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests
   - pip:  # dev, tests, docs
-      - DFO-LS  # dev, tests
+      - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests

diff --git a/.tools/envs/testenv-numpy.yml b/.tools/envs/testenv-numpy.yml
@@ -0,0 +1,37 @@
+---
+name: optimagic
+channels:
+  - conda-forge
+  - nodefaults
+dependencies:
+  - pandas>=2
+  - numpy<2
+  - cyipopt>=1.4.0  # dev, tests
+  - pygmo>=2.19.0  # dev, tests
+  - nlopt  # dev, tests
+  - pip  # dev, tests, docs
+  - pytest  # dev, tests
+  - pytest-cov  # tests
+  - pytest-xdist  # dev, tests
+  - statsmodels  # dev, tests
+  - cloudpickle  # run, tests
+  - joblib  # run, tests
+  - plotly  # run, tests
+  - pybaum >= 0.1.2  # run, tests
+  - scipy>=1.2.1  # run, tests
+  - sqlalchemy  # run, tests
+  - seaborn  # dev, tests
+  - mypy=1.13  # dev, tests
+  - pyyaml  # dev, tests
+  - jinja2  # dev, tests
+  - annotated-types  # dev, tests
+  - pip:  # dev, tests, docs
+      - DFO-LS>=1.5.3  # dev, tests
+      - Py-BOBYQA  # dev, tests
+      - fides==0.7.4  # dev, tests
+      - kaleido  # dev, tests
+      - types-cffi  # dev, tests
+      - types-openpyxl  # dev, tests
+      - types-jinja2  # dev, tests
+      - sqlalchemy-stubs  # dev, tests
+      - -e ../../
diff --git a/.tools/envs/testenv-others.yml b/.tools/envs/testenv-others.yml
@@ -14,19 +14,19 @@ dependencies:
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum >= 0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests
   - pip:  # dev, tests, docs
-      - DFO-LS  # dev, tests
+      - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests

diff --git a/.tools/envs/testenv-pandas.yml b/.tools/envs/testenv-pandas.yml
@@ -4,7 +4,8 @@ channels:
   - conda-forge
   - nodefaults
 dependencies:
-  - pandas<2.0.0
+  - pandas<2
+  - numpy<2
   - cyipopt>=1.4.0  # dev, tests
   - pygmo>=2.19.0  # dev, tests
   - nlopt  # dev, tests
@@ -15,18 +16,17 @@ dependencies:
   - statsmodels  # dev, tests
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
   - plotly  # run, tests
   - pybaum >= 0.1.2  # run, tests
   - scipy>=1.2.1  # run, tests
   - sqlalchemy  # run, tests
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - annotated-types  # dev, tests
   - pip:  # dev, tests, docs
-      - DFO-LS  # dev, tests
+      - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests

diff --git a/.tools/update_envs.py b/.tools/update_envs.py
@@ -33,20 +33,27 @@ def main():
     ## test environment others
     test_env_others = deepcopy(test_env)
 
-    ## test environment for pandas version 1
+    ## test environment for pandas version < 2 (requires numpy < 2)
     test_env_pandas = deepcopy(test_env)
-    test_env_pandas = [line for line in test_env_pandas if "pandas" not in line]
-    test_env_pandas.insert(_insert_idx, "  - pandas<2.0.0")
-
-    # create docs testing environment
-
+    for pkg in ["numpy", "pandas"]:
+        test_env_pandas = [line for line in test_env_pandas if pkg not in line]
+        test_env_pandas.insert(_insert_idx, f"  - {pkg}<2")
+
+    ## test environment for numpy version < 2 (with pandas >= 2)
+    test_env_numpy = deepcopy(test_env)
+    for pkg in ["numpy", "pandas"]:
+        test_env_numpy = [line for line in test_env_numpy if pkg not in line]
+    test_env_numpy.insert(_insert_idx, "  - numpy<2")
+    test_env_numpy.insert(_insert_idx, "  - pandas>=2")
+
+    # test environment for documentation
     docs_env = [line for line in lines if _keep_line(line, "docs")]
     docs_env.append("      - -e ../../")  # add local installation
 
     # write environments
     for name, env in zip(
-        ["linux", "others", "pandas"],
-        [test_env_linux, test_env_others, test_env_pandas],
+        ["linux", "others", "pandas", "numpy"],
+        [test_env_linux, test_env_others, test_env_pandas, test_env_numpy],
         strict=False,
     ):
         # Specify newline to avoid wrong line endings on Windows.

diff --git a/docs/source/algorithms.md b/docs/source/algorithms.md
@@ -1083,6 +1083,8 @@ install each of them separately:
 ```{eval-rst}
 .. dropdown::  nag_dfols
 
+    *Note*: We recommend to install `DFO-LS` version 1.5.3 or higher. Versions of 1.5.0 or lower also work but the versions `1.5.1` and `1.5.2` contain bugs that can lead to errors being raised.
+
     .. code-block::
 
         "nag_dfols"

diff --git a/docs/source/how_to/how_to_multistart.ipynb b/docs/source/how_to/how_to_multistart.ipynb
@@ -436,7 +436,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "np.row_stack(res.multistart_info.exploration_sample).shape"
+    "np.vstack(res.multistart_info.exploration_sample).shape"
    ]
   },
   {

diff --git a/docs/source/installation.md b/docs/source/installation.md
@@ -40,6 +40,10 @@ pip install Py-BOBYQA
 pip install DFO-LS
 ```
 
+*Note*: We recommend to install `DFO-LS` version 1.5.3 or higher. Versions of 1.5.0 or
+lower also work but the versions `1.5.1` and `1.5.2` contain bugs that can lead to
+errors being raised.
+
 ```
 conda install -c conda-forge petsc4py
 ```

diff --git a/environment.yml b/environment.yml
@@ -19,7 +19,7 @@ dependencies:
   - toml  # dev
   - cloudpickle  # run, tests
   - joblib  # run, tests
-  - numpy<2.0  # run, tests
+  - numpy >= 2  # run, tests
   - pandas  # run, tests
   - plotly  # run, tests
   - pybaum >= 0.1.2  # run, tests
@@ -32,17 +32,17 @@ dependencies:
   - sphinx-panels  # docs
   - sphinxcontrib-bibtex  # docs
   - seaborn  # dev, tests
-  - mypy>=1.11  # dev, tests
+  - mypy=1.13  # dev, tests
   - pyyaml  # dev, tests
   - jinja2  # dev, tests
   - furo  # dev, docs
   - annotated-types  # dev, tests
   - pip:  # dev, tests, docs
-      - DFO-LS  # dev, tests
+      - DFO-LS>=1.5.3  # dev, tests
       - Py-BOBYQA  # dev, tests
       - fides==0.7.4  # dev, tests
       - kaleido  # dev, tests
-      - pre-commit  # dev
+      - pre-commit>=4  # dev
       - -e .  # dev
       # type stubs
       - pandas-stubs  # dev, tests

diff --git a/pyproject.toml b/pyproject.toml
@@ -8,7 +8,7 @@ requires-python = ">=3.10"
 dependencies = [
     "cloudpickle",
     "joblib",
-    "numpy<2.0",
+    "numpy",
     "pandas",
     "plotly",
     "pybaum>=0.1.2",

diff --git a/src/estimagic/estimate_ml.py b/src/estimagic/estimate_ml.py
@@ -95,8 +95,8 @@ def estimate_ml(
             optimize_options to False. Pytrees can be a numpy array, a pandas Series, a
             DataFrame with "value" column, a float and any kind of (nested) dictionary
             or list containing these elements. See :ref:`params` for examples.
-        optimize_options (dict, str or False): Keyword arguments that govern the
-            numerical optimization. Valid entries are all arguments of
+        optimize_options (dict, Algorithm, str or False): Keyword arguments that govern
+            the numerical optimization. Valid entries are all arguments of
             :func:`~estimagic.optimization.optimize.minimize` except for those that are
             passed explicilty to ``estimate_ml``. If you pass False as optimize_options
             you signal that ``params`` are already the optimal parameters and no
@@ -199,7 +199,10 @@ def estimate_ml(
     is_optimized = optimize_options is False
 
     if not is_optimized:
-        if isinstance(optimize_options, str):
+        # If optimize_options is not a dictionary and not False, we assume it represents
+        # an algorithm. The actual testing of whether it is a valid algorithm is done
+        # when `maximize` is called.
+        if not isinstance(optimize_options, dict):
             optimize_options = {"algorithm": optimize_options}
 
         check_optimization_options(

diff --git a/src/estimagic/estimate_msm.py b/src/estimagic/estimate_msm.py
@@ -107,8 +107,8 @@ def estimate_msm(
             optimize_options to False. Pytrees can be a numpy array, a pandas Series, a
             DataFrame with "value" column, a float and any kind of (nested) dictionary
             or list containing these elements. See :ref:`params` for examples.
-        optimize_options (dict, str or False): Keyword arguments that govern the
-            numerical optimization. Valid entries are all arguments of
+        optimize_options (dict, Algorithm, str or False): Keyword arguments that govern
+            the numerical optimization. Valid entries are all arguments of
             :func:`~estimagic.optimization.optimize.minimize` except for those that can
             be passed explicitly to ``estimate_msm``.  If you pass False as
             ``optimize_options`` you signal that ``params`` are already
@@ -199,7 +199,10 @@ def estimate_msm(
     is_optimized = optimize_options is False
 
     if not is_optimized:
-        if isinstance(optimize_options, str):
+        # If optimize_options is not a dictionary and not False, we assume it represents
+        # an algorithm. The actual testing of whether it is a valid algorithm is done
+        # when `minimize` is called.
+        if not isinstance(optimize_options, dict):
             optimize_options = {"algorithm": optimize_options}
 
         check_optimization_options(

diff --git a/src/optimagic/algorithms.py b/src/optimagic/algorithms.py
@@ -40,10 +40,12 @@
         name = candidate.__algo_info__.name
         if issubclass(candidate, Algorithm) and candidate is not Algorithm:
             ALL_ALGORITHMS[name] = candidate
-            if candidate.__algo_info__.is_available:
+            if candidate.__algo_info__.is_available:  # type: ignore[attr-defined]
                 AVAILABLE_ALGORITHMS[name] = candidate
 
 
 GLOBAL_ALGORITHMS = [
-    name for name, algo in ALL_ALGORITHMS.items() if algo.__algo_info__.is_global
+    name
+    for name, algo in ALL_ALGORITHMS.items()
+    if algo.__algo_info__.is_global  # type: ignore[attr-defined]
 ]
diff --git a/src/optimagic/differentiation/richardson_extrapolation.py b/src/optimagic/differentiation/richardson_extrapolation.py
@@ -272,7 +272,7 @@ def _compute_step_ratio(steps):
 
     """
     ratios = steps[1:, :] / steps[:-1, :]
-    ratios = ratios[np.isfinite(ratios)]
+    finite_ratios = ratios[np.isfinite(ratios)]
 
-    step_ratio = ratios.flat[0]
+    step_ratio = finite_ratios.item(0)
     return step_ratio