Merge branch 'research' into feature_calculate_forcings

.cirun.yml

@@ -0,0 +1,16 @@
+# setup for using github runners via https://cirun.io/
+runners:
+  - name: "aws-runner"
+    # Cloud Provider: AWS
+    cloud: "aws"
+    # https://aws.amazon.com/ec2/instance-types/g4/
+    instance_type: "g4ad.xlarge"
+    # Deep Learning Base OSS Nvidia Driver GPU AMI (Ubuntu 22.04), Frankfurt region
+    machine_image: "ami-0ba41b554b28d24a4"
+    # use Frankfurt region
+    region: "eu-central-1"
+    preemptible: false
+    # Add this label in the "runs-on" param in .github/workflows/<workflow-name>.yml
+    # So that this runner is created for running the workflow
+    labels:
+      - "cirun-aws-runner"

.flake8

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 88
+ignore = E203, F811, I002, W503

.github/pull_request_template.md

@@ -0,0 +1,52 @@
+## Describe your changes
+
+< Summary of the changes.>
+
+< Please also include relevant motivation and context. >
+
+< List any dependencies that are required for this change. >
+
+## Issue Link
+
+< Link to the relevant issue or task. > (e.g. `closes #00` or `solves #00`)
+
+## Type of change
+
+- [ ] 🐛 Bug fix (non-breaking change that fixes an issue)
+- [ ] ✨ New feature (non-breaking change that adds functionality)
+- [ ] 💥 Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ] 📖 Documentation (Addition or improvements to documentation)
+
+## Checklist before requesting a review
+
+- [ ] My branch is up-to-date with the target branch - if not update your fork with the changes from the target branch (use `pull` with `--rebase` option if possible).
+- [ ] I have performed a self-review of my code
+- [ ] For any new/modified functions/classes I have added docstrings that clearly describe its purpose, expected inputs and returned values
+- [ ] I have placed in-line comments to clarify the intent of any hard-to-understand passages of my code
+- [ ] I have updated the [README](README.MD) to cover introduced code changes
+- [ ] I have added tests that prove my fix is effective or that my feature works
+- [ ] I have given the PR a name that clearly describes the change, written in imperative form ([context](https://www.gitkraken.com/learn/git/best-practices/git-commit-message#using-imperative-verb-form)).
+- [ ] I have requested a reviewer and an assignee (assignee is responsible for merging). This applies only if you have write access to the repo, otherwise feel free to tag a maintainer to add a reviewer and assignee.
+
+## Checklist for reviewers
+
+Each PR comes with its own improvements and flaws. The reviewer should check the following:
+- [ ] the code is readable
+- [ ] the code is well tested
+- [ ] the code is documented (including return types and parameters)
+- [ ] the code is easy to maintain
+
+## Author checklist after completed review
+
+- [ ] I have added a line to the CHANGELOG describing this change, in a section
+  reflecting type of change (add section where missing):
+  - *added*: when you have added new functionality
+  - *changed*: when default behaviour of the code has been changed
+  - *fixes*: when your contribution fixes a bug
+
+## Checklist for assignee
+
+- [ ] PR is up to date with the base branch
+- [ ] the tests pass
+- [ ] author has added an entry to the changelog (and designated the change as *added*, *changed* or *fixed*)
+- Once the PR is ready to be merged, squash commits and merge the PR.

.github/workflows/ci-pdm-install-and-test-cpu.yml

@@ -0,0 +1,55 @@
+# cicd workflow for running tests with pytest
+# needs to first install pdm, then install torch cpu manually and then install the package
+# then run the tests
+
+name: test (pdm install, cpu)
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install pdm
+        run: |
+          python -m pip install pdm
+
+      - name: Create venv
+        run: |
+          pdm venv create --with-pip
+          pdm use --venv in-project
+
+      - name: Install torch (CPU)
+        run: |
+          pdm run python -m pip install torch  --index-url https://download.pytorch.org/whl/cpu
+          # check that the CPU version is installed
+
+      - name: Install package (including dev dependencies)
+        run: |
+          pdm install --group :all
+
+      - name: Print and check torch version
+        run: |
+          pdm run python -c "import torch; print(torch.__version__)"
+          pdm run python -c "import torch; assert torch.__version__.endswith('+cpu')"
+
+      - name: Load cache data
+        uses: actions/cache/restore@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+          restore-keys: |
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+
+      - name: Run tests
+        run: |
+          pdm run pytest -vv -s
+
+      - name: Save cache data
+        uses: actions/cache/save@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/ci-pdm-install-and-test-gpu.yml

@@ -0,0 +1,60 @@
+# cicd workflow for running tests with pytest
+# needs to first install pdm, then install torch cpu manually and then install the package
+# then run the tests
+
+name: test (pdm install, gpu)
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: "cirun-aws-runner--${{ github.run_id }}"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10
+
+      - name: Install pdm
+        run: |
+          python -m pip install pdm
+
+      - name: Create venv
+        run: |
+          pdm config venv.in_project False
+          pdm config venv.location /opt/dlami/nvme/venv
+          pdm venv create --with-pip
+
+      - name: Install torch (GPU CUDA 12.1)
+        run: |
+          pdm run python -m pip install torch  --index-url https://download.pytorch.org/whl/cu121
+
+      - name: Print and check torch version
+        run: |
+          pdm run python -c "import torch; print(torch.__version__)"
+          pdm run python -c "import torch; assert not torch.__version__.endswith('+cpu')"
+
+      - name: Install package (including dev dependencies)
+        run: |
+          pdm install --group :all
+
+      - name: Load cache data
+        uses: actions/cache/restore@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+          restore-keys: |
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+
+      - name: Run tests
+        run: |
+          pdm run pytest -vv -s
+
+      - name: Save cache data
+        uses: actions/cache/save@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/ci-pip-install-and-test-cpu.yml

@@ -0,0 +1,45 @@
+# cicd workflow for running tests with pytest
+# needs to first install pdm, then install torch cpu manually and then install the package
+# then run the tests
+
+name: test (pip install, cpu)
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Install torch (CPU)
+        run: |
+          python -m pip install torch  --index-url https://download.pytorch.org/whl/cpu
+
+      - name: Install package (including dev dependencies)
+        run: |
+          python -m pip install ".[dev]"
+
+      - name: Print and check torch version
+        run: |
+          python -c "import torch; print(torch.__version__)"
+          python -c "import torch; assert torch.__version__.endswith('+cpu')"
+
+      - name: Load cache data
+        uses: actions/cache/restore@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+          restore-keys: |
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+
+      - name: Run tests
+        run: |
+          python -m pytest -vv -s
+
+      - name: Save cache data
+        uses: actions/cache/save@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/ci-pip-install-and-test-gpu.yml

@@ -0,0 +1,50 @@
+# cicd workflow for running tests with pytest
+# needs to first install pdm, then install torch cpu manually and then install the package
+# then run the tests
+
+name: test (pip install, gpu)
+
+on: [push, pull_request]
+
+jobs:
+  tests:
+    runs-on: "cirun-aws-runner--${{ github.run_id }}"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.10
+
+      - name: Install torch (GPU CUDA 12.1)
+        run: |
+          python -m pip install torch  --index-url https://download.pytorch.org/whl/cu121
+
+      - name: Install package (including dev dependencies)
+        run: |
+          python -m pip install ".[dev]"
+
+      - name: Print and check torch version
+        run: |
+          python -c "import torch; print(torch.__version__)"
+          python -c "import torch; assert not torch.__version__.endswith('+cpu')"
+
+      - name: Load cache data
+        uses: actions/cache/restore@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+          restore-keys: |
+            ${{ runner.os }}-meps-reduced-example-data-v0.2.0
+
+      - name: Run tests
+        run: |
+          python -m pytest -vv -s
+
+      - name: Save cache data
+        uses: actions/cache/save@v4
+        with:
+          path: tests/datastore_examples/npyfilesmeps/meps_example_reduced.zip
+          key: ${{ runner.os }}-meps-reduced-example-data-v0.2.0

.github/workflows/pre-commit.yml

@@ -1,33 +1,23 @@
-name: Run pre-commit job
+name: Linting
 
 on:
- push:
+  # trigger on pushes to any branch
+  push:
+  # and also on PRs to main
+  pull_request:
     branches:
-    - main
- pull_request:
-    branches:
-    - main
+      - main
 
 jobs:
- pre-commit-job:
+  pre-commit-job:
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        shell: bash -l {0}
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11"]
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9
-    - name: Install pre-commit hooks
-      run: |
-        pip install torch==2.0.1 torchvision==0.15.2 torchaudio==2.0.2 \
-          --index-url https://download.pytorch.org/whl/cpu
-        pip install -r requirements.txt
-        pip install pyg-lib==0.2.0 torch-scatter==2.1.1 torch-sparse==0.6.17 \
-          torch-cluster==1.6.1 torch-geometric==2.3.1 \
-          -f https://pytorch-geometric.com/whl/torch-2.0.1+cpu.html
-    - name: Run pre-commit hooks
-      run: |
-        pre-commit run --all-files
+        python-version: ${{ matrix.python-version }}
+    - uses: pre-commit/action@v2.0.3

.gitignore

@@ -1,18 +1,15 @@
 ### Project Specific ###
 wandb
-slurm_log*
 saved_models
+lightning_logs
 data
 graphs
 *.sif
 sweeps
-test_*.sh
-cosmo_hilam.html
-.gitignore
 .vscode
+*.html
 *.zarr
-*.png
-shps
+*slurm*
 
 ### Python ###
 # Byte-compiled / optimized / DLL files
@@ -79,4 +76,14 @@ tags
 # Coc configuration directory
 .vim
 .vscode
-boundary_mask_donut.png
+
+# macos
+.DS_Store
+__MACOSX
+
+# pdm (https://pdm-project.org/en/stable/)
+.pdm-python
+.venv
+
+# exclude pdm.lock file so that both cpu and gpu versions of torch will be accepted by pdm
+pdm.lock