Merge branch 'develop' into develop

.github/actions/tr_post_test_run/action.yml

@@ -16,7 +16,7 @@ runs:
       if: ${{ always() }}
       run: |
         export PYTHONPATH="$PYTHONPATH:."
-        python tests/end_to_end/utils/summary_helper.py
+        python tests/end_to_end/utils/summary_helper.py --func_name "print_task_runner_score"
         echo "Test summary printed"
       shell: bash
 

.github/workflows/federated_runtime.yml

@@ -0,0 +1,66 @@
+#---------------------------------------------------------------------------
+# Workflow to run 301_MNIST_Watermarking notebook
+# Authors - Noopur, Payal Chaurasiya
+#---------------------------------------------------------------------------
+name: Federated Runtime 301 MNIST Watermarking
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test_federated_runtime_301_watermarking_notebook:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    steps:
+    - name: Checkout OpenFL repository
+      uses: actions/checkout@v4.1.1
+      with:
+        fetch-depth: 2 # needed for detecting changes
+        submodules: "true"
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+
+    - name: Install dependencies # Without this step, fx command will not work
+      id: install_dependencies
+      run: |
+        python -m pip install --upgrade pip ipython ipykernel
+        pip install .
+        pip install -r test-requirements.txt
+
+    - name: Run Federated Runtime 301 MNIST Watermarking via pytest
+      id: run_tests
+      run: |
+        python -m pytest -s tests/end_to_end/test_suites/wf_federated_runtime_tests.py -k test_federated_runtime_301_watermarking
+        echo "Federated Runtime 301 MNIST Watermarking test run completed"
+
+    - name: Print test summary
+      id: print_test_summary
+      if: ${{ always() }}
+      run: |
+        export PYTHONPATH="$PYTHONPATH:."
+        python tests/end_to_end/utils/summary_helper.py --func_name "print_federated_runtime_score"
+        echo "Test summary printed"
+
+    - name: Tar files
+      if: ${{ always() }}   # collect artifacts regardless of failures
+      run: |
+        tar -cvf notebook_301.tar --exclude="__pycache__" $HOME/results --ignore-failed-read
+        echo "TAR file created"
+
+    - name: Upload Artifacts
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}   # collect artifacts regardless of failures
+      with:
+        name: federated_runtime_301_watermarking_${{ github.run_id }}
+        path: notebook_301.tar

.github/workflows/lint.yml

@@ -25,6 +25,6 @@ jobs:
     - name: Install linters
       run: |
         python -m pip install --upgrade pip
-        pip install -r linters-requirements.txt 
+        pip install -r linters-requirements.txt
     - name: Lint with OpenFL-specific rules
       run: bash scripts/lint.sh

.github/workflows/straggler-handling.yml

@@ -21,7 +21,7 @@ jobs:
       matrix:
        os: ['ubuntu-latest', 'windows-latest']
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 15
+    timeout-minutes: 30
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/task_runner_basic_e2e.yml

@@ -31,7 +31,7 @@ jobs:
   test_with_tls:
     name: tr_tls
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # There are open issues for some of the models, so excluding them for now:
@@ -74,7 +74,7 @@ jobs:
   test_with_non_tls:
     name: tr_non_tls
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
@@ -117,7 +117,7 @@ jobs:
   test_with_no_client_auth:
     name: tr_no_client_auth
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
@@ -160,7 +160,7 @@ jobs:
   test_memory_logs:
     name: tr_tls_memory_logs
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10

.github/workflows/taskrunner.yml

@@ -17,29 +17,19 @@ env:
 jobs:
   build:
     if: github.event.pull_request.draft == false
-    strategy:
-      matrix:
-       os: ['ubuntu-latest', 'windows-latest']
-       python-version: ["3.10", "3.11", "3.12"]
-    runs-on: ${{ matrix.os }}
-    timeout-minutes: 15
-
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 
       uses: actions/setup-python@v4
       with:
-        python-version: ${{ matrix.python-version }}
+        python-version: "3.10"
     - name: Install dependencies ubuntu
-      if: matrix.os == 'ubuntu-latest'
-      run: |
-        python -m pip install --upgrade pip
-        pip install .
-    - name: Install dependencies windows
-      if: matrix.os == 'windows-latest'
       run: |
         python -m pip install --upgrade pip
         pip install .
-    - name: Test TaskRunner API
+    - name: Task Runner API
       run: |
-        python -m tests.github.test_hello_federation --template keras_cnn_mnist --fed_workspace aggregator --col1 col1 --col2 col2  --rounds-to-train 3 --save-model output_model
+          python -m tests.github.test_hello_federation --template torch_cnn_mnist --fed_workspace aggregator --col1 collaborator1 --col2 collaborator2  --rounds-to-train 3 --save-model output_model

.github/workflows/ubuntu.yml

@@ -13,6 +13,9 @@ env:
 
 jobs:
   pytest-coverage: # from pytest_coverage.yml
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
     runs-on: ubuntu-latest
     timeout-minutes: 15
 
@@ -21,7 +24,7 @@ jobs:
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/wf_functional_e2e.yml

@@ -29,9 +29,9 @@ env:
   NUM_COLLABORATORS: ${{ github.event.inputs.num_collaborators || '2' }}
 
 jobs:
-  test_wf_func:
+  test_wf_functional_local_runtime:
     if: github.event.pull_request.draft == false
-    name: wf_func
+    name: wf_functional_local_runtime
     runs-on: ubuntu-22.04
     timeout-minutes: 15
     strategy:
@@ -74,7 +74,7 @@ jobs:
         if: ${{ always() }}
         run: |
           export PYTHONPATH="$PYTHONPATH:."
-          python tests/end_to_end/utils/summary_helper.py
+          python tests/end_to_end/utils/summary_helper.py --func_name "print_local_runtime_score"
           echo "Test summary printed"
 
       - name: Create Tar (exclude cert and data folders)

.github/workflows/windows.yml

@@ -13,14 +13,18 @@ env:
 
 jobs:
   pytest-coverage: # from pytest_coverage.yml
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
     runs-on: windows-latest
     timeout-minutes: 15
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 3
       uses: actions/setup-python@v3
       with:
-        python-version: "3.10"
+        python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/workflow_interface_101_mnist.yml

@@ -17,7 +17,7 @@ jobs:
   run_notebook:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     steps:
     - name: Checkout OpenFL repository
       uses: actions/checkout@v4.1.1

docs/about/features_index/taskrunner.rst

@@ -44,8 +44,9 @@ Configurable Settings
  - :code:`best_state_path`: (str:path) Defines the weight protobuf file path that will be saved to for the highest accuracy model during the experiment.
  - :code:`last_state_path`: (str:path)  Defines the weight protobuf file path that will be saved to during the last round completed in each experiment.
  - :code:`rounds_to_train`: (int) Specifies the number of rounds in a federation. A federated learning round is defined as one complete iteration when the collaborators train the model and send the updated model weights back to the aggregator to form a new global model. Within a round, collaborators can train the model for multiple iterations called epochs.
- - :code:`write_logs`: (boolean) Metric logging callback feature. By default, logging is done through `tensorboard <https://www.tensorflow.org/tensorboard/get_started>`_ but users can also use custom metric logging function for each task.      
-
+ - :code:`write_logs`: (boolean) Metric logging callback feature. By default, logging is done through `tensorboard <https://www.tensorflow.org/tensorboard/get_started>`_ but users can also use custom metric logging function for each task.     
+ - :code:`persist_checkpoint`: (boolean) Specifies whether to enable the storage of a persistent checkpoint in non-volatile storage for recovery purposes. When enabled, the aggregator will restore its state to what it was prior to the restart, ensuring continuity after a restart. 
+ - :code:`persistent_db_path`: (str:path) Defines the persisted database path. 
 
 - :class:`Collaborator <openfl.component.Collaborator>`
     `openfl.component.Collaborator <https://github.com/intel/openfl/blob/develop/openfl/component/collaborator/collaborator.py>`_

linters-requirements.txt

@@ -1,2 +1,2 @@
 pre-commit
-ruff==0.8.1
+ruff==0.9.1

openfl-tutorials/experimental/workflow/105_Numpy_Linear_Regression_Workflow.ipynb

@@ -42,10 +42,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Below code will display the print statement output on screen as well\n",
-    "import sys\n",
-    "sys.stdout = open('/dev/stdout', 'w')\n",
-    "\n",
     "!pip install git+https://github.com/securefederatedai/openfl.git\n",
     "!pip install -r workflow_interface_requirements.txt\n",
     "!pip install matplotlib\n",
@@ -308,7 +304,7 @@
     "        self.current_round += 1\n",
     "        if self.current_round < self.rounds:\n",
     "            self.next(self.aggregated_model_validation,\n",
-    "                      foreach='collaborators', exclude=['private'])\n",
+    "                      foreach='collaborators')\n",
     "        else:\n",
     "            self.next(self.end)\n",
     "\n",

openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermaking/workspace/MNIST_Watermarking.ipynb → openfl-tutorials/experimental/workflow/FederatedRuntime/301_MNIST_Watermarking/workspace/MNIST_Watermarking.ipynb

@@ -39,7 +39,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "d79eacbd",
    "metadata": {},
    "outputs": [],
@@ -66,7 +66,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "f7475cba",
    "metadata": {},
    "outputs": [],
@@ -95,7 +95,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "9bd8ac2d",
    "metadata": {},
    "outputs": [],
@@ -193,7 +193,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "89cf4866",
    "metadata": {},
    "outputs": [],
@@ -245,7 +245,7 @@
     "        watermark_pretrain_optimizer=None,\n",
     "        watermark_retrain_optimizer=None,\n",
     "        round_number=0,\n",
-    "        n_rounds=1,\n",
+    "        n_rounds=3,\n",
     "        **kwargs,\n",
     "    ):\n",
     "        super().__init__(**kwargs)\n",
@@ -425,7 +425,20 @@
     "                  + f\" Acc: {self.watermark_retrain_validation_score:<.6f}\")\n",
     "            retrain_round += 1\n",
     "\n",
-    "        self.next(self.end)\n",
+    "        self.next(self.internal_loop)\n",
+    "    \n",
+    "    @aggregator\n",
+    "    def internal_loop(self):\n",
+    "        \"\"\"\n",
+    "        Internal loop to continue the Federated Learning process.\n",
+    "        \"\"\"\n",
+    "        if self.round_number == self.n_rounds - 1:\n",
+    "            print(f\"\\nCompleted training for all {self.n_rounds} round(s)\")\n",
+    "            self.next(self.end)\n",
+    "        else:\n",
+    "            self.round_number += 1\n",
+    "            print(f\"\\nCompleted round: {self.round_number}\")\n",
+    "            self.next(self.aggregated_model_validation, foreach='collaborators')\n",
     "\n",
     "    @aggregator\n",
     "    def end(self):\n",
@@ -449,7 +462,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "1715a373",
    "metadata": {},
    "outputs": [],
@@ -468,7 +481,7 @@
     "federated_runtime = FederatedRuntime(\n",
     "    collaborators=authorized_collaborators,\n",
     "    director=director_info, \n",
-    "    notebook_path='./MNIST_Watermarking.ipynb'\n",
+    "    notebook_path='./MNIST_Watermarking.ipynb',\n",
     ")"
    ]
   },
@@ -552,7 +565,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "fed_run",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -566,7 +579,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

openfl-workspace/workspace/plan/defaults/aggregator.yaml

@@ -1,3 +1,5 @@
 template : openfl.component.Aggregator
 settings :
     db_store_rounds : 2
+    persist_checkpoint: True
+    persistent_db_path: save/tensor.db