FederatedRuntime Workflow for CI Pipeline - 301 Watermarking notebook…

… run (securefederatedai#1267) * FederatedRuntime Workflow for CI Pipeline - 301 Watermarking notebook run Signed-off-by: noopur <noopur@intel.com> * Display output on screen Signed-off-by: noopur <noopur@intel.com> * 5 Rounds Signed-off-by: noopur <noopur@intel.com> * Removed extra ) bracket Signed-off-by: noopur <noopur@intel.com> * Timeout of 30 min due to 5 rounds Signed-off-by: noopur <noopur@intel.com> * End the loop after all rounds Signed-off-by: noopur <noopur@intel.com> * Review comments incor Signed-off-by: noopur <noopur@intel.com> * Retry the envoy fetch Signed-off-by: noopur <noopur@intel.com> * Added invalid code just to verify negative scenario Signed-off-by: noopur <noopur@intel.com> * Added invalid code just to verify negative scenario Signed-off-by: noopur <noopur@intel.com> * 20m job timeout Signed-off-by: noopur <noopur@intel.com> * Revert invalid code and stdout notebook run Signed-off-by: noopur <noopur@intel.com> * Use markdown with stdout Signed-off-by: noopur <noopur@intel.com> * Induced error for testing Signed-off-by: noopur <noopur@intel.com> * Reverted error, added 10s sleep in github fetch logic Signed-off-by: noopur <noopur@intel.com> * Code format check Signed-off-by: noopur <noopur@intel.com> * pytest for the notebook Signed-off-by: noopur <noopur@intel.com> * Pip install ipython ipykernel Signed-off-by: noopur <noopur@intel.com> * Minor changes Signed-off-by: noopur <noopur@intel.com> * Test summary step corrected for wf_functional_e2e workflow Signed-off-by: noopur <noopur@intel.com> * 3 rounds instead of 5 Signed-off-by: noopur <noopur@intel.com> * Job name change Signed-off-by: noopur <noopur@intel.com> * Review comments incorp Signed-off-by: noopur <noopur@intel.com> * Increased timeout to 30m for CI pipeline jobs Signed-off-by: noopur <noopur@intel.com> * Increased timeout to 30m for CI pipeline jobs Signed-off-by: noopur <noopur@intel.com> --------- Signed-off-by: noopur <noopur@intel.com>
nambi21 · Jan 16, 2025 · b8e2c70 · b8e2c70
1 parent 9658c03
commit b8e2c70
Show file tree

Hide file tree

Showing 28 changed files with 479 additions and 40 deletions.
diff --git a/.github/actions/tr_post_test_run/action.yml b/.github/actions/tr_post_test_run/action.yml
@@ -16,7 +16,7 @@ runs:
       if: ${{ always() }}
       run: |
         export PYTHONPATH="$PYTHONPATH:."
-        python tests/end_to_end/utils/summary_helper.py
+        python tests/end_to_end/utils/summary_helper.py --func_name "print_task_runner_score"
         echo "Test summary printed"
       shell: bash
 

diff --git a/.github/workflows/federated_runtime.yml b/.github/workflows/federated_runtime.yml
@@ -0,0 +1,66 @@
+#---------------------------------------------------------------------------
+# Workflow to run 301_MNIST_Watermarking notebook
+# Authors - Noopur, Payal Chaurasiya
+#---------------------------------------------------------------------------
+name: Federated Runtime 301 MNIST Watermarking
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test_federated_runtime_301_watermarking_notebook:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-22.04
+    timeout-minutes: 20
+    steps:
+    - name: Checkout OpenFL repository
+      uses: actions/checkout@v4.1.1
+      with:
+        fetch-depth: 2 # needed for detecting changes
+        submodules: "true"
+        token: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.10"
+
+    - name: Install dependencies # Without this step, fx command will not work
+      id: install_dependencies
+      run: |
+        python -m pip install --upgrade pip ipython ipykernel
+        pip install .
+        pip install -r test-requirements.txt
+
+    - name: Run Federated Runtime 301 MNIST Watermarking via pytest
+      id: run_tests
+      run: |
+        python -m pytest -s tests/end_to_end/test_suites/wf_federated_runtime_tests.py -k test_federated_runtime_301_watermarking
+        echo "Federated Runtime 301 MNIST Watermarking test run completed"
+
+    - name: Print test summary
+      id: print_test_summary
+      if: ${{ always() }}
+      run: |
+        export PYTHONPATH="$PYTHONPATH:."
+        python tests/end_to_end/utils/summary_helper.py --func_name "print_federated_runtime_score"
+        echo "Test summary printed"
+
+    - name: Tar files
+      if: ${{ always() }}   # collect artifacts regardless of failures
+      run: |
+        tar -cvf notebook_301.tar --exclude="__pycache__" $HOME/results --ignore-failed-read
+        echo "TAR file created"
+
+    - name: Upload Artifacts
+      uses: actions/upload-artifact@v4
+      if: ${{ always() }}   # collect artifacts regardless of failures
+      with:
+        name: federated_runtime_301_watermarking_${{ github.run_id }}
+        path: notebook_301.tar
diff --git a/.github/workflows/straggler-handling.yml b/.github/workflows/straggler-handling.yml
@@ -21,7 +21,7 @@ jobs:
       matrix:
        os: ['ubuntu-latest', 'windows-latest']
     runs-on: ${{ matrix.os }}
-    timeout-minutes: 15
+    timeout-minutes: 30
 
     steps:
     - uses: actions/checkout@v3

diff --git a/.github/workflows/task_runner_basic_e2e.yml b/.github/workflows/task_runner_basic_e2e.yml
@@ -31,7 +31,7 @@ jobs:
   test_with_tls:
     name: tr_tls
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # There are open issues for some of the models, so excluding them for now:
@@ -74,7 +74,7 @@ jobs:
   test_with_non_tls:
     name: tr_non_tls
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
@@ -117,7 +117,7 @@ jobs:
   test_with_no_client_auth:
     name: tr_no_client_auth
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10
@@ -160,7 +160,7 @@ jobs:
   test_memory_logs:
     name: tr_tls_memory_logs
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     strategy:
       matrix:
         # Testing non TLS scenario only for torch_cnn_mnist model and python 3.10

diff --git a/.github/workflows/taskrunner.yml b/.github/workflows/taskrunner.yml
@@ -18,8 +18,8 @@ jobs:
   build:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-latest
-    timeout-minutes: 15
-    
+    timeout-minutes: 30
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python 

diff --git a/.github/workflows/wf_functional_e2e.yml b/.github/workflows/wf_functional_e2e.yml
@@ -29,9 +29,9 @@ env:
   NUM_COLLABORATORS: ${{ github.event.inputs.num_collaborators || '2' }}
 
 jobs:
-  test_wf_func:
+  test_wf_functional_local_runtime:
     if: github.event.pull_request.draft == false
-    name: wf_func
+    name: wf_functional_local_runtime
     runs-on: ubuntu-22.04
     timeout-minutes: 15
     strategy:
@@ -74,7 +74,7 @@ jobs:
         if: ${{ always() }}
         run: |
           export PYTHONPATH="$PYTHONPATH:."
-          python tests/end_to_end/utils/summary_helper.py
+          python tests/end_to_end/utils/summary_helper.py --func_name "print_local_runtime_score"
           echo "Test summary printed"
 
       - name: Create Tar (exclude cert and data folders)

diff --git a/.github/workflows/workflow_interface_101_mnist.yml b/.github/workflows/workflow_interface_101_mnist.yml
@@ -17,7 +17,7 @@ jobs:
   run_notebook:
     if: github.event.pull_request.draft == false
     runs-on: ubuntu-22.04
-    timeout-minutes: 15
+    timeout-minutes: 30
     steps:
     - name: Checkout OpenFL repository
       uses: actions/checkout@v4.1.1

diff --git a/...termaking/Bangalore/Bangalore_config.yaml → ...ermarking/Bangalore/Bangalore_config.yaml b/...termaking/Bangalore/Bangalore_config.yaml → ...ermarking/Bangalore/Bangalore_config.yaml
diff --git a/...termaking/Bangalore/private_attributes.py → ...ermarking/Bangalore/private_attributes.py b/...termaking/Bangalore/private_attributes.py → ...ermarking/Bangalore/private_attributes.py
diff --git a/...ST_Watermaking/Bangalore/requirements.txt → ...T_Watermarking/Bangalore/requirements.txt b/...ST_Watermaking/Bangalore/requirements.txt → ...T_Watermarking/Bangalore/requirements.txt
diff --git a/...NIST_Watermaking/Bangalore/start_envoy.sh → ...IST_Watermarking/Bangalore/start_envoy.sh b/...NIST_Watermaking/Bangalore/start_envoy.sh → ...IST_Watermarking/Bangalore/start_envoy.sh
diff --git a/...Watermaking/Chandler/Chandler_config.yaml → ...atermarking/Chandler/Chandler_config.yaml b/...Watermaking/Chandler/Chandler_config.yaml → ...atermarking/Chandler/Chandler_config.yaml
diff --git a/...atermaking/Chandler/private_attributes.py → ...termarking/Chandler/private_attributes.py b/...atermaking/Chandler/private_attributes.py → ...termarking/Chandler/private_attributes.py
diff --git a/...IST_Watermaking/Chandler/requirements.txt → ...ST_Watermarking/Chandler/requirements.txt b/...IST_Watermaking/Chandler/requirements.txt → ...ST_Watermarking/Chandler/requirements.txt
diff --git a/...MNIST_Watermaking/Chandler/start_envoy.sh → ...NIST_Watermarking/Chandler/start_envoy.sh b/...MNIST_Watermaking/Chandler/start_envoy.sh → ...NIST_Watermarking/Chandler/start_envoy.sh
diff --git a/...edRuntime/301_MNIST_Watermaking/README.md → ...dRuntime/301_MNIST_Watermarking/README.md b/...edRuntime/301_MNIST_Watermaking/README.md → ...dRuntime/301_MNIST_Watermarking/README.md
diff --git a/...Watermaking/director/director_config.yaml → ...atermarking/director/director_config.yaml b/...Watermaking/director/director_config.yaml → ...atermarking/director/director_config.yaml
diff --git a/...atermaking/director/private_attributes.py → ...termarking/director/private_attributes.py b/...atermaking/director/private_attributes.py → ...termarking/director/private_attributes.py
diff --git a/...ST_Watermaking/director/start_director.sh → ...T_Watermarking/director/start_director.sh b/...ST_Watermaking/director/start_director.sh → ...T_Watermarking/director/start_director.sh
diff --git a/...making/workspace/MNIST_Watermarking.ipynb → ...arking/workspace/MNIST_Watermarking.ipynb b/...making/workspace/MNIST_Watermarking.ipynb → ...arking/workspace/MNIST_Watermarking.ipynb
@@ -39,7 +39,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 1,
    "id": "d79eacbd",
    "metadata": {},
    "outputs": [],
@@ -66,7 +66,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "id": "f7475cba",
    "metadata": {},
    "outputs": [],
@@ -95,7 +95,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "id": "9bd8ac2d",
    "metadata": {},
    "outputs": [],
@@ -193,7 +193,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "id": "89cf4866",
    "metadata": {},
    "outputs": [],
@@ -245,7 +245,7 @@
     "        watermark_pretrain_optimizer=None,\n",
     "        watermark_retrain_optimizer=None,\n",
     "        round_number=0,\n",
-    "        n_rounds=1,\n",
+    "        n_rounds=3,\n",
     "        **kwargs,\n",
     "    ):\n",
     "        super().__init__(**kwargs)\n",
@@ -425,7 +425,20 @@
     "                  + f\" Acc: {self.watermark_retrain_validation_score:<.6f}\")\n",
     "            retrain_round += 1\n",
     "\n",
-    "        self.next(self.end)\n",
+    "        self.next(self.internal_loop)\n",
+    "    \n",
+    "    @aggregator\n",
+    "    def internal_loop(self):\n",
+    "        \"\"\"\n",
+    "        Internal loop to continue the Federated Learning process.\n",
+    "        \"\"\"\n",
+    "        if self.round_number == self.n_rounds - 1:\n",
+    "            print(f\"\\nCompleted training for all {self.n_rounds} round(s)\")\n",
+    "            self.next(self.end)\n",
+    "        else:\n",
+    "            self.round_number += 1\n",
+    "            print(f\"\\nCompleted round: {self.round_number}\")\n",
+    "            self.next(self.aggregated_model_validation, foreach='collaborators')\n",
     "\n",
     "    @aggregator\n",
     "    def end(self):\n",
@@ -449,7 +462,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "id": "1715a373",
    "metadata": {},
    "outputs": [],
@@ -468,7 +481,7 @@
     "federated_runtime = FederatedRuntime(\n",
     "    collaborators=authorized_collaborators,\n",
     "    director=director_info, \n",
-    "    notebook_path='./MNIST_Watermarking.ipynb'\n",
+    "    notebook_path='./MNIST_Watermarking.ipynb',\n",
     ")"
    ]
   },
@@ -552,7 +565,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "fed_run",
+   "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
   },
@@ -566,7 +579,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.10.15"
+   "version": "3.10.12"
   }
  },
  "nbformat": 4,

diff --git a/openfl/experimental/workflow/runtime/federated_runtime.py b/openfl/experimental/workflow/runtime/federated_runtime.py
@@ -193,8 +193,12 @@ def get_flow_state(self) -> Tuple[bool, Any]:
 
         return status, flow_object
 
-    def get_envoys(self) -> None:
-        """Prints the status of Envoys in a formatted way."""
+    def get_envoys(self) -> List[str]:
+        """
+        Prints the status of Envoys in a formatted way.
+        Returns:
+            online_envoys (List[str]): List of online envoys.
+        """
         # Fetch envoy data
         envoys = self._dir_client.get_envoys()
         DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S"
@@ -204,6 +208,7 @@ def get_envoys(self) -> None:
         headers = ["Name", "Online", "Last Updated", "Experiment Running", "Experiment Name"]
         # Prepare the table rows
         rows = []
+        online_envoys = []
         for envoy in envoys.envoy_infos:
             rows.append(
                 [
@@ -214,11 +219,15 @@ def get_envoys(self) -> None:
                     envoy.experiment_name if envoy.experiment_name else "None",
                 ]
             )
+            if envoy.is_online:
+                online_envoys.append(envoy.envoy_name)
+
         # Use tabulate to format the table
         result = tabulate(rows, headers=headers, tablefmt="grid")
         # Display the current timestamp
         print(f"Status of Envoys connected to Federation at: {now}\n")
         print(result)
+        return online_envoys
 
     def stream_experiment_stdout(self, experiment_name) -> None:
         """Stream experiment stdout.

diff --git a/test-requirements.txt b/test-requirements.txt
@@ -6,4 +6,5 @@ pytest-asyncio==0.25.2
 pytest-mock==3.14.0
 defusedxml==0.7.1
 matplotlib==3.10.0
-fpdf==1.7.2
+fpdf==1.7.2
+papermill==2.6.0
diff --git a/tests/end_to_end/pytest.ini b/tests/end_to_end/pytest.ini
@@ -8,5 +8,6 @@ markers =
     log_memory_usage: mark a test as a log memory usage test.
     task_runner_basic: mark a test as a task runner basic test.
     task_runner_dockerized_ws: mark a test as a task runner dockerized workspace test.
+    federated_runtime_301_watermarking: mark a test as a federated runtime 301 watermarking test.
 asyncio_mode=auto
 asyncio_default_fixture_loop_scope="function"