Add scheduled tests for compositional tasks

.github/workflows/unit_tests.yml

@@ -5,6 +5,8 @@ on:
     branches:
       - main
   pull_request:
+  schedule:
+    - cron: '0 0 * * 5'  # Runs at midnight on Friday
 
 jobs:
 
@@ -101,4 +103,32 @@ jobs:
           SNOW_INSTANCE_URL: ${{ secrets.SNOW_INSTANCE_URL }}
           SNOW_INSTANCE_UNAME: ${{ secrets.SNOW_INSTANCE_UNAME }}
           SNOW_INSTANCE_PWD: ${{ secrets.SNOW_INSTANCE_PWD }}
-        run: pytest -n 5 --durations=10 -m 'slow and not pricy' --slowmo 1000 -v tests
+        run: pytest -n 5 --durations=10 -m 'slow and not pricy' --slowmo 1000 -v tests
+
+  end-to-end-tests:
+    runs-on: ubuntu-latest
+    if: github.event_name == 'schedule'
+    defaults:
+      run:
+        shell: bash -l {0}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+      - name: Pip install
+        working-directory: ./dev
+        run: pip install -r requirements.txt
+      - name: Pip list
+        run: pip list
+      - name: Install Playwright
+        run: playwright install --with-deps
+      - name: Run E2E Tests
+        env:
+          SNOW_INSTANCE_URL: ${{ secrets.SNOW_INSTANCE_URL }}
+          SNOW_INSTANCE_UNAME: ${{ secrets.SNOW_INSTANCE_UNAME }}
+          SNOW_INSTANCE_PWD: ${{ secrets.SNOW_INSTANCE_PWD }}
+        run: pytest -n 10 --durations=10 -m 'pricy' --slowmo 1800 -v tests

workarena_test.py → src/workarena_test.py

@@ -1,5 +1,3 @@
-import random
-
 from browsergym.core.env import BrowserEnv
 from browsergym.workarena import get_all_tasks_agents
 

tests/test_compositional.py

@@ -49,7 +49,7 @@
 @pytest.mark.parametrize("random_seed", range(1))
 @pytest.mark.parametrize("level", range(2, 4))
 @pytest.mark.slow
-@pytest.mark.skip(reason="Tests are too slow")
+# @pytest.mark.skip(reason="Tests are too slow")
 def test_cheat_compositional(task_entrypoint, random_seed, level, page: Page):
     task = task_entrypoint(seed=random_seed, level=level)
     goal, info = task.setup(page=page)
@@ -74,8 +74,7 @@ def test_cheat_compositional(task_entrypoint, random_seed, level, page: Page):
     before_sleep=lambda _: logging.info("Retrying due to a TimeoutError..."),
 )
 @pytest.mark.parametrize("task_entrypoint, seed", zip(AGENT_L2_SAMPLED_TASKS, AGENT_L2_SEEDS))
-@pytest.mark.slow
-@pytest.mark.skip(reason="Tests are too slow")
+@pytest.mark.pricy
 def test_cheat_compositional_sampled_agent_set_l2(task_entrypoint, seed, page: Page):
     task = task_entrypoint(seed=seed)
     goal, info = task.setup(page=page)