Update screenshot params

Kappa-Dev · Nov 7, 2024 · 77ddd95 · 77ddd95
1 parent 0cf2a10
commit 77ddd95
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 17 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -228,7 +228,7 @@ jobs:
         lighttpd -f lighttpd.conf -D &
         sleep 10 && curl -i http://127.0.0.1:12345/index.html # test 
     - name: Run Playwright tests
-      run: DEBUG=pw:webserver npx playwright test --retries=3 --trace retain-on-failure
+      run: DEBUG=pw:webserver npx playwright test --retries=3 --trace retain-on-first-failure 
     - uses: actions/upload-artifact@v4
       if: ${{ !cancelled() }}
       with:

diff --git a/tests/playwright/procedure.spec.ts b/tests/playwright/procedure.spec.ts
@@ -10,7 +10,7 @@ import * as utils from './webapp_utils';
 test.describe.configure({ mode: 'parallel' });
 
 test.beforeEach(async ({ page }, testInfo) => {
-  testInfo.setTimeout(testInfo.timeout + 30000);
+  testInfo.setTimeout(60000);
   // Wait on fonts, might be useful on CI
   await page.waitForFunction(() => document.fonts.ready);
 });
@@ -69,7 +69,7 @@ test.describe('Editor tab', () => {
     await utils.expect_no_error(page);
   });
 
-  test('contact_map', async ({ browserName, page }) => {
+  test('contact_map', async ({ page, browserName }) => {
     await utils.open_app_with_model(page, abc_ka);
     const contact_map = page.locator('#map-container');
     await expect.soft(contact_map).toHaveScreenshot();
@@ -97,8 +97,9 @@ test.describe('Editor tab', () => {
   });
 
   test('influences', async ({ page }) => {
-    const opts_screen = { threshold: 0.2, maxDiffPixels: 60 }
-    const opts_screen_lenient = { threshold: 0.4, maxDiffPixels: 150 }
+    const opts_screen = { maxDiffPixels: 60 }
+    const opts_screen_lenient = { maxDiffPixels: 150, threshold: 0.4 }
+
     await utils.open_app_with_model(page, abc_ka);
     await page.locator('#navinfluences').click();
     const table = page.locator('#influences-table');
@@ -221,6 +222,7 @@ R(CN[C.R],CR[CR.R])  =>  R(CN[2],CR[1]),R(C[2],CR[1])
   });
 
   test('contact_map_accuracy', async ({ page }) => {
+    // TODO: find a smaller example so that execution is faster
     test.setTimeout(180000)
     await utils.open_app_with_model(page, minikai_counters_ka, false, 120000);
     const contact_map = page.locator('#map-container');
@@ -233,7 +235,7 @@ R(CN[C.R],CR[CR.R])  =>  R(CN[2],CR[1]),R(C[2],CR[1])
 
 test.describe('Simulation tools', () => {
 
-  test('Simulation, plot', async ({ browserName, page }) => {
+  test('Simulation, plot', async ({ page, browserName }) => {
     await utils.open_app_with_model(page, abc_ka);
     await utils.setSeed(page, 1);
     // Run simulation to 30, then 100, then test plot options
@@ -294,13 +296,24 @@ test.describe('Simulation tools', () => {
     await page.getByRole('button', { name: 'pause' }).click();
   });
 
-  test('DIN', async ({ page }) => {
+  test('DIN', async ({ page, browserName }) => {
     await utils.open_app_with_model(page, abc_ka);
     await utils.setSeed(page, 1);
 
-    async function expectScreenShotDINTable() {
+    async function expectScreenShotDINTable(chromium_maxDiffPixels: number = 0) {
       await expect.soft(page.getByRole('cell', { name: 'affects' })).toBeVisible();
-      await expect.soft(page.locator('#DIN div').first()).toHaveScreenshot({ mask: [page.locator('#export_din-export_form')] });
+      const DIN_table_locator = page.locator('#DIN div').first();
+
+      var opts_screen: utils.ScreenshotOptions = { mask: [page.locator('#export_din-export_form')] }
+      // Chromium spacing between _numbers_ can be different across machines for some reason, here between CI and local
+      // e.g. https://stackoverflow.com/questions/34814993/letter-spacing-is-different-with-same-browser-font-size-ect-in-chrome 
+      if (browserName == "chromium") {
+        opts_screen = {
+          mask: opts_screen.mask,
+          maxDiffPixels: chromium_maxDiffPixels
+        }
+      }
+      await expect.soft(DIN_table_locator).toHaveScreenshot(opts_screen);
     }
 
     // Run simulation to 30, then 100, then test plot options
@@ -311,7 +324,7 @@ test.describe('Simulation tools', () => {
     // await utils.wait_for_file_load(page, { timeout: 20000 }); // TODO: fix: glyphicon doesn't show properly
     //
     await page.locator('#navDIN').click();
-    await expectScreenShotDINTable();
+    await expectScreenShotDINTable(350);
 
     await utils.testExports(page, '#export_din-export', 'flux', ['json', 'dot', 'html']);
 
@@ -321,17 +334,17 @@ test.describe('Simulation tools', () => {
     await page.waitForTimeout(3000);
     // await utils.wait_for_file_load(page, { timeout: 20000 }); // TODO: fix: glyphicon doesn't show properly
     //
-    await expectScreenShotDINTable();
+    await expectScreenShotDINTable(350);
     await page.getByRole('combobox').first().selectOption('flux.json');
-    await expectScreenShotDINTable();
+    await expectScreenShotDINTable(3000);
 
     await utils.testExports(page, '#export_din-export', 'flux_json', ['json', 'dot', 'html']);
 
     await page.getByRole('combobox').first().selectOption('flux.html');
-    await expectScreenShotDINTable();
+    await expectScreenShotDINTable(350);
   });
 
-  test('snapshots', async ({ browserName, page }) => {
+  test('snapshots', async ({ page, browserName }) => {
     await utils.open_app_with_model(page, abc_ka);
     await utils.setSeed(page, 1);
 
@@ -400,7 +413,7 @@ test.describe('Simulation tools', () => {
     await page.locator('#format_select_id').selectOption('Graph');
     await expect.soft(snapshot_display_graph_loc).toHaveScreenshot();
     await page.locator('.navcontent-view > div:nth-child(3)').click();
-    await expect.soft(snapshot_display_graph_loc).toHaveScreenshot();
+    await expect.soft(snapshot_display_graph_loc).toHaveScreenshot({ maxDiffPixels: 10 });
     await page.getByRole('button', { name: 'Back to root' }).click();
     await expect.soft(snapshot_display_graph_loc).toHaveScreenshot();
     await page.getByRole('radio', { name: 'Count' }).check();
@@ -412,7 +425,7 @@ test.describe('Simulation tools', () => {
 
     // TODO: clicks in graph, link with contact map
     await page.locator('[id="root\\.mixture1"] rect').nth(1).click();
-    await expect.soft(snapshot_display_graph_loc).toHaveScreenshot();
+    await expect.soft(snapshot_display_graph_loc).toHaveScreenshot({ maxDiffPixels: 600 });
     // await page.locator('#force-container circle').nth(1).click();
     // await page.locator('#force-container circle').first().click();
 
@@ -801,7 +814,7 @@ test.describe('projects_and_files', () => {
     await utils.wait_for_file_load(page, { timeout: 10000 });
     const contact_map = page.locator('#map-container');
 
-    const opts_screen = { threshold: 0.2, maxDiffPixels: 150 }
+    const opts_screen = { maxDiffPixels: 150 }
     await expect.soft(contact_map).toHaveScreenshot(opts_screen);
 
     // TODO: fix this flaky test: sometimes the graph doesn't show, bug?

diff --git a/tests/playwright/webapp_utils.ts b/tests/playwright/webapp_utils.ts
@@ -12,6 +12,17 @@ const arg_set_model = '?model=https%3A'
 
 const referencesDir = 'tests/playwright/refs/'
 
+// Interface for screenshots options to be able to add them when needed.
+// Extend when needed 
+// The original type seems not to be accessible from here? 
+// ( https://github.com/microsoft/playwright/blob/910ecdf5566e29f2f7c021e75a46fba35a7d0cf1/packages/playwright/src/matchers/toMatchSnapshot.ts#L47-L63 )
+// TODO: add original type instead if possible
+export interface ScreenshotOptions {
+  maxDiffPixels?: number,
+  mask?: Array<Locator>;
+}
+
+
 function timeout_of_options(options: { timeout?: number | undefined; visible?: boolean | undefined; } | undefined, default_timeout: number = 5000) {
   var timeout = default_timeout;
   if (options !== undefined) {