From 77ddd9592e3010dd9bf6b7939e47c717103c7de0 Mon Sep 17 00:00:00 2001 From: Antoine Pouille Date: Thu, 7 Nov 2024 16:01:55 +0100 Subject: [PATCH] Update screenshot params --- .github/workflows/ci.yml | 2 +- tests/playwright/procedure.spec.ts | 45 +++++++++++++++++++----------- tests/playwright/webapp_utils.ts | 11 ++++++++ 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a735363fc..fc44db702 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -228,7 +228,7 @@ jobs: lighttpd -f lighttpd.conf -D & sleep 10 && curl -i http://127.0.0.1:12345/index.html # test - name: Run Playwright tests - run: DEBUG=pw:webserver npx playwright test --retries=3 --trace retain-on-failure + run: DEBUG=pw:webserver npx playwright test --retries=3 --trace retain-on-first-failure - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: diff --git a/tests/playwright/procedure.spec.ts b/tests/playwright/procedure.spec.ts index 94e81b875..96ccd82f2 100644 --- a/tests/playwright/procedure.spec.ts +++ b/tests/playwright/procedure.spec.ts @@ -10,7 +10,7 @@ import * as utils from './webapp_utils'; test.describe.configure({ mode: 'parallel' }); test.beforeEach(async ({ page }, testInfo) => { - testInfo.setTimeout(testInfo.timeout + 30000); + testInfo.setTimeout(60000); // Wait on fonts, might be useful on CI await page.waitForFunction(() => document.fonts.ready); }); @@ -69,7 +69,7 @@ test.describe('Editor tab', () => { await utils.expect_no_error(page); }); - test('contact_map', async ({ browserName, page }) => { + test('contact_map', async ({ page, browserName }) => { await utils.open_app_with_model(page, abc_ka); const contact_map = page.locator('#map-container'); await expect.soft(contact_map).toHaveScreenshot(); @@ -97,8 +97,9 @@ test.describe('Editor tab', () => { }); test('influences', async ({ page }) => { - const opts_screen = { threshold: 0.2, maxDiffPixels: 60 } - const opts_screen_lenient = { threshold: 0.4, maxDiffPixels: 150 } + const opts_screen = { maxDiffPixels: 60 } + const opts_screen_lenient = { maxDiffPixels: 150, threshold: 0.4 } + await utils.open_app_with_model(page, abc_ka); await page.locator('#navinfluences').click(); const table = page.locator('#influences-table'); @@ -221,6 +222,7 @@ R(CN[C.R],CR[CR.R]) => R(CN[2],CR[1]),R(C[2],CR[1]) }); test('contact_map_accuracy', async ({ page }) => { + // TODO: find a smaller example so that execution is faster test.setTimeout(180000) await utils.open_app_with_model(page, minikai_counters_ka, false, 120000); const contact_map = page.locator('#map-container'); @@ -233,7 +235,7 @@ R(CN[C.R],CR[CR.R]) => R(CN[2],CR[1]),R(C[2],CR[1]) test.describe('Simulation tools', () => { - test('Simulation, plot', async ({ browserName, page }) => { + test('Simulation, plot', async ({ page, browserName }) => { await utils.open_app_with_model(page, abc_ka); await utils.setSeed(page, 1); // Run simulation to 30, then 100, then test plot options @@ -294,13 +296,24 @@ test.describe('Simulation tools', () => { await page.getByRole('button', { name: 'pause' }).click(); }); - test('DIN', async ({ page }) => { + test('DIN', async ({ page, browserName }) => { await utils.open_app_with_model(page, abc_ka); await utils.setSeed(page, 1); - async function expectScreenShotDINTable() { + async function expectScreenShotDINTable(chromium_maxDiffPixels: number = 0) { await expect.soft(page.getByRole('cell', { name: 'affects' })).toBeVisible(); - await expect.soft(page.locator('#DIN div').first()).toHaveScreenshot({ mask: [page.locator('#export_din-export_form')] }); + const DIN_table_locator = page.locator('#DIN div').first(); + + var opts_screen: utils.ScreenshotOptions = { mask: [page.locator('#export_din-export_form')] } + // Chromium spacing between _numbers_ can be different across machines for some reason, here between CI and local + // e.g. https://stackoverflow.com/questions/34814993/letter-spacing-is-different-with-same-browser-font-size-ect-in-chrome + if (browserName == "chromium") { + opts_screen = { + mask: opts_screen.mask, + maxDiffPixels: chromium_maxDiffPixels + } + } + await expect.soft(DIN_table_locator).toHaveScreenshot(opts_screen); } // Run simulation to 30, then 100, then test plot options @@ -311,7 +324,7 @@ test.describe('Simulation tools', () => { // await utils.wait_for_file_load(page, { timeout: 20000 }); // TODO: fix: glyphicon doesn't show properly // await page.locator('#navDIN').click(); - await expectScreenShotDINTable(); + await expectScreenShotDINTable(350); await utils.testExports(page, '#export_din-export', 'flux', ['json', 'dot', 'html']); @@ -321,17 +334,17 @@ test.describe('Simulation tools', () => { await page.waitForTimeout(3000); // await utils.wait_for_file_load(page, { timeout: 20000 }); // TODO: fix: glyphicon doesn't show properly // - await expectScreenShotDINTable(); + await expectScreenShotDINTable(350); await page.getByRole('combobox').first().selectOption('flux.json'); - await expectScreenShotDINTable(); + await expectScreenShotDINTable(3000); await utils.testExports(page, '#export_din-export', 'flux_json', ['json', 'dot', 'html']); await page.getByRole('combobox').first().selectOption('flux.html'); - await expectScreenShotDINTable(); + await expectScreenShotDINTable(350); }); - test('snapshots', async ({ browserName, page }) => { + test('snapshots', async ({ page, browserName }) => { await utils.open_app_with_model(page, abc_ka); await utils.setSeed(page, 1); @@ -400,7 +413,7 @@ test.describe('Simulation tools', () => { await page.locator('#format_select_id').selectOption('Graph'); await expect.soft(snapshot_display_graph_loc).toHaveScreenshot(); await page.locator('.navcontent-view > div:nth-child(3)').click(); - await expect.soft(snapshot_display_graph_loc).toHaveScreenshot(); + await expect.soft(snapshot_display_graph_loc).toHaveScreenshot({ maxDiffPixels: 10 }); await page.getByRole('button', { name: 'Back to root' }).click(); await expect.soft(snapshot_display_graph_loc).toHaveScreenshot(); await page.getByRole('radio', { name: 'Count' }).check(); @@ -412,7 +425,7 @@ test.describe('Simulation tools', () => { // TODO: clicks in graph, link with contact map await page.locator('[id="root\\.mixture1"] rect').nth(1).click(); - await expect.soft(snapshot_display_graph_loc).toHaveScreenshot(); + await expect.soft(snapshot_display_graph_loc).toHaveScreenshot({ maxDiffPixels: 600 }); // await page.locator('#force-container circle').nth(1).click(); // await page.locator('#force-container circle').first().click(); @@ -801,7 +814,7 @@ test.describe('projects_and_files', () => { await utils.wait_for_file_load(page, { timeout: 10000 }); const contact_map = page.locator('#map-container'); - const opts_screen = { threshold: 0.2, maxDiffPixels: 150 } + const opts_screen = { maxDiffPixels: 150 } await expect.soft(contact_map).toHaveScreenshot(opts_screen); // TODO: fix this flaky test: sometimes the graph doesn't show, bug? diff --git a/tests/playwright/webapp_utils.ts b/tests/playwright/webapp_utils.ts index bf63c838b..9e0e7d929 100644 --- a/tests/playwright/webapp_utils.ts +++ b/tests/playwright/webapp_utils.ts @@ -12,6 +12,17 @@ const arg_set_model = '?model=https%3A' const referencesDir = 'tests/playwright/refs/' +// Interface for screenshots options to be able to add them when needed. +// Extend when needed +// The original type seems not to be accessible from here? +// ( https://github.com/microsoft/playwright/blob/910ecdf5566e29f2f7c021e75a46fba35a7d0cf1/packages/playwright/src/matchers/toMatchSnapshot.ts#L47-L63 ) +// TODO: add original type instead if possible +export interface ScreenshotOptions { + maxDiffPixels?: number, + mask?: Array; +} + + function timeout_of_options(options: { timeout?: number | undefined; visible?: boolean | undefined; } | undefined, default_timeout: number = 5000) { var timeout = default_timeout; if (options !== undefined) {