From 2c022a40df8c436d6e8a8a9f8aa9cb0541646ea1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C5=ABdolfs=20O=C5=A1i=C5=86=C5=A1?= Date: Wed, 14 Dec 2022 11:30:52 +0100 Subject: [PATCH] Add visual snapshot tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We use the cache Github action to store snapshots when pushing to master and then load them in pull requests and run the visual tests against it. This way we don't have to commit the images to the repository and always get diffs to the latest master. Signed-off-by: Rūdolfs Ošiņš --- .github/workflows/check-e2e.yml | 97 +++++++++++++++++--------------- playwright.config.ts | 23 +++++++- tests/support/fixtures.ts | 5 +- tests/visual/landingPage.spec.ts | 11 ++++ tests/visual/project.spec.ts | 25 ++++++++ tests/visual/seed.spec.ts | 6 ++ tests/visual/snapshots/.gitkeep | 0 7 files changed, 121 insertions(+), 46 deletions(-) create mode 100644 tests/visual/landingPage.spec.ts create mode 100644 tests/visual/project.spec.ts create mode 100644 tests/visual/seed.spec.ts create mode 100644 tests/visual/snapshots/.gitkeep diff --git a/.github/workflows/check-e2e.yml b/.github/workflows/check-e2e.yml index d33d5e253..9a4544904 100644 --- a/.github/workflows/check-e2e.yml +++ b/.github/workflows/check-e2e.yml @@ -2,57 +2,66 @@ name: check-e2e on: push: branches: - - master + - master pull_request: branches: - - master + - master jobs: check-e2e: + continue-on-error: true strategy: matrix: - browser: [chromium, firefox] + browser: [chromium, firefox, visual] timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: '18.12.1' - - name: Install dependencies - run: npm ci - - name: Get npm cache directory - id: npm-cache-dir - run: | - echo "::set-output name=dir::$(npm config get cache)" - - name: Get Playwright version - id: playwright-version - run: | - echo "::set-output name=version::$(node -p "require('@playwright/test/package.json').version")" - - uses: actions/cache@v3 - name: Check if Playwright browser is cached - id: playwright-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-Playwright-${{steps.playwright-version.outputs.version}} - - name: Install Playwright browser if not cached - if: steps.playwright-cache.outputs.cache-hit != 'true' - run: npx playwright install --with-deps - env: - PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}} - - name: Install OS dependencies of Playwright if cache hit - if: steps.playwright-cache.outputs.cache-hit == 'true' - run: npx playwright install-deps - - name: Start http-api test server - run: ./scripts/run-http-api-with-fixtures --non-interactive --detach - - name: Run Playwright tests - run: npm run test:e2e -- --project ${{ matrix.browser }} - env: - PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}} - - uses: actions/upload-artifact@v3 - if: always() - with: - name: test-artifacts-${{ runner.os }} - retention-days: 30 - path: | - tests/artifacts/**/* + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: "18.12.1" + + - name: Install dependencies + run: npm ci + + - name: Cache Playwright browsers + uses: actions/cache@v3 + id: playwright-dep-cache + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }} + + - name: Cache snapshots (generate on master, load on pull_request) + uses: actions/cache@v3 + id: playwright-snapshot-cache + with: + path: tests/visual/snapshots/**/* + key: ${{ runner.os }}-snapshots + + - name: Install Playwright with deps unless cached + if: steps.playwright-dep-cache.outputs.cache-hit != 'true' + run: npx playwright install --with-deps + + - name: Install only Playwright OS dependencies if cache hit + if: steps.playwright-dep-cache.outputs.cache-hit == 'true' + run: npx playwright install-deps + + - name: Start http-api test server + run: ./scripts/run-http-api-with-fixtures --non-interactive --detach + + - name: Run Playwright tests + run: | + if [ ${{ matrix.browser }} = "visual" ] && [ ${{ github.ref }} = "refs/heads/master" ]; then + npm run test:e2e -- --project ${{ matrix.browser }} --update-snapshots; + else + npm run test:e2e -- --project ${{ matrix.browser }}; + fi + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: test-artifacts-${{ runner.os }} + retention-days: 30 + path: | + tests/artifacts/**/* diff --git a/playwright.config.ts b/playwright.config.ts index bfc17586d..6eb04fcc9 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -3,12 +3,12 @@ import { devices } from "@playwright/test"; const config: PlaywrightTestConfig = { testDir: "./tests/e2e", + outputDir: "./tests/artifacts", timeout: 30_000, expect: { timeout: 8000, }, fullyParallel: true, - outputDir: "./tests/artifacts", workers: process.env.CI ? 1 : undefined, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, @@ -39,6 +39,27 @@ const config: PlaywrightTestConfig = { ...devices["Desktop Safari"], }, }, + { + name: "visual", + expect: { + timeout: 16_000, + toHaveScreenshot: { + threshold: 0.1, + scale: "device", + animations: "disabled", + }, + }, + testDir: "./tests/visual", + snapshotDir: "./tests/visual/snapshots", + retries: 0, + use: { + ...devices["Desktop Chrome"], + actionTimeout: 0, + deviceScaleFactor: 2, + baseURL: "http://localhost:3000", + trace: "off", + }, + }, ], webServer: { diff --git a/tests/support/fixtures.ts b/tests/support/fixtures.ts index 582f37e06..373098c26 100644 --- a/tests/support/fixtures.ts +++ b/tests/support/fixtures.ts @@ -139,7 +139,10 @@ export const test = base.extend<{ await Fs.mkdir(stateDir, { recursive: true }); await use(stateDir); - if (process.env.CI && testInfo.status === "passed") { + if ( + process.env.CI && + (testInfo.status === "passed" || testInfo.status === "skipped") + ) { await Fs.rm(stateDir, { recursive: true }); } }, diff --git a/tests/visual/landingPage.spec.ts b/tests/visual/landingPage.spec.ts new file mode 100644 index 000000000..357727d69 --- /dev/null +++ b/tests/visual/landingPage.spec.ts @@ -0,0 +1,11 @@ +import { test, expect, appConfigWithFixture } from "@tests/support/fixtures.js"; + +test.use({ + customAppConfig: true, +}); + +test("landing page", async ({ page }) => { + await page.addInitScript(appConfigWithFixture); + await page.goto("/"); + await expect(page).toHaveScreenshot(); +}); diff --git a/tests/visual/project.spec.ts b/tests/visual/project.spec.ts new file mode 100644 index 000000000..2667b55f2 --- /dev/null +++ b/tests/visual/project.spec.ts @@ -0,0 +1,25 @@ +import { test, expect, projectFixtureUrl } from "@tests/support/fixtures.js"; + +test("source tree page", async ({ page }) => { + await page.goto(projectFixtureUrl); + await expect(page).toHaveScreenshot(); +}); + +test("commits page", async ({ page }) => { + await page.goto( + `${projectFixtureUrl}/remotes/hyn1mjueopwzrmb18c3zmgg8ei8qunn5wpg76ouymytfqkfxqx7bun/history`, + ); + await expect(page).toHaveScreenshot({ fullPage: true }); +}); + +test("commit page", async ({ page }) => { + await page.goto( + `${projectFixtureUrl}/remotes/hyn1mjueopwzrmb18c3zmgg8ei8qunn5wpg76ouymytfqkfxqx7bun/commits/d6318f7f3d9c15b8ac6dd52267c53220d00f0982`, + ); + await expect(page).toHaveScreenshot({ fullPage: true }); +}); + +test("markdown rendering", async ({ page }) => { + await page.goto(`${projectFixtureUrl}/tree/main/markdown/cheatsheet.md`); + await expect(page).toHaveScreenshot({ fullPage: true }); +}); diff --git a/tests/visual/seed.spec.ts b/tests/visual/seed.spec.ts new file mode 100644 index 000000000..6e7000c98 --- /dev/null +++ b/tests/visual/seed.spec.ts @@ -0,0 +1,6 @@ +import { test, expect } from "@tests/support/fixtures.js"; + +test("seed page", async ({ page }) => { + await page.goto("/seeds/radicle.local"); + await expect(page).toHaveScreenshot(); +}); diff --git a/tests/visual/snapshots/.gitkeep b/tests/visual/snapshots/.gitkeep new file mode 100644 index 000000000..e69de29bb