Add visual snapshot tests

We use the cache Github action to store snapshots when pushing to master and
then load them in pull requests and run the visual tests against it.

This way we don't have to commit the images to the repository and always
get diffs to the latest master.

Signed-off-by: Rūdolfs Ošiņš <rudolfs@osins.org>

.github/workflows/check-e2e.yml

@@ -2,57 +2,66 @@ name: check-e2e
 on:
   push:
     branches:
-    - master
+      - master
   pull_request:
     branches:
-    - master
+      - master
 
 jobs:
   check-e2e:
+    continue-on-error: true
     strategy:
       matrix:
-        browser: [chromium, firefox]
+        browser: [chromium, firefox, visual]
     timeout-minutes: 30
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-node@v3
-      with:
-        node-version: '18.12.1'
-    - name: Install dependencies
-      run: npm ci
-    - name: Get npm cache directory
-      id: npm-cache-dir
-      run: |
-        echo "::set-output name=dir::$(npm config get cache)"
-    - name: Get Playwright version
-      id: playwright-version
-      run: |
-        echo "::set-output name=version::$(node -p "require('@playwright/test/package.json').version")"
-    - uses: actions/cache@v3
-      name: Check if Playwright browser is cached
-      id: playwright-cache
-      with:
-        path: ${{ steps.npm-cache-dir.outputs.dir }}
-        key: ${{ runner.os }}-Playwright-${{steps.playwright-version.outputs.version}}
-    - name: Install Playwright browser if not cached
-      if: steps.playwright-cache.outputs.cache-hit != 'true'
-      run: npx playwright install --with-deps
-      env:
-        PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}}
-    - name: Install OS dependencies of Playwright if cache hit
-      if: steps.playwright-cache.outputs.cache-hit == 'true'
-      run: npx playwright install-deps
-    - name: Start http-api test server
-      run: ./scripts/run-http-api-with-fixtures --non-interactive --detach
-    - name: Run Playwright tests
-      run: npm run test:e2e -- --project ${{ matrix.browser }}
-      env:
-        PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}}
-    - uses: actions/upload-artifact@v3
-      if: always()
-      with:
-        name: test-artifacts-${{ runner.os }}
-        retention-days: 30
-        path: |
-          tests/artifacts/**/*
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-node@v3
+        with:
+          node-version: "18.12.1"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Cache Playwright browsers
+        uses: actions/cache@v3
+        id: playwright-dep-cache
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }}
+
+      - name: Cache snapshots (generate on master, load on pull_request)
+        uses: actions/cache@v3
+        id: playwright-snapshot-cache
+        with:
+          path: tests/visual/snapshots/**/*
+          key: ${{ runner.os }}-snapshots
+
+      - name: Install Playwright with deps unless cached
+        if: steps.playwright-dep-cache.outputs.cache-hit != 'true'
+        run: npx playwright install --with-deps
+
+      - name: Install only Playwright OS dependencies if cache hit
+        if: steps.playwright-dep-cache.outputs.cache-hit == 'true'
+        run: npx playwright install-deps
+
+      - name: Start http-api test server
+        run: ./scripts/run-http-api-with-fixtures --non-interactive --detach
+
+      - name: Run Playwright tests
+        run: |
+              if [ ${{ matrix.browser }} = "visual" ] && [ ${{ github.ref }} = "refs/heads/master" ]; then
+                npm run test:e2e -- --project ${{ matrix.browser }} --update-snapshots;
+              else
+                npm run test:e2e -- --project ${{ matrix.browser }};
+              fi
+
+      - uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: test-artifacts-${{ runner.os }}
+          retention-days: 30
+          path: |
+            tests/artifacts/**/*

playwright.config.ts

@@ -3,12 +3,12 @@ import { devices } from "@playwright/test";
 
 const config: PlaywrightTestConfig = {
   testDir: "./tests/e2e",
+  outputDir: "./tests/artifacts",
   timeout: 30_000,
   expect: {
     timeout: 8000,
   },
   fullyParallel: true,
-  outputDir: "./tests/artifacts",
   workers: process.env.CI ? 1 : undefined,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 2 : 0,
@@ -39,6 +39,27 @@ const config: PlaywrightTestConfig = {
         ...devices["Desktop Safari"],
       },
     },
+    {
+      name: "visual",
+      expect: {
+        timeout: 16_000,
+        toHaveScreenshot: {
+          threshold: 0.1,
+          scale: "device",
+          animations: "disabled",
+        },
+      },
+      testDir: "./tests/visual",
+      snapshotDir: "./tests/visual/snapshots",
+      retries: 0,
+      use: {
+        ...devices["Desktop Chrome"],
+        actionTimeout: 0,
+        deviceScaleFactor: 2,
+        baseURL: "http://localhost:3000",
+        trace: "off",
+      },
+    },
   ],
 
   webServer: {

tests/support/fixtures.ts

@@ -139,7 +139,10 @@ export const test = base.extend<{
     await Fs.mkdir(stateDir, { recursive: true });
 
     await use(stateDir);
-    if (process.env.CI && testInfo.status === "passed") {
+    if (
+      process.env.CI &&
+      (testInfo.status === "passed" || testInfo.status === "skipped")
+    ) {
       await Fs.rm(stateDir, { recursive: true });
     }
   },

tests/visual/landingPage.spec.ts

@@ -0,0 +1,11 @@
+import { test, expect, appConfigWithFixture } from "@tests/support/fixtures.js";
+
+test.use({
+  customAppConfig: true,
+});
+
+test("landing page", async ({ page }) => {
+  await page.addInitScript(appConfigWithFixture);
+  await page.goto("/");
+  await expect(page).toHaveScreenshot();
+});

tests/visual/project.spec.ts

@@ -0,0 +1,25 @@
+import { test, expect, projectFixtureUrl } from "@tests/support/fixtures.js";
+
+test("source tree page", async ({ page }) => {
+  await page.goto(projectFixtureUrl);
+  await expect(page).toHaveScreenshot();
+});
+
+test("commits page", async ({ page }) => {
+  await page.goto(
+    `${projectFixtureUrl}/remotes/hyn1mjueopwzrmb18c3zmgg8ei8qunn5wpg76ouymytfqkfxqx7bun/history`,
+  );
+  await expect(page).toHaveScreenshot({ fullPage: true });
+});
+
+test("commit page", async ({ page }) => {
+  await page.goto(
+    `${projectFixtureUrl}/remotes/hyn1mjueopwzrmb18c3zmgg8ei8qunn5wpg76ouymytfqkfxqx7bun/commits/d6318f7f3d9c15b8ac6dd52267c53220d00f0982`,
+  );
+  await expect(page).toHaveScreenshot({ fullPage: true });
+});
+
+test("markdown rendering", async ({ page }) => {
+  await page.goto(`${projectFixtureUrl}/tree/main/markdown/cheatsheet.md`);
+  await expect(page).toHaveScreenshot({ fullPage: true });
+});

tests/visual/seed.spec.ts

@@ -0,0 +1,6 @@
+import { test, expect } from "@tests/support/fixtures.js";
+
+test("seed page", async ({ page }) => {
+  await page.goto("/seeds/radicle.local");
+  await expect(page).toHaveScreenshot();
+});