diff --git a/.github/workflows/check-e2e.yml b/.github/workflows/check-e2e.yml index d33d5e253..9a4544904 100644 --- a/.github/workflows/check-e2e.yml +++ b/.github/workflows/check-e2e.yml @@ -2,57 +2,66 @@ name: check-e2e on: push: branches: - - master + - master pull_request: branches: - - master + - master jobs: check-e2e: + continue-on-error: true strategy: matrix: - browser: [chromium, firefox] + browser: [chromium, firefox, visual] timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 - with: - node-version: '18.12.1' - - name: Install dependencies - run: npm ci - - name: Get npm cache directory - id: npm-cache-dir - run: | - echo "::set-output name=dir::$(npm config get cache)" - - name: Get Playwright version - id: playwright-version - run: | - echo "::set-output name=version::$(node -p "require('@playwright/test/package.json').version")" - - uses: actions/cache@v3 - name: Check if Playwright browser is cached - id: playwright-cache - with: - path: ${{ steps.npm-cache-dir.outputs.dir }} - key: ${{ runner.os }}-Playwright-${{steps.playwright-version.outputs.version}} - - name: Install Playwright browser if not cached - if: steps.playwright-cache.outputs.cache-hit != 'true' - run: npx playwright install --with-deps - env: - PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}} - - name: Install OS dependencies of Playwright if cache hit - if: steps.playwright-cache.outputs.cache-hit == 'true' - run: npx playwright install-deps - - name: Start http-api test server - run: ./scripts/run-http-api-with-fixtures --non-interactive --detach - - name: Run Playwright tests - run: npm run test:e2e -- --project ${{ matrix.browser }} - env: - PLAYWRIGHT_BROWSERS_PATH: ${{steps.npm-cache-dir.outputs.dir}} - - uses: actions/upload-artifact@v3 - if: always() - with: - name: test-artifacts-${{ runner.os }} - retention-days: 30 - path: | - tests/artifacts/**/* + - uses: actions/checkout@v3 + + - uses: actions/setup-node@v3 + with: + node-version: "18.12.1" + + - name: Install dependencies + run: npm ci + + - name: Cache Playwright browsers + uses: actions/cache@v3 + id: playwright-dep-cache + with: + path: ~/.cache/ms-playwright + key: ${{ runner.os }}-playwright-${{ hashFiles('package-lock.json') }} + + - name: Cache snapshots (generate on master, load on pull_request) + uses: actions/cache@v3 + id: playwright-snapshot-cache + with: + path: tests/visual/snapshots/**/* + key: ${{ runner.os }}-snapshots + + - name: Install Playwright with deps unless cached + if: steps.playwright-dep-cache.outputs.cache-hit != 'true' + run: npx playwright install --with-deps + + - name: Install only Playwright OS dependencies if cache hit + if: steps.playwright-dep-cache.outputs.cache-hit == 'true' + run: npx playwright install-deps + + - name: Start http-api test server + run: ./scripts/run-http-api-with-fixtures --non-interactive --detach + + - name: Run Playwright tests + run: | + if [ ${{ matrix.browser }} = "visual" ] && [ ${{ github.ref }} = "refs/heads/master" ]; then + npm run test:e2e -- --project ${{ matrix.browser }} --update-snapshots; + else + npm run test:e2e -- --project ${{ matrix.browser }}; + fi + + - uses: actions/upload-artifact@v3 + if: always() + with: + name: test-artifacts-${{ runner.os }} + retention-days: 30 + path: | + tests/artifacts/**/* diff --git a/playwright.config.ts b/playwright.config.ts index bfc17586d..6eb04fcc9 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -3,12 +3,12 @@ import { devices } from "@playwright/test"; const config: PlaywrightTestConfig = { testDir: "./tests/e2e", + outputDir: "./tests/artifacts", timeout: 30_000, expect: { timeout: 8000, }, fullyParallel: true, - outputDir: "./tests/artifacts", workers: process.env.CI ? 1 : undefined, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, @@ -39,6 +39,27 @@ const config: PlaywrightTestConfig = { ...devices["Desktop Safari"], }, }, + { + name: "visual", + expect: { + timeout: 16_000, + toHaveScreenshot: { + threshold: 0.1, + scale: "device", + animations: "disabled", + }, + }, + testDir: "./tests/visual", + snapshotDir: "./tests/visual/snapshots", + retries: 0, + use: { + ...devices["Desktop Chrome"], + actionTimeout: 0, + deviceScaleFactor: 2, + baseURL: "http://localhost:3000", + trace: "off", + }, + }, ], webServer: { diff --git a/tests/support/fixtures.ts b/tests/support/fixtures.ts index 582f37e06..373098c26 100644 --- a/tests/support/fixtures.ts +++ b/tests/support/fixtures.ts @@ -139,7 +139,10 @@ export const test = base.extend<{ await Fs.mkdir(stateDir, { recursive: true }); await use(stateDir); - if (process.env.CI && testInfo.status === "passed") { + if ( + process.env.CI && + (testInfo.status === "passed" || testInfo.status === "skipped") + ) { await Fs.rm(stateDir, { recursive: true }); } }, diff --git a/tests/visual/landingPage.spec.ts b/tests/visual/landingPage.spec.ts new file mode 100644 index 000000000..357727d69 --- /dev/null +++ b/tests/visual/landingPage.spec.ts @@ -0,0 +1,11 @@ +import { test, expect, appConfigWithFixture } from "@tests/support/fixtures.js"; + +test.use({ + customAppConfig: true, +}); + +test("landing page", async ({ page }) => { + await page.addInitScript(appConfigWithFixture); + await page.goto("/"); + await expect(page).toHaveScreenshot(); +}); diff --git a/tests/visual/project.spec.ts b/tests/visual/project.spec.ts new file mode 100644 index 000000000..2667b55f2 --- /dev/null +++ b/tests/visual/project.spec.ts @@ -0,0 +1,25 @@ +import { test, expect, projectFixtureUrl } from "@tests/support/fixtures.js"; + +test("source tree page", async ({ page }) => { + await page.goto(projectFixtureUrl); + await expect(page).toHaveScreenshot(); +}); + +test("commits page", async ({ page }) => { + await page.goto( + `${projectFixtureUrl}/remotes/hyn1mjueopwzrmb18c3zmgg8ei8qunn5wpg76ouymytfqkfxqx7bun/history`, + ); + await expect(page).toHaveScreenshot({ fullPage: true }); +}); + +test("commit page", async ({ page }) => { + await page.goto( + `${projectFixtureUrl}/remotes/hyn1mjueopwzrmb18c3zmgg8ei8qunn5wpg76ouymytfqkfxqx7bun/commits/d6318f7f3d9c15b8ac6dd52267c53220d00f0982`, + ); + await expect(page).toHaveScreenshot({ fullPage: true }); +}); + +test("markdown rendering", async ({ page }) => { + await page.goto(`${projectFixtureUrl}/tree/main/markdown/cheatsheet.md`); + await expect(page).toHaveScreenshot({ fullPage: true }); +}); diff --git a/tests/visual/seed.spec.ts b/tests/visual/seed.spec.ts new file mode 100644 index 000000000..6e7000c98 --- /dev/null +++ b/tests/visual/seed.spec.ts @@ -0,0 +1,6 @@ +import { test, expect } from "@tests/support/fixtures.js"; + +test("seed page", async ({ page }) => { + await page.goto("/seeds/radicle.local"); + await expect(page).toHaveScreenshot(); +}); diff --git a/tests/visual/snapshots/.gitkeep b/tests/visual/snapshots/.gitkeep new file mode 100644 index 000000000..e69de29bb