radicle-explorer/.buildkite/visual.yml

# Visual snapshot tests.
#
# tests/visual/snapshots is gitignored, so the baselines are not in the checkout.
# Master regenerates them with --update-snapshots and uploads them as build
# artifacts. Every other branch downloads the set from the newest passed master
# build and compares against it.
#
# Artifacts rather than a cache volume, because a volume publishes a new version
# only when a job exits 0. That made every passing branch job a writer, so the
# branch step had to fail on purpose to keep its copy out, and it left the volume
# at its empty first version until a master job went green. Artifacts have
# neither problem, and they free the one cache volume a step can attach for the
# shared `deps` volume from node-checks.yml.
#
# A job knows its own build id and no other, so the branch step asks the REST API
# for the newest passed master build. No credential a job already holds can do
# that: the agent access token authenticates the agent API, not the REST API. So
# the step reads an API token with the read_builds scope from the cluster secret
# `read_builds_api_token`. A secret key cannot start with `buildkite` or `bk`,
# which is why the name does not say which API it is for.
#
# Passed is the right filter for more than freshness: the agent uploads artifacts
# whether or not the job succeeded, so a run that died halfway leaves a partial
# set behind, but its build never reaches that state.

env:
  CI: "true"
  NPM_CONFIG_CACHE: ".npm-cache"

steps:
  - label: "📸 visual update snapshots"
    key: visual-update-snapshots
    branches: "master"
    # The check step in node-checks.yml seeds the shared volume.
    depends_on: check
    timeout_in_minutes: 30
    # Keep these paths the same as the ones in node-checks.yml and e2e.yml.
    cache: &deps_cache
      name: deps
      paths:
        - ".npm-cache"
        - "tests/tmp/bin"
        - ".lintcache"
    artifact_paths:
      # The baseline set that every branch build compares against.
      - "tests/visual/snapshots/**/*.png"
      - "tests/artifacts/**/*"
    plugins:
      # The image ships WebKit as well as Chromium, which this suite needs: the
      # visual-mobile project uses the iPhone 13 Mini device descriptor, whose
      # defaultBrowserType is webkit. The tag must match @playwright/test in
      # package.json — Playwright refuses browsers built for another version.
      - &playwright_docker
        docker#v5.14.0:
          image: "mcr.microsoft.com/playwright:v1.61.1-noble"
          # Chromium exhausts Docker's default 64MB /dev/shm.
          ipc: host
          # Puts buildkite-agent in the container, with the job id, the build id
          # and the access token it needs.
          mount-buildkite-agent: true
          # Without this the cache volume links point at nothing.
          volumes:
            - "/cache/bkcache:/cache/bkcache"
          environment:
            - CI
            - NPM_CONFIG_CACHE
            # The two slugs that address this pipeline in the REST API.
            # mount-buildkite-agent does not pass them.
            - BUILDKITE_ORGANIZATION_SLUG
            - BUILDKITE_PIPELINE_SLUG
    commands:
      - npm ci
      # The image ships no decompressors: install-binaries unpacks .tar.xz
      # releases, and the repo fixtures are .tar.bz2 archives.
      - apt-get update -qq && apt-get install -y -qq xz-utils bzip2
      - mkdir -p tests/artifacts
      - ./scripts/install-binaries
      # --update-snapshots reports a re-generated snapshot as a pass, so a
      # non-zero exit means the run itself broke. --retries overrides the
      # retries: 0 of the visual projects, so that one unstable screenshot
      # cannot keep the build out of the passed state that publishes the set.
      - npm run test:e2e -- --project visual-desktop visual-mobile --update-snapshots --retries=2

  - label: "📸 visual"
    key: visual
    branches: "!master"
    depends_on: check
    timeout_in_minutes: 30
    cache: *deps_cache
    artifact_paths:
      - "tests/artifacts/**/*"
    plugins:
      - *playwright_docker
    commands:
      # jq reads the build id out of the API response.
      - apt-get update -qq && apt-get install -y -qq xz-utils bzip2 jq
      - |
        token=$$(buildkite-agent secret get read_builds_api_token)
        builds=$$(curl -sfS -H "Authorization: Bearer $$token" \
          "https://api.buildkite.com/v2/organizations/$$BUILDKITE_ORGANIZATION_SLUG/pipelines/$$BUILDKITE_PIPELINE_SLUG/builds?branch=master&state=passed&per_page=1")
        baseline=$$(printf '%s' "$$builds" | jq -r '.[0].id // empty')
        # A master build older than this file uploaded no baselines, so let a
        # download that finds none pass and judge the result below instead.
        if [ -n "$$baseline" ]; then
          buildkite-agent artifact download "tests/visual/snapshots/**/*.png" . --build "$$baseline" || true
        fi
        # Comparing with no baselines makes Playwright report all 89 screenshots
        # as missing, which reads like a total regression. Report the real state
        # instead, and stop before the expensive part of the step.
        if ! find tests/visual/snapshots -name '*.png' 2>/dev/null | grep -q .; then
          buildkite-agent annotate "📸 visual: no baselines yet — a master build must publish them first" \
            --style warning --context visual
          exit 0
        fi
      - npm ci
      - mkdir -p tests/artifacts
      - ./scripts/install-binaries
      - |
        if npm run test:e2e -- --project visual-desktop visual-mobile; then
          buildkite-agent annotate "📸 visual: no differences" \
            --style success --context visual
        else
          buildkite-agent annotate "📸 visual: differences found, see the uploaded images" \
            --style warning --context visual
          exit 1
        fi