From 8f2ddcf489ec8af4350af3cd746bc6203bd1ad4a Mon Sep 17 00:00:00 2001 From: Daniel Norman Date: Wed, 12 Aug 2026 11:16:21 +0200 Subject: [PATCH] Cache npm dependencies in Buildkite - Add twemoji as npm dependency instead of build download - Package build assets with a Vite plugin instead of scripts - Keep the eslint and prettier caches in one directory --- .buildkite/README.md | 57 ------------------------- .buildkite/e2e.yml | 17 ++++++++ .buildkite/node-checks.yml | 53 +++++++++++++++++++++-- .buildkite/pipeline.yml | 7 +++ .buildkite/rust.yml | 17 +++++++- .buildkite/visual.yml | 24 ++++++++--- .gitignore | 6 ++- Dockerfile.e2e | 4 +- eslint.config.js | 1 + flake.nix | 12 +----- package-lock.json | 8 +++- package.json | 4 +- public/twemoji/.gitkeep | 0 scripts/check | 9 +++- scripts/copy-katex-assets | 7 --- scripts/install-twemoji-assets | 9 ---- vite-plugin-assets.ts | 78 ++++++++++++++++++++++++++++++++++ vite.config.ts | 3 ++ 18 files changed, 213 insertions(+), 103 deletions(-) delete mode 100644 .buildkite/README.md delete mode 100644 public/twemoji/.gitkeep delete mode 100755 scripts/copy-katex-assets delete mode 100755 scripts/install-twemoji-assets create mode 100644 vite-plugin-assets.ts diff --git a/.buildkite/README.md b/.buildkite/README.md deleted file mode 100644 index a40569071..000000000 --- a/.buildkite/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Buildkite CI - -The Garden Buildkite integration stores a bare `buildkite-agent pipeline -upload`, so the agent finds `pipeline.yml` by its default name. The integration -makes one pipeline per repo, so all suites land in the same build. - -## Cache volumes and artifacts - -A **cache volume** carries state between builds. It is an input: content that a -step needs, but that the repo does not contain. Volume names are scoped to the -pipeline. - -An **artifact** is a file that one build produces. It is an output: evidence for -a person to read in the Buildkite UI. The agent uploads artifacts whatever the -exit code of the step, so a failed step still reports its results. - -The visual suite uses both. The volume `visual-snapshots` holds the baseline -images, which are gitignored and exist nowhere else. `artifact_paths` uploads -the actual, expected, and diff images that a failed comparison writes. The two -cannot be swapped: a volume is not visible in the UI, and an artifact does not -persist as an input to the next build. - -## One writer per volume - -A job does not read its volume in place. It gets a private copy at start. That -copy becomes the version that the next job sees, but only when the job exits 0. -Every successful job is therefore a writer, whether or not it changed a file. - -Two consequences follow. - -Keep the volume names distinct across the uploaded files. A step that commits a -version discards what a concurrent step wrote to the same volume. This is why -`rust.yml` gives test, doc, and lint a separate volume each. - -The visual suite keeps exactly one writer. `master` updates the baselines, and -`concurrency: 1` serializes the group. Other branches always exit non-zero, so -Buildkite discards their copy and no branch can republish a stale baseline set. -The branch step then cannot use the exit code to report the test result, so two -codes carry both: 100 means no differences, which `soft_fail` keeps green, and -any other non-zero code fails the build. An annotation reports the outcome in -both cases. - -## Docker images - -Two images cover all Node steps: - -- `node:24-bookworm` tracks the major version in `.nvmrc` and the `engines` - field in `package.json`. -- `mcr.microsoft.com/playwright:v1.61.1-noble` ships the browsers at - `/ms-playwright`, so no step installs them. The tag must match - `@playwright/test` in `package.json`. Playwright refuses browsers built for - another version. - -Node steps use no cache volume. Cache paths are relative to the step working -directory, and the agent home is not visible inside a container, so npm -downloads its dependencies on every run. The Rust steps run on the agent -instead, so their `~`-rooted paths work. diff --git a/.buildkite/e2e.yml b/.buildkite/e2e.yml index ff6bd4aa9..2564d23bc 100644 --- a/.buildkite/e2e.yml +++ b/.buildkite/e2e.yml @@ -1,14 +1,27 @@ # No explicit build step: tests/support/globalSetup.ts runs `npm run build` in # parallel with fixture creation, and asserts the pinned rad and radicle-httpd # versions are on PATH. +# +# This step shares the `deps` volume with the steps in node-checks.yml. Keep the +# cache paths the same as the ones there, and see that file for how the volume +# works and why this step waits for check. env: CI: "true" + NPM_CONFIG_CACHE: ".npm-cache" steps: - label: "๐ŸŽญ e2e" key: e2e + # The check step in node-checks.yml seeds the shared volume. + depends_on: check timeout_in_minutes: 30 + cache: + name: deps + paths: + - ".npm-cache" + - "tests/tmp/bin" + - ".lintcache" artifact_paths: - "tests/artifacts/**/*" plugins: @@ -19,8 +32,12 @@ steps: image: "mcr.microsoft.com/playwright:v1.61.1-noble" # Chromium exhausts Docker's default 64MB /dev/shm. ipc: host + # Without this the cache volume links point at nothing. + volumes: + - "/cache/bkcache:/cache/bkcache" environment: - CI + - NPM_CONFIG_CACHE commands: - npm ci # The image ships no decompressors: install-binaries unpacks .tar.xz diff --git a/.buildkite/node-checks.yml b/.buildkite/node-checks.yml index c0dbcff56..75a7296b0 100644 --- a/.buildkite/node-checks.yml +++ b/.buildkite/node-checks.yml @@ -1,17 +1,38 @@ -# No cache volume: cache paths are relative to the step working directory, and -# the agent's home is not visible inside a container, so npm downloads its -# dependencies on every run. +# One `deps` cache volume serves every step here and the e2e step in e2e.yml. +# The volume is mounted at /cache/bkcache on the agent, and the configured paths +# become links into it. A container step must therefore mount that directory, or +# every link dangles and the job persists nothing. +# +# A job does not write the volume in place. It gets a private copy at start, and +# that copy becomes the version the next job sees if the job exits 0. Concurrent +# jobs are never merged: the last one to exit 0 publishes its whole copy, and +# whatever a job that finished earlier committed is gone. +# +# check therefore runs first, and is the only step that writes content no other +# step produces: the lint caches, and the pinned binaries. The four steps that +# follow all copy its version and add nothing unique to it โ€” `npm ci` against the +# same lockfile fetches the same tarballs, and install-binaries does nothing once +# the pinned version is there. Whichever of them commits last keeps everything. +# +# Stale content is safe. The npm cache is content-addressed, and install-binaries +# keys its download on the pinned version. env: CI: "true" + # Applies to every npm and npx call in the step, not just to `npm ci`. + NPM_CONFIG_CACHE: ".npm-cache" __defs: # Tracks the major version in .nvmrc and the engines field in package.json. node_image: &node_image docker#v5.14.0: image: "node:24-bookworm" + # Without this the cache volume links point at nothing. + volumes: + - "/cache/bkcache:/cache/bkcache" environment: - CI + - NPM_CONFIG_CACHE # Ships the browsers at /ms-playwright, which is why no step installs them. # The tag must match @playwright/test in package.json โ€” Playwright refuses to @@ -21,22 +42,44 @@ __defs: image: "mcr.microsoft.com/playwright:v1.61.1-noble" # Chromium exhausts Docker's default 64MB /dev/shm. ipc: host + volumes: + - "/cache/bkcache:/cache/bkcache" environment: - CI + - NPM_CONFIG_CACHE + + # Keep this list the same in every step that attaches the volume, here and in + # e2e.yml. + deps_cache: &deps_cache + name: deps + paths: + - ".npm-cache" + - "tests/tmp/bin" + # The eslint and prettier caches, which scripts/check keeps here rather + # than under node_modules so that `npm ci` cannot delete them. A cache + # path is always a directory, so the two files share one. + - ".lintcache" steps: - label: "๐Ÿ”Ž check" key: check timeout_in_minutes: 10 + cache: *deps_cache plugins: - *node_image commands: - npm ci + # check does not use the binaries. It installs them while it is the only + # writer of the volume, so that no later step has to win a race to keep + # them. The node image ships curl and xz, so nothing is installed first. + - ./scripts/install-binaries - npm run check - label: "๐Ÿงช unit-tests" key: unit-tests + depends_on: check timeout_in_minutes: 10 + cache: *deps_cache plugins: - *node_image commands: @@ -47,7 +90,9 @@ steps: # binaries on PATH even though it runs under vitest rather than Playwright. - label: "๐Ÿ”Œ http-client-tests" key: http-client-tests + depends_on: check timeout_in_minutes: 15 + cache: *deps_cache artifact_paths: - "tests/artifacts/**/*" plugins: @@ -62,7 +107,9 @@ steps: # needs a browser but no Radicle binaries or fixtures. - label: "๐Ÿ— build-smoke" key: build-smoke + depends_on: check timeout_in_minutes: 30 + cache: *deps_cache artifact_paths: - "tests/artifacts/**/*" plugins: diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 612b33253..55bd82258 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -6,6 +6,13 @@ # suite lands in this single build. Keep cache volume names distinct across the # uploaded files: volumes are scoped to the pipeline, and a step that commits a # volume version discards what a concurrent step wrote to the same volume. +# +# Upload node-checks.yml before e2e.yml: the e2e step depends on the `check` key +# defined there, and Buildkite rejects a step whose dependency is not yet part of +# the build. +# +# Superseded builds are not cancelled. Buildkite has no YAML equivalent of the +# GitHub Actions cancel-in-progress. steps: - label: ":pipeline: upload" diff --git a/.buildkite/rust.yml b/.buildkite/rust.yml index cd9c66043..56a830912 100644 --- a/.buildkite/rust.yml +++ b/.buildkite/rust.yml @@ -22,11 +22,21 @@ __defs: if ! command -v rustup >/dev/null 2>&1; then curl -fsSL https://sh.rustup.rs | sh -s -- -y --no-modify-path --profile minimal fi - rustup component add rustfmt clippy rustc --version + # Only the parts of ~/.cargo that are expensive to fetch again. registry/src + # and git/checkouts are left out on purpose: cargo re-creates both from the + # cache and the git db, so caching them would double the size of the volume + # for no gain. + # + # Nothing prunes the target directory, so it keeps the artifacts of every + # dependency version the branch has ever built. Deleting the volume is the + # only cleanup, and it costs one full rebuild. cargo_cache: &cargo_cache - - "~/.cargo" + - "~/.cargo/bin" + - "~/.cargo/registry/index" + - "~/.cargo/registry/cache" + - "~/.cargo/git/db" - "~/.rustup" - "~/.cache/cargo-target" @@ -62,5 +72,8 @@ steps: paths: *cargo_cache commands: - *install_rust + # The minimal profile ships neither, and rust-toolchain names no + # components. Only this step needs them. + - rustup component add rustfmt clippy - RUSTFLAGS="-D warnings" cargo clippy --workspace --all-features --tests - cargo fmt --all -- --check diff --git a/.buildkite/visual.yml b/.buildkite/visual.yml index df34ccf12..ec978182c 100644 --- a/.buildkite/visual.yml +++ b/.buildkite/visual.yml @@ -4,6 +4,10 @@ # volume. Master updates them with --update-snapshots; every other branch # compares against them. # +# A step can attach only one volume, so these two steps cannot also read the +# `deps` volume from node-checks.yml. This volume carries the npm cache and the +# Radicle binaries as well as the baselines. +# # A job does not read the volume in place: it gets a private copy at start, and # that copy becomes the version the next job sees if the job exits 0. So every # successful job is a writer, whether or not it changed a file. The two steps @@ -16,9 +20,15 @@ # --update-snapshots the mode is `missing`, under which Playwright still # writes a baseline that does not exist yet. A branch therefore needs no # concurrency limit, and any number of branch builds can run at once. +# +# The cost of that trick is that a branch build never adds to this volume, and +# so never warms the npm cache it shares with the baselines. A branch that +# changes package-lock.json therefore downloads the full dependency set on every +# build until it lands on master. env: CI: "true" + NPM_CONFIG_CACHE: ".npm-cache" steps: - label: "๐Ÿ“ธ visual update snapshots" @@ -29,10 +39,12 @@ steps: # jobs were created, so the newest master build commits last. concurrency: 1 concurrency_group: "radicle-explorer/visual-snapshots" - cache: + cache: &visual_cache name: visual-snapshots paths: - "tests/visual/snapshots" + - ".npm-cache" + - "tests/tmp/bin" artifact_paths: - "tests/artifacts/**/*.png" plugins: @@ -47,8 +59,13 @@ steps: ipc: host # buildkite-agent annotate runs inside the container. mount-buildkite-agent: true + # Without this the cache volume links point at nothing, which is why + # the baselines never persisted before. + volumes: + - "/cache/bkcache:/cache/bkcache" environment: - CI + - NPM_CONFIG_CACHE commands: - npm ci # The image ships no decompressors: install-binaries unpacks .tar.xz @@ -78,10 +95,7 @@ steps: # produces, so only the success path below is soft. soft_fail: - exit_status: 100 - cache: - name: visual-snapshots - paths: - - "tests/visual/snapshots" + cache: *visual_cache artifact_paths: - "tests/artifacts/**/*.png" plugins: diff --git a/.gitignore b/.gitignore index 87319a439..26c37d2e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ /build/ node_modules/ +.npm-cache/ +.lintcache/ NOTES config/local* crates/radicle-httpd/build/artifacts @@ -12,8 +14,8 @@ KaTeX_**.ttf KaTeX_**.woff KaTeX_**.woff2 -# Twemoji Assets -public/twemoji/*.svg +# Twemoji Assets, copied in by vite-plugin-assets +public/twemoji/ # Integration Tests tests/tmp/**/* diff --git a/Dockerfile.e2e b/Dockerfile.e2e index 145361fb5..c2e5830be 100644 --- a/Dockerfile.e2e +++ b/Dockerfile.e2e @@ -27,8 +27,8 @@ WORKDIR /work # Playwright version in package-lock.json so the baked browser matches the # @playwright/test that npm ci installs into the volume at runtime. Copying # only the lockfiles keeps this layer cached across source changes. -# --ignore-scripts: the repo's postinstall (copy-katex/twemoji assets) needs -# scripts/ which isn't in this layer; those run at runtime instead. +# --ignore-scripts: this layer only warms the browser cache, so dependency +# install scripts are not necessary. COPY package.json package-lock.json ./ RUN npm ci --ignore-scripts \ && npx playwright install --with-deps chromium \ diff --git a/eslint.config.js b/eslint.config.js index eba67dc3b..758e98d5c 100644 --- a/eslint.config.js +++ b/eslint.config.js @@ -161,6 +161,7 @@ export default [ { ignores: [ "node_modules/**/*", + ".npm-cache/**/*", "build/**/*", "public/**/*", "radicle-httpd/**/*", diff --git a/flake.nix b/flake.nix index 1f235b364..dcc31bca0 100644 --- a/flake.nix +++ b/flake.nix @@ -81,13 +81,6 @@ packages = { default = self.packages.${system}.radicle-explorer; - twemoji-assets = pkgs.fetchFromGitHub { - owner = "twitter"; - repo = "twemoji"; - rev = "v14.0.2"; - hash = "sha256-YoOnZ5uVukzi/6bLi22Y8U5TpplPzB7ji42l+/ys5xI="; - }; - radicle-explorer = pkgs.callPackage ({ lib, buildNpmPackage, @@ -97,12 +90,9 @@ pname = "radicle-explorer"; version = (builtins.fromJSON (builtins.readFile ./package.json)).version; src = ./.; - npmDepsHash = "sha256-RC+QQXtvXC48uM0oOAFA0ni5AU/l9m8k1LgrxykSu5M="; + npmDepsHash = "sha256-pUyB56EJjuMYU//oMaFNV27Fg+Ga5jD0HQOB/v+gHa8="; postPatch = '' patchShebangs --build ./scripts - mkdir -p "public/twemoji" - cp -t public/twemoji -r -- ${self.packages.${system}.twemoji-assets}/assets/svg/* - : >scripts/install-twemoji-assets ''; dontConfigure = true; diff --git a/package-lock.json b/package-lock.json index bc2cff6a6..87eee495d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,8 +7,8 @@ "": { "name": "radicle-explorer", "version": "1.0.0", - "hasInstallScript": true, "dependencies": { + "@twemoji/svg": "^15.0.0", "@wooorm/starry-night": "^3.10.0", "async-mutex": "^0.5.0", "baconjs": "^3.0.23", @@ -1183,6 +1183,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@twemoji/svg": { + "version": "15.0.0", + "resolved": "https://registry.npmjs.org/@twemoji/svg/-/svg-15.0.0.tgz", + "integrity": "sha512-ZSPef2B6nBaYnfgdTbAy4jgW95o7pi2xPGwGCU+WMTxo7J6B1lMPTWwSq/wTuiMq+N0khQ90CcvYp1wFoQpo/w==", + "license": "MIT" + }, "node_modules/@tybys/wasm-util": { "version": "0.10.3", "resolved": "https://registry.npmjs.org/@tybys/wasm-util/-/wasm-util-0.10.3.tgz", diff --git a/package.json b/package.json index d18e4c684..8e4895634 100644 --- a/package.json +++ b/package.json @@ -6,8 +6,7 @@ "start": "vite", "start:httpd": "cargo run -p radicle-httpd --release -- --listen 127.0.0.1:8080", "serve": "vite preview", - "build": "vite build && scripts/copy-katex-assets && scripts/install-twemoji-assets", - "postinstall": "scripts/copy-katex-assets && scripts/install-twemoji-assets", + "build": "vite build", "check": "scripts/check", "check:radicle-httpd": "cargo clippy --all --tests && cargo fmt --all --check", "format": "npx prettier '**/*.@(ts|js|svelte|json|css|html|yml)' --write", @@ -63,6 +62,7 @@ "wait-on": "^9.0.10" }, "dependencies": { + "@twemoji/svg": "^15.0.0", "@wooorm/starry-night": "^3.10.0", "async-mutex": "^0.5.0", "baconjs": "^3.0.23", diff --git a/public/twemoji/.gitkeep b/public/twemoji/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/scripts/check b/scripts/check index 5673783d2..583a1ab9b 100755 --- a/scripts/check +++ b/scripts/check @@ -3,5 +3,10 @@ set -e npx tsc --noEmit npx svelte-check --tsconfig tsconfig.json --config ./vite.config.ts --fail-on-warnings --compiler-warnings options_missing_custom_element:ignore -npx eslint --cache --cache-location node_modules/.cache/eslint --max-warnings 0 . -npx prettier "**/*.@(ts|js|svelte|json|css|html|yml)" --ignore-path .gitignore --ignore-path .prettierignore --check --cache +# Both caches sit outside node_modules, which `npm ci` deletes, and both are +# keyed on file contents rather than mtimes, which a fresh checkout changes. +# CI can then carry them between builds. They share one directory because a +# Buildkite cache path is always a directory, never a file. +mkdir -p .lintcache +npx eslint --cache --cache-location .lintcache/eslint --cache-strategy content --max-warnings 0 . +npx prettier "**/*.@(ts|js|svelte|json|css|html|yml)" --ignore-path .gitignore --ignore-path .prettierignore --check --cache --cache-location .lintcache/prettier diff --git a/scripts/copy-katex-assets b/scripts/copy-katex-assets deleted file mode 100755 index aa83b2535..000000000 --- a/scripts/copy-katex-assets +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -Eeuo pipefail - -echo "Copying katex assets into bundle directory" - -cp -r node_modules/katex/dist/katex.min.css public/katex.min.css -cp -r node_modules/katex/dist/fonts/* public/fonts/ diff --git a/scripts/install-twemoji-assets b/scripts/install-twemoji-assets deleted file mode 100755 index 96a48b219..000000000 --- a/scripts/install-twemoji-assets +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -Eeou pipefail - -version="$(node -e 'console.log(require("twemoji/package.json").version)')" - -echo "Installing Twemoji SVG assets v${version}" - -curl -sSL "https://github.com/twitter/twemoji/archive/refs/tags/v${version}.tar.gz" \ - | tar -x -z -C public/twemoji/ --strip-components=3 "twemoji-${version}/assets/svg" diff --git a/vite-plugin-assets.ts b/vite-plugin-assets.ts new file mode 100644 index 000000000..184037689 --- /dev/null +++ b/vite-plugin-assets.ts @@ -0,0 +1,78 @@ +import type { Plugin } from "vite"; + +import fs from "node:fs/promises"; +import path from "node:path"; +import { createRequire } from "node:module"; + +const require = createRequire(import.meta.url); + +function packageDir(name: string): string { + return path.dirname(require.resolve(`${name}/package.json`)); +} + +function packageVersion(name: string): string { + const { version } = require(`${name}/package.json`) as { version: string }; + return version; +} + +// index.html loads /katex.min.css, so the stylesheet and the fonts it +// references must be in the public directory. +async function copyKatexAssets(publicDir: string) { + const dist = path.join(packageDir("katex"), "dist"); + + await fs.copyFile( + path.join(dist, "katex.min.css"), + path.join(publicDir, "katex.min.css"), + ); + await fs.cp(path.join(dist, "fonts"), path.join(publicDir, "fonts"), { + recursive: true, + }); +} + +// Emoji URLs are built at runtime, so the SVGs cannot be bundled and must be +// in the public directory. The version stamp skips the copy of ~3700 files +// when the assets are already installed. It lives outside the public directory +// because Vite copies that directory into the build as it is. node_modules is +// the right place for it: an install is what changes the package, and it is +// also what discards the stamp. +// +// The stamp is not enough on its own, because it and the assets sit in +// different places and either one can go without the other. The Docker e2e +// setup keeps node_modules in a volume that outlives the working tree, so a +// `git clean` there removes the assets and leaves the stamp behind. Checking +// the target as well stops a stamp that has outlived its assets from producing +// a build with no emoji in it, which nothing else would report. +async function copyTwemojiAssets(root: string, publicDir: string) { + const version = packageVersion("@twemoji/svg"); + const target = path.join(publicDir, "twemoji"); + const stamp = path.join(root, "node_modules", ".cache", "twemoji-version"); + + const installed = await fs.readFile(stamp, "utf8").catch(() => undefined); + const present = await fs.stat(target).catch(() => undefined); + if (installed === version && present) { + return; + } + + const source = packageDir("@twemoji/svg"); + await fs.cp(source, target, { + recursive: true, + // The package also holds a readme, a license and package.json. + filter: entry => entry === source || entry.endsWith(".svg"), + }); + await fs.mkdir(path.dirname(stamp), { recursive: true }); + await fs.writeFile(stamp, version); +} + +export function assets(): Plugin { + return { + name: "assets", + // Vitest loads this config too, but does not need the assets. + apply: () => !process.env.VITEST, + // The dev server reads the public directory right after the config is + // resolved, so the assets have to be in place before that. + async configResolved(config) { + await copyKatexAssets(config.publicDir); + await copyTwemojiAssets(config.root, config.publicDir); + }, + }; +} diff --git a/vite.config.ts b/vite.config.ts index 6effa64b1..a3a2c86a7 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -3,6 +3,8 @@ import path from "node:path"; import { defineConfig } from "vitest/config"; import { svelte } from "@sveltejs/vite-plugin-svelte"; +import { assets } from "./vite-plugin-assets"; + export default defineConfig({ define: { buildTimeConfig: JSON.stringify(config.util.toObject()), @@ -13,6 +15,7 @@ export default defineConfig({ reporters: "verbose", }, plugins: [ + assets(), svelte({ extensions: [".svelte", ".md"], // Reference: https://github.com/sveltejs/vite-plugin-svelte/issues/270#issuecomment-1033190138