httpd: Make release builds reproducible

- Pin the rust and alpine images by digest - Check that the rust image matches rust-toolchain - Install pinned zig 0.17.0 and cargo-zigbuild 0.23.4 - Install cargo-zigbuild before the source copy, to cache it - Build on linux/arm64 on every host, with docker or podman - Use 8-character commit IDs and read RADICLE_HTTPD_VERSION - Name the tagged commit in the announcement, not a build ID - Link macOS binaries with -dead_strip_dylibs and -rtlib=none - Move signing out of build into build/sign, which checks checksums - Fix Dockerfile lint warnings
2026-11-15 11:20:36 +08:00 · 2026-11-06 10:45:37 +01:00
parent b522155767
commit 63a5bb2266
6 changed files with 182 additions and 78 deletions
@@ -93,8 +93,9 @@ Do these steps after the tag is canonical.
   git switch --detach releases/httpd/<version>
   ```

-2. Run the reproducible build (Linux x86_64 or macOS). A single run compiles the
-   workspace and packages **both** binaries, one archive per binary per target:
+2. Run the reproducible build. A single run compiles the workspace and packages
+   **both** binaries, one archive per binary per target. It uses podman; set
+   `CONTAINER=docker` to use Docker:

   ```sh
   crates/radicle-httpd/build/build
@@ -104,9 +105,16 @@ Do these steps after the tag is canonical.
   - `radicle-httpd-<version>-<target>.tar.xz` (binary + man page)
   - `radicle-search-<version>-<target>.tar.xz` (binary)

-   each with a `.sig` and `.sha256` alongside.
+   each with a `.sha256` alongside.

-3. Upload. This publishes each binary into its own sibling folder on
+3. Sign the archives with your Radicle key. This checks each archive against its
+   `.sha256` and writes a `.sig` alongside:
+
+   ```sh
+   crates/radicle-httpd/build/sign
+   ```
+
+4. Upload. This publishes each binary into its own sibling folder on
   [files.radicle.dev][files] and creates the version-less and `latest` symlinks
   for both:

@@ -120,7 +128,10 @@ Do these steps after the tag is canonical.

 1. Mark the previous release as resolved in Zulip `#Announcements`

-2. Create a new announcement with the following template:
+2. Create a new announcement with the following template. `<commit>` is the
+   8-character ID of the tagged commit, which the build also writes to
+   `radicle-httpd.json`:
+   `git rev-parse --short=8 'releases/httpd/<version>^{commit}'`

   **Topic:** `radicle-httpd + radicle-search <version>`

@@ -129,7 +140,7 @@ Do these steps after the tag is canonical.
   ````markdown
   # 👾 radicle-httpd + radicle-search <version>

-   `radicle-httpd` and `radicle-search` <version> (<build-id>) are released.
+   `radicle-httpd` and `radicle-search` <version> (<commit>) are released.

   ## Installation

@@ -158,7 +169,9 @@ Do these steps after the tag is canonical.
 > **Important:**
 >
 > - macOS binaries are unsigned and not notarized; download via CLI only
-> - Build reproducibility across platforms is not guaranteed
+> - The build always runs in a `linux/arm64` container, with the base images and
+>   zig pinned, so that builds on different hosts give the same bytes. It runs
+>   natively on Apple silicon, and under emulation on amd64 hosts

 ## Podman Setup

@@ -1,9 +1,27 @@
 # Builds release binaries for Radicle.
+#
+# Base images and zig are pinned by digest, so that builds on different hosts
+# and days give the same bytes. When you bump `rust-toolchain`, also update the
+# digest of the `rust` image: the digest wins over the tag.
 ARG RUST_VERSION="1.97.1"
 ARG ALPINE_VERSION="3.21"
-FROM rust:${RUST_VERSION}-alpine${ALPINE_VERSION} as builder
+FROM rust:${RUST_VERSION}-alpine${ALPINE_VERSION}@sha256:7bae7c67364dad5ebbd4060923b34d734fbed66d7c1cf3af72aa2f062af93eb6 AS builder
 LABEL maintainer="Radicle Team <team@radicle.xyz>"
 WORKDIR /src
+
+ARG RUST_VERSION
+# Fail if the pinned image does not match `rust-toolchain`. Check before
+# copying the source: with `rust-toolchain` in place, rustup installs the
+# version it names, and the check always passes.
+RUN test "$(rustc --version | cut -d' ' -f2)" = "$RUST_VERSION"
+
+# Install dependencies for cross-compiling to macOS.
+# We use Zig as the linker to perform the compilation from a Linux host.
+# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`.
+# Install it before copying the source, so that `rust-toolchain` does not
+# override the default toolchain and the layer stays cached.
+RUN cargo install cargo-zigbuild@0.23.4 --locked
+
 COPY . .

 ARG TZ
@@ -15,7 +33,16 @@ ARG GIT_HEAD
 # Copy cargo configuration we're going to use to specify compiler options.
 RUN mkdir -p .cargo && cp crates/radicle-httpd/build/config.toml .cargo/config.toml
 # Install dependencies.
-RUN apk update && apk add --no-cache git musl-dev xz asciidoctor zig
+RUN apk update && apk add --no-cache git musl-dev xz asciidoctor
+# Install a pinned zig, since the version in apk changes over time.
+ARG ZIG_VERSION="0.17.0"
+ARG ZIG_SHA256="9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8"
+RUN wget -q -O /tmp/zig.tar.xz \
+    "https://ziglang.org/download/${ZIG_VERSION}/zig-aarch64-linux-${ZIG_VERSION}.tar.xz" && \
+    echo "${ZIG_SHA256}  /tmp/zig.tar.xz" | sha256sum -c - && \
+    mkdir /opt/zig && tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 && \
+    rm /tmp/zig.tar.xz
+ENV PATH="/opt/zig:${PATH}"
 # Build man pages and strip metadata. Removes all comments, since they include
 # non-reproducible information, such as version numbers.
 RUN asciidoctor --doctype manpage --backend manpage --destination-dir . \
@@ -30,17 +57,11 @@ RUN rustup target add \
    x86_64-apple-darwin \
    aarch64-apple-darwin

-# Install dependencies for cross-compiling to macOS.
-# We use Zig as the linker to perform the compilation from a Linux host.
-# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`.
-RUN cargo install cargo-zigbuild@0.20.0 --locked
-
-
 # Parts of the macOS SDK are required to build Radicle, we make these available
 # here. So far only `CoreFoundation` and `Security` frameworks are needed.
 RUN xz -d -c crates/radicle-httpd/build/macos-sdk-11.3.tar.xz | tar -x
 # This env var is used by `cargo-zigbuild` to find the SDK.
-ENV SDKROOT /src/macos-sdk-11.3
+ENV SDKROOT=/src/macos-sdk-11.3

 # Build binaries.
 RUN cargo zigbuild --locked --release \
@@ -53,7 +74,7 @@ RUN cargo zigbuild --locked --release \

 # Now copy the files to a new image without all the intermediary artifacts to
 # save some space.
-FROM alpine:3.21 as packager
+FROM alpine:3.21@sha256:ee16d92f96db08d9d2d07da48555a27f3a69d8f0759021dbc601195ed0d2b5bd AS packager

 ARG RADICLE_HTTPD_VERSION
 ARG SOURCE_DATE_EPOCH
@@ -1,25 +1,38 @@
 #!/bin/sh
 set -e

+# The container engine: `podman` (default) or `docker`.
+CONTAINER=${CONTAINER:-podman}
+
 main() {
  # Use UTC time for everything.
  export TZ=UTC0
  # Set minimal locale.
  export LC_ALL=C
  # Set source date. This is honored by `asciidoctor` and other tools.
-  export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
-  # Define user OS for podman
-  export OS=$(uname)
-  # Define architecture for podman
-  export ARCH=$(arch)
+  SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
+  export SOURCE_DATE_EPOCH

-  if ! command -v rad >/dev/null; then
-    echo "fatal: rad is not installed" >&2
+  case "$CONTAINER" in
+  podman)
+    # Rootless podman on Linux needs cgroupfs.
+    if [ "$(uname)" = Linux ]; then
+      engine="podman --cgroup-manager=cgroupfs"
+    else
+      engine=podman
+    fi
+    ;;
+  docker)
+    engine=docker
+    ;;
+  *)
+    echo "fatal: CONTAINER must be podman or docker, not '$CONTAINER'" >&2
    exit 1
-  fi
+    ;;
+  esac

-  if ! command -v podman >/dev/null; then
-    echo "fatal: podman is not installed" >&2
+  if ! command -v "$CONTAINER" >/dev/null; then
+    echo "fatal: $CONTAINER is not installed" >&2
    exit 1
  fi

@@ -28,21 +41,15 @@ main() {
    exit 1
  fi

-  rev="$(git rev-parse --short HEAD)"
+  # Use a fixed length: the default of `--short` depends on the clone.
+  rev="$(git rev-parse HEAD | cut -c1-8)"
  tempdir="$(mktemp -d)"
  context="$tempdir/heartwood-$rev"
-  keypath="$(rad path)/keys/radicle.pub"
  version="$("$(dirname "$0")/version")"
  image=radicle-build-$version
+  container=radicle-build-container
  rust_version="$(cat "$(dirname "$0")/../../../rust-toolchain")"

-  if [ ! -f "$keypath" ]; then
-    echo "fatal: no key found at $keypath" >&2
-    exit 1
-  fi
-  # Authenticate user for signing
-  rad auth
-
  echo "Building Radicle $version.."
  echo "Exporting repository at $rev to $context.."
  # Only explorer/ is part of the build, with paths relative to it. Run from
@@ -52,67 +59,49 @@ main() {
  mkdir "$context"
  git -C "$(git rev-parse --show-toplevel)" archive HEAD:explorer | tar -x -C "$context"

-  echo "Building image ($image).."
+  echo "Building image ($image) with $CONTAINER.."

-  case "$OS" in
-  Darwin)
-    podman build \
-      --build-arg "RUST_VERSION=$rust_version" \
-      --build-arg SOURCE_DATE_EPOCH \
-      --build-arg TZ \
-      --build-arg LC_ALL \
-      --build-arg RADICLE_HTTPD_VERSION=$version \
-      --build-arg GIT_HEAD=$rev \
-      --arch $ARCH --tag $image -f "$(dirname "$0")/Dockerfile" "$context"
+  # Build on linux/arm64 on every host, so that builds on different hosts
+  # give the same bytes. Apple silicon runs it natively; amd64 hosts run it
+  # under emulation.
+  $engine build \
+    --build-arg "RUST_VERSION=$rust_version" \
+    --build-arg SOURCE_DATE_EPOCH \
+    --build-arg TZ \
+    --build-arg LC_ALL \
+    --build-arg "RADICLE_HTTPD_VERSION=$version" \
+    --build-arg "GIT_HEAD=$rev" \
+    --platform linux/arm64 --tag "$image" -f "$(dirname "$0")/Dockerfile" "$context"

-    echo "Creating container (radicle-build-container).."
-    podman create --ulimit=host --replace --name radicle-build-container $image
-    ;;
-  *)
-    podman --cgroup-manager=cgroupfs build \
-      --build-arg "RUST_VERSION=$rust_version" \
-      --build-arg SOURCE_DATE_EPOCH \
-      --build-arg TZ \
-      --build-arg LC_ALL \
-      --build-arg GIT_HEAD=$rev \
-      --build-arg RADICLE_HTTPD_VERSION=$version \
-      --arch $ARCH --tag $image -f "$(dirname "$0")/Dockerfile" "$context"
-
-    echo "Creating container (radicle-build-container).."
-    podman --cgroup-manager=cgroupfs create --ulimit=host --replace --name radicle-build-container $image
-    ;;
-  esac
+  echo "Creating container ($container).."
+  $engine rm --force "$container" >/dev/null 2>&1 || true
+  $engine create --name "$container" "$image" >/dev/null

  # Copy build artifacts to output folder.
  outdir="$(dirname "$0")/artifacts"
-  mkdir -p $outdir
-  podman cp --overwrite radicle-build-container:/builds/. $outdir/
+  mkdir -p "$outdir"
+  $engine cp "$container:/builds/." "$outdir/"

  for target in $(cat "$(dirname "$0")/TARGETS"); do
    # `radicle-httpd` and `radicle-search` are released together: each ships as
    # its own per-target archive.
    for bin in radicle-httpd radicle-search; do
-      echo "Signing $bin artifacts for $target.."
-
      filename="$bin-$version-$target.tar.xz"
      filepath="$outdir/$filename"

      # Output SHA256 digest of archive.
-      checksum="$(cd $outdir && sha256sum $filename)"
+      checksum="$(cd "$outdir" && sha256sum "$filename")"
      echo "Checksum of $filepath is $(echo "$checksum" | cut -d' ' -f1)"
-      echo "$checksum" >$filepath.sha256
-
-      # Sign archive and verify archive.
-      rm -f $filepath.sig # Delete existing signature
-      ssh-keygen -Y sign -n file -f $keypath $filepath
-      ssh-keygen -Y check-novalidate -n file -s $filepath.sig <$filepath
+      echo "$checksum" >"$filepath.sha256"
+      # A signature from an earlier build no longer matches.
+      rm -f "$filepath.sig"
    done
  done

  # Remove build artifacts that aren't needed anymore.
  rm -rf "$tempdir"
-  podman rm radicle-build-container >/dev/null
-  podman rmi --ignore localhost/$image
+  $engine rm "$container" >/dev/null
+  $engine rmi "$image" >/dev/null || true
 }

 # Run build.
@@ -11,3 +11,27 @@ rustflags = [
    "-C", "incremental=false",
    "-C", "opt-level=3",
 ]
+
+# Several crates link the same libraries and frameworks on Apple targets, so
+# `rustc` passes them more than once. Apple's `ld` deduplicates, but Zig does
+# not, and emits duplicate `LC_LOAD_DYLIB` commands, e.g. for CoreFoundation.
+# `dyld` on macOS 26 refuses to load such a binary ("duplicate linked dylib"),
+# while older versions tolerate it. `-dead_strip_dylibs` drops the redundant
+# entries.
+#
+# Zig also links its own compiler_rt, which it compiles on many threads. It
+# names constants there after an index that depends on the thread, e.g.
+# `___anon_671098611`, so the symbol table changes from one build to the next.
+# `-rtlib=none` leaves compiler_rt out: Rust's compiler_builtins and libSystem
+# provide everything it did.
+[target.x86_64-apple-darwin]
+rustflags = [
+    "-C", "link-arg=-Wl,-dead_strip_dylibs",
+    "-C", "link-arg=-rtlib=none",
+]
+
+[target.aarch64-apple-darwin]
+rustflags = [
+    "-C", "link-arg=-Wl,-dead_strip_dylibs",
+    "-C", "link-arg=-rtlib=none",
+]
@@ -0,0 +1,50 @@
+#!/bin/sh
+set -e
+
+# Signs the archives in `artifacts/` with the delegate's Radicle key, whether
+# `build` or `fetch` put them there.
+
+outdir="$(dirname "$0")/artifacts"
+version="$("$(dirname "$0")/version")"
+
+if ! command -v rad >/dev/null; then
+  echo "fatal: rad is not installed" >&2
+  exit 1
+fi
+
+if ! command -v sha256sum >/dev/null; then
+  echo "fatal: sha256sum is not installed" >&2
+  exit 1
+fi
+
+keypath="$(rad path)/keys/radicle.pub"
+
+if [ ! -f "$keypath" ]; then
+  echo "fatal: no key found at $keypath" >&2
+  exit 1
+fi
+# Authenticate user for signing
+rad auth
+
+for target in $(cat "$(dirname "$0")/TARGETS"); do
+  for bin in radicle-httpd radicle-search; do
+    filename="$bin-$version-$target.tar.xz"
+    filepath="$outdir/$filename"
+
+    if [ ! -f "$filepath" ]; then
+      echo "fatal: $filepath is missing" >&2
+      exit 1
+    fi
+
+    echo "Signing $filename.."
+
+    # Never sign an archive that differs from its checksum.
+    (cd "$outdir" && sha256sum --check --quiet "$filename.sha256")
+
+    rm -f "$filepath.sig"
+    ssh-keygen -Y sign -n file -f "$keypath" "$filepath"
+    ssh-keygen -Y check-novalidate -n file -s "$filepath.sig" <"$filepath"
+  done
+done
+
+echo "Signing successful."
@@ -1,6 +1,13 @@
 #!/bin/sh

-if ! version="$(git describe --match='releases/httpd/*' --candidates=1 2>/dev/null)"; then
+# CI sets the version from the tag it builds, since its checkout can lack the
+# tag ref that `git describe` needs.
+if [ -n "$RADICLE_HTTPD_VERSION" ]; then
+  echo "$RADICLE_HTTPD_VERSION"
+  exit
+fi
+
+if ! version="$(git describe --match='releases/httpd/*' --candidates=1 --abbrev=8 2>/dev/null)"; then
  echo "fatal: no version tag found by 'git describe'" >&2 ; exit 1
 fi
 # Remove `releases/httpd/` prefix from version.