diff --git a/explorer/RELEASE.md b/explorer/RELEASE.md index f6e7b4573..13a88d60e 100644 --- a/explorer/RELEASE.md +++ b/explorer/RELEASE.md @@ -93,8 +93,9 @@ Do these steps after the tag is canonical. git switch --detach releases/httpd/ ``` -2. Run the reproducible build (Linux x86_64 or macOS). A single run compiles the - workspace and packages **both** binaries, one archive per binary per target: +2. Run the reproducible build. A single run compiles the workspace and packages + **both** binaries, one archive per binary per target. It uses podman; set + `CONTAINER=docker` to use Docker: ```sh crates/radicle-httpd/build/build @@ -104,9 +105,16 @@ Do these steps after the tag is canonical. - `radicle-httpd--.tar.xz` (binary + man page) - `radicle-search--.tar.xz` (binary) - each with a `.sig` and `.sha256` alongside. + each with a `.sha256` alongside. -3. Upload. This publishes each binary into its own sibling folder on +3. Sign the archives with your Radicle key. This checks each archive against its + `.sha256` and writes a `.sig` alongside: + + ```sh + crates/radicle-httpd/build/sign + ``` + +4. Upload. This publishes each binary into its own sibling folder on [files.radicle.dev][files] and creates the version-less and `latest` symlinks for both: @@ -120,7 +128,10 @@ Do these steps after the tag is canonical. 1. Mark the previous release as resolved in Zulip `#Announcements` -2. Create a new announcement with the following template: +2. Create a new announcement with the following template. `` is the + 8-character ID of the tagged commit, which the build also writes to + `radicle-httpd.json`: + `git rev-parse --short=8 'releases/httpd/^{commit}'` **Topic:** `radicle-httpd + radicle-search ` @@ -129,7 +140,7 @@ Do these steps after the tag is canonical. ````markdown # 👾 radicle-httpd + radicle-search - `radicle-httpd` and `radicle-search` () are released. + `radicle-httpd` and `radicle-search` () are released. ## Installation @@ -158,7 +169,9 @@ Do these steps after the tag is canonical. > **Important:** > > - macOS binaries are unsigned and not notarized; download via CLI only -> - Build reproducibility across platforms is not guaranteed +> - The build always runs in a `linux/arm64` container, with the base images and +> zig pinned, so that builds on different hosts give the same bytes. It runs +> natively on Apple silicon, and under emulation on amd64 hosts ## Podman Setup diff --git a/explorer/crates/radicle-httpd/build/Dockerfile b/explorer/crates/radicle-httpd/build/Dockerfile index 3cf2a8a1b..68e93bcf7 100644 --- a/explorer/crates/radicle-httpd/build/Dockerfile +++ b/explorer/crates/radicle-httpd/build/Dockerfile @@ -1,9 +1,27 @@ # Builds release binaries for Radicle. +# +# Base images and zig are pinned by digest, so that builds on different hosts +# and days give the same bytes. When you bump `rust-toolchain`, also update the +# digest of the `rust` image: the digest wins over the tag. ARG RUST_VERSION="1.97.1" ARG ALPINE_VERSION="3.21" -FROM rust:${RUST_VERSION}-alpine${ALPINE_VERSION} as builder +FROM rust:${RUST_VERSION}-alpine${ALPINE_VERSION}@sha256:7bae7c67364dad5ebbd4060923b34d734fbed66d7c1cf3af72aa2f062af93eb6 AS builder LABEL maintainer="Radicle Team " WORKDIR /src + +ARG RUST_VERSION +# Fail if the pinned image does not match `rust-toolchain`. Check before +# copying the source: with `rust-toolchain` in place, rustup installs the +# version it names, and the check always passes. +RUN test "$(rustc --version | cut -d' ' -f2)" = "$RUST_VERSION" + +# Install dependencies for cross-compiling to macOS. +# We use Zig as the linker to perform the compilation from a Linux host. +# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`. +# Install it before copying the source, so that `rust-toolchain` does not +# override the default toolchain and the layer stays cached. +RUN cargo install cargo-zigbuild@0.23.4 --locked + COPY . . ARG TZ @@ -15,7 +33,16 @@ ARG GIT_HEAD # Copy cargo configuration we're going to use to specify compiler options. RUN mkdir -p .cargo && cp crates/radicle-httpd/build/config.toml .cargo/config.toml # Install dependencies. -RUN apk update && apk add --no-cache git musl-dev xz asciidoctor zig +RUN apk update && apk add --no-cache git musl-dev xz asciidoctor +# Install a pinned zig, since the version in apk changes over time. +ARG ZIG_VERSION="0.17.0" +ARG ZIG_SHA256="9e8d11661d4ae3bd57702a3832781e23ad151dde5798e16a5ccd503f65234ff8" +RUN wget -q -O /tmp/zig.tar.xz \ + "https://ziglang.org/download/${ZIG_VERSION}/zig-aarch64-linux-${ZIG_VERSION}.tar.xz" && \ + echo "${ZIG_SHA256} /tmp/zig.tar.xz" | sha256sum -c - && \ + mkdir /opt/zig && tar -xJf /tmp/zig.tar.xz -C /opt/zig --strip-components=1 && \ + rm /tmp/zig.tar.xz +ENV PATH="/opt/zig:${PATH}" # Build man pages and strip metadata. Removes all comments, since they include # non-reproducible information, such as version numbers. RUN asciidoctor --doctype manpage --backend manpage --destination-dir . \ @@ -30,17 +57,11 @@ RUN rustup target add \ x86_64-apple-darwin \ aarch64-apple-darwin -# Install dependencies for cross-compiling to macOS. -# We use Zig as the linker to perform the compilation from a Linux host. -# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`. -RUN cargo install cargo-zigbuild@0.20.0 --locked - - # Parts of the macOS SDK are required to build Radicle, we make these available # here. So far only `CoreFoundation` and `Security` frameworks are needed. RUN xz -d -c crates/radicle-httpd/build/macos-sdk-11.3.tar.xz | tar -x # This env var is used by `cargo-zigbuild` to find the SDK. -ENV SDKROOT /src/macos-sdk-11.3 +ENV SDKROOT=/src/macos-sdk-11.3 # Build binaries. RUN cargo zigbuild --locked --release \ @@ -53,7 +74,7 @@ RUN cargo zigbuild --locked --release \ # Now copy the files to a new image without all the intermediary artifacts to # save some space. -FROM alpine:3.21 as packager +FROM alpine:3.21@sha256:ee16d92f96db08d9d2d07da48555a27f3a69d8f0759021dbc601195ed0d2b5bd AS packager ARG RADICLE_HTTPD_VERSION ARG SOURCE_DATE_EPOCH diff --git a/explorer/crates/radicle-httpd/build/build b/explorer/crates/radicle-httpd/build/build index 852dd801c..865c320b7 100755 --- a/explorer/crates/radicle-httpd/build/build +++ b/explorer/crates/radicle-httpd/build/build @@ -1,25 +1,38 @@ #!/bin/sh set -e +# The container engine: `podman` (default) or `docker`. +CONTAINER=${CONTAINER:-podman} + main() { # Use UTC time for everything. export TZ=UTC0 # Set minimal locale. export LC_ALL=C # Set source date. This is honored by `asciidoctor` and other tools. - export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) - # Define user OS for podman - export OS=$(uname) - # Define architecture for podman - export ARCH=$(arch) + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + export SOURCE_DATE_EPOCH - if ! command -v rad >/dev/null; then - echo "fatal: rad is not installed" >&2 + case "$CONTAINER" in + podman) + # Rootless podman on Linux needs cgroupfs. + if [ "$(uname)" = Linux ]; then + engine="podman --cgroup-manager=cgroupfs" + else + engine=podman + fi + ;; + docker) + engine=docker + ;; + *) + echo "fatal: CONTAINER must be podman or docker, not '$CONTAINER'" >&2 exit 1 - fi + ;; + esac - if ! command -v podman >/dev/null; then - echo "fatal: podman is not installed" >&2 + if ! command -v "$CONTAINER" >/dev/null; then + echo "fatal: $CONTAINER is not installed" >&2 exit 1 fi @@ -28,21 +41,15 @@ main() { exit 1 fi - rev="$(git rev-parse --short HEAD)" + # Use a fixed length: the default of `--short` depends on the clone. + rev="$(git rev-parse HEAD | cut -c1-8)" tempdir="$(mktemp -d)" context="$tempdir/heartwood-$rev" - keypath="$(rad path)/keys/radicle.pub" version="$("$(dirname "$0")/version")" image=radicle-build-$version + container=radicle-build-container rust_version="$(cat "$(dirname "$0")/../../../rust-toolchain")" - if [ ! -f "$keypath" ]; then - echo "fatal: no key found at $keypath" >&2 - exit 1 - fi - # Authenticate user for signing - rad auth - echo "Building Radicle $version.." echo "Exporting repository at $rev to $context.." # Only explorer/ is part of the build, with paths relative to it. Run from @@ -52,67 +59,49 @@ main() { mkdir "$context" git -C "$(git rev-parse --show-toplevel)" archive HEAD:explorer | tar -x -C "$context" - echo "Building image ($image).." + echo "Building image ($image) with $CONTAINER.." - case "$OS" in - Darwin) - podman build \ - --build-arg "RUST_VERSION=$rust_version" \ - --build-arg SOURCE_DATE_EPOCH \ - --build-arg TZ \ - --build-arg LC_ALL \ - --build-arg RADICLE_HTTPD_VERSION=$version \ - --build-arg GIT_HEAD=$rev \ - --arch $ARCH --tag $image -f "$(dirname "$0")/Dockerfile" "$context" + # Build on linux/arm64 on every host, so that builds on different hosts + # give the same bytes. Apple silicon runs it natively; amd64 hosts run it + # under emulation. + $engine build \ + --build-arg "RUST_VERSION=$rust_version" \ + --build-arg SOURCE_DATE_EPOCH \ + --build-arg TZ \ + --build-arg LC_ALL \ + --build-arg "RADICLE_HTTPD_VERSION=$version" \ + --build-arg "GIT_HEAD=$rev" \ + --platform linux/arm64 --tag "$image" -f "$(dirname "$0")/Dockerfile" "$context" - echo "Creating container (radicle-build-container).." - podman create --ulimit=host --replace --name radicle-build-container $image - ;; - *) - podman --cgroup-manager=cgroupfs build \ - --build-arg "RUST_VERSION=$rust_version" \ - --build-arg SOURCE_DATE_EPOCH \ - --build-arg TZ \ - --build-arg LC_ALL \ - --build-arg GIT_HEAD=$rev \ - --build-arg RADICLE_HTTPD_VERSION=$version \ - --arch $ARCH --tag $image -f "$(dirname "$0")/Dockerfile" "$context" - - echo "Creating container (radicle-build-container).." - podman --cgroup-manager=cgroupfs create --ulimit=host --replace --name radicle-build-container $image - ;; - esac + echo "Creating container ($container).." + $engine rm --force "$container" >/dev/null 2>&1 || true + $engine create --name "$container" "$image" >/dev/null # Copy build artifacts to output folder. outdir="$(dirname "$0")/artifacts" - mkdir -p $outdir - podman cp --overwrite radicle-build-container:/builds/. $outdir/ + mkdir -p "$outdir" + $engine cp "$container:/builds/." "$outdir/" for target in $(cat "$(dirname "$0")/TARGETS"); do # `radicle-httpd` and `radicle-search` are released together: each ships as # its own per-target archive. for bin in radicle-httpd radicle-search; do - echo "Signing $bin artifacts for $target.." - filename="$bin-$version-$target.tar.xz" filepath="$outdir/$filename" # Output SHA256 digest of archive. - checksum="$(cd $outdir && sha256sum $filename)" + checksum="$(cd "$outdir" && sha256sum "$filename")" echo "Checksum of $filepath is $(echo "$checksum" | cut -d' ' -f1)" - echo "$checksum" >$filepath.sha256 - - # Sign archive and verify archive. - rm -f $filepath.sig # Delete existing signature - ssh-keygen -Y sign -n file -f $keypath $filepath - ssh-keygen -Y check-novalidate -n file -s $filepath.sig <$filepath + echo "$checksum" >"$filepath.sha256" + # A signature from an earlier build no longer matches. + rm -f "$filepath.sig" done done # Remove build artifacts that aren't needed anymore. rm -rf "$tempdir" - podman rm radicle-build-container >/dev/null - podman rmi --ignore localhost/$image + $engine rm "$container" >/dev/null + $engine rmi "$image" >/dev/null || true } # Run build. diff --git a/explorer/crates/radicle-httpd/build/config.toml b/explorer/crates/radicle-httpd/build/config.toml index b50ab6768..1bc60983b 100644 --- a/explorer/crates/radicle-httpd/build/config.toml +++ b/explorer/crates/radicle-httpd/build/config.toml @@ -11,3 +11,27 @@ rustflags = [ "-C", "incremental=false", "-C", "opt-level=3", ] + +# Several crates link the same libraries and frameworks on Apple targets, so +# `rustc` passes them more than once. Apple's `ld` deduplicates, but Zig does +# not, and emits duplicate `LC_LOAD_DYLIB` commands, e.g. for CoreFoundation. +# `dyld` on macOS 26 refuses to load such a binary ("duplicate linked dylib"), +# while older versions tolerate it. `-dead_strip_dylibs` drops the redundant +# entries. +# +# Zig also links its own compiler_rt, which it compiles on many threads. It +# names constants there after an index that depends on the thread, e.g. +# `___anon_671098611`, so the symbol table changes from one build to the next. +# `-rtlib=none` leaves compiler_rt out: Rust's compiler_builtins and libSystem +# provide everything it did. +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-Wl,-dead_strip_dylibs", + "-C", "link-arg=-rtlib=none", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-Wl,-dead_strip_dylibs", + "-C", "link-arg=-rtlib=none", +] diff --git a/explorer/crates/radicle-httpd/build/sign b/explorer/crates/radicle-httpd/build/sign new file mode 100755 index 000000000..bc7d3d021 --- /dev/null +++ b/explorer/crates/radicle-httpd/build/sign @@ -0,0 +1,50 @@ +#!/bin/sh +set -e + +# Signs the archives in `artifacts/` with the delegate's Radicle key, whether +# `build` or `fetch` put them there. + +outdir="$(dirname "$0")/artifacts" +version="$("$(dirname "$0")/version")" + +if ! command -v rad >/dev/null; then + echo "fatal: rad is not installed" >&2 + exit 1 +fi + +if ! command -v sha256sum >/dev/null; then + echo "fatal: sha256sum is not installed" >&2 + exit 1 +fi + +keypath="$(rad path)/keys/radicle.pub" + +if [ ! -f "$keypath" ]; then + echo "fatal: no key found at $keypath" >&2 + exit 1 +fi +# Authenticate user for signing +rad auth + +for target in $(cat "$(dirname "$0")/TARGETS"); do + for bin in radicle-httpd radicle-search; do + filename="$bin-$version-$target.tar.xz" + filepath="$outdir/$filename" + + if [ ! -f "$filepath" ]; then + echo "fatal: $filepath is missing" >&2 + exit 1 + fi + + echo "Signing $filename.." + + # Never sign an archive that differs from its checksum. + (cd "$outdir" && sha256sum --check --quiet "$filename.sha256") + + rm -f "$filepath.sig" + ssh-keygen -Y sign -n file -f "$keypath" "$filepath" + ssh-keygen -Y check-novalidate -n file -s "$filepath.sig" <"$filepath" + done +done + +echo "Signing successful." diff --git a/explorer/crates/radicle-httpd/build/version b/explorer/crates/radicle-httpd/build/version index 089d5ee46..1709e8de9 100755 --- a/explorer/crates/radicle-httpd/build/version +++ b/explorer/crates/radicle-httpd/build/version @@ -1,6 +1,13 @@ #!/bin/sh -if ! version="$(git describe --match='releases/httpd/*' --candidates=1 2>/dev/null)"; then +# CI sets the version from the tag it builds, since its checkout can lack the +# tag ref that `git describe` needs. +if [ -n "$RADICLE_HTTPD_VERSION" ]; then + echo "$RADICLE_HTTPD_VERSION" + exit +fi + +if ! version="$(git describe --match='releases/httpd/*' --candidates=1 --abbrev=8 2>/dev/null)"; then echo "fatal: no version tag found by 'git describe'" >&2 ; exit 1 fi # Remove `releases/httpd/` prefix from version.