diff --git a/.gitignore b/.gitignore index 3bdd7284b..2b47c0cde 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules/ NOTES config/local* http-server/target +http-server/build/artifacts # KaTeX files *.min.css diff --git a/http-server/build/Dockerfile b/http-server/build/Dockerfile new file mode 100644 index 000000000..8ae30d1fd --- /dev/null +++ b/http-server/build/Dockerfile @@ -0,0 +1,90 @@ +# Builds release binaries for Radicle. +FROM rust:1.77.2-alpine3.19 as builder +LABEL maintainer="Radicle Team " +WORKDIR /src +COPY . . + +# Copy cargo configuration we're going to use to specify compiler options. +RUN mkdir -p .cargo && cp build/config.toml .cargo/config.toml +# Install dependencies. +RUN apk update && apk add --no-cache git musl-dev minisign curl xz asciidoctor +# Build man pages and strip metadata. Removes all comments, since they include +# non-reproducible information, such as version numbers. +RUN asciidoctor --doctype manpage --backend manpage --destination-dir . *.1.adoc && \ + find . -maxdepth 1 -type f -name '*.1' -exec sed -i '/^.\\\"/d' '{}' \; +# Add cargo targets. +RUN rustup target add \ + x86_64-unknown-linux-musl \ + aarch64-unknown-linux-musl \ + x86_64-apple-darwin \ + aarch64-apple-darwin + +# Install dependencies for cross-compiling to macOS. +# We use Zig as the linker to perform the compilation from a Linux host. +# Zig is not yet available on Debian, so we download the official binary. +# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`. +RUN curl -sSf -o zig.tar.xz https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz && \ + curl -sSf -o zig.tar.xz.minisig https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz.minisig && \ + minisign -Vm zig.tar.xz -P RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U && \ + xz -d -c zig.tar.xz | tar -x && \ + mv zig-linux-x86_64-0.12.0/zig /usr/bin/zig && \ + mv zig-linux-x86_64-0.12.0/lib /usr/lib/zig && \ + cargo install cargo-zigbuild@0.18.3 + + +# Parts of the macOS SDK are required to build Radicle, we make these available +# here. So far only `CoreFoundation` and `Security` frameworks are needed. +RUN xz -d -c build/macos-sdk-11.3.tar.xz | tar -x +# This env var is used by `cargo-zigbuild` to find the SDK. +ENV SDKROOT /src/macos-sdk-11.3 + +# Build binaries. +RUN cargo zigbuild --locked --release \ + --target=x86_64-apple-darwin \ + --target=aarch64-apple-darwin \ + --target=aarch64-unknown-linux-musl \ + --target=x86_64-unknown-linux-musl \ + -p radicle-httpd + +# Now copy the files to a new image without all the intermediary artifacts to +# save some space. +FROM alpine:3.19 as packager +COPY --from=builder \ + /src/target/x86_64-unknown-linux-musl/release/rad-web \ + /src/target/x86_64-unknown-linux-musl/release/radicle-httpd \ + /builds/x86_64-unknown-linux-musl/bin/ +COPY --from=builder \ + /src/target/aarch64-unknown-linux-musl/release/rad-web \ + /src/target/aarch64-unknown-linux-musl/release/radicle-httpd \ + /builds/aarch64-unknown-linux-musl/bin/ +COPY --from=builder \ + /src/target/aarch64-apple-darwin/release/rad-web \ + /src/target/aarch64-apple-darwin/release/radicle-httpd \ + /builds/aarch64-apple-darwin/bin/ +COPY --from=builder \ + /src/target/x86_64-apple-darwin/release/rad-web \ + /src/target/x86_64-apple-darwin/release/radicle-httpd \ + /builds/x86_64-apple-darwin/bin/ +COPY --from=builder /src/*.1 /builds/x86_64-unknown-linux-musl/man/man1/ +COPY --from=builder /src/*.1 /builds/aarch64-unknown-linux-musl/man/man1/ +COPY --from=builder /src/*.1 /builds/aarch64-apple-darwin/man/man1/ +COPY --from=builder /src/*.1 /builds/x86_64-apple-darwin/man/man1/ + +# Create and compress reproducible archive. +WORKDIR /builds +RUN apk update && apk add --no-cache tar xz +RUN find * -maxdepth 0 -type d -exec mv '{}' "radicle-$RADICLE_VERSION-{}" \; && \ + find * -maxdepth 0 -type d -exec tar \ + --sort=name \ + --verbose \ + --mtime="@$GIT_COMMIT_TIME" \ + --owner=0 \ + --group=0 \ + --numeric-owner \ + --format=posix \ + --pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \ + --mode='go+u,go-w' \ + --remove-files \ + --create --xz \ + --file="{}.tar.xz" \ + '{}' \; diff --git a/http-server/build/README.md b/http-server/build/README.md new file mode 100644 index 000000000..51ec10768 --- /dev/null +++ b/http-server/build/README.md @@ -0,0 +1,70 @@ +# Builds + +Radicle uses a [reproducible build][rb] pipeline to make binary verification +easier and more secure. + +[rb]: https://reproducible-builds.org/ + +This build pipeline is designed to be run on an x86_64 machine running Linux. +The output is a set of `.tar.xz` archives containing binaries for the supported +platforms and signed by the user's Radicle key. + +These binaries are statically linked to be maximally portable, and designed to +be reproducible, byte for byte. + +To run the build, simply enter the following command from the repository root: + + build/build + +This will build all targets and place the output in `build/artifacts` with +one sub-directory per build target. + +Note that it will use `git describe` to get a version number for the build. +You *must* have a commit tagged with a version in your history or the build +will fail, eg. `v1.0.0`. + +When the build completes, the SHA-256 checksums of the artifacts are output. +For a given Radicle version and source tree, the same set of checksums should +always be output, no matter where or when the build is run. If they do not +match, either the build pipeline has a bug, making it non-reproducible, or one +of the machines is compromised. + +Here's an example output for a development version of Radicle: + + b9aa75bba175e18e05df4f6b39ec097414bbf56ccdeb4a2229b557f8b8e05404 radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-apple-darwin.tar.xz + c7070806bf2d17a8a0d3b329e4d57b1e544b7b82cb58e2863074d96348a2ab0d radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-unknown-linux-musl.tar.xz + 1a8327854f16ea90491fb90e0c3291a63c4b2ab01742c8435faec7d370cacb79 radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-apple-darwin.tar.xz + 709ac67541ff0c0c570ac22ab2de9f98320e0cc2cc9b67f1909c014a2bb5bd49 radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-unknown-linux-musl.tar.xz + +A script is included in `build/checksums` to output these checksums after +the artifacts are built. + +## Requirements + +The following software is required for the build: + + * `podman` + * `rad` (The Radicle CLI) + * `sha256sum` + +## macOS + +macOS binaries are not signed or notarized, so they have to be downloaded via +the CLI to avoid issues. A copy of a small subset of the Apple SDK is included +here to be able to cross-compile. + +## Podman + +We use `podman` to make the build reproducible on any machine by controlling +the build environment. We prefer `podman` to `docker` because it doesn't +require a background process to run and can be run without root access out of +the box. + +The first time you run `podman`, you may have to give yourself some extra UIDs +for `podman` to use, with: + + sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USER + +Then update `podman` with: + + podman system migrate diff --git a/http-server/build/TARGETS b/http-server/build/TARGETS new file mode 100644 index 000000000..eec922875 --- /dev/null +++ b/http-server/build/TARGETS @@ -0,0 +1,4 @@ +x86_64-unknown-linux-musl +aarch64-unknown-linux-musl +x86_64-apple-darwin +aarch64-apple-darwin diff --git a/http-server/build/build b/http-server/build/build new file mode 100755 index 000000000..0c8ed44b8 --- /dev/null +++ b/http-server/build/build @@ -0,0 +1,115 @@ +#!/bin/sh +set -e + +main() { + # Use UTC time for everything. + export TZ=UTC0 + # Set minimal locale. + export LC_ALL=C + # Set source date. This is honored by `asciidoctor` and other tools. + export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + # Define user OS for podman + export OS=$(uname) + + if ! command -v rad >/dev/null; then + echo "fatal: rad is not installed" >&2 + exit 1 + fi + + if ! command -v podman >/dev/null; then + echo "fatal: podman is not installed" >&2 + exit 1 + fi + + if ! command -v sha256sum >/dev/null; then + echo "fatal: sha256sum is not installed" >&2 + exit 1 + fi + + rev="$(git rev-parse --short HEAD)" + tempdir="$(mktemp -d)" + gitarchive="$tempdir/heartwood-$rev.tar.gz" + keypath="$(rad path)/keys/radicle.pub" + version="$(build/version)" + image=radicle-build-$version + + if [ ! -f "$keypath" ]; then + echo "fatal: no key found at $keypath" >&2 + exit 1 + fi + # Authenticate user for signing + rad auth + + echo "Building Radicle $version.." + echo "Creating archive of repository at $rev in $gitarchive.." + git archive --format tar.gz -o "$gitarchive" HEAD + + echo "Building image ($image).." + + case "$OS" in + Darwin) + podman build \ + --env SOURCE_DATE_EPOCH \ + --env TZ \ + --env LC_ALL \ + --env GIT_COMMIT_TIME=$SOURCE_DATE_EPOCH \ + --env GIT_HEAD=$rev \ + --env RADICLE_VERSION=$version \ + --arch aarch64 --tag $image -f ./build/Dockerfile - <$gitarchive + + echo "Creating container (radicle-build-container).." + podman create --ulimit=host --replace --name radicle-build-container $image + ;; + *) + podman --cgroup-manager=cgroupfs build \ + --env SOURCE_DATE_EPOCH \ + --env TZ \ + --env LC_ALL \ + --env GIT_COMMIT_TIME=$SOURCE_DATE_EPOCH \ + --env GIT_HEAD=$rev \ + --env RADICLE_VERSION=$version \ + --arch amd64 --tag $image -f ./build/Dockerfile - <$gitarchive + + echo "Creating container (radicle-build-container).." + podman --cgroup-manager=cgroupfs create --ulimit=host --replace --name radicle-build-container $image + ;; + esac + + # Copy build artifacts to output folder. + outdir=build/artifacts + mkdir -p $outdir + podman cp --overwrite radicle-build-container:/builds/. $outdir/ + + for target in $(cat build/TARGETS); do + echo "Signing artifacts for $target.." + + filename="radicle-$version-$target.tar.xz" + filepath="$outdir/$filename" + + # Output SHA256 digest of archive. + checksum="$(cd $outdir && sha256sum $filename)" + echo "Checksum of $filepath is $(echo "$checksum" | cut -d' ' -f1)" + echo "$checksum" >$filepath.sha256 + + # Sign archive and verify archive. + rm -f $filepath.sig # Delete existing signature + ssh-keygen -Y sign -n file -f $keypath $filepath + ssh-keygen -Y check-novalidate -n file -s $filepath.sig <$filepath + done + + # Remove build artifacts that aren't needed anymore. + rm -f $gitarchive + podman rm radicle-build-container >/dev/null + podman rmi --ignore localhost/$image +} + +# Run build. +echo "Running build.." +main "$@" + +# Show artifact checksums. +echo +build/checksums +echo + +echo "Build successful." diff --git a/http-server/build/checksums b/http-server/build/checksums new file mode 100755 index 000000000..a0225cf56 --- /dev/null +++ b/http-server/build/checksums @@ -0,0 +1,2 @@ +#!/bin/sh +find build/artifacts -type f -name '*.sha256' -exec cat {} + diff --git a/http-server/build/config.toml b/http-server/build/config.toml new file mode 100644 index 000000000..b50ab6768 --- /dev/null +++ b/http-server/build/config.toml @@ -0,0 +1,13 @@ +[target.x86_64-unknown-linux-musl] +rustflags = [ + "-C", "codegen-units=1", + "-C", "incremental=false", + "-C", "opt-level=3", +] + +[target.aarch64-unknown-linux-musl] +rustflags = [ + "-C", "codegen-units=1", + "-C", "incremental=false", + "-C", "opt-level=3", +] diff --git a/http-server/build/macos-sdk-11.3.tar.xz b/http-server/build/macos-sdk-11.3.tar.xz new file mode 100644 index 000000000..02fad1f94 Binary files /dev/null and b/http-server/build/macos-sdk-11.3.tar.xz differ diff --git a/http-server/build/tag b/http-server/build/tag new file mode 100755 index 000000000..7c8b3dbf5 --- /dev/null +++ b/http-server/build/tag @@ -0,0 +1,38 @@ +#!/bin/sh +set -e + +if [ $# -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +version="$1" +tag="v$version" +commit="$(git rev-parse HEAD)" +signing_key=$(git config user.signingKey) + +git show "$commit" + +if [ "$signing_key" != "$(rad self --ssh-key)" ]; then + echo "The Git signing key does not match the output of 'rad self --ssh-key'." + exit 1 +fi + +printf "\n" +printf "Tag the above commit with \033[35m$tag\033[0m, using \033[35m$(rad self --did)\033[0m? [y/N] " +read confirmation +rad auth + +case "$confirmation" in + [Yy]*) + if git tag --annotate --sign "$tag" -m "Release $version" "$commit"; then + echo "Tag $tag created and signed over $commit." + else + echo "Failed to create tag." + exit 1 + fi ;; + *) + echo "Operation aborted." + exit 1 ;; +esac + diff --git a/http-server/build/upload b/http-server/build/upload new file mode 100755 index 000000000..a4e17efb4 --- /dev/null +++ b/http-server/build/upload @@ -0,0 +1,47 @@ +#!/bin/sh +set -e + +SSH_LOGIN=${SSH_LOGIN:-release} +SSH_ADDRESS=${SSH_ADDRESS:-$SSH_LOGIN@files.radicle.xyz} +SSH_KEY="$(rad path)/keys/radicle" + +main() { + version="$(build/version)" + + echo "Uploading Radicle $version..." + + if [ -z "$version" ]; then + echo "fatal: empty version number" >&2 + exit 1 + fi + + # Create remote folder. + ssh -i $SSH_KEY $SSH_ADDRESS mkdir -p /mnt/radicle/files/releases/radicle-httpd/$version + # Copy files over. + scp -i $SSH_KEY build/artifacts/radicle-$version* $SSH_ADDRESS:/mnt/radicle/files/releases/radicle-httpd/$version + + for target in $(cat build/TARGETS); do + archive=/mnt/radicle/files/releases/radicle-httpd/$version/radicle-$version-$target.tar.xz + symlink=/mnt/radicle/files/releases/radicle-httpd/$version/radicle-$target.tar.xz + + echo "Creating symlinks for $target.." + + ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive $symlink + ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive.sig $symlink.sig + ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive.sha256 $symlink.sha256 + done + + if git describe --exact-match --match='v*' 2>/dev/null; then + echo "Creating 'latest' symlink.." + ssh -i $SSH_KEY $SSH_ADDRESS ln -snf /mnt/radicle/files/releases/radicle-httpd/$version /mnt/radicle/files/releases/radicle-httpd/latest + else + echo "Skipping 'latest' symlink creation for development build." + fi + + echo "Pushing tags.." + git push rad --tags + + echo "Done." +} + +main "$@" diff --git a/http-server/build/version b/http-server/build/version new file mode 100755 index 000000000..4383147c1 --- /dev/null +++ b/http-server/build/version @@ -0,0 +1,9 @@ +#!/bin/sh + +if ! version="$(git describe --match='v*' --candidates=1 2>/dev/null)"; then + echo "fatal: no version tag found by 'git describe'" >&2 ; exit 1 +fi +# Remove `v` prefix from version. +version=${version#v} + +echo $version