mirror of
https://seed.radicle.dev/rad:z4V1sjrXqjvFdnCUbxPFqd5p4DtH5.git
synced 2026-11-15 19:30:38 +08:00
Add reproducible build pipeline
This commit is contained in:
Sebastian Martinez
@@ -3,6 +3,7 @@ node_modules/
|
||||
NOTES
|
||||
config/local*
|
||||
http-server/target
|
||||
http-server/build/artifacts
|
||||
|
||||
# KaTeX files
|
||||
*.min.css
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
# Builds release binaries for Radicle.
|
||||
FROM rust:1.77.2-alpine3.19 as builder
|
||||
LABEL maintainer="Radicle Team <team@radicle.xyz>"
|
||||
WORKDIR /src
|
||||
COPY . .
|
||||
|
||||
# Copy cargo configuration we're going to use to specify compiler options.
|
||||
RUN mkdir -p .cargo && cp build/config.toml .cargo/config.toml
|
||||
# Install dependencies.
|
||||
RUN apk update && apk add --no-cache git musl-dev minisign curl xz asciidoctor
|
||||
# Build man pages and strip metadata. Removes all comments, since they include
|
||||
# non-reproducible information, such as version numbers.
|
||||
RUN asciidoctor --doctype manpage --backend manpage --destination-dir . *.1.adoc && \
|
||||
find . -maxdepth 1 -type f -name '*.1' -exec sed -i '/^.\\\"/d' '{}' \;
|
||||
# Add cargo targets.
|
||||
RUN rustup target add \
|
||||
x86_64-unknown-linux-musl \
|
||||
aarch64-unknown-linux-musl \
|
||||
x86_64-apple-darwin \
|
||||
aarch64-apple-darwin
|
||||
|
||||
# Install dependencies for cross-compiling to macOS.
|
||||
# We use Zig as the linker to perform the compilation from a Linux host.
|
||||
# Zig is not yet available on Debian, so we download the official binary.
|
||||
# Compilation is done via `cargo-zigbuild` which is a wrapper around `zig`.
|
||||
RUN curl -sSf -o zig.tar.xz https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz && \
|
||||
curl -sSf -o zig.tar.xz.minisig https://ziglang.org/download/0.12.0/zig-linux-x86_64-0.12.0.tar.xz.minisig && \
|
||||
minisign -Vm zig.tar.xz -P RWSGOq2NVecA2UPNdBUZykf1CCb147pkmdtYxgb3Ti+JO/wCYvhbAb/U && \
|
||||
xz -d -c zig.tar.xz | tar -x && \
|
||||
mv zig-linux-x86_64-0.12.0/zig /usr/bin/zig && \
|
||||
mv zig-linux-x86_64-0.12.0/lib /usr/lib/zig && \
|
||||
cargo install cargo-zigbuild@0.18.3
|
||||
|
||||
|
||||
# Parts of the macOS SDK are required to build Radicle, we make these available
|
||||
# here. So far only `CoreFoundation` and `Security` frameworks are needed.
|
||||
RUN xz -d -c build/macos-sdk-11.3.tar.xz | tar -x
|
||||
# This env var is used by `cargo-zigbuild` to find the SDK.
|
||||
ENV SDKROOT /src/macos-sdk-11.3
|
||||
|
||||
# Build binaries.
|
||||
RUN cargo zigbuild --locked --release \
|
||||
--target=x86_64-apple-darwin \
|
||||
--target=aarch64-apple-darwin \
|
||||
--target=aarch64-unknown-linux-musl \
|
||||
--target=x86_64-unknown-linux-musl \
|
||||
-p radicle-httpd
|
||||
|
||||
# Now copy the files to a new image without all the intermediary artifacts to
|
||||
# save some space.
|
||||
FROM alpine:3.19 as packager
|
||||
COPY --from=builder \
|
||||
/src/target/x86_64-unknown-linux-musl/release/rad-web \
|
||||
/src/target/x86_64-unknown-linux-musl/release/radicle-httpd \
|
||||
/builds/x86_64-unknown-linux-musl/bin/
|
||||
COPY --from=builder \
|
||||
/src/target/aarch64-unknown-linux-musl/release/rad-web \
|
||||
/src/target/aarch64-unknown-linux-musl/release/radicle-httpd \
|
||||
/builds/aarch64-unknown-linux-musl/bin/
|
||||
COPY --from=builder \
|
||||
/src/target/aarch64-apple-darwin/release/rad-web \
|
||||
/src/target/aarch64-apple-darwin/release/radicle-httpd \
|
||||
/builds/aarch64-apple-darwin/bin/
|
||||
COPY --from=builder \
|
||||
/src/target/x86_64-apple-darwin/release/rad-web \
|
||||
/src/target/x86_64-apple-darwin/release/radicle-httpd \
|
||||
/builds/x86_64-apple-darwin/bin/
|
||||
COPY --from=builder /src/*.1 /builds/x86_64-unknown-linux-musl/man/man1/
|
||||
COPY --from=builder /src/*.1 /builds/aarch64-unknown-linux-musl/man/man1/
|
||||
COPY --from=builder /src/*.1 /builds/aarch64-apple-darwin/man/man1/
|
||||
COPY --from=builder /src/*.1 /builds/x86_64-apple-darwin/man/man1/
|
||||
|
||||
# Create and compress reproducible archive.
|
||||
WORKDIR /builds
|
||||
RUN apk update && apk add --no-cache tar xz
|
||||
RUN find * -maxdepth 0 -type d -exec mv '{}' "radicle-$RADICLE_VERSION-{}" \; && \
|
||||
find * -maxdepth 0 -type d -exec tar \
|
||||
--sort=name \
|
||||
--verbose \
|
||||
--mtime="@$GIT_COMMIT_TIME" \
|
||||
--owner=0 \
|
||||
--group=0 \
|
||||
--numeric-owner \
|
||||
--format=posix \
|
||||
--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime \
|
||||
--mode='go+u,go-w' \
|
||||
--remove-files \
|
||||
--create --xz \
|
||||
--file="{}.tar.xz" \
|
||||
'{}' \;
|
||||
@@ -0,0 +1,70 @@
|
||||
# Builds
|
||||
|
||||
Radicle uses a [reproducible build][rb] pipeline to make binary verification
|
||||
easier and more secure.
|
||||
|
||||
[rb]: https://reproducible-builds.org/
|
||||
|
||||
This build pipeline is designed to be run on an x86_64 machine running Linux.
|
||||
The output is a set of `.tar.xz` archives containing binaries for the supported
|
||||
platforms and signed by the user's Radicle key.
|
||||
|
||||
These binaries are statically linked to be maximally portable, and designed to
|
||||
be reproducible, byte for byte.
|
||||
|
||||
To run the build, simply enter the following command from the repository root:
|
||||
|
||||
build/build
|
||||
|
||||
This will build all targets and place the output in `build/artifacts` with
|
||||
one sub-directory per build target.
|
||||
|
||||
Note that it will use `git describe` to get a version number for the build.
|
||||
You *must* have a commit tagged with a version in your history or the build
|
||||
will fail, eg. `v1.0.0`.
|
||||
|
||||
When the build completes, the SHA-256 checksums of the artifacts are output.
|
||||
For a given Radicle version and source tree, the same set of checksums should
|
||||
always be output, no matter where or when the build is run. If they do not
|
||||
match, either the build pipeline has a bug, making it non-reproducible, or one
|
||||
of the machines is compromised.
|
||||
|
||||
Here's an example output for a development version of Radicle:
|
||||
|
||||
b9aa75bba175e18e05df4f6b39ec097414bbf56ccdeb4a2229b557f8b8e05404 radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-apple-darwin.tar.xz
|
||||
c7070806bf2d17a8a0d3b329e4d57b1e544b7b82cb58e2863074d96348a2ab0d radicle-1.0.0-rc.4-3-gb299f3b5-aarch64-unknown-linux-musl.tar.xz
|
||||
1a8327854f16ea90491fb90e0c3291a63c4b2ab01742c8435faec7d370cacb79 radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-apple-darwin.tar.xz
|
||||
709ac67541ff0c0c570ac22ab2de9f98320e0cc2cc9b67f1909c014a2bb5bd49 radicle-1.0.0-rc.4-3-gb299f3b5-x86_64-unknown-linux-musl.tar.xz
|
||||
|
||||
A script is included in `build/checksums` to output these checksums after
|
||||
the artifacts are built.
|
||||
|
||||
## Requirements
|
||||
|
||||
The following software is required for the build:
|
||||
|
||||
* `podman`
|
||||
* `rad` (The Radicle CLI)
|
||||
* `sha256sum`
|
||||
|
||||
## macOS
|
||||
|
||||
macOS binaries are not signed or notarized, so they have to be downloaded via
|
||||
the CLI to avoid issues. A copy of a small subset of the Apple SDK is included
|
||||
here to be able to cross-compile.
|
||||
|
||||
## Podman
|
||||
|
||||
We use `podman` to make the build reproducible on any machine by controlling
|
||||
the build environment. We prefer `podman` to `docker` because it doesn't
|
||||
require a background process to run and can be run without root access out of
|
||||
the box.
|
||||
|
||||
The first time you run `podman`, you may have to give yourself some extra UIDs
|
||||
for `podman` to use, with:
|
||||
|
||||
sudo usermod --add-subuids 100000-165535 --add-subgids 100000-165535 $USER
|
||||
|
||||
Then update `podman` with:
|
||||
|
||||
podman system migrate
|
||||
@@ -0,0 +1,4 @@
|
||||
x86_64-unknown-linux-musl
|
||||
aarch64-unknown-linux-musl
|
||||
x86_64-apple-darwin
|
||||
aarch64-apple-darwin
|
||||
Executable
+115
@@ -0,0 +1,115 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
main() {
|
||||
# Use UTC time for everything.
|
||||
export TZ=UTC0
|
||||
# Set minimal locale.
|
||||
export LC_ALL=C
|
||||
# Set source date. This is honored by `asciidoctor` and other tools.
|
||||
export SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct)
|
||||
# Define user OS for podman
|
||||
export OS=$(uname)
|
||||
|
||||
if ! command -v rad >/dev/null; then
|
||||
echo "fatal: rad is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v podman >/dev/null; then
|
||||
echo "fatal: podman is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v sha256sum >/dev/null; then
|
||||
echo "fatal: sha256sum is not installed" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rev="$(git rev-parse --short HEAD)"
|
||||
tempdir="$(mktemp -d)"
|
||||
gitarchive="$tempdir/heartwood-$rev.tar.gz"
|
||||
keypath="$(rad path)/keys/radicle.pub"
|
||||
version="$(build/version)"
|
||||
image=radicle-build-$version
|
||||
|
||||
if [ ! -f "$keypath" ]; then
|
||||
echo "fatal: no key found at $keypath" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Authenticate user for signing
|
||||
rad auth
|
||||
|
||||
echo "Building Radicle $version.."
|
||||
echo "Creating archive of repository at $rev in $gitarchive.."
|
||||
git archive --format tar.gz -o "$gitarchive" HEAD
|
||||
|
||||
echo "Building image ($image).."
|
||||
|
||||
case "$OS" in
|
||||
Darwin)
|
||||
podman build \
|
||||
--env SOURCE_DATE_EPOCH \
|
||||
--env TZ \
|
||||
--env LC_ALL \
|
||||
--env GIT_COMMIT_TIME=$SOURCE_DATE_EPOCH \
|
||||
--env GIT_HEAD=$rev \
|
||||
--env RADICLE_VERSION=$version \
|
||||
--arch aarch64 --tag $image -f ./build/Dockerfile - <$gitarchive
|
||||
|
||||
echo "Creating container (radicle-build-container).."
|
||||
podman create --ulimit=host --replace --name radicle-build-container $image
|
||||
;;
|
||||
*)
|
||||
podman --cgroup-manager=cgroupfs build \
|
||||
--env SOURCE_DATE_EPOCH \
|
||||
--env TZ \
|
||||
--env LC_ALL \
|
||||
--env GIT_COMMIT_TIME=$SOURCE_DATE_EPOCH \
|
||||
--env GIT_HEAD=$rev \
|
||||
--env RADICLE_VERSION=$version \
|
||||
--arch amd64 --tag $image -f ./build/Dockerfile - <$gitarchive
|
||||
|
||||
echo "Creating container (radicle-build-container).."
|
||||
podman --cgroup-manager=cgroupfs create --ulimit=host --replace --name radicle-build-container $image
|
||||
;;
|
||||
esac
|
||||
|
||||
# Copy build artifacts to output folder.
|
||||
outdir=build/artifacts
|
||||
mkdir -p $outdir
|
||||
podman cp --overwrite radicle-build-container:/builds/. $outdir/
|
||||
|
||||
for target in $(cat build/TARGETS); do
|
||||
echo "Signing artifacts for $target.."
|
||||
|
||||
filename="radicle-$version-$target.tar.xz"
|
||||
filepath="$outdir/$filename"
|
||||
|
||||
# Output SHA256 digest of archive.
|
||||
checksum="$(cd $outdir && sha256sum $filename)"
|
||||
echo "Checksum of $filepath is $(echo "$checksum" | cut -d' ' -f1)"
|
||||
echo "$checksum" >$filepath.sha256
|
||||
|
||||
# Sign archive and verify archive.
|
||||
rm -f $filepath.sig # Delete existing signature
|
||||
ssh-keygen -Y sign -n file -f $keypath $filepath
|
||||
ssh-keygen -Y check-novalidate -n file -s $filepath.sig <$filepath
|
||||
done
|
||||
|
||||
# Remove build artifacts that aren't needed anymore.
|
||||
rm -f $gitarchive
|
||||
podman rm radicle-build-container >/dev/null
|
||||
podman rmi --ignore localhost/$image
|
||||
}
|
||||
|
||||
# Run build.
|
||||
echo "Running build.."
|
||||
main "$@"
|
||||
|
||||
# Show artifact checksums.
|
||||
echo
|
||||
build/checksums
|
||||
echo
|
||||
|
||||
echo "Build successful."
|
||||
Executable
+2
@@ -0,0 +1,2 @@
|
||||
#!/bin/sh
|
||||
find build/artifacts -type f -name '*.sha256' -exec cat {} +
|
||||
@@ -0,0 +1,13 @@
|
||||
[target.x86_64-unknown-linux-musl]
|
||||
rustflags = [
|
||||
"-C", "codegen-units=1",
|
||||
"-C", "incremental=false",
|
||||
"-C", "opt-level=3",
|
||||
]
|
||||
|
||||
[target.aarch64-unknown-linux-musl]
|
||||
rustflags = [
|
||||
"-C", "codegen-units=1",
|
||||
"-C", "incremental=false",
|
||||
"-C", "opt-level=3",
|
||||
]
|
||||
Binary file not shown.
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
echo "Usage: $0 <version-number>"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
version="$1"
|
||||
tag="v$version"
|
||||
commit="$(git rev-parse HEAD)"
|
||||
signing_key=$(git config user.signingKey)
|
||||
|
||||
git show "$commit"
|
||||
|
||||
if [ "$signing_key" != "$(rad self --ssh-key)" ]; then
|
||||
echo "The Git signing key does not match the output of 'rad self --ssh-key'."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf "\n"
|
||||
printf "Tag the above commit with \033[35m$tag\033[0m, using \033[35m$(rad self --did)\033[0m? [y/N] "
|
||||
read confirmation
|
||||
rad auth
|
||||
|
||||
case "$confirmation" in
|
||||
[Yy]*)
|
||||
if git tag --annotate --sign "$tag" -m "Release $version" "$commit"; then
|
||||
echo "Tag $tag created and signed over $commit."
|
||||
else
|
||||
echo "Failed to create tag."
|
||||
exit 1
|
||||
fi ;;
|
||||
*)
|
||||
echo "Operation aborted."
|
||||
exit 1 ;;
|
||||
esac
|
||||
|
||||
Executable
+47
@@ -0,0 +1,47 @@
|
||||
#!/bin/sh
|
||||
set -e
|
||||
|
||||
SSH_LOGIN=${SSH_LOGIN:-release}
|
||||
SSH_ADDRESS=${SSH_ADDRESS:-$SSH_LOGIN@files.radicle.xyz}
|
||||
SSH_KEY="$(rad path)/keys/radicle"
|
||||
|
||||
main() {
|
||||
version="$(build/version)"
|
||||
|
||||
echo "Uploading Radicle $version..."
|
||||
|
||||
if [ -z "$version" ]; then
|
||||
echo "fatal: empty version number" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Create remote folder.
|
||||
ssh -i $SSH_KEY $SSH_ADDRESS mkdir -p /mnt/radicle/files/releases/radicle-httpd/$version
|
||||
# Copy files over.
|
||||
scp -i $SSH_KEY build/artifacts/radicle-$version* $SSH_ADDRESS:/mnt/radicle/files/releases/radicle-httpd/$version
|
||||
|
||||
for target in $(cat build/TARGETS); do
|
||||
archive=/mnt/radicle/files/releases/radicle-httpd/$version/radicle-$version-$target.tar.xz
|
||||
symlink=/mnt/radicle/files/releases/radicle-httpd/$version/radicle-$target.tar.xz
|
||||
|
||||
echo "Creating symlinks for $target.."
|
||||
|
||||
ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive $symlink
|
||||
ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive.sig $symlink.sig
|
||||
ssh -i $SSH_KEY $SSH_ADDRESS ln -snf $archive.sha256 $symlink.sha256
|
||||
done
|
||||
|
||||
if git describe --exact-match --match='v*' 2>/dev/null; then
|
||||
echo "Creating 'latest' symlink.."
|
||||
ssh -i $SSH_KEY $SSH_ADDRESS ln -snf /mnt/radicle/files/releases/radicle-httpd/$version /mnt/radicle/files/releases/radicle-httpd/latest
|
||||
else
|
||||
echo "Skipping 'latest' symlink creation for development build."
|
||||
fi
|
||||
|
||||
echo "Pushing tags.."
|
||||
git push rad --tags
|
||||
|
||||
echo "Done."
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+9
@@ -0,0 +1,9 @@
|
||||
#!/bin/sh
|
||||
|
||||
if ! version="$(git describe --match='v*' --candidates=1 2>/dev/null)"; then
|
||||
echo "fatal: no version tag found by 'git describe'" >&2 ; exit 1
|
||||
fi
|
||||
# Remove `v` prefix from version.
|
||||
version=${version#v}
|
||||
|
||||
echo $version
|
||||
Reference in New Issue
Block a user