httpd: implement git fetch-only server

Introduces a new `radicle-httpd` crate which will serve as the basis
for heartwood's HTTP backend.

The first part of this is a git server which allows projects to be
fetched from storage using plain `git`.

Signed-off-by: Alexis Sellier <alexis@radicle.xyz>

radicle-httpd/Cargo.toml

@@ -0,0 +1,33 @@
+[package]
+name = "radicle-httpd"
+license = "MIT OR Apache-2.0"
+version = "0.1.0"
+authors = ["Alexis Sellier <alexis@radicle.xyz>"]
+edition = "2021"
+default-run = "radicle-httpd"
+build = "../build.rs"
+
+[features]
+default = []
+logfmt = [
+  "tracing-logfmt",
+  "tracing-subscriber/env-filter"
+]
+
+[dependencies]
+anyhow = { version = "1" }
+flate2 = { version = "1" }
+lexopt = { version = "0.2.1" }
+thiserror = { version = "1" }
+tokio = { version = "1.21", default-features = false, features = ["macros", "rt-multi-thread"] }
+tracing = { version = "0.1.37", default-features = false, features = ["std", "log"] }
+tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "ansi", "fmt"] }
+tracing-logfmt = { version = "0.2", optional = true }
+axum = { version = "0.5.16", default-features = false }
+axum-server = { version = "0.4.2", default-features = false }
+hyper = { version = "0.14.17", default-features = false }
+tower-http = { version = "0.3.4", default-features = false, features = ["trace"] }
+
+[dependencies.radicle]
+path = "../radicle"
+version = "0.2.0"

radicle-httpd/src/error.rs

@@ -0,0 +1,52 @@
+use axum::http;
+use axum::response::{IntoResponse, Response};
+
+/// Errors relating to the HTTP backend.
+#[derive(Debug, thiserror::Error)]
+pub enum Error {
+    /// I/O error.
+    #[error("i/o error: {0}")]
+    Io(#[from] std::io::Error),
+
+    /// The service is not available.
+    #[error("service '{0}' not available")]
+    ServiceUnavailable(&'static str),
+
+    /// Invalid identifier.
+    #[error("invalid radicle identifier: {0}")]
+    Id(#[from] radicle::identity::IdError),
+
+    /// Git backend error.
+    #[error("backend error")]
+    Backend,
+
+    /// Id is not valid.
+    #[error("id is not valid")]
+    InvalidId,
+
+    /// HeaderName error.
+    #[error(transparent)]
+    InvalidHeaderName(#[from] axum::http::header::InvalidHeaderName),
+
+    /// HeaderValue error.
+    #[error(transparent)]
+    InvalidHeaderValue(#[from] axum::http::header::InvalidHeaderValue),
+}
+
+impl Error {
+    pub fn status(&self) -> http::StatusCode {
+        match self {
+            Error::ServiceUnavailable(_) => http::StatusCode::SERVICE_UNAVAILABLE,
+            Error::InvalidId => http::StatusCode::NOT_FOUND,
+            _ => http::StatusCode::INTERNAL_SERVER_ERROR,
+        }
+    }
+}
+
+impl IntoResponse for Error {
+    fn into_response(self) -> Response {
+        tracing::error!("{}", self);
+
+        self.status().into_response()
+    }
+}

radicle-httpd/src/lib.rs

@@ -0,0 +1,251 @@
+#![allow(clippy::type_complexity)]
+#![allow(clippy::too_many_arguments)]
+pub mod error;
+
+use std::collections::HashMap;
+use std::io::prelude::*;
+use std::net::SocketAddr;
+use std::path::Path;
+use std::process::{Command, Stdio};
+use std::sync::Arc;
+use std::time::Duration;
+use std::{io, net, str};
+
+use anyhow::Context as _;
+use axum::body::Body;
+use axum::body::{BoxBody, Bytes};
+use axum::extract::{ConnectInfo, Path as AxumPath, RawQuery};
+use axum::http::header::HeaderName;
+use axum::http::HeaderMap;
+use axum::http::{Method, StatusCode};
+use axum::http::{Request, Response};
+use axum::response::IntoResponse;
+use axum::routing::any;
+use axum::{Extension, Router};
+use flate2::write::GzDecoder;
+use hyper::body::Buf as _;
+use tower_http::trace::TraceLayer;
+use tracing::Span;
+
+use radicle::identity::Id;
+use radicle::profile::Profile;
+
+use error::Error;
+
+pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+
+#[derive(Debug, Clone)]
+pub struct Options {
+    pub listen: net::SocketAddr,
+}
+
+/// Run the Git Server.
+pub async fn run(options: Options) -> anyhow::Result<()> {
+    let git_version = Command::new("git")
+        .arg("version")
+        .output()
+        .context("'git' command must be available")?
+        .stdout;
+    tracing::info!("{}", str::from_utf8(&git_version)?.trim());
+
+    let profile = radicle::Profile::load()?;
+    tracing::info!("using radicle home at {}", profile.home.display());
+
+    let app = Router::new()
+        .route("/:project/*request", any(git_handler))
+        .layer(Extension(Arc::new(profile)))
+        .layer(
+            TraceLayer::new_for_http()
+                .make_span_with(|request: &Request<Body>| {
+                    tracing::info_span!(
+                        "request",
+                        method = %request.method(),
+                        uri = %request.uri(),
+                        status = tracing::field::Empty,
+                        latency = tracing::field::Empty,
+                    )
+                })
+                .on_response(
+                    |response: &Response<BoxBody>, latency: Duration, span: &Span| {
+                        span.record("status", &tracing::field::debug(response.status()));
+                        span.record("latency", &tracing::field::debug(latency));
+
+                        tracing::info!("Processed");
+                    },
+                ),
+        )
+        .into_make_service_with_connect_info::<SocketAddr>();
+
+    tracing::info!("listening on http://{}", options.listen);
+
+    axum::Server::bind(&options.listen)
+        .serve(app)
+        .await
+        .map_err(anyhow::Error::from)
+}
+
+async fn git_handler(
+    Extension(profile): Extension<Arc<Profile>>,
+    AxumPath((project, request)): AxumPath<(String, String)>,
+    method: Method,
+    headers: HeaderMap,
+    body: Bytes,
+    ConnectInfo(remote): ConnectInfo<SocketAddr>,
+    query: RawQuery,
+) -> impl IntoResponse {
+    let query = query.0.unwrap_or_default();
+    let id: Id = project.strip_suffix(".git").unwrap_or(&project).parse()?;
+
+    let (status, headers, body) =
+        git_http_backend(&profile, method, headers, body, remote, id, &request, query).await?;
+
+    let mut response_headers = HeaderMap::new();
+    for (name, vec) in headers.iter() {
+        for value in vec {
+            let header: HeaderName = name.try_into()?;
+            response_headers.insert(header, value.parse()?);
+        }
+    }
+
+    Ok::<_, Error>((status, response_headers, body))
+}
+
+async fn git_http_backend(
+    profile: &Profile,
+    method: Method,
+    headers: HeaderMap,
+    mut body: Bytes,
+    remote: net::SocketAddr,
+    id: Id,
+    path: &str,
+    query: String,
+) -> Result<(StatusCode, HashMap<String, Vec<String>>, Vec<u8>), Error> {
+    let git_dir = radicle::storage::git::paths::repository(&profile.storage, &id);
+    let content_type =
+        if let Some(Ok(content_type)) = headers.get("Content-Type").map(|h| h.to_str()) {
+            content_type
+        } else {
+            ""
+        };
+
+    // Reject push requests.
+    match (path, query.as_str()) {
+        ("git-receive-pack", _) | (_, "service=git-receive-pack") => {
+            return Err(Error::ServiceUnavailable("git-receive-pack"));
+        }
+        _ => {}
+    };
+
+    tracing::debug!("id: {:?}", id);
+    tracing::debug!("headers: {:?}", headers);
+    tracing::debug!("path: {:?}", path);
+    tracing::debug!("method: {:?}", method.as_str());
+    tracing::debug!("remote: {:?}", remote.to_string());
+
+    let mut cmd = Command::new("git");
+    let mut child = cmd
+        .arg("http-backend")
+        .env("REQUEST_METHOD", method.as_str())
+        .env("GIT_PROJECT_ROOT", git_dir)
+        // "The GIT_HTTP_EXPORT_ALL environmental variable may be passed to git-http-backend to bypass
+        // the check for the "git-daemon-export-ok" file in each repository before allowing export of
+        // that repository."
+        .env("GIT_HTTP_EXPORT_ALL", String::default())
+        .env("PATH_INFO", Path::new("/").join(path))
+        .env("CONTENT_TYPE", content_type)
+        .env("QUERY_STRING", query)
+        .stderr(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stdin(Stdio::piped())
+        .spawn()?;
+
+    // Whether the request body is compressed.
+    let gzip = matches!(
+        headers.get("Content-Encoding").map(|h| h.to_str()),
+        Some(Ok("gzip"))
+    );
+
+    {
+        // This is safe because we captured the child's stdin.
+        let mut stdin = child.stdin.take().unwrap();
+
+        // Copy the request body to git-http-backend's stdin.
+        if gzip {
+            let mut decoder = GzDecoder::new(&mut stdin);
+            let mut reader = body.reader();
+
+            io::copy(&mut reader, &mut decoder)?;
+            decoder.finish()?;
+        } else {
+            while body.has_remaining() {
+                let mut chunk = body.chunk();
+                let count = chunk.len();
+
+                io::copy(&mut chunk, &mut stdin)?;
+                body.advance(count);
+            }
+        }
+    }
+
+    match child.wait_with_output() {
+        Ok(output) if output.status.success() => {
+            tracing::info!("git-http-backend: exited successfully for {}", id);
+
+            let mut reader = std::io::Cursor::new(output.stdout);
+            let mut headers = HashMap::new();
+
+            // Parse headers returned by git so that we can use them in the client response.
+            for line in io::Read::by_ref(&mut reader).lines() {
+                let line = line?;
+
+                if line.is_empty() || line == "\r" {
+                    break;
+                }
+
+                let mut parts = line.splitn(2, ':');
+                let key = parts.next();
+                let value = parts.next();
+
+                if let (Some(key), Some(value)) = (key, value) {
+                    let value = &value[1..];
+
+                    headers
+                        .entry(key.to_string())
+                        .or_insert_with(Vec::new)
+                        .push(value.to_string());
+                } else {
+                    return Err(Error::Backend);
+                }
+            }
+
+            let status = {
+                tracing::debug!("git-http-backend: {:?}", &headers);
+
+                let line = headers.remove("Status").unwrap_or_default();
+                let line = line.into_iter().next().unwrap_or_default();
+                let mut parts = line.split(' ');
+
+                parts
+                    .next()
+                    .and_then(|p| p.parse().ok())
+                    .unwrap_or(StatusCode::OK)
+            };
+
+            let position = reader.position() as usize;
+            let body = reader.into_inner().split_off(position);
+
+            Ok((status, headers, body))
+        }
+        Ok(output) => {
+            tracing::error!("git-http-backend: exited with code {}", output.status);
+
+            if let Ok(output) = std::str::from_utf8(&output.stderr) {
+                tracing::error!("git-http-backend: stderr: {}", output.trim_end());
+            }
+            Err(Error::Backend)
+        }
+        Err(err) => {
+            panic!("failed to wait for git-http-backend: {}", err);
+        }
+    }
+}

radicle-httpd/src/main.rs

@@ -0,0 +1,82 @@
+use std::{net, process};
+
+use tracing::dispatcher::Dispatch;
+
+use radicle_httpd as httpd;
+
+#[derive(Debug)]
+pub struct Options {
+    pub listen: net::SocketAddr,
+}
+
+impl Options {
+    fn from_env() -> Result<Self, lexopt::Error> {
+        use lexopt::prelude::*;
+
+        let mut parser = lexopt::Parser::from_env();
+        let mut listen = None;
+
+        while let Some(arg) = parser.next()? {
+            match arg {
+                Long("listen") => {
+                    let addr = parser.value()?.parse()?;
+                    listen = Some(addr);
+                }
+                Long("help") => {
+                    println!("usage: radicle-httpd [--listen <addr>]");
+                    process::exit(0);
+                }
+                _ => return Err(arg.unexpected()),
+            }
+        }
+        Ok(Self {
+            listen: listen.unwrap_or_else(|| ([0, 0, 0, 0], 8080).into()),
+        })
+    }
+}
+
+impl From<Options> for httpd::Options {
+    fn from(other: Options) -> Self {
+        Self {
+            listen: other.listen,
+        }
+    }
+}
+
+#[cfg(feature = "logfmt")]
+mod logger {
+    use tracing_subscriber::layer::SubscriberExt as _;
+    use tracing_subscriber::EnvFilter;
+
+    pub fn subscriber() -> impl tracing::Subscriber {
+        tracing_subscriber::Registry::default()
+            .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info")))
+            .with(tracing_logfmt::layer())
+    }
+}
+
+#[cfg(not(feature = "logfmt"))]
+mod logger {
+    pub fn subscriber() -> impl tracing::Subscriber {
+        tracing_subscriber::FmtSubscriber::new()
+    }
+}
+
+#[tokio::main]
+async fn main() -> anyhow::Result<()> {
+    let options = Options::from_env()?;
+
+    tracing::dispatcher::set_global_default(Dispatch::new(logger::subscriber()))
+        .expect("Global logger hasn't already been set");
+
+    tracing::info!("version {}-{}", env!("CARGO_PKG_VERSION"), env!("GIT_HEAD"));
+
+    match httpd::run(options.into()).await {
+        Ok(()) => {}
+        Err(err) => {
+            tracing::error!("Fatal: {:#}", err);
+            process::exit(1);
+        }
+    }
+    Ok(())
+}