commit ba2a36b983265a134d47880cb19aecfd10238f91 Author: Alexis Sellier Date: Wed Oct 19 08:47:40 2022 +0200 httpd: implement `git` fetch-only server Introduces a new `radicle-httpd` crate which will serve as the basis for heartwood's HTTP backend. The first part of this is a git server which allows projects to be fetched from storage using plain `git`. Signed-off-by: Alexis Sellier diff --git a/radicle-httpd/Cargo.toml b/radicle-httpd/Cargo.toml new file mode 100644 index 000000000..36f41aafa --- /dev/null +++ b/radicle-httpd/Cargo.toml @@ -0,0 +1,33 @@ +[package] +name = "radicle-httpd" +license = "MIT OR Apache-2.0" +version = "0.1.0" +authors = ["Alexis Sellier "] +edition = "2021" +default-run = "radicle-httpd" +build = "../build.rs" + +[features] +default = [] +logfmt = [ + "tracing-logfmt", + "tracing-subscriber/env-filter" +] + +[dependencies] +anyhow = { version = "1" } +flate2 = { version = "1" } +lexopt = { version = "0.2.1" } +thiserror = { version = "1" } +tokio = { version = "1.21", default-features = false, features = ["macros", "rt-multi-thread"] } +tracing = { version = "0.1.37", default-features = false, features = ["std", "log"] } +tracing-subscriber = { version = "0.3", default-features = false, features = ["std", "ansi", "fmt"] } +tracing-logfmt = { version = "0.2", optional = true } +axum = { version = "0.5.16", default-features = false } +axum-server = { version = "0.4.2", default-features = false } +hyper = { version = "0.14.17", default-features = false } +tower-http = { version = "0.3.4", default-features = false, features = ["trace"] } + +[dependencies.radicle] +path = "../radicle" +version = "0.2.0" diff --git a/radicle-httpd/src/error.rs b/radicle-httpd/src/error.rs new file mode 100644 index 000000000..01f11f6a4 --- /dev/null +++ b/radicle-httpd/src/error.rs @@ -0,0 +1,52 @@ +use axum::http; +use axum::response::{IntoResponse, Response}; + +/// Errors relating to the HTTP backend. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// I/O error. + #[error("i/o error: {0}")] + Io(#[from] std::io::Error), + + /// The service is not available. + #[error("service '{0}' not available")] + ServiceUnavailable(&'static str), + + /// Invalid identifier. + #[error("invalid radicle identifier: {0}")] + Id(#[from] radicle::identity::IdError), + + /// Git backend error. + #[error("backend error")] + Backend, + + /// Id is not valid. + #[error("id is not valid")] + InvalidId, + + /// HeaderName error. + #[error(transparent)] + InvalidHeaderName(#[from] axum::http::header::InvalidHeaderName), + + /// HeaderValue error. + #[error(transparent)] + InvalidHeaderValue(#[from] axum::http::header::InvalidHeaderValue), +} + +impl Error { + pub fn status(&self) -> http::StatusCode { + match self { + Error::ServiceUnavailable(_) => http::StatusCode::SERVICE_UNAVAILABLE, + Error::InvalidId => http::StatusCode::NOT_FOUND, + _ => http::StatusCode::INTERNAL_SERVER_ERROR, + } + } +} + +impl IntoResponse for Error { + fn into_response(self) -> Response { + tracing::error!("{}", self); + + self.status().into_response() + } +} diff --git a/radicle-httpd/src/lib.rs b/radicle-httpd/src/lib.rs new file mode 100644 index 000000000..554cbf15b --- /dev/null +++ b/radicle-httpd/src/lib.rs @@ -0,0 +1,251 @@ +#![allow(clippy::type_complexity)] +#![allow(clippy::too_many_arguments)] +pub mod error; + +use std::collections::HashMap; +use std::io::prelude::*; +use std::net::SocketAddr; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::sync::Arc; +use std::time::Duration; +use std::{io, net, str}; + +use anyhow::Context as _; +use axum::body::Body; +use axum::body::{BoxBody, Bytes}; +use axum::extract::{ConnectInfo, Path as AxumPath, RawQuery}; +use axum::http::header::HeaderName; +use axum::http::HeaderMap; +use axum::http::{Method, StatusCode}; +use axum::http::{Request, Response}; +use axum::response::IntoResponse; +use axum::routing::any; +use axum::{Extension, Router}; +use flate2::write::GzDecoder; +use hyper::body::Buf as _; +use tower_http::trace::TraceLayer; +use tracing::Span; + +use radicle::identity::Id; +use radicle::profile::Profile; + +use error::Error; + +pub const VERSION: &str = env!("CARGO_PKG_VERSION"); + +#[derive(Debug, Clone)] +pub struct Options { + pub listen: net::SocketAddr, +} + +/// Run the Git Server. +pub async fn run(options: Options) -> anyhow::Result<()> { + let git_version = Command::new("git") + .arg("version") + .output() + .context("'git' command must be available")? + .stdout; + tracing::info!("{}", str::from_utf8(&git_version)?.trim()); + + let profile = radicle::Profile::load()?; + tracing::info!("using radicle home at {}", profile.home.display()); + + let app = Router::new() + .route("/:project/*request", any(git_handler)) + .layer(Extension(Arc::new(profile))) + .layer( + TraceLayer::new_for_http() + .make_span_with(|request: &Request| { + tracing::info_span!( + "request", + method = %request.method(), + uri = %request.uri(), + status = tracing::field::Empty, + latency = tracing::field::Empty, + ) + }) + .on_response( + |response: &Response, latency: Duration, span: &Span| { + span.record("status", &tracing::field::debug(response.status())); + span.record("latency", &tracing::field::debug(latency)); + + tracing::info!("Processed"); + }, + ), + ) + .into_make_service_with_connect_info::(); + + tracing::info!("listening on http://{}", options.listen); + + axum::Server::bind(&options.listen) + .serve(app) + .await + .map_err(anyhow::Error::from) +} + +async fn git_handler( + Extension(profile): Extension>, + AxumPath((project, request)): AxumPath<(String, String)>, + method: Method, + headers: HeaderMap, + body: Bytes, + ConnectInfo(remote): ConnectInfo, + query: RawQuery, +) -> impl IntoResponse { + let query = query.0.unwrap_or_default(); + let id: Id = project.strip_suffix(".git").unwrap_or(&project).parse()?; + + let (status, headers, body) = + git_http_backend(&profile, method, headers, body, remote, id, &request, query).await?; + + let mut response_headers = HeaderMap::new(); + for (name, vec) in headers.iter() { + for value in vec { + let header: HeaderName = name.try_into()?; + response_headers.insert(header, value.parse()?); + } + } + + Ok::<_, Error>((status, response_headers, body)) +} + +async fn git_http_backend( + profile: &Profile, + method: Method, + headers: HeaderMap, + mut body: Bytes, + remote: net::SocketAddr, + id: Id, + path: &str, + query: String, +) -> Result<(StatusCode, HashMap>, Vec), Error> { + let git_dir = radicle::storage::git::paths::repository(&profile.storage, &id); + let content_type = + if let Some(Ok(content_type)) = headers.get("Content-Type").map(|h| h.to_str()) { + content_type + } else { + "" + }; + + // Reject push requests. + match (path, query.as_str()) { + ("git-receive-pack", _) | (_, "service=git-receive-pack") => { + return Err(Error::ServiceUnavailable("git-receive-pack")); + } + _ => {} + }; + + tracing::debug!("id: {:?}", id); + tracing::debug!("headers: {:?}", headers); + tracing::debug!("path: {:?}", path); + tracing::debug!("method: {:?}", method.as_str()); + tracing::debug!("remote: {:?}", remote.to_string()); + + let mut cmd = Command::new("git"); + let mut child = cmd + .arg("http-backend") + .env("REQUEST_METHOD", method.as_str()) + .env("GIT_PROJECT_ROOT", git_dir) + // "The GIT_HTTP_EXPORT_ALL environmental variable may be passed to git-http-backend to bypass + // the check for the "git-daemon-export-ok" file in each repository before allowing export of + // that repository." + .env("GIT_HTTP_EXPORT_ALL", String::default()) + .env("PATH_INFO", Path::new("/").join(path)) + .env("CONTENT_TYPE", content_type) + .env("QUERY_STRING", query) + .stderr(Stdio::piped()) + .stdout(Stdio::piped()) + .stdin(Stdio::piped()) + .spawn()?; + + // Whether the request body is compressed. + let gzip = matches!( + headers.get("Content-Encoding").map(|h| h.to_str()), + Some(Ok("gzip")) + ); + + { + // This is safe because we captured the child's stdin. + let mut stdin = child.stdin.take().unwrap(); + + // Copy the request body to git-http-backend's stdin. + if gzip { + let mut decoder = GzDecoder::new(&mut stdin); + let mut reader = body.reader(); + + io::copy(&mut reader, &mut decoder)?; + decoder.finish()?; + } else { + while body.has_remaining() { + let mut chunk = body.chunk(); + let count = chunk.len(); + + io::copy(&mut chunk, &mut stdin)?; + body.advance(count); + } + } + } + + match child.wait_with_output() { + Ok(output) if output.status.success() => { + tracing::info!("git-http-backend: exited successfully for {}", id); + + let mut reader = std::io::Cursor::new(output.stdout); + let mut headers = HashMap::new(); + + // Parse headers returned by git so that we can use them in the client response. + for line in io::Read::by_ref(&mut reader).lines() { + let line = line?; + + if line.is_empty() || line == "\r" { + break; + } + + let mut parts = line.splitn(2, ':'); + let key = parts.next(); + let value = parts.next(); + + if let (Some(key), Some(value)) = (key, value) { + let value = &value[1..]; + + headers + .entry(key.to_string()) + .or_insert_with(Vec::new) + .push(value.to_string()); + } else { + return Err(Error::Backend); + } + } + + let status = { + tracing::debug!("git-http-backend: {:?}", &headers); + + let line = headers.remove("Status").unwrap_or_default(); + let line = line.into_iter().next().unwrap_or_default(); + let mut parts = line.split(' '); + + parts + .next() + .and_then(|p| p.parse().ok()) + .unwrap_or(StatusCode::OK) + }; + + let position = reader.position() as usize; + let body = reader.into_inner().split_off(position); + + Ok((status, headers, body)) + } + Ok(output) => { + tracing::error!("git-http-backend: exited with code {}", output.status); + + if let Ok(output) = std::str::from_utf8(&output.stderr) { + tracing::error!("git-http-backend: stderr: {}", output.trim_end()); + } + Err(Error::Backend) + } + Err(err) => { + panic!("failed to wait for git-http-backend: {}", err); + } + } +} diff --git a/radicle-httpd/src/main.rs b/radicle-httpd/src/main.rs new file mode 100644 index 000000000..ab6704750 --- /dev/null +++ b/radicle-httpd/src/main.rs @@ -0,0 +1,82 @@ +use std::{net, process}; + +use tracing::dispatcher::Dispatch; + +use radicle_httpd as httpd; + +#[derive(Debug)] +pub struct Options { + pub listen: net::SocketAddr, +} + +impl Options { + fn from_env() -> Result { + use lexopt::prelude::*; + + let mut parser = lexopt::Parser::from_env(); + let mut listen = None; + + while let Some(arg) = parser.next()? { + match arg { + Long("listen") => { + let addr = parser.value()?.parse()?; + listen = Some(addr); + } + Long("help") => { + println!("usage: radicle-httpd [--listen ]"); + process::exit(0); + } + _ => return Err(arg.unexpected()), + } + } + Ok(Self { + listen: listen.unwrap_or_else(|| ([0, 0, 0, 0], 8080).into()), + }) + } +} + +impl From for httpd::Options { + fn from(other: Options) -> Self { + Self { + listen: other.listen, + } + } +} + +#[cfg(feature = "logfmt")] +mod logger { + use tracing_subscriber::layer::SubscriberExt as _; + use tracing_subscriber::EnvFilter; + + pub fn subscriber() -> impl tracing::Subscriber { + tracing_subscriber::Registry::default() + .with(EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"))) + .with(tracing_logfmt::layer()) + } +} + +#[cfg(not(feature = "logfmt"))] +mod logger { + pub fn subscriber() -> impl tracing::Subscriber { + tracing_subscriber::FmtSubscriber::new() + } +} + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let options = Options::from_env()?; + + tracing::dispatcher::set_global_default(Dispatch::new(logger::subscriber())) + .expect("Global logger hasn't already been set"); + + tracing::info!("version {}-{}", env!("CARGO_PKG_VERSION"), env!("GIT_HEAD")); + + match httpd::run(options.into()).await { + Ok(()) => {} + Err(err) => { + tracing::error!("Fatal: {:#}", err); + process::exit(1); + } + } + Ok(()) +}