From 18641de76a34e2b580ad6cf3a6d51dcd3cb4ded2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C5=ABdolfs=20O=C5=A1i=C5=86=C5=A1?= Date: Thu, 16 Jul 2026 13:53:26 +0200 Subject: [PATCH] httpd: Speed up opening files in large repositories surf's blob() walks the whole history to find the commit that last touched a file, taking seconds to minutes on large repositories. Read the blob directly and resolve its last commit with `git rev-list -1`, which uses the commit-graph, falling back to surf. --- crates/radicle-httpd/src/api/json/commit.rs | 41 +++--- crates/radicle-httpd/src/api/v1/repos.rs | 132 +++++++++++++++++--- 2 files changed, 138 insertions(+), 35 deletions(-) diff --git a/crates/radicle-httpd/src/api/json/commit.rs b/crates/radicle-httpd/src/api/json/commit.rs index 9ad23a0c9..9c2759e5a 100644 --- a/crates/radicle-httpd/src/api/json/commit.rs +++ b/crates/radicle-httpd/src/api/json/commit.rs @@ -31,27 +31,28 @@ impl<'a> Commit<'a> { } } -pub(crate) struct Blob<'a, T: AsRef<[u8]>>(&'a surf::blob::Blob); +/// Serialize a blob to JSON from its raw parts. +/// +/// The blob is read directly via git, bypassing +/// `radicle_surf::Repository::blob` and its history walk. +pub(crate) fn blob_json( + is_binary: bool, + content: &[u8], + path: &str, + last_commit: &surf::Commit, +) -> Value { + let content = match str::from_utf8(content) { + Ok(s) => s.to_owned(), + Err(_) => BASE64_STANDARD.encode(content), + }; -impl<'a, T: AsRef<[u8]>> Blob<'a, T> { - pub fn new(blob: &'a surf::blob::Blob) -> Self { - Self(blob) - } - - pub fn as_json(&self, path: &str) -> Value { - let content = match str::from_utf8(self.0.content()) { - Ok(s) => s.to_owned(), - Err(_) => BASE64_STANDARD.encode(self.0.content()), - }; - - json!({ - "binary": self.0.is_binary(), - "name": name_in_path(path), - "content": content, - "path": path, - "lastCommit": Commit(self.0.commit()).as_json() - }) - } + json!({ + "binary": is_binary, + "name": name_in_path(path), + "content": content, + "path": path, + "lastCommit": Commit::new(last_commit).as_json() + }) } pub(crate) struct Tree<'a>(&'a surf::tree::Tree); diff --git a/crates/radicle-httpd/src/api/v1/repos.rs b/crates/radicle-httpd/src/api/v1/repos.rs index 44f89f26a..8b7371017 100644 --- a/crates/radicle-httpd/src/api/v1/repos.rs +++ b/crates/radicle-httpd/src/api/v1/repos.rs @@ -825,6 +825,88 @@ fn remote_info( .set_delegate(delegates.contains(&id.into())) } +/// A blob read from the repository, or a marker that it exceeds the size limit. +enum BlobData { + Blob { + is_binary: bool, + content: Vec, + last_commit: Box, + }, + TooLarge, +} + +/// Read the blob at `path` in the tree of `commit`. +/// +/// Resolves content via a direct tree lookup and the last-modifying commit via +/// [`last_path_commit`], avoiding `radicle_surf::Repository::blob`, which walks +/// the full history to find that commit. +fn read_blob( + storage: &radicle::storage::git::Repository, + surf_repo: &Repository, + commit: Oid, + path: &str, +) -> Result { + let tree = storage.backend.find_commit(commit.into())?.tree()?; + let entry = tree.get_path(std::path::Path::new(path))?; + let blob = entry + .to_object(&storage.backend)? + .into_blob() + .map_err(|_| radicle::git::raw::Error::from_str("path does not point to a blob"))?; + + if blob.content().len() > MAX_BODY_LIMIT { + return Ok(BlobData::TooLarge); + } + + let last_commit = last_path_commit(surf_repo, storage.path(), commit, path)?; + + Ok(BlobData::Blob { + is_binary: blob.is_binary(), + content: blob.content().to_vec(), + last_commit: Box::new(last_commit), + }) +} + +/// Resolve the most recent commit that modified `path`, reachable from `head`. +/// +/// Fast path: `git rev-list -1` walks history using the commit-graph (when +/// present), skipping the per-commit tree diff that a libgit2 pathspec walk +/// performs. On large histories this is near-instant even for a file last +/// touched long ago. Falls back to the surf history walk if the git binary is +/// unavailable or errors. +fn last_path_commit( + surf_repo: &Repository, + repo_path: &std::path::Path, + head: Oid, + path: &str, +) -> Result { + let fast = std::process::Command::new("git") + .current_dir(repo_path) + .arg("rev-list") + .arg("-1") + .arg(head.to_string()) + .arg("--") + // `:(literal)` disables pathspec glob matching so file names containing + // `[`, `*` or `?` (e.g. `src/pages/[id].ts`) are looked up verbatim. + .arg(format!(":(literal){path}")) + .output() + .ok() + .filter(|output| output.status.success()) + .and_then(|output| { + String::from_utf8_lossy(&output.stdout) + .trim() + .parse::() + .ok() + }); + + match fast { + Some(oid) => Ok(surf_repo.commit(oid)?), + None => { + let path = std::path::Path::new(path); + surf_repo.last_commit(&path, head)?.ok_or(Error::NotFound) + } + } +} + /// Get repo source file. /// `GET /repos/:rid/blob/:sha/*path` async fn blob_handler( @@ -832,22 +914,31 @@ async fn blob_handler( Path((rid, sha, path)): Path<(RepoId, Oid, String)>, ) -> impl IntoResponse { let (repo, _) = ctx.repo(rid)?; - let repo = Repository::open(repo.path())?; - let blob = repo.blob(sha, &path)?; + let surf_repo = Repository::open(repo.path())?; - if blob.size() > MAX_BODY_LIMIT { - return Ok::<_, Error>( + match read_blob(&repo, &surf_repo, sha, &path)? { + BlobData::TooLarge => Ok::<_, Error>( ( StatusCode::PAYLOAD_TOO_LARGE, [(header::CACHE_CONTROL, "no-cache")], Json(json!([])), ) .into_response(), - ); + ), + BlobData::Blob { + is_binary, + content, + last_commit, + } => Ok::<_, Error>( + immutable_response(api::json::commit::blob_json( + is_binary, + &content, + &path, + &last_commit, + )) + .into_response(), + ), } - Ok::<_, Error>( - immutable_response(api::json::commit::Blob::new(&blob).as_json(&path)).into_response(), - ) } /// Get repo readme. @@ -857,7 +948,7 @@ async fn readme_handler( Path((rid, sha)): Path<(RepoId, Oid)>, ) -> impl IntoResponse { let (repo, _) = ctx.repo(rid)?; - let repo = Repository::open(repo.path())?; + let surf_repo = Repository::open(repo.path())?; let paths = [ "README", "README.md", @@ -873,8 +964,8 @@ async fn readme_handler( .map(ToString::to_string) .chain(paths.iter().map(|p| p.to_lowercase())) { - if let Ok(blob) = repo.blob(sha, &path) { - if blob.size() > MAX_BODY_LIMIT { + match read_blob(&repo, &surf_repo, sha, &path) { + Ok(BlobData::TooLarge) => { return Ok::<_, Error>( ( StatusCode::PAYLOAD_TOO_LARGE, @@ -884,11 +975,22 @@ async fn readme_handler( .into_response(), ); } - - return Ok::<_, Error>( - immutable_response(api::json::commit::Blob::new(&blob).as_json(&path)) + Ok(BlobData::Blob { + is_binary, + content, + last_commit, + }) => { + return Ok::<_, Error>( + immutable_response(api::json::commit::blob_json( + is_binary, + &content, + &path, + &last_commit, + )) .into_response(), - ); + ); + } + Err(_) => continue, } }