mirror of
https://seed.radicle.dev/rad:z6cFWeWpnZNHh9rUW8phgA3b5yGt.git
synced 2026-11-15 07:50:35 +08:00
radicle-git init
Port over all git relevant parts of the radicle-link repository[0]. The packages that are kept are: * git-ext * git-ref-format * git-trailers * link-git * macros * std-ext * test -- only the portions that test the above The development and testing scripts are also added for local and CI testing. [0]: https://github.com/radicle-dev/radicle-link/ Signed-off-by: Fintan Halpenny <fintan.halpenny@gmail.com>
This commit is contained in:
Fintan Halpenny
@@ -0,0 +1,36 @@
|
||||
// Copyright © 2021 The Radicle Link Contributors
|
||||
//
|
||||
// This file is part of radicle-link, distributed under the GPLv3 with Radicle
|
||||
// Linking Exception. For full terms see the included LICENSE file.
|
||||
|
||||
use git_hash::oid;
|
||||
use git_pack::{cache::DecodeEntry, data::Object};
|
||||
|
||||
use super::{index, pack, window};
|
||||
|
||||
pub type Loose = git_odb::loose::Store;
|
||||
|
||||
pub struct Packed<I, D> {
|
||||
pub index: I,
|
||||
pub data: D,
|
||||
}
|
||||
|
||||
impl<I, D> Packed<I, D>
|
||||
where
|
||||
I: index::Index,
|
||||
D: window::Cache,
|
||||
{
|
||||
pub fn contains(&self, id: impl AsRef<oid>) -> bool {
|
||||
self.index.contains(id)
|
||||
}
|
||||
|
||||
pub fn find<'a>(
|
||||
&self,
|
||||
id: impl AsRef<oid>,
|
||||
buf: &'a mut Vec<u8>,
|
||||
cache: &mut impl DecodeEntry,
|
||||
) -> Result<Option<Object<'a>>, index::error::Lookup<pack::error::Data>> {
|
||||
self.index
|
||||
.lookup(|info| self.data.get(info), id, buf, cache)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
// Copyright © 2021 The Radicle Link Contributors
|
||||
//
|
||||
// This file is part of radicle-link, distributed under the GPLv3 with Radicle
|
||||
// Linking Exception. For full terms see the included LICENSE file.
|
||||
|
||||
use std::{
|
||||
fs,
|
||||
io,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use git_hash::oid;
|
||||
use git_pack::{
|
||||
cache::DecodeEntry,
|
||||
data::{Object, ResolvedBase},
|
||||
};
|
||||
use parking_lot::Mutex;
|
||||
use tracing::trace;
|
||||
|
||||
use super::pack;
|
||||
|
||||
pub use git_pack::index::File as IndexFile;
|
||||
|
||||
mod metrics;
|
||||
pub use metrics::{Metrics, Stats, StatsView};
|
||||
|
||||
pub mod error {
|
||||
use super::*;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Discover {
|
||||
#[error(transparent)]
|
||||
Index(#[from] pack::error::Index),
|
||||
|
||||
#[error(transparent)]
|
||||
Io(#[from] io::Error),
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum Lookup<E> {
|
||||
#[error(transparent)]
|
||||
Lookup(E),
|
||||
|
||||
#[error(transparent)]
|
||||
Reload(#[from] Discover),
|
||||
|
||||
#[error(transparent)]
|
||||
Decode(#[from] git_pack::data::decode_entry::Error),
|
||||
}
|
||||
}
|
||||
|
||||
pub trait Index {
|
||||
fn contains(&self, id: impl AsRef<oid>) -> bool;
|
||||
|
||||
fn lookup<'a, F, E>(
|
||||
&self,
|
||||
pack_cache: F,
|
||||
id: impl AsRef<oid>,
|
||||
buf: &'a mut Vec<u8>,
|
||||
cache: &mut impl DecodeEntry,
|
||||
) -> Result<Option<Object<'a>>, error::Lookup<E>>
|
||||
where
|
||||
F: FnOnce(&pack::Info) -> Result<Arc<pack::Data>, E>;
|
||||
}
|
||||
|
||||
/// An [`Index`] which can be shared between threads.
|
||||
///
|
||||
/// [`Shared`] assumes that:
|
||||
///
|
||||
/// * newer packs are likely to contain recent objects
|
||||
/// * lookups tend to favour recent objects
|
||||
/// * lookups tend to expect the object to be found (the object id is either
|
||||
/// pointed to by a ref, or linked to by an existing object)
|
||||
///
|
||||
/// Thus, it:
|
||||
///
|
||||
/// * orders indices found in `GIT_DIR/objects/pack` by modification time, and
|
||||
/// queries the more recent ones first
|
||||
/// * attempts to rescan `GIT_DIR/objects/pack` when an object id was _not_
|
||||
/// found (assuming that this is due to a compaction)
|
||||
///
|
||||
/// Unless a reload occurs, lookups are lock-free and mostly wait-free. Writes
|
||||
/// ([`Shared::push`], [`Shared::reload`]) are guarded by a [`Mutex`].
|
||||
// TODO: consecutive lookups also tend to resolve to the same pack, so we could
|
||||
// remember the index into the `im::Vector` where we found a match and look
|
||||
// there first. This is what libgit2 does, but the heuristic is not necessarily
|
||||
// true when `Shared` is shared across multiple concurrent link replication
|
||||
// tasks; per-namespace packs are independent pre-compaction.
|
||||
pub struct Shared<M> {
|
||||
pack_dir: PathBuf,
|
||||
indices: ArcSwap<im::Vector<Arc<pack::Index>>>,
|
||||
write: Mutex<()>,
|
||||
stats: M,
|
||||
}
|
||||
|
||||
impl Shared<()> {
|
||||
pub fn open(git_dir: impl AsRef<Path>) -> Result<Self, error::Discover> {
|
||||
let pack_dir = git_dir.as_ref().join("objects").join("pack");
|
||||
let indices = discover(&pack_dir)?;
|
||||
|
||||
Ok(Self {
|
||||
pack_dir,
|
||||
indices: ArcSwap::new(Arc::new(indices)),
|
||||
write: Mutex::new(()),
|
||||
stats: (),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<M> Shared<M>
|
||||
where
|
||||
M: Metrics,
|
||||
{
|
||||
pub fn with_stats(self) -> Shared<Stats> {
|
||||
self.with_metrics(Stats::default())
|
||||
}
|
||||
|
||||
pub fn with_metrics<N: Metrics>(self, m: N) -> Shared<N> {
|
||||
Shared {
|
||||
pack_dir: self.pack_dir,
|
||||
indices: self.indices,
|
||||
write: self.write,
|
||||
stats: m,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> M::Snapshot {
|
||||
self.stats.snapshot(self.len())
|
||||
}
|
||||
|
||||
/// Add a newly discovered [`pack::Index`].
|
||||
///
|
||||
/// This index will be considered first by subsequent lookups. Note that it
|
||||
/// is only guaranteed that the index will be visible to readers if it
|
||||
/// resides in the `git_dir` this [`Shared`] was initialised with.
|
||||
pub fn push(&self, idx: pack::Index) {
|
||||
let lock = self.write.lock();
|
||||
let mut new = self.indices.load_full();
|
||||
Arc::make_mut(&mut new).push_front(Arc::new(idx));
|
||||
self.indices.store(new);
|
||||
drop(lock);
|
||||
|
||||
self.stats.record_push()
|
||||
}
|
||||
|
||||
/// Re-scan the packs directory and replace the in-memory indices with the
|
||||
/// result.
|
||||
///
|
||||
/// If the application can intercept compaction events, this method can be
|
||||
/// used to release memory early. Otherwise it is not required to call this
|
||||
/// method, as [`Shared`] manages reloads automatically.
|
||||
pub fn reload(&self) -> Result<(), error::Discover> {
|
||||
let lock = self.write.lock();
|
||||
let indices = discover(&self.pack_dir)?;
|
||||
self.indices.store(Arc::new(indices));
|
||||
drop(lock);
|
||||
|
||||
self.stats.record_reload();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.indices.load().is_empty()
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.indices.load().len()
|
||||
}
|
||||
|
||||
fn contains(&self, id: impl AsRef<oid>) -> bool {
|
||||
for i in 0..2 {
|
||||
for idx in self.indices.load().iter() {
|
||||
if idx.contains(&id) {
|
||||
self.stats.record_hit();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
if i == 0 && self.reload().is_err() {
|
||||
self.stats.record_miss();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
self.stats.record_miss();
|
||||
false
|
||||
}
|
||||
|
||||
fn lookup<'a, F, E>(
|
||||
&self,
|
||||
pack_cache: F,
|
||||
id: impl AsRef<oid>,
|
||||
buf: &'a mut Vec<u8>,
|
||||
cache: &mut impl DecodeEntry,
|
||||
) -> Result<Option<Object<'a>>, error::Lookup<E>>
|
||||
where
|
||||
F: FnOnce(&pack::Info) -> Result<Arc<pack::Data>, E>,
|
||||
{
|
||||
for i in 0..2 {
|
||||
for idx in self.indices.load().iter() {
|
||||
if let Some(ofs) = idx.ofs(&id) {
|
||||
self.stats.record_hit();
|
||||
return load_obj(ofs, idx, pack_cache, buf, cache).map(Some);
|
||||
}
|
||||
}
|
||||
|
||||
if i == 0 {
|
||||
self.reload()?;
|
||||
}
|
||||
}
|
||||
|
||||
self.stats.record_miss();
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
fn load_obj<'a, F, E>(
|
||||
ofs: u64,
|
||||
idx: &pack::Index,
|
||||
pack_cache: F,
|
||||
buf: &'a mut Vec<u8>,
|
||||
cache: &mut impl DecodeEntry,
|
||||
) -> Result<Object<'a>, error::Lookup<E>>
|
||||
where
|
||||
F: FnOnce(&pack::Info) -> Result<Arc<pack::Data>, E>,
|
||||
{
|
||||
let data = pack_cache(&idx.info).map_err(error::Lookup::Lookup)?;
|
||||
let pack = data.file();
|
||||
let entry = pack.entry(ofs);
|
||||
let obj = pack
|
||||
.decode_entry(
|
||||
entry,
|
||||
buf,
|
||||
|id, _| idx.ofs(id).map(|ofs| ResolvedBase::InPack(pack.entry(ofs))),
|
||||
cache,
|
||||
)
|
||||
.map(move |out| Object {
|
||||
kind: out.kind,
|
||||
data: buf.as_slice(),
|
||||
pack_location: None,
|
||||
})?;
|
||||
|
||||
Ok(obj)
|
||||
}
|
||||
|
||||
fn discover(pack_dir: impl AsRef<Path>) -> Result<im::Vector<Arc<pack::Index>>, error::Discover> {
|
||||
let pack_dir = pack_dir.as_ref();
|
||||
let pack_dir_disp = pack_dir.display();
|
||||
trace!("discovering packs at {}", pack_dir_disp);
|
||||
match fs::read_dir(&pack_dir) {
|
||||
Ok(iter) => {
|
||||
let mut paths = Vec::new();
|
||||
for entry in iter {
|
||||
let entry = entry?;
|
||||
let path = entry.path();
|
||||
trace!("{}", path.display());
|
||||
let meta = entry.metadata()?;
|
||||
if meta.file_type().is_file() && path.extension().unwrap_or_default() == "idx" {
|
||||
let mtime = meta.modified()?;
|
||||
paths.push((path, mtime));
|
||||
}
|
||||
}
|
||||
paths.sort_by(|(_, mtime_a), (_, mtime_b)| mtime_a.cmp(mtime_b));
|
||||
|
||||
let indices = paths
|
||||
.into_iter()
|
||||
.rev()
|
||||
.map(|(path, _)| Ok(pack::Index::open(path).map(Arc::new)?))
|
||||
.collect::<Result<_, error::Discover>>()?;
|
||||
|
||||
Ok(indices)
|
||||
},
|
||||
// It's not an error if the directory doesn't exist, the repository
|
||||
// could contain only loose objects
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => {
|
||||
trace!("not a directory: {}", pack_dir_disp);
|
||||
Ok(im::Vector::new())
|
||||
},
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
impl<M> Index for Shared<M>
|
||||
where
|
||||
M: Metrics,
|
||||
{
|
||||
fn contains(&self, id: impl AsRef<oid>) -> bool {
|
||||
self.contains(id)
|
||||
}
|
||||
|
||||
fn lookup<'a, F, E>(
|
||||
&self,
|
||||
pack_cache: F,
|
||||
id: impl AsRef<oid>,
|
||||
buf: &'a mut Vec<u8>,
|
||||
cache: &mut impl DecodeEntry,
|
||||
) -> Result<Option<Object<'a>>, error::Lookup<E>>
|
||||
where
|
||||
F: FnOnce(&pack::Info) -> Result<Arc<pack::Data>, E>,
|
||||
{
|
||||
self.lookup(pack_cache, id, buf, cache)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,80 @@
|
||||
// Copyright © 2021 The Radicle Link Contributors
|
||||
//
|
||||
// This file is part of radicle-link, distributed under the GPLv3 with Radicle
|
||||
// Linking Exception. For full terms see the included LICENSE file.
|
||||
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
pub struct StatsView {
|
||||
/// Total number of times a lookup was successful.
|
||||
pub hits: usize,
|
||||
/// Total number of times a lookup was unsuccessful.
|
||||
pub misses: usize,
|
||||
/// Total number of times an index was added explicitly via
|
||||
/// [`super::Shared::push`].
|
||||
pub pushes: usize,
|
||||
/// Total number of reloads via [`super::Shared::reload`].
|
||||
pub reloads: usize,
|
||||
/// Number of [`crate::odb::pack::Index`]es currently held.
|
||||
pub indices: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Stats {
|
||||
hits: AtomicUsize,
|
||||
misses: AtomicUsize,
|
||||
pushes: AtomicUsize,
|
||||
reloads: AtomicUsize,
|
||||
}
|
||||
|
||||
pub trait Metrics {
|
||||
type Snapshot;
|
||||
|
||||
fn record_hit(&self);
|
||||
fn record_miss(&self);
|
||||
fn record_push(&self);
|
||||
fn record_reload(&self);
|
||||
|
||||
fn snapshot(&self, indices: usize) -> Self::Snapshot;
|
||||
}
|
||||
|
||||
impl Metrics for Stats {
|
||||
type Snapshot = StatsView;
|
||||
|
||||
fn record_hit(&self) {
|
||||
self.hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn record_miss(&self) {
|
||||
self.misses.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn record_push(&self) {
|
||||
self.pushes.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn record_reload(&self) {
|
||||
self.reloads.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn snapshot(&self, indices: usize) -> Self::Snapshot {
|
||||
StatsView {
|
||||
hits: self.hits.load(Ordering::Relaxed),
|
||||
misses: self.misses.load(Ordering::Relaxed),
|
||||
pushes: self.pushes.load(Ordering::Relaxed),
|
||||
reloads: self.reloads.load(Ordering::Relaxed),
|
||||
indices,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Metrics for () {
|
||||
type Snapshot = ();
|
||||
|
||||
fn record_hit(&self) {}
|
||||
fn record_miss(&self) {}
|
||||
fn record_push(&self) {}
|
||||
fn record_reload(&self) {}
|
||||
|
||||
fn snapshot(&self, _: usize) -> Self::Snapshot {}
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright © 2021 The Radicle Link Contributors
|
||||
//
|
||||
// This file is part of radicle-link, distributed under the GPLv3 with Radicle
|
||||
// Linking Exception. For full terms see the included LICENSE file.
|
||||
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
|
||||
use git_hash::{oid, ObjectId};
|
||||
use git_pack::{data, index};
|
||||
use rustc_hash::FxHasher;
|
||||
use tracing::warn;
|
||||
|
||||
pub mod error {
|
||||
use super::*;
|
||||
use thiserror::Error;
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
#[error("failed to load pack data from {path:?}")]
|
||||
pub struct Data {
|
||||
pub path: PathBuf,
|
||||
pub source: data::header::decode::Error,
|
||||
}
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
#[error("failed to load pack index from {path:?}")]
|
||||
pub struct Index {
|
||||
pub path: PathBuf,
|
||||
pub source: index::init::Error,
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Data {
|
||||
pub hash: u64,
|
||||
hits: AtomicUsize,
|
||||
file: data::File,
|
||||
}
|
||||
|
||||
impl Data {
|
||||
pub fn hit(&self) {
|
||||
self.hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub fn hits(&self) -> usize {
|
||||
self.hits.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
pub fn file(&self) -> &data::File {
|
||||
&self.file
|
||||
}
|
||||
}
|
||||
|
||||
impl AsRef<data::File> for Data {
|
||||
fn as_ref(&self) -> &data::File {
|
||||
self.file()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct Info {
|
||||
pub(super) hash: u64,
|
||||
pub data_path: PathBuf,
|
||||
}
|
||||
|
||||
impl Info {
|
||||
pub fn data(&self) -> Result<Data, error::Data> {
|
||||
let file = data::File::at(&self.data_path).map_err(|source| error::Data {
|
||||
path: self.data_path.clone(),
|
||||
source,
|
||||
})?;
|
||||
Ok(Data {
|
||||
hash: self.hash,
|
||||
hits: AtomicUsize::new(0),
|
||||
file,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Index {
|
||||
pub info: Info,
|
||||
file: index::File,
|
||||
}
|
||||
|
||||
impl Index {
|
||||
pub fn open(path: impl AsRef<Path>) -> Result<Self, error::Index> {
|
||||
let path = path.as_ref();
|
||||
let file = index::File::at(path).map_err(|source| error::Index {
|
||||
path: path.to_path_buf(),
|
||||
source,
|
||||
})?;
|
||||
let data_path = path.with_extension("pack");
|
||||
let hash = {
|
||||
let file_name = path
|
||||
.file_name()
|
||||
.expect("must have a file name, we opened it")
|
||||
.to_string_lossy();
|
||||
// XXX: inexplicably, gitoxide omits the "pack-" prefix
|
||||
let sha_hex = file_name.strip_prefix("pack-").unwrap_or(&file_name);
|
||||
match ObjectId::from_hex(&sha_hex.as_bytes()[..40]) {
|
||||
Err(e) => {
|
||||
warn!(
|
||||
"unconventional pack name {:?}, falling back to fxhash: {}",
|
||||
path, e
|
||||
);
|
||||
hash(path)
|
||||
},
|
||||
Ok(oid) => {
|
||||
let mut buf = [0u8; 8];
|
||||
buf.copy_from_slice(&oid.sha1()[..8]);
|
||||
u64::from_be_bytes(buf)
|
||||
},
|
||||
}
|
||||
};
|
||||
let info = Info { hash, data_path };
|
||||
|
||||
Ok(Self { file, info })
|
||||
}
|
||||
|
||||
pub fn contains(&self, id: impl AsRef<oid>) -> bool {
|
||||
self.file.lookup(id).is_some()
|
||||
}
|
||||
|
||||
pub fn ofs(&self, id: impl AsRef<oid>) -> Option<u64> {
|
||||
self.file
|
||||
.lookup(id)
|
||||
.map(|idx| self.file.pack_offset_at_index(idx))
|
||||
}
|
||||
}
|
||||
|
||||
fn hash(p: &Path) -> u64 {
|
||||
use std::hash::{Hash as _, Hasher as _};
|
||||
|
||||
let mut hasher = FxHasher::default();
|
||||
p.hash(&mut hasher);
|
||||
hasher.finish()
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
// Copyright © 2021 The Radicle Link Contributors
|
||||
//
|
||||
// This file is part of radicle-link, distributed under the GPLv3 with Radicle
|
||||
// Linking Exception. For full terms see the included LICENSE file.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use arc_swap::{ArcSwap, Guard};
|
||||
use parking_lot::Mutex;
|
||||
|
||||
use super::pack;
|
||||
|
||||
mod metrics;
|
||||
pub use metrics::{Metrics, Stats, StatsView};
|
||||
|
||||
/// A threadsafe, shareable cache of packfiles.
|
||||
pub trait Cache {
|
||||
type Stats;
|
||||
|
||||
fn stats(&self) -> Self::Stats;
|
||||
|
||||
fn get(&self, info: &pack::Info) -> Result<Arc<pack::Data>, pack::error::Data>;
|
||||
}
|
||||
|
||||
impl<M, const B: usize, const S: usize> Cache for Fixed<M, B, S>
|
||||
where
|
||||
M: Metrics,
|
||||
{
|
||||
type Stats = M::Snapshot;
|
||||
|
||||
fn stats(&self) -> Self::Stats {
|
||||
self.stats()
|
||||
}
|
||||
|
||||
fn get(&self, info: &pack::Info) -> Result<Arc<pack::Data>, pack::error::Data> {
|
||||
self.get(info)
|
||||
}
|
||||
}
|
||||
|
||||
/// 128 open files
|
||||
pub type Small<S> = Fixed<S, 16, 8>;
|
||||
/// 512 open files
|
||||
pub type Medium<S> = Fixed<S, 32, 16>;
|
||||
/// 1024 open files
|
||||
pub type Large<S> = Fixed<S, 64, 16>;
|
||||
/// 2048 open files
|
||||
pub type XLarge<S> = Fixed<S, 128, 16>;
|
||||
|
||||
/// A fixed-size [`Cache`].
|
||||
///
|
||||
/// [`Fixed`] is essentially a very simple, fixed-capacity hashtable. When a
|
||||
/// pack (data-) file is requested via [`Cache::get`], the file is loaded
|
||||
/// (typically `mmap`ed) from disk if it is not already in the cache. Otherwise,
|
||||
/// a pointer to the already loaded file is returned. Old entries are replaced
|
||||
/// on an approximate LRU basis when the cache becomes full (this means that old
|
||||
/// entries are **not** evicted when there is still space).
|
||||
///
|
||||
/// The implementation is a somewhat dumbed-down version of JGit's
|
||||
/// `WindowCache`. The main differences are that the table buckets are of fixed
|
||||
/// size (`SLOTS`), instead of a linked list. This means that the cache does not
|
||||
/// allow to (temporarily) commit more entries than its nominal capacity.
|
||||
///
|
||||
/// Reading cached values is lock-free and mostly wait-free. Modifications are
|
||||
/// guarded by locks on individual buckets; if a cache miss occurs, multiple
|
||||
/// threads requesting the same entry will be blocked until one of them
|
||||
/// succeeded loading the data and updating the cache. Writers will _not_,
|
||||
/// however, contend with readers (unlike `RwLock`).
|
||||
///
|
||||
/// This favours usage patterns where different threads tend to request disjoint
|
||||
/// sets of packfiles, and of course their hashes colliding relatively
|
||||
/// infrequently.
|
||||
pub struct Fixed<M, const BUCKETS: usize, const SLOTS: usize> {
|
||||
entries: [ArcSwap<[Option<Arc<pack::Data>>; SLOTS]>; BUCKETS],
|
||||
locks: [Mutex<()>; BUCKETS],
|
||||
stats: M,
|
||||
}
|
||||
|
||||
trait AssertSendSync: Send + Sync {}
|
||||
impl<M, const B: usize, const S: usize> AssertSendSync for Fixed<M, B, S> where M: Send + Sync {}
|
||||
|
||||
impl<M, const B: usize, const S: usize> AsRef<Fixed<M, B, S>> for Fixed<M, B, S> {
|
||||
fn as_ref(&self) -> &Fixed<M, B, S> {
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<const B: usize, const S: usize> Default for Fixed<(), B, S> {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
entries: [(); B].map(|_| ArcSwap::new(Arc::new([(); S].map(|_| None)))),
|
||||
locks: [(); B].map(|_| Mutex::new(())),
|
||||
stats: (),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<M, const B: usize, const S: usize> Fixed<M, B, S>
|
||||
where
|
||||
M: Metrics,
|
||||
{
|
||||
pub fn with_stats(self) -> Fixed<Stats, B, S> {
|
||||
self.with_metrics(Stats::default())
|
||||
}
|
||||
|
||||
pub fn with_metrics<N: Metrics>(self, m: N) -> Fixed<N, B, S> {
|
||||
Fixed {
|
||||
entries: self.entries,
|
||||
locks: self.locks,
|
||||
stats: m,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn stats(&self) -> M::Snapshot {
|
||||
let open_files = self
|
||||
.entries
|
||||
.iter()
|
||||
.map(|bucket| bucket.load().iter().flatten().count())
|
||||
.sum();
|
||||
self.stats.snapshot(open_files)
|
||||
}
|
||||
|
||||
pub fn get(&self, info: &pack::Info) -> Result<Arc<pack::Data>, pack::error::Data> {
|
||||
let idx = info.hash as usize % self.entries.len();
|
||||
|
||||
let bucket = self.entries[idx].load();
|
||||
for entry in bucket.iter().flatten() {
|
||||
if entry.hash == info.hash {
|
||||
self.stats.record_hit();
|
||||
entry.hit();
|
||||
return Ok(Arc::clone(entry));
|
||||
}
|
||||
}
|
||||
drop(bucket);
|
||||
|
||||
self.stats.record_miss();
|
||||
|
||||
// Cache miss, try to load the data file
|
||||
let lock = self.locks[idx].lock();
|
||||
// Did someone else win the race for the lock?
|
||||
let bucket = self.entries[idx].load();
|
||||
for entry in bucket.iter().flatten() {
|
||||
if entry.hash == info.hash {
|
||||
self.stats.record_hit();
|
||||
entry.hit();
|
||||
return Ok(Arc::clone(entry));
|
||||
}
|
||||
}
|
||||
// No, proceed
|
||||
self.stats.record_load();
|
||||
let data = Arc::new(info.data()?);
|
||||
|
||||
// Find an empty slot, or swap with the least popular
|
||||
let mut access = usize::MAX;
|
||||
let mut evict = 0;
|
||||
for (i, e) in bucket.iter().enumerate() {
|
||||
match e {
|
||||
Some(entry) => {
|
||||
let hits = entry.hits();
|
||||
if hits < access {
|
||||
access = hits;
|
||||
evict = i;
|
||||
}
|
||||
},
|
||||
None => {
|
||||
evict = i;
|
||||
break;
|
||||
},
|
||||
}
|
||||
}
|
||||
let mut entries = Guard::into_inner(bucket);
|
||||
{
|
||||
// This costs `SLOTS` refcount increments if the slot is currently
|
||||
// borrowed.
|
||||
let mutti = Arc::make_mut(&mut entries);
|
||||
mutti[evict] = Some(Arc::clone(&data));
|
||||
}
|
||||
self.entries[idx].store(entries);
|
||||
drop(lock);
|
||||
|
||||
data.hit();
|
||||
Ok(data)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// Copyright © 2021 The Radicle Link Contributors
|
||||
//
|
||||
// This file is part of radicle-link, distributed under the GPLv3 with Radicle
|
||||
// Linking Exception. For full terms see the included LICENSE file.
|
||||
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use tracing::trace;
|
||||
|
||||
pub struct StatsView {
|
||||
/// Total number of times the requested data was found in the cache.
|
||||
pub cache_hits: usize,
|
||||
/// Total number of times the requested data was not found in the cache.
|
||||
///
|
||||
/// Note that a cache hit can occur after a miss if another thread was
|
||||
/// faster to fill in the missing entry. Thus, `cache_hits + cache_misses`
|
||||
/// does not necessarily sum up to the number of cache accesses.
|
||||
pub cache_misses: usize,
|
||||
/// Total number of times a pack file was attempted to be loaded from disk
|
||||
/// (incl. failed attempts).
|
||||
pub file_loads: usize,
|
||||
/// Total number of pack files the cache holds on to.
|
||||
pub open_files: usize,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Stats {
|
||||
hits: AtomicUsize,
|
||||
miss: AtomicUsize,
|
||||
load: AtomicUsize,
|
||||
}
|
||||
|
||||
pub trait Metrics {
|
||||
type Snapshot;
|
||||
|
||||
fn record_hit(&self);
|
||||
fn record_miss(&self);
|
||||
fn record_load(&self);
|
||||
|
||||
fn snapshot(&self, open_files: usize) -> Self::Snapshot;
|
||||
}
|
||||
|
||||
impl Metrics for Stats {
|
||||
type Snapshot = StatsView;
|
||||
|
||||
fn record_hit(&self) {
|
||||
trace!("cache hit");
|
||||
self.hits.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn record_miss(&self) {
|
||||
trace!("cache miss");
|
||||
self.miss.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn record_load(&self) {
|
||||
trace!("pack load");
|
||||
self.load.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn snapshot(&self, open_files: usize) -> Self::Snapshot {
|
||||
StatsView {
|
||||
cache_hits: self.hits.load(Ordering::Relaxed),
|
||||
cache_misses: self.miss.load(Ordering::Relaxed),
|
||||
file_loads: self.load.load(Ordering::Relaxed),
|
||||
open_files,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Metrics for () {
|
||||
type Snapshot = ();
|
||||
|
||||
fn record_hit(&self) {}
|
||||
fn record_miss(&self) {}
|
||||
fn record_load(&self) {}
|
||||
|
||||
fn snapshot(&self, _: usize) -> Self::Snapshot {}
|
||||
}
|
||||
Reference in New Issue
Block a user