diff --git a/crates/radicle-search/README.md b/crates/radicle-search/README.md index 7603ce59c..d15696b09 100644 --- a/crates/radicle-search/README.md +++ b/crates/radicle-search/README.md @@ -35,9 +35,15 @@ Only public repos this node seeds are indexed. `rad node events`) and reacts to: - `RefsFetched`, `LocalRefsAnnounced`, `CanonicalRefUpdated`, `RefsSynced` → re-index the affected repo. - - `SeedDiscovered`, `SeedDropped`, `RefsAnnounced` → re-index only if - the rid is one we locally seed (filtered against an in-memory cache, - so gossip about repos we don't host is dropped at near-zero cost). + - `RefsAnnounced` → re-index only if the rid is one we locally seed + (filtered against an in-memory cache, so gossip about repos we don't + host is dropped at near-zero cost). + - `SeedDiscovered` and `SeedDropped` are ignored; the seeding count they + affect is refreshed by the periodic rescan. + + Affected repos are collected and re-indexed together every + `RADICLE_SEARCH_BATCH_SECS` (default 30s), so a burst of events for the + same repo costs a single re-index. 3. **Periodic rescan.** Every `RADICLE_SEARCH_RESCAN_SECS` (default 1h) as a safety net for missed events. @@ -107,6 +113,7 @@ All via environment variables: | `RADICLE_SEARCH_INDEX_NAME` | `repos` | Index name this daemon maintains. | | `RADICLE_SEARCH_RESCAN_SECS` | `3600` | Interval between safety-net full rescans. | | `RADICLE_SEARCH_RECONNECT_BACKOFF_SECS` | `5` | Delay before reconnecting after an event-stream disconnect. | +| `RADICLE_SEARCH_BATCH_SECS` | `30` | Interval at which repos touched by node events are re-indexed. | | `RAD_HOME` | `~/.radicle` | Standard Radicle profile path. | | `RAD_SOCKET` | `$RAD_HOME/node/control.sock` | Standard node control socket. | diff --git a/crates/radicle-search/src/config.rs b/crates/radicle-search/src/config.rs index 5d895e6cb..2389fba57 100644 --- a/crates/radicle-search/src/config.rs +++ b/crates/radicle-search/src/config.rs @@ -7,6 +7,7 @@ const DEFAULT_MEILI_URL: &str = "http://localhost:7700"; const DEFAULT_INDEX_NAME: &str = "repos"; const DEFAULT_RESCAN_INTERVAL_SECS: u64 = 3600; const DEFAULT_RECONNECT_BACKOFF_SECS: u64 = 5; +const DEFAULT_BATCH_SECS: u64 = 30; #[derive(Debug, Clone)] pub struct Config { @@ -15,6 +16,7 @@ pub struct Config { pub index_name: String, pub rescan_interval: Duration, pub reconnect_backoff: Duration, + pub batch_interval: Duration, } impl Config { @@ -32,6 +34,7 @@ impl Config { "RADICLE_SEARCH_RECONNECT_BACKOFF_SECS", DEFAULT_RECONNECT_BACKOFF_SECS, )?; + let batch_interval = duration_from_env("RADICLE_SEARCH_BATCH_SECS", DEFAULT_BATCH_SECS)?; Ok(Self { meili_url, @@ -39,6 +42,7 @@ impl Config { index_name, rescan_interval, reconnect_backoff, + batch_interval, }) } } diff --git a/crates/radicle-search/src/indexer.rs b/crates/radicle-search/src/indexer.rs index 8f585d24c..7c1ca2e08 100644 --- a/crates/radicle-search/src/indexer.rs +++ b/crates/radicle-search/src/indexer.rs @@ -270,6 +270,11 @@ impl Indexer { let mut rescan_timer = tokio::time::interval(self.config.rescan_interval); rescan_timer.tick().await; + let mut pending = HashSet::new(); + let mut batch_timer = tokio::time::interval(self.config.batch_interval); + batch_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + batch_timer.tick().await; + loop { tokio::select! { _ = shutdown.changed() => break, @@ -295,9 +300,22 @@ impl Indexer { tracing::info!("discovered new local seed: {rid}"); } } - tracing::info!("reindex {rid} (event: {})", event::event_kind(&event)); - if let Err(e) = self.reindex(rid).await { - tracing::warn!("reindex {} failed: {e:#}", rid); + if pending.insert(rid) { + tracing::debug!( + "queued reindex of {rid} (event: {})", + event::event_kind(&event) + ); + } + } + _ = batch_timer.tick() => { + if pending.is_empty() { + continue; + } + tracing::info!("reindexing {} repositories (batched)", pending.len()); + for rid in pending.drain().collect::>() { + if let Err(e) = self.reindex(rid).await { + tracing::warn!("reindex {} failed: {e:#}", rid); + } } } _ = rescan_timer.tick() => { diff --git a/crates/radicle-search/src/indexer/event.rs b/crates/radicle-search/src/indexer/event.rs index 9435323c7..bf02d60b2 100644 --- a/crates/radicle-search/src/indexer/event.rs +++ b/crates/radicle-search/src/indexer/event.rs @@ -14,9 +14,11 @@ pub(super) fn classify_event(event: &Event) -> Option<(RepoId, EventCategory)> { | Event::CanonicalRefUpdated { rid, .. } | Event::RefsFetched { rid, .. } | Event::RefsSynced { rid, .. } => Some((*rid, EventCategory::Replication)), - Event::SeedDiscovered { rid, .. } - | Event::SeedDropped { rid, .. } - | Event::RefsAnnounced { rid, .. } => Some((*rid, EventCategory::Gossip)), + Event::RefsAnnounced { rid, .. } => Some((*rid, EventCategory::Gossip)), + // Seed events only feed the seedingCount ranking field, which the + // periodic rescan refreshes anyway. Any peer can emit them at will + // by flapping its connection, so reindexing on them is a DoS vector. + Event::SeedDiscovered { .. } | Event::SeedDropped { .. } => None, _ => None, } } @@ -62,6 +64,16 @@ pub(super) fn event_action(category: EventCategory, is_locally_seeded: bool) -> mod test { use super::*; + #[test] + fn seed_events_are_ignored() { + let rid = "rad:z4V1sjrXqjvFdnCUbxPFqd5p4DtH5".parse().unwrap(); + let nid = "z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi" + .parse() + .unwrap(); + assert!(classify_event(&Event::SeedDiscovered { rid, nid }).is_none()); + assert!(classify_event(&Event::SeedDropped { rid, nid }).is_none()); + } + #[test] fn gossip_not_seeded_is_skipped() { assert_eq!( diff --git a/crates/radicle-search/src/main.rs b/crates/radicle-search/src/main.rs index 481ca1581..3b9f5b61b 100644 --- a/crates/radicle-search/src/main.rs +++ b/crates/radicle-search/src/main.rs @@ -23,6 +23,7 @@ Environment RADICLE_SEARCH_INDEX_NAME Meilisearch index name (default: repos) RADICLE_SEARCH_RESCAN_SECS Periodic full rescan interval in seconds (default: 3600) RADICLE_SEARCH_RECONNECT_BACKOFF_SECS Reconnect delay after a dropped event stream (default: 5) + RADICLE_SEARCH_BATCH_SECS Interval for re-indexing repos touched by events (default: 30) RAD_HOME, RAD_SOCKET Standard Radicle paths RUST_LOG Log level, e.g. RUST_LOG=warn or RUST_LOG=radicle_search=debug (default: info)