From fe29d7275b944c34f31dac2e7862e59d1370abc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Tue, 18 Aug 2026 10:05:56 +0200 Subject: [PATCH] httpd: Batch search reindexing, ignore seed events A peer with a bad connection kept reconnecting and re-announcing all its seeded repos, ~27k SeedDiscovered/SeedDropped events per hour on my node. Every event triggered a meili upsert until meilisearch sat at 100% cpu and its task queue timed out. Seed events only affect the seedingCount ranking field and the periodic rescan updates that anyway, so drop them. Collect the remaining events and reindex each touched repo once every 30s (RADICLE_SEARCH_BATCH_SECS). --- crates/radicle-search/README.md | 13 +++++++++--- crates/radicle-search/src/config.rs | 4 ++++ crates/radicle-search/src/indexer.rs | 24 +++++++++++++++++++--- crates/radicle-search/src/indexer/event.rs | 18 +++++++++++++--- crates/radicle-search/src/main.rs | 1 + 5 files changed, 51 insertions(+), 9 deletions(-) diff --git a/crates/radicle-search/README.md b/crates/radicle-search/README.md index 7603ce59c..d15696b09 100644 --- a/crates/radicle-search/README.md +++ b/crates/radicle-search/README.md @@ -35,9 +35,15 @@ Only public repos this node seeds are indexed. `rad node events`) and reacts to: - `RefsFetched`, `LocalRefsAnnounced`, `CanonicalRefUpdated`, `RefsSynced` → re-index the affected repo. - - `SeedDiscovered`, `SeedDropped`, `RefsAnnounced` → re-index only if - the rid is one we locally seed (filtered against an in-memory cache, - so gossip about repos we don't host is dropped at near-zero cost). + - `RefsAnnounced` → re-index only if the rid is one we locally seed + (filtered against an in-memory cache, so gossip about repos we don't + host is dropped at near-zero cost). + - `SeedDiscovered` and `SeedDropped` are ignored; the seeding count they + affect is refreshed by the periodic rescan. + + Affected repos are collected and re-indexed together every + `RADICLE_SEARCH_BATCH_SECS` (default 30s), so a burst of events for the + same repo costs a single re-index. 3. **Periodic rescan.** Every `RADICLE_SEARCH_RESCAN_SECS` (default 1h) as a safety net for missed events. @@ -107,6 +113,7 @@ All via environment variables: | `RADICLE_SEARCH_INDEX_NAME` | `repos` | Index name this daemon maintains. | | `RADICLE_SEARCH_RESCAN_SECS` | `3600` | Interval between safety-net full rescans. | | `RADICLE_SEARCH_RECONNECT_BACKOFF_SECS` | `5` | Delay before reconnecting after an event-stream disconnect. | +| `RADICLE_SEARCH_BATCH_SECS` | `30` | Interval at which repos touched by node events are re-indexed. | | `RAD_HOME` | `~/.radicle` | Standard Radicle profile path. | | `RAD_SOCKET` | `$RAD_HOME/node/control.sock` | Standard node control socket. | diff --git a/crates/radicle-search/src/config.rs b/crates/radicle-search/src/config.rs index 5d895e6cb..2389fba57 100644 --- a/crates/radicle-search/src/config.rs +++ b/crates/radicle-search/src/config.rs @@ -7,6 +7,7 @@ const DEFAULT_MEILI_URL: &str = "http://localhost:7700"; const DEFAULT_INDEX_NAME: &str = "repos"; const DEFAULT_RESCAN_INTERVAL_SECS: u64 = 3600; const DEFAULT_RECONNECT_BACKOFF_SECS: u64 = 5; +const DEFAULT_BATCH_SECS: u64 = 30; #[derive(Debug, Clone)] pub struct Config { @@ -15,6 +16,7 @@ pub struct Config { pub index_name: String, pub rescan_interval: Duration, pub reconnect_backoff: Duration, + pub batch_interval: Duration, } impl Config { @@ -32,6 +34,7 @@ impl Config { "RADICLE_SEARCH_RECONNECT_BACKOFF_SECS", DEFAULT_RECONNECT_BACKOFF_SECS, )?; + let batch_interval = duration_from_env("RADICLE_SEARCH_BATCH_SECS", DEFAULT_BATCH_SECS)?; Ok(Self { meili_url, @@ -39,6 +42,7 @@ impl Config { index_name, rescan_interval, reconnect_backoff, + batch_interval, }) } } diff --git a/crates/radicle-search/src/indexer.rs b/crates/radicle-search/src/indexer.rs index 8f585d24c..7c1ca2e08 100644 --- a/crates/radicle-search/src/indexer.rs +++ b/crates/radicle-search/src/indexer.rs @@ -270,6 +270,11 @@ impl Indexer { let mut rescan_timer = tokio::time::interval(self.config.rescan_interval); rescan_timer.tick().await; + let mut pending = HashSet::new(); + let mut batch_timer = tokio::time::interval(self.config.batch_interval); + batch_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + batch_timer.tick().await; + loop { tokio::select! { _ = shutdown.changed() => break, @@ -295,9 +300,22 @@ impl Indexer { tracing::info!("discovered new local seed: {rid}"); } } - tracing::info!("reindex {rid} (event: {})", event::event_kind(&event)); - if let Err(e) = self.reindex(rid).await { - tracing::warn!("reindex {} failed: {e:#}", rid); + if pending.insert(rid) { + tracing::debug!( + "queued reindex of {rid} (event: {})", + event::event_kind(&event) + ); + } + } + _ = batch_timer.tick() => { + if pending.is_empty() { + continue; + } + tracing::info!("reindexing {} repositories (batched)", pending.len()); + for rid in pending.drain().collect::>() { + if let Err(e) = self.reindex(rid).await { + tracing::warn!("reindex {} failed: {e:#}", rid); + } } } _ = rescan_timer.tick() => { diff --git a/crates/radicle-search/src/indexer/event.rs b/crates/radicle-search/src/indexer/event.rs index 9435323c7..bf02d60b2 100644 --- a/crates/radicle-search/src/indexer/event.rs +++ b/crates/radicle-search/src/indexer/event.rs @@ -14,9 +14,11 @@ pub(super) fn classify_event(event: &Event) -> Option<(RepoId, EventCategory)> { | Event::CanonicalRefUpdated { rid, .. } | Event::RefsFetched { rid, .. } | Event::RefsSynced { rid, .. } => Some((*rid, EventCategory::Replication)), - Event::SeedDiscovered { rid, .. } - | Event::SeedDropped { rid, .. } - | Event::RefsAnnounced { rid, .. } => Some((*rid, EventCategory::Gossip)), + Event::RefsAnnounced { rid, .. } => Some((*rid, EventCategory::Gossip)), + // Seed events only feed the seedingCount ranking field, which the + // periodic rescan refreshes anyway. Any peer can emit them at will + // by flapping its connection, so reindexing on them is a DoS vector. + Event::SeedDiscovered { .. } | Event::SeedDropped { .. } => None, _ => None, } } @@ -62,6 +64,16 @@ pub(super) fn event_action(category: EventCategory, is_locally_seeded: bool) -> mod test { use super::*; + #[test] + fn seed_events_are_ignored() { + let rid = "rad:z4V1sjrXqjvFdnCUbxPFqd5p4DtH5".parse().unwrap(); + let nid = "z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi" + .parse() + .unwrap(); + assert!(classify_event(&Event::SeedDiscovered { rid, nid }).is_none()); + assert!(classify_event(&Event::SeedDropped { rid, nid }).is_none()); + } + #[test] fn gossip_not_seeded_is_skipped() { assert_eq!( diff --git a/crates/radicle-search/src/main.rs b/crates/radicle-search/src/main.rs index 481ca1581..3b9f5b61b 100644 --- a/crates/radicle-search/src/main.rs +++ b/crates/radicle-search/src/main.rs @@ -23,6 +23,7 @@ Environment RADICLE_SEARCH_INDEX_NAME Meilisearch index name (default: repos) RADICLE_SEARCH_RESCAN_SECS Periodic full rescan interval in seconds (default: 3600) RADICLE_SEARCH_RECONNECT_BACKOFF_SECS Reconnect delay after a dropped event stream (default: 5) + RADICLE_SEARCH_BATCH_SECS Interval for re-indexing repos touched by events (default: 30) RAD_HOME, RAD_SOCKET Standard Radicle paths RUST_LOG Log level, e.g. RUST_LOG=warn or RUST_LOG=radicle_search=debug (default: info)