mirror of
https://seed.radicle.dev/rad:z4V1sjrXqjvFdnCUbxPFqd5p4DtH5.git
synced 2026-11-21 22:30:41 +08:00
httpd: Batch search reindexing, ignore seed events
A peer with a bad connection kept reconnecting and re-announcing all its seeded repos, ~27k SeedDiscovered/SeedDropped events per hour on my node. Every event triggered a meili upsert until meilisearch sat at 100% cpu and its task queue timed out. Seed events only affect the seedingCount ranking field and the periodic rescan updates that anyway, so drop them. Collect the remaining events and reindex each touched repo once every 30s (RADICLE_SEARCH_BATCH_SECS).
This commit is contained in:
committed by
Rūdolfs Ošiņš
parent
86b5a3d1ce
commit
fe29d7275b
@@ -35,9 +35,15 @@ Only public repos this node seeds are indexed.
|
||||
`rad node events`) and reacts to:
|
||||
- `RefsFetched`, `LocalRefsAnnounced`, `CanonicalRefUpdated`, `RefsSynced`
|
||||
→ re-index the affected repo.
|
||||
- `SeedDiscovered`, `SeedDropped`, `RefsAnnounced` → re-index only if
|
||||
the rid is one we locally seed (filtered against an in-memory cache,
|
||||
so gossip about repos we don't host is dropped at near-zero cost).
|
||||
- `RefsAnnounced` → re-index only if the rid is one we locally seed
|
||||
(filtered against an in-memory cache, so gossip about repos we don't
|
||||
host is dropped at near-zero cost).
|
||||
- `SeedDiscovered` and `SeedDropped` are ignored; the seeding count they
|
||||
affect is refreshed by the periodic rescan.
|
||||
|
||||
Affected repos are collected and re-indexed together every
|
||||
`RADICLE_SEARCH_BATCH_SECS` (default 30s), so a burst of events for the
|
||||
same repo costs a single re-index.
|
||||
3. **Periodic rescan.** Every `RADICLE_SEARCH_RESCAN_SECS` (default 1h)
|
||||
as a safety net for missed events.
|
||||
|
||||
@@ -107,6 +113,7 @@ All via environment variables:
|
||||
| `RADICLE_SEARCH_INDEX_NAME` | `repos` | Index name this daemon maintains. |
|
||||
| `RADICLE_SEARCH_RESCAN_SECS` | `3600` | Interval between safety-net full rescans. |
|
||||
| `RADICLE_SEARCH_RECONNECT_BACKOFF_SECS` | `5` | Delay before reconnecting after an event-stream disconnect. |
|
||||
| `RADICLE_SEARCH_BATCH_SECS` | `30` | Interval at which repos touched by node events are re-indexed. |
|
||||
| `RAD_HOME` | `~/.radicle` | Standard Radicle profile path. |
|
||||
| `RAD_SOCKET` | `$RAD_HOME/node/control.sock` | Standard node control socket. |
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ const DEFAULT_MEILI_URL: &str = "http://localhost:7700";
|
||||
const DEFAULT_INDEX_NAME: &str = "repos";
|
||||
const DEFAULT_RESCAN_INTERVAL_SECS: u64 = 3600;
|
||||
const DEFAULT_RECONNECT_BACKOFF_SECS: u64 = 5;
|
||||
const DEFAULT_BATCH_SECS: u64 = 30;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Config {
|
||||
@@ -15,6 +16,7 @@ pub struct Config {
|
||||
pub index_name: String,
|
||||
pub rescan_interval: Duration,
|
||||
pub reconnect_backoff: Duration,
|
||||
pub batch_interval: Duration,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -32,6 +34,7 @@ impl Config {
|
||||
"RADICLE_SEARCH_RECONNECT_BACKOFF_SECS",
|
||||
DEFAULT_RECONNECT_BACKOFF_SECS,
|
||||
)?;
|
||||
let batch_interval = duration_from_env("RADICLE_SEARCH_BATCH_SECS", DEFAULT_BATCH_SECS)?;
|
||||
|
||||
Ok(Self {
|
||||
meili_url,
|
||||
@@ -39,6 +42,7 @@ impl Config {
|
||||
index_name,
|
||||
rescan_interval,
|
||||
reconnect_backoff,
|
||||
batch_interval,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -270,6 +270,11 @@ impl Indexer {
|
||||
let mut rescan_timer = tokio::time::interval(self.config.rescan_interval);
|
||||
rescan_timer.tick().await;
|
||||
|
||||
let mut pending = HashSet::new();
|
||||
let mut batch_timer = tokio::time::interval(self.config.batch_interval);
|
||||
batch_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
batch_timer.tick().await;
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = shutdown.changed() => break,
|
||||
@@ -295,9 +300,22 @@ impl Indexer {
|
||||
tracing::info!("discovered new local seed: {rid}");
|
||||
}
|
||||
}
|
||||
tracing::info!("reindex {rid} (event: {})", event::event_kind(&event));
|
||||
if let Err(e) = self.reindex(rid).await {
|
||||
tracing::warn!("reindex {} failed: {e:#}", rid);
|
||||
if pending.insert(rid) {
|
||||
tracing::debug!(
|
||||
"queued reindex of {rid} (event: {})",
|
||||
event::event_kind(&event)
|
||||
);
|
||||
}
|
||||
}
|
||||
_ = batch_timer.tick() => {
|
||||
if pending.is_empty() {
|
||||
continue;
|
||||
}
|
||||
tracing::info!("reindexing {} repositories (batched)", pending.len());
|
||||
for rid in pending.drain().collect::<Vec<_>>() {
|
||||
if let Err(e) = self.reindex(rid).await {
|
||||
tracing::warn!("reindex {} failed: {e:#}", rid);
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = rescan_timer.tick() => {
|
||||
|
||||
@@ -14,9 +14,11 @@ pub(super) fn classify_event(event: &Event) -> Option<(RepoId, EventCategory)> {
|
||||
| Event::CanonicalRefUpdated { rid, .. }
|
||||
| Event::RefsFetched { rid, .. }
|
||||
| Event::RefsSynced { rid, .. } => Some((*rid, EventCategory::Replication)),
|
||||
Event::SeedDiscovered { rid, .. }
|
||||
| Event::SeedDropped { rid, .. }
|
||||
| Event::RefsAnnounced { rid, .. } => Some((*rid, EventCategory::Gossip)),
|
||||
Event::RefsAnnounced { rid, .. } => Some((*rid, EventCategory::Gossip)),
|
||||
// Seed events only feed the seedingCount ranking field, which the
|
||||
// periodic rescan refreshes anyway. Any peer can emit them at will
|
||||
// by flapping its connection, so reindexing on them is a DoS vector.
|
||||
Event::SeedDiscovered { .. } | Event::SeedDropped { .. } => None,
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -62,6 +64,16 @@ pub(super) fn event_action(category: EventCategory, is_locally_seeded: bool) ->
|
||||
mod test {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn seed_events_are_ignored() {
|
||||
let rid = "rad:z4V1sjrXqjvFdnCUbxPFqd5p4DtH5".parse().unwrap();
|
||||
let nid = "z6MknSLrJoTcukLrE435hVNQT4JUhbvWLX4kUzqkEStBU8Vi"
|
||||
.parse()
|
||||
.unwrap();
|
||||
assert!(classify_event(&Event::SeedDiscovered { rid, nid }).is_none());
|
||||
assert!(classify_event(&Event::SeedDropped { rid, nid }).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn gossip_not_seeded_is_skipped() {
|
||||
assert_eq!(
|
||||
|
||||
@@ -23,6 +23,7 @@ Environment
|
||||
RADICLE_SEARCH_INDEX_NAME Meilisearch index name (default: repos)
|
||||
RADICLE_SEARCH_RESCAN_SECS Periodic full rescan interval in seconds (default: 3600)
|
||||
RADICLE_SEARCH_RECONNECT_BACKOFF_SECS Reconnect delay after a dropped event stream (default: 5)
|
||||
RADICLE_SEARCH_BATCH_SECS Interval for re-indexing repos touched by events (default: 30)
|
||||
RAD_HOME, RAD_SOCKET Standard Radicle paths
|
||||
RUST_LOG Log level, e.g. RUST_LOG=warn or
|
||||
RUST_LOG=radicle_search=debug (default: info)
|
||||
|
||||
Reference in New Issue
Block a user