Files
radicle-ci-broker/src/pages.rs
T
2024-08-16 11:45:15 +03:00

641 lines
22 KiB
Rust

//! Status and report pages for CI broker.
//!
//! This module generates an HTML status page for the CI broker, as
//! well as per-repository pages for any repository for which the CI
//! broker has mediated to run CI. The status page gives the latest
//! known status of the broker, plus lists the repositories that CI
//! has run for. The per-repository pages lists all the runs for that
//! repository.
use std::{
collections::{HashMap, HashSet},
fs::write,
path::{Path, PathBuf},
sync::mpsc::RecvTimeoutError,
thread::{spawn, JoinHandle},
time::Duration,
};
use html_page::{Element, HtmlPage, Tag};
use log::{debug, info, trace, warn};
use serde::Serialize;
use time::{macros::format_description, OffsetDateTime};
use radicle::prelude::RepoId;
use crate::{
db::{Db, DbError},
event::BrokerEvent,
msg::{RunId, RunResult},
notif::NotificationReceiver,
run::{Run, RunState, Whence},
};
const CSS: &str = include_str!("radicle-ci.css");
const REFERESH_INTERVAL: &str = "300";
const UPDATE_INTERVAL: Duration = Duration::from_secs(60);
const STATUS_JSON: &str = "status.json";
/// All possible errors returned from the status page module.
#[derive(Debug, thiserror::Error)]
pub enum PageError {
/// Error formatting a time as a string.
#[error(transparent)]
Timeformat(#[from] time::error::Format),
#[error("failed to write status page to {0}")]
Write(PathBuf, #[source] std::io::Error),
#[error("no node alias has been set for builder")]
NoAlias,
#[error("no status data has been set for builder")]
NoStatusData,
#[error(transparent)]
Db(#[from] DbError),
#[error("failed to lock page data structure")]
Lock(&'static str),
#[error("failed to represent status page data as JSON")]
StatusToJson(#[source] serde_json::Error),
}
/// A builder for constructing a [`StatusPage`] value. It will only
/// construct a valid value.
#[derive(Default)]
pub struct PageBuilder {
node_alias: Option<String>,
runs: Vec<Run>,
}
impl PageBuilder {
pub fn node_alias(mut self, alias: &str) -> Self {
self.node_alias = Some(alias.into());
self
}
pub fn runs(mut self, runs: Vec<Run>) -> Self {
self.runs = runs;
self
}
pub fn build(self) -> Result<StatusPage, PageError> {
let mut runs = HashMap::new();
for run in self.runs.iter() {
runs.insert(run.adapter_run_id().unwrap().clone(), run.clone());
}
debug!("broker database has {} CI runs", runs.len());
Ok(StatusPage::new())
}
}
fn now() -> Result<String, time::error::Format> {
let fmt = format_description!("[year]-[month]-[day] [hour]:[minute]:[second]Z");
OffsetDateTime::now_utc().format(fmt)
}
struct PageData {
timestamp: String,
ci_broker_version: &'static str,
ci_broker_git_commit: &'static str,
node_alias: String,
runs: HashMap<RunId, Run>,
broker_event_counter: usize,
latest_broker_event: Option<BrokerEvent>,
latest_ci_run: Option<Run>,
}
impl PageData {
fn status_page_as_json(&self) -> Result<String, PageError> {
StatusData::from(self).as_json()
}
fn status_page_as_html(&self) -> Result<HtmlPage, PageError> {
let mut doc = HtmlPage::default();
Self::head(
&mut doc,
&format!("CI for Radicle node {}", self.node_alias),
);
doc.push_to_body(
Element::new(Tag::H1)
.with_text("CI for Radicle node ")
.with_text(&self.node_alias),
);
doc.push_to_body(Element::new(Tag::H2).with_text("Broker status"));
doc.push_to_body(
Element::new(Tag::P)
.with_text("Last updated: ")
.with_text(&self.timestamp)
.with_child(Element::new(Tag::Br))
.with_text("CI broker version: ")
.with_text(self.ci_broker_version)
.with_text(" (commit ")
.with_child(Element::new(Tag::Code).with_text(self.ci_broker_git_commit))
.with_text(")"),
);
doc.push_to_body(Element::new(Tag::H2).with_text("Repositories"));
doc.push_to_body(Element::new(Tag::P).with_text("Latest CI run for each repository."));
let mut list = Element::new(Tag::Ul).with_class("repolist");
for (alias, rid) in self.repos() {
let mut item = Element::new(Tag::Li);
let runs = self.runs(rid);
item.push_child(
Element::new(Tag::Span).with_child(
Element::new(Tag::A)
.with_attribute("href", &format!("{}.html", rid_to_basename(rid)))
.with_text("Repository ")
.with_child(
Element::new(Tag::Span)
.with_class("alias)")
.with_text(&alias),
)
.with_text(" (")
.with_child(
Element::new(Tag::Code)
.with_class("repoid)")
.with_text(&rid.to_string()),
)
.with_text(&format!("), {} runs", runs.len())),
),
);
if let Some(run) = self.latest_run(rid) {
item.push_child(Element::new(Tag::Br));
item.push_child(
Element::new(Tag::Span)
.with_text("latest run at ")
.with_text(run.timestamp())
.with_child(Element::new(Tag::Br))
.with_text("broker run ID: ")
.with_child(
Element::new(Tag::Code)
.with_class("broker_run_id")
.with_text(run.broker_run_id().as_str()),
)
.with_child(Element::new(Tag::Br))
.with_text("adapter run ID: ")
.with_child(if let Some(id) = run.adapter_run_id() {
Element::new(Tag::Code)
.with_class("adapter_run_id")
.with_text(id.as_str())
} else {
Element::new(Tag::Span).with_text("unknown")
})
.with_child(Element::new(Tag::Br))
.with_text(" ")
.with_child(Self::whence_as_html(run.whence())),
);
item.push_child(Element::new(Tag::Br));
let state = run.state().to_string();
item.push_child(Element::new(Tag::Span).with_class(&state).with_text(&state));
let result = match run.result() {
None => Element::new(Tag::Span)
.with_text(" ")
.with_class("unknown")
.with_text("unknown result"),
Some(RunResult::Success) => Element::new(Tag::Span)
.with_text(" ")
.with_class("success")
.with_text("success"),
Some(_) => Element::new(Tag::Span)
.with_text(" ")
.with_class("failure")
.with_text("failure"),
};
item.push_child(result);
if let Some(url) = run.adapter_info_url() {
item.push_child(Element::new(Tag::Br));
let log = Element::new(Tag::Span).with_child(
Element::new(Tag::A)
.with_attribute("href", url)
.with_text("log"),
);
item.push_child(log);
}
}
list.push_child(item);
}
doc.push_to_body(list);
doc.push_to_body(Element::new(Tag::H2).with_text("Recent status"));
let status = StatusData::from(self).as_json()?;
doc.push_to_body(
Element::new(Tag::P)
.with_text("See also as a separate file ")
.with_child(
Element::new(Tag::A)
.with_attribute("href", STATUS_JSON)
.with_child(Element::new(Tag::Code).with_text(STATUS_JSON)),
)
.with_text(": "),
);
doc.push_to_body(
Element::new(Tag::Blockquote).with_child(Element::new(Tag::Pre).with_text(&status)),
);
Ok(doc)
}
fn head(page: &mut HtmlPage, title: &str) {
page.push_to_head(Element::new(Tag::Title).with_text(title));
page.push_to_head(Element::new(Tag::Style).with_text(CSS));
page.push_to_head(
Element::new(Tag::Meta)
.with_attribute("http-equiv", "refresh")
.with_attribute("content", REFERESH_INTERVAL),
);
}
fn whence_as_html(whence: &Whence) -> Element {
match whence {
Whence::Branch {
name,
commit,
who: _,
} => Element::new(Tag::Span)
.with_text("branch ")
.with_child(
Element::new(Tag::Code)
.with_class("branch)")
.with_text(name),
)
.with_text(", commit ")
.with_child(
Element::new(Tag::Code)
.with_class("commit)")
.with_text(&commit.to_string()),
)
.with_child(Element::new(Tag::Br))
.with_text("from ")
.with_child(
Element::new(Tag::Span)
.with_class("who")
.with_text(whence.who().unwrap_or("<commit author not known>")),
),
Whence::Patch {
patch,
commit,
revision,
who: _,
} => Element::new(Tag::Span)
.with_text("patch ")
.with_child(
Element::new(Tag::Code)
.with_class("branch")
.with_text(&patch.to_string()),
)
.with_child(Element::new(Tag::Br))
.with_text("revision ")
.with_child(Element::new(Tag::Code).with_class("revision)").with_text(&{
if let Some(rev) = &revision {
rev.to_string()
} else {
"<unknown patch revision>".to_string()
}
}))
.with_child(Element::new(Tag::Br))
.with_text("commit ")
.with_child(
Element::new(Tag::Code)
.with_class("commit)")
.with_text(&commit.to_string()),
)
.with_child(Element::new(Tag::Br))
.with_text("from ")
.with_child(
Element::new(Tag::Span)
.with_class("who")
.with_text(whence.who().unwrap_or("<patch author not known>")),
),
}
}
fn per_repo_page_as_html(&self, rid: RepoId, alias: &str, timestamp: &str) -> HtmlPage {
let mut doc = HtmlPage::default();
doc.push_to_head(
Element::new(Tag::Title)
.with_text("CI runs for repository ")
.with_text(alias),
);
doc.push_to_head(Element::new(Tag::Style).with_text(CSS));
doc.push_to_body(
Element::new(Tag::H1)
.with_text("CI runs for repository ")
.with_text(alias),
);
doc.push_to_body(
Element::new(Tag::P)
.with_text("Last updated: ")
.with_text(timestamp),
);
doc.push_to_body(
Element::new(Tag::P)
.with_text("Repository ID ")
.with_child(Element::new(Tag::Code).with_text(&rid.to_string())),
);
let mut runs = self.runs(rid);
runs.sort_by_cached_key(|run| run.timestamp());
runs.reverse();
let mut list = Element::new(Tag::Ol).with_class("runlist");
for run in runs {
let current = match run.state() {
RunState::Triggered => Element::new(Tag::Span)
.with_attribute("state", "triggered")
.with_text("triggered"),
RunState::Running => Element::new(Tag::Span)
.with_class("running)")
.with_text("running"),
RunState::Finished => {
let result = if let Some(result) = run.result() {
result.to_string()
} else {
"unknown".into()
};
Element::new(Tag::Span)
.with_class(&result)
.with_text(&result)
}
};
let info_url = if let Some(url) = run.adapter_info_url() {
Element::new(Tag::Span).with_child(
Element::new(Tag::A)
.with_attribute("href", url)
.with_text("info from adapter"),
)
} else {
Element::new(Tag::Span)
.with_child(Element::new(Tag::Span).with_text("no info from adapter"))
};
list.push_child(
Element::new(Tag::Li)
.with_text("at ")
.with_text(run.timestamp())
.with_text("; currently ")
.with_child(current)
.with_text("; ")
.with_child(info_url)
.with_child(Element::new(Tag::Br))
.with_text("broker run ID: ")
.with_child(
Element::new(Tag::Code)
.with_class("broker_run_id")
.with_text(run.broker_run_id().as_str()),
)
.with_child(Element::new(Tag::Br))
.with_text("adapter run ID: ")
.with_child(if let Some(id) = run.adapter_run_id() {
Element::new(Tag::Code)
.with_class("adapter_run_id")
.with_text(id.as_str())
} else {
Element::new(Tag::Span).with_text("unknown")
})
.with_child(Element::new(Tag::Br))
.with_child(Self::whence_as_html(run.whence())),
);
}
doc.push_to_body(list);
doc
}
fn repos(&self) -> Vec<(String, RepoId)> {
let rids: HashSet<(String, RepoId)> = self
.runs
.values()
.map(|run| (run.repo_alias().to_string(), run.repo_id()))
.collect();
let mut repos: Vec<(String, RepoId)> = rids.iter().cloned().collect();
repos.sort();
repos
}
fn repo_alias(&self, wanted: RepoId) -> Option<String> {
self.repos().iter().find_map(|(alias, rid)| {
if *rid == wanted {
Some(alias.into())
} else {
None
}
})
}
fn runs(&self, repoid: RepoId) -> Vec<&Run> {
self.runs
.iter()
.filter_map(|(_, run)| {
if run.repo_id() == repoid {
Some(run)
} else {
None
}
})
.collect()
}
fn latest_run(&self, repoid: RepoId) -> Option<&Run> {
let mut value: Option<&Run> = None;
for run in self.runs(repoid) {
if let Some(latest) = value {
if run.timestamp() > latest.timestamp() {
value = Some(run);
}
} else {
value = Some(run);
}
}
value
}
}
/// Data for status pages for CI broker.
///
/// There is a "front page" with status about the broker, and a list
/// of repositories for which the broker has run CI. Then there is a
/// page per such repository, with a list of CI runs for that
/// repository.
pub struct StatusPage {
node_alias: String,
dirname: Option<PathBuf>,
}
impl StatusPage {
fn new() -> Self {
Self {
node_alias: String::new(),
dirname: None,
}
}
pub fn set_output_dir(&mut self, dirname: &Path) {
self.dirname = Some(dirname.into());
}
pub fn update_in_thread(
mut self,
run_rx: NotificationReceiver,
db: Db,
node_alias: &str,
once: bool,
) -> JoinHandle<Result<(), PageError>> {
if self.dirname.is_none() {
warn!("not writing HTML report pages as output directory has not been set");
}
self.node_alias = node_alias.into();
info!(
"wait about {} seconds to update HTML report pages again",
UPDATE_INTERVAL.as_secs()
);
spawn(move || {
'processing_loop: loop {
self.update_and_write(&db)?;
if once {
return Ok(());
}
trace!("wait for run notification or timeout");
match run_rx.recv_timeout(UPDATE_INTERVAL) {
Ok(_) => trace!("page updater: got a run notification"),
Err(RecvTimeoutError::Timeout) => {
trace!("page updater: timeout on run notification channel")
}
Err(RecvTimeoutError::Disconnected) => {
debug!("page updater: run notification channel disconnected");
break 'processing_loop;
}
}
}
// Make sure we update reports and status JSON at least once.
info!("make sure we update reports and status JSON at least once");
self.update_and_write(&db)?;
info!("end page updater thread");
Ok(())
})
}
fn update_and_write(&mut self, db: &Db) -> Result<(), PageError> {
if let Some(dirname) = &self.dirname {
info!(
"write HTML report pages and {STATUS_JSON} to {}",
dirname.display()
);
let runs = db.get_all_runs()?;
let data = PageData {
timestamp: now()?,
ci_broker_version: env!("CARGO_PKG_VERSION"),
ci_broker_git_commit: env!("GIT_HEAD"),
node_alias: self.node_alias.clone(),
runs: HashMap::from_iter(
runs.iter()
.map(|run| (run.broker_run_id().clone(), run.clone())),
),
broker_event_counter: 0,
latest_broker_event: None,
latest_ci_run: None,
};
let nameless = String::from("nameless repo");
// We avoid writing while keeping the lock, to reduce
// contention.
let (status, repos) = {
let status = data.status_page_as_html()?.to_string();
let mut repos = vec![];
for (_, rid) in data.repos() {
let basename = rid_to_basename(rid);
let filename = dirname.join(format!("{basename}.html"));
let alias = data.repo_alias(rid).unwrap_or(nameless.clone());
let repopage = data.per_repo_page_as_html(rid, &alias, &data.timestamp);
repos.push((filename, repopage.to_string()));
}
(status, repos)
};
let filename = dirname.join("index.html");
Self::write_file(&filename, &status)?;
for (filename, repopage) in repos {
Self::write_file(&filename, &repopage)?;
}
let filename = dirname.join(STATUS_JSON);
let json = data.status_page_as_json()?;
Self::write_file(&filename, &json)?;
}
Ok(())
}
fn write_file(filename: &Path, text: &str) -> Result<(), PageError> {
debug!("write file {}", filename.display());
write(filename, text).map_err(|e| PageError::Write(filename.into(), e))?;
Ok(())
}
}
#[derive(Debug, Clone, Serialize)]
struct StatusData {
timestamp: String,
broker_event_counter: usize,
ci_broker_version: &'static str,
ci_broker_git_commit: &'static str,
latest_broker_event: Option<BrokerEvent>,
latest_ci_run: Option<Run>,
}
impl StatusData {
fn as_json(&self) -> Result<String, PageError> {
serde_json::to_string_pretty(self).map_err(PageError::StatusToJson)
}
}
impl From<&PageData> for StatusData {
fn from(page: &PageData) -> Self {
Self {
timestamp: page.timestamp.clone(),
broker_event_counter: page.broker_event_counter,
ci_broker_version: page.ci_broker_version,
ci_broker_git_commit: page.ci_broker_git_commit,
latest_broker_event: page.latest_broker_event.clone(),
latest_ci_run: page.latest_ci_run.clone(),
}
}
}
fn rid_to_basename(repoid: RepoId) -> String {
let mut basename = repoid.to_string();
assert!(basename.starts_with("rad:"));
basename.drain(..4);
basename
}