feat: produce an index.html with list of, and links to, run logs

This is super-simplistic, but better than nothing.

Signed-off-by: Lars Wirzenius <liw@liw.fi>
This commit is contained in:
Lars Wirzenius
2023-12-29 15:18:51 +02:00
parent 28a93105a3
commit d5b88b6596
4 changed files with 382 additions and 11 deletions
Generated
+97
View File
@@ -342,6 +342,15 @@ dependencies = [
"zeroize",
]
[[package]]
name = "deranged"
version = "0.3.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8eb30d70a07a3b04884d2677f06bec33509dc67ca60d92949e5535352d3191dc"
dependencies = [
"powerfmt",
]
[[package]]
name = "digest"
version = "0.9.0"
@@ -591,6 +600,25 @@ dependencies = [
"digest 0.10.7",
]
[[package]]
name = "html-escape"
version = "0.2.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6d1ad449764d627e22bfd7cd5e8868264fc9236e07c752972b4080cd351cb476"
dependencies = [
"utf8-width",
]
[[package]]
name = "html-page"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f8314b0ea57e9e3fc648213a02315e8a16154bb86da7516fec7a09ec4d7417c"
dependencies = [
"html-escape",
"line-col",
]
[[package]]
name = "humantime"
version = "2.1.0"
@@ -706,6 +734,12 @@ dependencies = [
"vcpkg",
]
[[package]]
name = "line-col"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e69cdf6b85b5c8dce514f694089a2cf8b1a702f6cd28607bcb3cf296c9778db"
[[package]]
name = "linux-raw-sys"
version = "0.4.12"
@@ -887,6 +921,12 @@ version = "0.3.27"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964"
[[package]]
name = "powerfmt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
version = "0.2.17"
@@ -1049,6 +1089,7 @@ dependencies = [
name = "radicle-native-ci"
version = "0.1.0"
dependencies = [
"html-page",
"log",
"pretty_env_logger",
"radicle",
@@ -1059,7 +1100,9 @@ dependencies = [
"subprocess",
"tempfile",
"thiserror",
"time",
"uuid",
"walkdir",
]
[[package]]
@@ -1238,6 +1281,15 @@ version = "1.0.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741"
[[package]]
name = "same-file"
version = "1.0.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
dependencies = [
"winapi-util",
]
[[package]]
name = "sec1"
version = "0.3.0"
@@ -1521,6 +1573,35 @@ dependencies = [
"syn 2.0.39",
]
[[package]]
name = "time"
version = "0.3.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e"
dependencies = [
"deranged",
"itoa",
"powerfmt",
"serde",
"time-core",
"time-macros",
]
[[package]]
name = "time-core"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3"
[[package]]
name = "time-macros"
version = "0.2.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f"
dependencies = [
"time-core",
]
[[package]]
name = "tinyvec"
version = "1.6.0"
@@ -1580,6 +1661,12 @@ dependencies = [
"percent-encoding",
]
[[package]]
name = "utf8-width"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "86bd8d4e895da8537e5315b8254664e6b769c4ff3db18321b297a1e7004392e3"
[[package]]
name = "uuid"
version = "1.6.1"
@@ -1601,6 +1688,16 @@ version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
[[package]]
name = "walkdir"
version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee"
dependencies = [
"same-file",
"winapi-util",
]
[[package]]
name = "wasi"
version = "0.9.0+wasi-snapshot-preview1"
+3
View File
@@ -4,6 +4,7 @@ version = "0.1.0"
edition = "2021"
[dependencies]
html-page = "0.1.0"
log = "0.4.20"
pretty_env_logger = "0.5.0"
radicle-git-ext = "0.7.0"
@@ -12,7 +13,9 @@ serde_yaml = "0.9.27"
subprocess = "0.2.9"
tempfile = "3.8.1"
thiserror = "1.0.50"
time = { version = "0.3.31", features = ["formatting", "macros"] }
uuid = { version = "1.6.1", features = ["v4"] }
walkdir = "2.4.0"
[dependencies.radicle]
git = "https://seed.radicle.xyz/z3gqcJUoA1n9HaHKufZs5FCSGazv5.git"
+158 -11
View File
@@ -21,13 +21,16 @@ use std::{
};
use log::{debug, info};
use serde::Deserialize;
use serde::{Deserialize, Serialize};
use subprocess::{Popen, PopenConfig, Redirection};
use time::{macros::format_description, OffsetDateTime};
use uuid::Uuid;
use radicle::prelude::Profile;
use radicle_ci_broker::msg::{Id, MessageError, Oid, Request, Response, RunId, RunResult};
mod report;
/// Path to the repository's CI run specification. This is relative to
/// the root of the repository.
const RUNSPEC_PATH: &str = ".radicle/native.yaml";
@@ -54,22 +57,42 @@ fn fallible_main() -> Result<(), NativeError> {
logfile.write("radicle-native-ci starts\n".into())?;
let (run_id, run_dir) = mkdir_run(&config)?;
let run_id = RunId::from(format!("{}", run_id).as_str());
logfile.write(format!("run directory {}\n", run_dir.display()))?;
let src = run_dir.join("src");
let run_log = run_dir.join("log.txt");
let run_info_file = run_dir.join("run.yaml");
let profile = Profile::load().map_err(NativeError::LoadProfile)?;
let storage = profile.storage.path();
let req = read_request()?;
if let Request::Trigger { repo, commit } = req {
let run_info = if let Request::Trigger { repo, commit } = req {
info!("Request to trigger CI on {}, {}", repo, commit);
run(run_id, storage, repo, commit, &src, &mut logfile, &run_log)?;
run(
run_id,
storage,
repo,
commit,
&src,
&mut logfile,
&run_log,
&config.state,
)?
} else {
write_response(&Response::error("first request was not Trigger\n"))?;
}
RunInfo::builder()
.id(run_id)
.log(&config.state, run_log)
.result(RunResult::Error("first request was not Trigger".into()))
.build()?
};
run_info.write(&run_info_file)?;
report::build_report(&config.state)?;
logfile.write("radicle-native-ci ends successfully".into())?;
info!("radicle-native-ci ends");
@@ -106,15 +129,17 @@ fn write_response(resp: &Response) -> Result<(), NativeError> {
}
/// Perform the CI run.
#[allow(clippy::too_many_arguments)]
fn run(
run_id: Uuid,
run_id: RunId,
storage: &Path,
repo: Id,
commit: Oid,
src: &Path,
log: &mut LogFile,
run_log: &Path,
) -> Result<(), NativeError> {
state: &Path,
) -> Result<RunInfo, NativeError> {
let mut run_log = LogFile::open(run_log)?;
log.write(format!("CI run on {}, {}\n", repo, commit))?;
@@ -127,8 +152,8 @@ fn run(
run_id.to_string().as_str(),
)))?;
let repo = storage.join(repo.canonical());
debug!("repo path: {}", repo.display());
let repo_path = storage.join(repo.canonical());
debug!("repo path: {}", repo_path.display());
debug!("cloning repository to {}", src.display());
runcmd(
@@ -136,7 +161,7 @@ fn run(
&[
"git",
"clone",
repo.to_str().unwrap(),
repo_path.to_str().unwrap(),
src.to_str().unwrap(),
],
storage,
@@ -152,13 +177,23 @@ fn run(
let snippet = format!("set -xeuo pipefail\n{}", &runspec.shell);
runcmd(&mut run_log, &["bash", "-c", &snippet], src)?;
write_response(&Response::finished(RunResult::Success))?;
let result = RunResult::Success;
write_response(&Response::finished(result.clone()))?;
run_log.write("CI run finished successfully\n".into())?;
std::fs::remove_dir_all(src).map_err(|e| NativeError::RemoveDir(src.into(), e))?;
Ok(())
let info = RunInfo::builder()
.repo(repo)
.commit(commit)
.id(run_id)
.result(result)
.log(state, run_log.filename().into())
.build()?;
Ok(info)
}
/// Run a command in a directory.
@@ -239,6 +274,10 @@ struct LogFile {
}
impl LogFile {
fn filename(&self) -> &Path {
self.filename.as_path()
}
fn open(filename: &Path) -> Result<Self, NativeError> {
let file = std::fs::OpenOptions::new()
.append(true)
@@ -291,6 +330,99 @@ impl RunSpec {
}
}
/// Metadata about a run.
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
#[allow(dead_code)]
pub struct RunInfo {
/// Repository ID.
repo: String,
/// Commit ID.
commit: String,
/// Identifier for the CI run.
id: String,
/// Result of the CI run.
result: RunResult,
/// Name of log file.
log: PathBuf,
/// Timestamp of when the run ended (the value was created).
/// ISO8601 format.
timestamp: String,
}
impl RunInfo {
fn builder() -> RunInfoBuilder {
RunInfoBuilder::default()
}
fn write(&self, filename: &Path) -> Result<(), NativeError> {
let yaml = serde_yaml::to_string(&self).map_err(NativeError::SerializeRunInfo)?;
std::fs::write(filename, yaml.as_bytes())
.map_err(|e| NativeError::WriteRunInfo(filename.into(), e))?;
Ok(())
}
}
#[derive(Debug, Default)]
#[allow(dead_code)]
struct RunInfoBuilder {
repo: Option<Id>,
commit: Option<Oid>,
id: Option<String>,
result: Option<RunResult>,
log: Option<PathBuf>,
}
impl RunInfoBuilder {
fn repo(mut self, repo: Id) -> Self {
self.repo = Some(repo);
self
}
fn commit(mut self, commit: Oid) -> Self {
self.commit = Some(commit);
self
}
fn id(mut self, id: RunId) -> Self {
self.id = Some(format!("{}", id));
self
}
fn result(mut self, result: RunResult) -> Self {
self.result = Some(result);
self
}
fn log(mut self, state: &Path, log: PathBuf) -> Self {
self.log = Some(log.strip_prefix(state).unwrap().into());
self
}
fn build(self) -> Result<RunInfo, NativeError> {
let fmt = format_description!("[year]-[month]-[day] [hour]:[minute]:[second]");
let now = OffsetDateTime::now_utc()
.format(fmt)
.map_err(NativeError::TimeFormat)?;
Ok(RunInfo {
repo: self.repo.map(|x| x.to_string()).unwrap_or("".into()),
commit: self.commit.map(|x| x.to_string()).unwrap_or("".into()),
id: self.id.ok_or(NativeError::MissingInfo("id".into()))?,
result: self
.result
.ok_or(NativeError::MissingInfo("result".into()))?,
log: self.log.ok_or(NativeError::MissingInfo("log".into()))?,
timestamp: now,
})
}
}
#[derive(Debug, thiserror::Error)]
enum NativeError {
#[error("failed to read configuration file {0}")]
@@ -340,4 +472,19 @@ enum NativeError {
#[error("failed to remove {0}")]
RemoveDir(PathBuf, #[source] std::io::Error),
#[error("programming error: field {0} is not set")]
MissingInfo(String),
#[error("failed to write file for run metadata: {0}")]
WriteRunInfo(PathBuf, #[source] std::io::Error),
#[error("failed to serialize run metadata: programming error")]
SerializeRunInfo(#[source] serde_yaml::Error),
#[error(transparent)]
Report(#[from] report::ReportError),
#[error("failed to format current time as a string")]
TimeFormat(#[source] time::error::Format),
}
+124
View File
@@ -0,0 +1,124 @@
use std::{
collections::HashSet,
path::{Path, PathBuf},
};
use html_page::{Document, Element, Tag};
use radicle_ci_broker::msg::RunResult;
use walkdir::WalkDir;
use crate::RunInfo;
const CSS: &str = include_str!("native-ci.css");
pub fn build_report(state: &Path) -> Result<(), ReportError> {
let mut run_infos = collect_run_infos(state)?;
run_infos.sort_by(|a, b| a.timestamp.partial_cmp(&b.timestamp).unwrap());
let repos = repositories(&run_infos);
let doc = list_runs(&repos, &run_infos);
let index = state.join("index.html");
std::fs::write(&index, doc.to_string().as_bytes())
.map_err(|e| ReportError::WriteHtml(index.clone(), e))?;
Ok(())
}
fn collect_run_infos(state: &Path) -> Result<Vec<RunInfo>, ReportError> {
let mut infos = vec![];
for entry in WalkDir::new(state) {
let entry = entry?;
if entry.path().ends_with("run.yaml") {
infos.push(read_file_info(entry.path())?);
}
}
Ok(infos)
}
fn repositories(infos: &[RunInfo]) -> Vec<&str> {
let mut ids: Vec<&str> = infos
.iter()
.map(|ri| ri.repo.as_str())
.collect::<HashSet<&str>>()
.iter()
.copied()
.collect();
ids.sort_by(|a, b| a.partial_cmp(b).unwrap());
ids
}
fn runs_for_repo<'a>(repo: &str, infos: &'a [RunInfo]) -> Vec<&'a RunInfo> {
infos.iter().filter(|ri| ri.repo == repo).collect()
}
fn read_file_info(filename: &Path) -> Result<RunInfo, ReportError> {
let yaml = std::fs::read(filename).map_err(|e| ReportError::ReadRunInfo(filename.into(), e))?;
serde_yaml::from_slice(&yaml).map_err(|e| ReportError::DeserializeRunInfo(filename.into(), e))
}
fn list_runs(repos: &[&str], run_infos: &[RunInfo]) -> Document {
let mut doc = Document::default();
doc.push_to_head(&Element::new(Tag::Title).with_text("CI run logs"));
doc.push_to_head(&Element::new(Tag::Style).with_text(CSS));
for repo in repos {
let section = Element::new(Tag::H1)
.with_text("Repository ")
.with_child(Element::new(Tag::Code).with_text(repo));
doc.push_to_body(&section);
let mut list = Element::new(Tag::Ol).with_class("runlist");
for ri in runs_for_repo(repo, run_infos) {
let commit = Element::new(Tag::Code)
.with_class("commit")
.with_text(&ri.commit);
let timestamp = Element::new(Tag::Span)
.with_class("timestamp")
.with_text(&ri.timestamp);
let (result_text, class) = match &ri.result {
RunResult::Success => ("success".into(), "success"),
RunResult::Failure => ("failure".into(), "failure"),
RunResult::Error(msg) => (format!("error: {}", msg), "failure"),
_ => (format!("unknown: {:?}", ri.result), "unknown"),
};
let result = Element::new(Tag::Span)
.with_class(class)
.with_text(&result_text);
let br = Element::new(Tag::Br);
let href = format!("{}", ri.log.display());
let link = Element::new(Tag::A)
.with_attribute("href", &href)
.with_child(timestamp)
.with_text(" ")
.with_child(result)
.with_child(br)
.with_child(commit);
let item = Element::new(Tag::Li).with_child(link);
list.push_child(&item);
}
doc.push_to_body(&list);
}
doc
}
#[derive(Debug, thiserror::Error)]
pub enum ReportError {
#[error(transparent)]
WalkDir(#[from] walkdir::Error),
#[error("failed to read file for run metadata: {0}")]
ReadRunInfo(PathBuf, #[source] std::io::Error),
#[error("failed to parse run metadata as YAML: {0}")]
DeserializeRunInfo(PathBuf, #[source] serde_yaml::Error),
#[error("failed to write HTML file {0}")]
WriteHtml(PathBuf, #[source] std::io::Error),
}