From ba301ead4ed4a213ad90cf341dedee998bc3a05c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Jan 2024 11:51:59 +0200 Subject: [PATCH] tests: add test suite implemented as a Python program Signed-off-by: Lars Wirzenius --- src/main.rs | 8 +- src/report.rs | 2 +- test-suite | 339 ++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 343 insertions(+), 6 deletions(-) create mode 100755 test-suite diff --git a/src/main.rs b/src/main.rs index 01c94de..89ede33 100644 --- a/src/main.rs +++ b/src/main.rs @@ -448,14 +448,12 @@ impl RunInfoBuilder { Ok(RunInfo { repo: self.repo.map(|x| x.to_string()).unwrap_or("".into()), commit: self.commit.map(|x| x.to_string()).unwrap_or("".into()), - id: self.id.ok_or(NativeError::MissingInfo("id".into()))?, + id: self.id.unwrap_or("".into()), result: self .result .ok_or(NativeError::MissingInfo("result".into()))?, - log: self.log.ok_or(NativeError::MissingInfo("log".into()))?, - run_info: self - .run_info - .ok_or(NativeError::MissingInfo("run_info".into()))?, + log: self.log.unwrap_or("".into()), + run_info: self.run_info.unwrap_or("".into()), timestamp: now, }) } diff --git a/src/report.rs b/src/report.rs index 60a4096..78b4233 100644 --- a/src/report.rs +++ b/src/report.rs @@ -11,7 +11,7 @@ use crate::{LogFile, RunInfo}; const CSS: &str = include_str!("native-ci.css"); -pub fn build_report(log: &mut LogFile, state: &Path) -> Result<(), ReportError> { +pub fn build_report(_log: &mut LogFile, state: &Path) -> Result<(), ReportError> { let mut run_infos = collect_run_infos(state)?; run_infos.sort_by(|a, b| a.timestamp.partial_cmp(&b.timestamp).unwrap()); diff --git a/test-suite b/test-suite new file mode 100755 index 0000000..cea41f0 --- /dev/null +++ b/test-suite @@ -0,0 +1,339 @@ +#!/usr/bin/python3 + +import argparse +import json +import logging +import os +import shutil +import subprocess +import sys +import tempfile +import yaml + + +class Suite: + def __init__(self, tmp): + self.tmp = tmp + self.rad = Rad(os.path.join(tmp, "dot-radicle")) + self.rad.auth() + + self.config = Config(tmp) + self.config.write() + + def _test_case(self, repo_name, shell): + git = Git(os.path.join(tmp, repo_name)) + git.init() + git.write("README.md", "README") + native = NativeYaml(shell) + debug(f"native.yaml: {native.yaml()}") + git.write(".radicle/native.yaml", native.yaml()) + git.commit("first commit") + commit = git.head() + self.rad.init(git) + rid = self.rad.rid(git) + trigger = Trigger(rid, commit) + return self._run_test_case(trigger) + + def _run_test_case(self, trigger): + ci = NativeCI(self.rad, self.config) + exit, resps = ci.run(trigger) + debug(f"exit: {exit}") + debug(f"responses: {resps}") + return exit, resps + + def assert_triggered(self, msg): + assert msg["response"] == "triggered" + + def assert_success(self, msg): + assert msg == {"response": "finished", "result": "success"} + + def assert_error(self, msg): + assert msg["response"] == "finished" + res = msg["result"] + assert isinstance(res, dict) + assert "error" in res + + def run_all_test_cases(self, tests): + methods = self._test_methods() + chosen = self._chosen(methods, tests) + for meth in methods: + if meth in chosen: + info(f"test case {meth}") + getattr(self, meth)() + else: + info(f"skipping test {meth}") + + def _test_methods(self): + return [meth for meth in dir(self) if meth.startswith("test_")] + + def _chosen(self, methods, tests): + if tests: + chosen = set() + for test in tests: + for meth in methods: + if test in meth: + chosen.add(meth) + else: + chosen = set(methods) + return chosen + + def test_happy_path(self): + exit, resps = self._test_case("happy-path", "echo hello, world") + assert exit == 0 + assert len(resps) == 2 + self.assert_triggered(resps[0]) + self.assert_success(resps[1]) + + def test_command_fails(self): + exit, resps = self._test_case("cmd-fails", "false") + assert exit != 0 + assert len(resps) == 2 + self.assert_triggered(resps[0]) + self.assert_error(resps[1]) + + def test_repository_does_not_exist(self): + trigger = Trigger( + "rad:z3qg5TKmN83afz2fj9z3fQjU8vaYE", + "8d947e182b096ec009e1c9eda9e6a67f5eef83d9", + ) + exit, resps = self._run_test_case(trigger) + + assert exit != 0 + assert len(resps) == 2 + self.assert_triggered(resps[0]) + self.assert_error(resps[1]) + error = resps[1]["result"]["error"] + assert "git" in error + assert "clone" in error + + def test_native_yaml_has_no_shell(self): + exit, resps = self._test_case("no-shell", None) + assert exit != 0 + assert len(resps) == 1 + self.assert_triggered(resps[0]) + # FIXME: verify that stderr or build log contains complaint about shell + # missing + + def test_native_yaml_shell_is_not_string(self): + exit, resps = self._test_case("shell-not-string", {"foo": "bar"}) + assert exit != 0 + assert len(resps) == 1 + self.assert_triggered(resps[0]) + # FIXME: verify that stderr or build log contains complaint about shell + # not being a string + + +class Git: + def __init__(self, path): + self.path = path + + def filename(self, relative): + return os.path.join(self.path, "./" + relative) + + def write(self, relative, data): + write(self.filename(relative), data) + + def _git(self, argv): + return run(argv, cwd=self.path).stdout + + def init(self): + assert not os.path.exists(self.path) + run(["git", "init", self.path]) + debug(f"created git repository at {self.path}") + + def commit(self, msg): + run(["git", "add", "."], cwd=self.path) + run(["git", "commit", "-m", msg], cwd=self.path) + + def head(self): + return self._git(["git", "rev-parse", "HEAD"]).strip() + + def ls_files(self): + out = self._git(["git", "ls-files"]) + debug(f"ls-files:\n{out}") + + +class Rad: + PASSPHRASE = "xyzzy" + + def __init__(self, rad_home): + assert rad_home is not None + self.rad_home = rad_home + + def _rad(self, argv, env=None, cwd=None): + if env is None: + env = {} + env["RAD_HOME"] = self.rad_home + env["RAD_PASSPHRASE"] = self.PASSPHRASE + return run(argv, env=env, cwd=cwd).stdout + + def auth(self): + self._rad(["rad", "auth", "--alias=test-node"]) + debug(f"created Radicle node at {self.rad_home}") + + def init(self, git): + name = os.path.basename(git.path) + self._rad( + [ + "rad", + "init", + f"--name={name}", + f"--description=test-repo", + f"--public", + f"--no-confirm", + ], + cwd=git.path, + ) + debug(f"added git repository {git.path} to node") + + def rid(self, git): + return self._rad(["rad", "."], cwd=git.path).strip() + + +class Config: + def __init__(self, tmp): + self.path = os.path.join(tmp, "config.yaml") + self.dict = { + "state": os.path.join(tmp, "state"), + "log": os.path.join(tmp, "node-log.txt"), + } + + def write(self): + write(self.path, self.yaml()) + + def yaml(self): + return yaml.safe_dump(self.dict, indent=4) + + +class NativeYaml: + def __init__(self, shell): + self.dict = {} + if shell is not None: + self.dict["shell"] = shell + + def yaml(self): + return yaml.safe_dump(self.dict, indent=4) + + +class Trigger: + def __init__(self, rid, commit): + self.rid = rid + self.commit = commit + + def json(self): + return json.dumps( + {"request": "trigger", "repo": self.rid, "commit": self.commit} + ) + + +class NativeCI: + def __init__(self, rad, config): + self.rad_home = rad.rad_home + self.config = config.path + + def env(self): + return { + "RAD_HOME": self.rad_home, + "RADICLE_NATIVE_CI": self.config, + "RADICLE_NATIVE_CI_LOG": "debug", + } + + def run(self, request): + p = run( + [ + "cargo", + "run", + "-q", + ], + input=request.json(), + env=self.env(), + may_fail=True, + ) + return p.returncode, [ + json.loads(line.strip()) for line in p.stdout.splitlines() + ] + + +def debug(msg): + logging.debug(msg) + + +def info(msg): + logging.info(msg) + + +def die(msg, exc_info=None): + logging.critical(msg, exc_info=exc_info) + sys.exit(1) + + +def run(argv, env=None, cwd=None, input=None, may_fail=False): + debug(f"run {argv} with env={env}, cwd={cwd} input={input!r}") + if env is not None: + for name, value in os.environ.items(): + if name not in env: + env[name] = value + p = subprocess.run( + argv, capture_output=True, env=env, cwd=cwd, input=input, text=True + ) + + debug(f"stdout:\n{indent(p.stdout)}") + debug(f"stderr:\n{indent(p.stderr)}") + if p.returncode != 0 and not may_fail: + die(f"command failed: {argv}") + + return p + + +def indent(s): + return "".join([f" {line}\n" for line in s.splitlines()]) + + +def write(filename, data): + dirname = os.path.dirname(filename) + if not os.path.exists(dirname): + os.makedirs(dirname) + with open(filename, "w") as f: + f.write(data) + + +def main(tmp, tests): + suite = Suite(tmp) + suite.run_all_test_cases(tests) + + +p = argparse.ArgumentParser() +p.add_argument("--verbose", action="store_true") +p.add_argument("--test", action="append", nargs="?", dest="tests") +args = p.parse_args() + +if args.verbose: + level = logging.DEBUG + print(args) +else: + level = logging.INFO + + +logging.basicConfig( + level=level, + stream=sys.stderr, + format="%(levelname)s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) + +tmp = tempfile.mkdtemp() +debug(f"created temporary directory {tmp}") + +try: + main(tmp, args.tests) +except AssertionError as e: + debug(f"removing temporary directory {tmp}") + shutil.rmtree(tmp) + die(f"ERROR: assertion failed: {e}", exc_info=True) +except Exception as e: + debug(f"removing temporary directory {tmp}") + shutil.rmtree(tmp) + die(f"ERROR: uncaught exception {e}", exc_info=True) +else: + debug(f"removing temporary directory {tmp}") + shutil.rmtree(tmp)