git ents: effect: add ents-effect, the Executor trait and Docker/Sprite run loop

Cargo.lock @@ -383,6 +383,26 @@ "thiserror", ] +[[package]] +name = "ents-effect" +version = "0.0.0" +dependencies = [ + "ents-gate", + "ents-model", + "ents-query", + "ents-receive", + "ents-testutil", + "facet", + "facet-git-tree", + "gix", + "gix-hash", + "gix-object", + "gix-ref-store", + "rstest", + "tempfile", + "thiserror", +] + [[package]] name = "ents-gate" version = "0.0.0"

Cargo.toml @@ -2,6 +2,7 @@ resolver = "3" members = [ "crates/ents-anchor", + "crates/ents-effect", "crates/ents-gate", "crates/ents-model", "crates/ents-query",

crates/ents-effect/Cargo.toml @@ -1,0 +1,36 @@ +[package] +name = "ents-effect" +version = "0.0.0" +edition.workspace = true +publish.workspace = true +license.workspace = true + +[features] +# Neither backend needs an extra crate dependency — both shell out to an +# already-installed CLI (`docker`, `sprite`), the same pattern `pre-redo` +# used and this phase ports. The features exist so a composition root pulls +# in only the backend(s) it deploys (`roots.local`: Docker only; +# `roots.hosted`: Sprite only), never both in one binary by accident. +docker = [] +sprite = [] + +[dependencies] +ents-model = { workspace = true } +ents-query = { workspace = true } +ents-receive = { workspace = true } +facet-git-tree = { workspace = true } +gix = { workspace = true } +gix-hash = { workspace = true } +gix-object = { workspace = true } +gix-ref-store = { workspace = true } +thiserror = { workspace = true } + +[dev-dependencies] +ents-gate = { workspace = true } +ents-testutil = { workspace = true } +facet = { workspace = true } +rstest = { workspace = true } +tempfile = { workspace = true } + +[lints] +workspace = true

crates/ents-effect/src/definition.rs @@ -1,0 +1,103 @@ +//! Write-time validation of an effect definition (`effect.validation`). +//! +//! `ents-receive` cannot call this (`arch.query-effect-split`: no push path +//! may link executor code), so a future frontend that builds an effect +//! definition's commit (`git effect add`, `git-ents` bin, phase 6) calls +//! [`validate`] itself before ever proposing the write — `receive` still +//! admits or refuses the *push* on its own terms (the gate, `receive.unit`); +//! this only keeps a frontend from proposing a definition that could never +//! usefully run. + +use ents_model::{Effect, namespace}; +use ents_query::Query; + +use crate::error::{Error, Result}; + +/// Reject `effect` before it is ever proposed to `receive`, per +/// `effect.validation`: every name in `toolchains` must be a valid +/// ref-path segment, and `trigger` must parse as a `CommitQuery` +/// (`query.grammar`) — which already rejects a `rev(expr)` naming a +/// `refs/meta/*` pattern (`query.rev`) and a `meta(glob)` naming an +/// effect-written namespace (`query.meta`), since the parser enforces +/// both. +/// +/// # Errors +/// +/// [`Error::Trigger`] if `trigger` does not parse; [`Error::InvalidToolchainName`] +/// for the first toolchain name that is not a valid ref-path segment. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::definition::validate; +/// use ents_model::Effect; +/// +/// let good = Effect { +/// trigger: "rev(refs/heads/main)".into(), +/// toolchains: vec!["rust-stable".into()], +/// run: "cargo test".into(), +/// }; +/// assert!(validate(&good).is_ok()); +/// +/// let bad_trigger = Effect { trigger: "not a query".into(), ..good.clone() }; +/// assert!(validate(&bad_trigger).is_err()); +/// +/// let bad_toolchain = Effect { toolchains: vec!["../escape".into()], ..good }; +/// assert!(validate(&bad_toolchain).is_err()); +/// ``` +// @relation(effect.validation, scope=function) +pub fn validate(effect: &Effect) -> Result<()> { + effect.trigger.parse::<Query>().map_err(Error::from)?; + for name in &effect.toolchains { + namespace::toolchain_ref(name) + .map_err(|_invalid| Error::InvalidToolchainName(name.clone()))?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use rstest::rstest; + + use super::*; + + fn effect(trigger: &str, toolchains: &[&str]) -> Effect { + Effect { + trigger: trigger.to_owned(), + toolchains: toolchains.iter().map(|s| (*s).to_owned()).collect(), + run: "true".to_owned(), + } + } + + #[rstest] + // @relation(effect.validation, scope=function, role=Verifies) + fn validate_accepts_a_well_formed_definition() { + validate(&effect("rev(refs/heads/main)", &["rust-stable"])).expect("well-formed"); + } + + #[rstest] + // @relation(effect.validation, scope=function, role=Verifies) + fn validate_rejects_an_unparsable_trigger() { + assert!(validate(&effect("not a query", &[])).is_err()); + } + + #[rstest] + // @relation(effect.validation, query.rev, scope=function, role=Verifies) + fn validate_rejects_a_rev_naming_a_meta_pattern() { + assert!(validate(&effect("rev(refs/meta/effects/*)", &[])).is_err()); + } + + #[rstest] + // @relation(effect.validation, query.meta, scope=function, role=Verifies) + fn validate_rejects_a_meta_glob_naming_an_effect_written_namespace() { + assert!(validate(&effect("meta(refs/meta/results/*)", &[])).is_err()); + } + + #[rstest] + // @relation(effect.validation, scope=function, role=Verifies) + fn validate_rejects_an_invalid_toolchain_name() { + assert!(validate(&effect("rev(refs/heads/main)", &["../escape"])).is_err()); + } +}

crates/ents-effect/src/docker.rs @@ -1,0 +1,180 @@ +//! The Docker [`Executor`] backend (`effect.execution`, `roots.local`): +//! shells out to the `docker` CLI (no docker API crate — the same +//! rationale [`crate::sprite`] uses for the `sprite` CLI), running each +//! effect in a throwaway `--rm` container with the materialized workdir and +//! toolchains bind-mounted in. +//! +//! Ported from `pre-redo`'s `git-effect::docker` module: the readiness +//! probe ([`ensure_docker`]) and the pure argv assembly ([`run_args`]) carry +//! over verbatim (minus the cache-directory bind mount — this design has no +//! effect-level cache, `model.effect-definition`); the rest is rewritten +//! against this phase's [`crate::Executor`] trait. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use crate::error::{Error, Result}; +use crate::executor::{Executor, RunOutput, RunStatus, SandboxInputs, activate}; + +/// The minimal base image every effect runs in — no toolchain of its own; +/// everything the command needs comes from the bind-mounted, host-exported +/// toolchains. +pub const IMAGE: &str = "debian:stable-slim"; + +/// Where the workdir is bind-mounted in the container. +pub const WORKDIR: &str = "/work"; + +/// Where a toolchain's `bin/` directory is bind-mounted, per toolchain +/// name: `{TOOLCHAINS_DIR}/<name>/bin`. +pub const TOOLCHAINS_DIR: &str = "/toolchains"; + +/// Confirm `docker` is on `PATH` and the daemon answers, with a clean error +/// — rather than a raw "os error 2" — when it is not. The one place this +/// backend can fail before an effect ever runs. +/// +/// # Errors +/// +/// [`Error::Spawn`] if `docker` could not be started at all; +/// [`Error::Process`] if it ran but the daemon did not respond. +pub fn ensure_docker() -> Result<()> { + let status = Command::new("docker") + .arg("version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map_err(|e| Error::Spawn { + program: "docker".to_owned(), + detail: e.to_string(), + })?; + if status.success() { + Ok(()) + } else { + Err(Error::Process { + program: "docker".to_owned(), + detail: "the daemon did not respond to `docker version`; is it running?".to_owned(), + }) + } +} + +/// Assemble `docker run`'s argv for one sandboxed run — pure, so the exact +/// invocation is unit tested without a daemon. The command runs under +/// `sh -c`, stderr folded into stdout so the captured recording is one +/// interleaved stream. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::docker::run_args; +/// use std::path::Path; +/// +/// let args = run_args(Path::new("/tmp/s/work"), &[], "cargo test"); +/// assert_eq!( +/// args, +/// vec![ +/// "run", "--rm", "-v", "/tmp/s/work:/work", "-w", "/work", +/// "debian:stable-slim", "sh", "-c", "cargo test 2>&1", +/// ] +/// ); +/// ``` +#[must_use] +pub fn run_args(workdir: &Path, toolchains: &[(String, PathBuf)], command: &str) -> Vec<String> { + let mut args = vec![ + "run".to_owned(), + "--rm".to_owned(), + "-v".to_owned(), + format!("{}:{WORKDIR}", workdir.display()), + ]; + let mut sandbox_dirs = Vec::with_capacity(toolchains.len()); + for (name, host_dir) in toolchains { + let sandbox_dir = format!("{TOOLCHAINS_DIR}/{name}/bin"); + args.push("-v".to_owned()); + args.push(format!( + "{}:{TOOLCHAINS_DIR}/{name}/bin:ro", + host_dir.display() + )); + sandbox_dirs.push((name.clone(), sandbox_dir)); + } + args.push("-w".to_owned()); + args.push(WORKDIR.to_owned()); + args.push(IMAGE.to_owned()); + args.push("sh".to_owned()); + args.push("-c".to_owned()); + args.push(format!("{} 2>&1", activate(command, &sandbox_dirs))); + args +} + +/// [`Executor`] running each effect in a throwaway local Docker container +/// (`roots.local`). +#[derive(Debug, Clone, Copy, Default)] +pub struct DockerExecutor; + +impl Executor for DockerExecutor { + fn run(&self, inputs: &SandboxInputs<'_>) -> Result<RunOutput> { + ensure_docker()?; + let args = run_args(inputs.workdir, inputs.toolchains, inputs.command); + let output = Command::new("docker") + .args(&args) + .output() + .map_err(|e| Error::Spawn { + program: "docker".to_owned(), + detail: e.to_string(), + })?; + let log = String::from_utf8_lossy(&output.stdout).into_owned(); + let status = if output.status.success() { + RunStatus::Pass + } else { + RunStatus::Fail + }; + Ok(RunOutput { status, log }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use rstest::rstest; + + use super::*; + + #[rstest] + // @relation(effect.execution, scope=function, role=Verifies) + fn run_args_binds_the_workdir() { + let args = run_args(Path::new("/tmp/s/work"), &[], "cargo test"); + assert_eq!( + args, + vec![ + "run", + "--rm", + "-v", + "/tmp/s/work:/work", + "-w", + "/work", + IMAGE, + "sh", + "-c", + "cargo test 2>&1", + ] + ); + } + + #[rstest] + // @relation(effect.execution, effect.toolchains, scope=function, role=Verifies) + fn run_args_binds_each_toolchain_read_only_and_activates_it() { + let toolchains = vec![("rust".to_owned(), PathBuf::from("/cache/rust/bin"))]; + let args = run_args(Path::new("/w"), &toolchains, "cargo test"); + assert!(args.contains(&"/cache/rust/bin:/toolchains/rust/bin:ro".to_owned())); + let last = args.last().expect("has a command"); + assert!(last.starts_with("export PATH=/toolchains/rust/bin:$PATH; cargo test")); + } + + #[rstest] + // @relation(effect.execution, scope=function, role=Verifies) + fn run_args_uses_the_minimal_base_image() { + let args = run_args(Path::new("/w"), &[], "true"); + assert_eq!( + args.get(args.len().saturating_sub(4)).map(String::as_str), + Some(IMAGE) + ); + } +}

crates/ents-effect/src/error.rs @@ -1,0 +1,168 @@ +//! `ents-effect`'s error type: everything that can prevent a run from +//! reaching a recorded outcome. +//! +//! Mirrors `ents-receive`'s split: an [`Error`] means the run never reached +//! a judgment at all (a store or object read failed, a sandbox never +//! started, `curl`/`tar`/`docker`/`sprite` was not on `PATH`) — as opposed +//! to a completed run reporting `pass` or `fail` +//! (`effect.result-taxonomy`), which is a reached judgment, not an `Err`. +//! Per `effect.result-taxonomy`, an [`Error`] here is exactly the "queue +//! concern with bounded retry" case: this crate never turns one into a +//! `Status::Error` result itself — only a caller that has exhausted its own +//! retry bound does that, by calling [`crate::write_result`] with +//! `Status::Error` explicitly. + +use std::path::PathBuf; + +use gix_hash::ObjectId; + +/// Everything that can prevent an `ents-effect` operation from reaching a +/// result. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The ref store's read or write half failed. + #[error("ref store operation failed: {0}")] + Refs(#[from] gix_ref_store::Error), + + /// `receive` (the write-back path, `effect.results-writeback`) could + /// not reach an outcome. + /// + /// Boxed rather than `#[from]`-derived inline (see [`From`] below): + /// `ents_receive::Error` embeds `ents_gate::Error`, which is large + /// enough on its own to trip `clippy::result_large_err` for every + /// fallible function in this crate if stored inline. + #[error("receive failed: {0}")] + Receive(Box<ents_receive::Error>), + + /// The query evaluator could not compute an effect's work set. + #[error("query evaluation failed: {0}")] + Eval(#[from] ents_query::EvalError), + + /// An effect's `trigger` failed to parse as a `CommitQuery` + /// (`effect.validation`). + #[error("trigger does not parse as a CommitQuery: {0}")] + Trigger(#[from] ents_query::ParseError), + + /// A typed-tree entity (a [`ents_model::Toolchain`] or + /// [`ents_model::Status`]) could not be (de)serialized. + #[error("typed-tree operation failed: {0}")] + Facet(#[from] facet_git_tree::Error), + + /// An object could not be read or decoded. + #[error("object {oid} could not be read: {detail}")] + Decode { + /// The undecodable object. + oid: ObjectId, + /// What failed, human-readable. + detail: String, + }, + + /// An object referenced by a tree or commit is missing from the object + /// store. + #[error("object {oid} is missing")] + Missing { + /// The missing object. + oid: ObjectId, + }, + + /// `refs/meta/toolchains/<name>` does not exist. + #[error("no toolchain named {0:?}")] + UnknownToolchain(String), + + /// A toolchain's `recipe` field did not parse as a [`crate::Recipe`] + /// (`effect.toolchains`: "a manifest's declared components MUST be + /// resolved during effect execution"). + #[error("toolchain {name:?} has an unreadable recipe: {detail}")] + InvalidRecipe { + /// The toolchain's name. + name: String, + /// What failed, human-readable. + detail: String, + }, + + /// An effect's `toolchains` list named something that is not a valid + /// ref-path segment (`effect.validation`). + #[error("{0:?} is not a valid toolchain name")] + InvalidToolchainName(String), + + /// A materialized tree entry was a git submodule (a commit entry). + /// Gitlinks retain nothing in this design (no embedded submodule + /// content), so materializing one is refused rather than silently + /// skipped. + #[error("cannot materialize {path:?}: it is a git submodule (gitlink)")] + Submodule { + /// The offending path, relative to the materialization root. + path: String, + }, + + /// A tree entry's filename, or a downloaded component's extracted file + /// name, was not valid UTF-8. + #[error("{0:?} is not valid UTF-8")] + NotUtf8(PathBuf), + + /// A path under a materialization destination could not be read or + /// written. + #[error("could not access {path}: {source}")] + Io { + /// The path being accessed. + path: PathBuf, + /// The underlying I/O error. + #[source] + source: std::io::Error, + }, + + /// A [`crate::Component`] carried a `dest` unsafe to use as a path + /// segment (not empty and not a single safe component), or a `url` or + /// `sha256` unsafe to interpolate into a shell command. + #[error("invalid toolchain component: {0}")] + InvalidComponent(String), + + /// Running an external program (`docker`, `sprite`, `curl`, `tar`, + /// `sha256sum`/`shasum`) failed to start at all — the readiness probes + /// this phase ports from `pre-redo` exist precisely to turn this into + /// an actionable message instead of a raw "os error 2". + #[error("could not run `{program}`: {detail}")] + Spawn { + /// The program that could not be started. + program: String, + /// What failed, human-readable. + detail: String, + }, + + /// An external program ran but reported failure (nonzero exit, or + /// output this crate could not parse). + #[error("{program} failed: {detail}")] + Process { + /// The program that failed. + program: String, + /// What failed, human-readable. + detail: String, + }, + + /// A downloaded component's content did not match its recorded + /// sha256 — refused rather than extracted anyway. + #[error("{url}: expected sha256 {expected}, got {actual}")] + HashMismatch { + /// The component's source URL. + url: String, + /// The recorded sha256. + expected: String, + /// The sha256 actually computed. + actual: String, + }, + + /// The sandbox reported an infrastructure failure rather than a + /// completed run — never itself a `Status::Error` result + /// (`effect.result-taxonomy`); see this type's own doc. + #[error("the sandbox did not complete a run: {0}")] + Sandbox(String), +} + +impl From<ents_receive::Error> for Error { + fn from(source: ents_receive::Error) -> Self { + Self::Receive(Box::new(source)) + } +} + +/// The `Result` alias every fallible `ents-effect` operation returns. +pub type Result<T> = std::result::Result<T, Error>;

crates/ents-effect/src/executor.rs @@ -1,0 +1,151 @@ +//! The `Executor` seam: one trait, multiple sandbox backends +//! (`effect.execution`). +//! +//! No execution logic is duplicated per backend: [`Executor::run`] is +//! handed a fully materialized workdir and a fully materialized set of +//! toolchain directories (both produced by `crate::materialize::checkout` +//! and [`crate::toolchain::materialize`] — the same code the run loop and +//! `git effect run` share, `effect.local-run`), and does only the +//! backend-specific part: get those bytes into the sandbox, run the +//! command, and report what happened. + +use std::path::{Path, PathBuf}; + +use crate::error::Result; + +/// The inputs one sandboxed run needs, already materialized on the host — +/// a backend's only job is to get these into its sandbox and run +/// [`SandboxInputs::command`]. +#[derive(Debug, Clone)] +pub struct SandboxInputs<'a> { + /// The host directory holding the tested commit's checked-out tree. + pub workdir: &'a Path, + /// Each declared toolchain's name and the host directory holding its + /// activated `bin/` (`crate::toolchain::materialize`'s return value), + /// in the effect's declared order — the order [`activate`] honors when + /// two toolchains would otherwise collide on `PATH`. + pub toolchains: &'a [(String, PathBuf)], + /// The run command, exactly as stored on the effect definition + /// (`model.effect-definition`). + pub command: &'a str, +} + +/// What a completed run reported. Only `Pass` or `Fail` +/// (`effect.result-taxonomy`: "a completed command's exit status MUST +/// always be recorded as a result"); an infrastructure failure — the +/// sandbox never started — is [`crate::Error`], not a variant here. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RunStatus { + /// The command exited zero. + Pass, + /// The command exited nonzero. + Fail, +} + +/// The output of one completed sandboxed run. +#[derive(Debug, Clone)] +pub struct RunOutput { + /// Whether the command passed or failed. + pub status: RunStatus, + /// The command's combined stdout/stderr. + pub log: String, +} + +/// One sandbox backend (`effect.execution`): Docker, Sprite, or +/// unsandboxed host-direct — selected only at a composition root +/// (`roots.local`, `roots.hosted`), never by effect data +/// (`effect.deployment-property`). +/// +/// # Errors +/// +/// [`Executor::run`] returns `Err` only for an infrastructure failure — +/// the sandbox never started, or crashed before the command could report +/// an exit status. A command that ran to completion and merely exited +/// nonzero is `Ok(RunOutput { status: RunStatus::Fail, .. })`, never an +/// `Err` (`effect.result-taxonomy`). +/// +/// # Examples +/// +/// A minimal executor for tests: runs the command directly on the host, no +/// sandbox at all (this is deliberately *not* [`crate::UnsandboxedExecutor`] +/// — it ignores `toolchains` entirely — so it demonstrates only the trait +/// shape, not the `--unsandboxed` contract). +/// +/// ``` +/// use ents_effect::{Executor, RunStatus, SandboxInputs}; +/// +/// struct AlwaysPass; +/// impl Executor for AlwaysPass { +/// fn run(&self, _inputs: &SandboxInputs<'_>) -> ents_effect::Result<ents_effect::RunOutput> { +/// Ok(ents_effect::RunOutput { status: RunStatus::Pass, log: String::new() }) +/// } +/// } +/// +/// let dir = tempfile::tempdir().expect("tempdir"); +/// let inputs = SandboxInputs { workdir: dir.path(), toolchains: &[], command: "true" }; +/// let output = AlwaysPass.run(&inputs).expect("infallible"); +/// assert_eq!(output.status, RunStatus::Pass); +/// ``` +pub trait Executor: Send + Sync { + /// Run `inputs.command` in this backend's sandbox, materialized from + /// `inputs.workdir` and `inputs.toolchains`. + fn run(&self, inputs: &SandboxInputs<'_>) -> Result<RunOutput>; +} + +/// Prefix `command` with a `PATH` export activating `dirs`' entries, +/// declared order first (so the first-listed toolchain's `bin` wins on a +/// name collision) — ported from `pre-redo`'s `engine::activate`. `dirs` +/// holds each toolchain's *in-sandbox* path (a backend maps its host +/// [`SandboxInputs::toolchains`] entries to sandbox paths before calling +/// this), so it is a plain string, not a [`Path`]. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::executor::activate; +/// +/// let dirs = vec![("rust".to_owned(), "/toolchains/rust/bin".to_owned())]; +/// assert_eq!( +/// activate("cargo test", &dirs), +/// "export PATH=/toolchains/rust/bin:$PATH; cargo test" +/// ); +/// assert_eq!(activate("true", &[]), "true"); +/// ``` +#[must_use] +pub fn activate(command: &str, dirs: &[(String, String)]) -> String { + if dirs.is_empty() { + return command.to_owned(); + } + let path = dirs + .iter() + .map(|(_, dir)| dir.as_str()) + .collect::<Vec<_>>() + .join(":"); + format!("export PATH={path}:$PATH; {command}") +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + // @relation(effect.execution, scope=function, role=Verifies) + fn activate_prefixes_path_in_declared_order() { + let dirs = vec![ + ("a".to_owned(), "/t/a/bin".to_owned()), + ("b".to_owned(), "/t/b/bin".to_owned()), + ]; + assert_eq!( + activate("run", &dirs), + "export PATH=/t/a/bin:/t/b/bin:$PATH; run" + ); + } + + #[rstest] + // @relation(effect.execution, scope=function, role=Verifies) + fn activate_is_identity_with_no_toolchains() { + assert_eq!(activate("run", &[]), "run"); + } +}

crates/ents-effect/src/lib.rs @@ -1,0 +1,162 @@ +//! Effect execution, results, and toolchains at run time (`docs/spec/effect.sdoc`): +//! the `Executor` trait, its Docker and Sprite backends, toolchain +//! materialization, and the run loop that ties them to +//! [`ents_receive::receive`] as the sole path a result re-enters the +//! repository. +//! +//! This crate closes the loop `ents-gate` and `ents-receive` open +//! (`docs/abstractions.adoc`, "The loop"): an effect's trigger is +//! evaluated by `ents-query` (already linked by `ents-receive` for +//! footprint matching, never by this crate's own dependents in the other +//! direction — `arch.query-effect-split`), its run happens behind one +//! [`Executor`] seam with multiple backends, and its outcome returns as an +//! ordinary signed commit through [`write_result`], a `receive` client +//! exactly like the CLI or a web edit. +//! +//! # Spec coverage +//! +//! From `docs/spec/effect.sdoc`: +//! +//! - `effect.definition`, `effect.admin-only` — already carried by +//! `ents-model`'s [`ents_model::Effect`] and `ents-gate`'s default +//! authorization arm; nothing new here. +//! - `effect.validation` — [`definition::validate`]. `ents-receive` cannot +//! call this itself (`arch.query-effect-split`); a future frontend that +//! builds an effect-definition commit does, before ever proposing the +//! write. +//! - `effect.execution`, `effect.deployment-property` — [`Executor`], +//! [`SandboxInputs`], [`RunOutput`]; [`docker::DockerExecutor`] (feature +//! `docker`), [`sprite::SpriteExecutor`] (feature `sprite`), +//! [`UnsandboxedExecutor`]. No executor, sandbox, or retry choice is +//! readable from an [`ents_model::Effect`] — every backend is +//! constructed and selected only by a composition root. +//! - `effect.local-run` — [`run::run_one`] is the single code path +//! [`run::run_effect`] (the boot-time/on-demand form, no queue) and a +//! future hosted worker (a queue drain feeding the same [`run::run_one`] +//! calls) both use. +//! - `effect.results-writeback`, `effect.result-taxonomy` — +//! [`write_result`]: an ordinary [`ents_receive::receive`] client, +//! landing exactly `pass`/`fail`/`error` on +//! `refs/meta/results/<effect>/<short-oid>` +//! ([`run::short_oid`]). This crate never writes `Status::Error` itself +//! — see [`Error`]'s own doc for why an infrastructure failure is always +//! an `Err`, never a taxonomy value this crate chooses on a caller's +//! behalf. +//! - `effect.identity` — [`write_result`] takes a `sign` closure the +//! composition root injects (mirrors `ents_sync::resolve::merge_heads`); +//! this crate never holds key material. +//! - `effect.official` — a refname-authorization rule on canonical +//! `refs/meta/results/<effect>/*`, owned by `ents-gate`'s future +//! Config-driven worker-key narrowing (see that crate's own doc); this +//! crate only chooses *which* refname to target +//! ([`run::run_one`]'s `results_ref`), never judges authorization. +//! - `effect.self-run` — [`write_result`] and [`run::run_effect`] accept +//! any results refname, canonical or +//! [`ents_model::namespace::self_result_ref`]; adopting a self-run +//! result onto the canonical ref is `ents-sync`'s adoption merge +//! (`gate.adoption-merge`, `sync.adoption-machinery`), unchanged by this +//! crate. +//! - `effect.toolchains`, `model.toolchain` — [`Recipe`], [`Component`], +//! [`toolchain::resolve`], [`toolchain::materialize`]; only +//! [`Executor::run`]'s sandbox ever touches the materialized bytes this +//! crate hands it. +//! - `effect.fanout-index` — structurally satisfied, no dedicated code: a +//! fanout-index rebuild is an ordinary effect (`run`ning `git index +//! rebuild` or similar, a later, unbuilt command), so it uses exactly +//! the same [`Executor`] and [`write_result`] path as any other effect. +//! +//! # Examples +//! +//! An end-to-end local run: enroll a worker, define an effect and a +//! (trivial, embedded-empty) toolchain, advance a code ref, and run the +//! effect with a stub executor — the shape `effect.local-run` names, minus +//! only a real sandbox. +//! +//! ``` +//! use ents_effect::run::{run_effect, short_oid}; +//! use ents_effect::{Executor, Recipe, RunOutput, RunStatus, SandboxInputs}; +//! use ents_model::{Effect, Provenance, Toolchain, namespace}; +//! use ents_receive::{Mode, NullEventSink}; +//! use ents_testutil::{ +//! Keypair, MemRefStore, ObjectStore, advance_ref, empty_tree, enroll_member, write_meta_entity, +//! }; +//! use gix_ref_store::RefStoreRead as _; +//! +//! struct AlwaysPass; +//! impl Executor for AlwaysPass { +//! fn run(&self, _inputs: &SandboxInputs<'_>) -> ents_effect::Result<RunOutput> { +//! Ok(RunOutput { status: RunStatus::Pass, log: "ok".into() }) +//! } +//! } +//! +//! let refs = MemRefStore::default(); +//! let objects = ObjectStore::default(); +//! let worker = Keypair::from_seed(1); +//! enroll_member(&refs, &objects, "worker", &worker, Provenance::AdminRegistered, 100); +//! +//! // A toolchain over the empty tree — nothing to extract, but it still +//! // exercises resolve + materialize. +//! let toolchain = Toolchain { +//! name: "none".into(), +//! recipe: Recipe::Embedded { tree: empty_tree(&objects) }.render(), +//! }; +//! let toolchain_ref = namespace::toolchain_ref("none").expect("valid"); +//! write_meta_entity(&refs, &objects, toolchain_ref, &toolchain, None, 150); +//! +//! let effect = Effect { +//! trigger: "rev(refs/heads/main)".into(), +//! toolchains: vec!["none".into()], +//! run: "true".into(), +//! }; +//! assert!(ents_effect::definition::validate(&effect).is_ok()); +//! +//! let commits = advance_ref(&refs, &objects, "refs/heads/main", 1, 200); +//! +//! let author = gix::actor::Signature { +//! name: "worker".into(), email: "worker@ents.test".into(), +//! time: gix::date::Time { seconds: 300, offset: 0 }, +//! }; +//! let scratch = tempfile::tempdir().expect("tempdir"); +//! let cache = tempfile::tempdir().expect("tempdir"); +//! +//! let outcomes = run_effect( +//! &refs, &objects, &NullEventSink, &AlwaysPass, scratch.path(), cache.path(), +//! "unit", &effect, None, +//! |short| Ok(namespace::result_ref("unit", short).expect("valid")), +//! &author, &|payload| worker.sign(payload), Mode::Advisory, +//! ).expect("runs"); +//! +//! assert_eq!(outcomes.len(), 1); +//! assert_eq!(outcomes[0].0, commits[0]); +//! assert_eq!(outcomes[0].1.result, ents_receive::TxResult::Applied); +//! +//! // The canonical results ref now carries a pass. +//! let name = namespace::result_ref("unit", &short_oid(commits[0])).expect("valid"); +//! assert!(refs.get(name.as_ref()).expect("readable").is_some()); +//! ``` + +pub mod definition; +mod error; +pub mod executor; +mod materialize; +mod results; +pub mod run; +pub mod toolchain; +mod unsandboxed; + +#[cfg(feature = "docker")] +pub mod docker; +#[cfg(feature = "sprite")] +pub mod sprite; + +pub use error::{Error, Result}; +pub use executor::{Executor, RunOutput, RunStatus, SandboxInputs}; +pub use results::write_result; +pub use run::{run_effect, run_one}; +pub use toolchain::{Component, Recipe}; +pub use unsandboxed::UnsandboxedExecutor; + +#[cfg(feature = "docker")] +pub use docker::DockerExecutor; +#[cfg(feature = "sprite")] +pub use sprite::SpriteExecutor;

crates/ents-effect/src/materialize.rs @@ -1,0 +1,258 @@ +//! Checking out a git tree onto disk through gitoxide's `Find` seam alone +//! (`arch.no-object-store-trait`) — no dependency on a real on-disk `.git` +//! directory or a `git archive` subprocess, so this works identically +//! against the in-memory fixture store in tests and a real odb in +//! production. +//! +//! This is the one code path both the run loop's pushed-tree checkout and +//! [`crate::toolchain::materialize`]'s `Embedded` case share +//! (`effect.local-run`: "identical code path"). + +use std::path::Path; + +use gix_hash::ObjectId; +use gix_object::bstr::ByteSlice as _; +use gix_object::tree::EntryKind; +use gix_object::{Find, Kind, TreeRef}; + +use crate::error::{Error, Result}; + +/// Recursively write `tree`'s entries under `dest`, which must already +/// exist. Blob entries are written verbatim, with the executable bit set +/// per the entry's mode; tree entries recurse into a created subdirectory. +/// +/// # Errors +/// +/// [`Error::Submodule`] for a gitlink entry (this design embeds no +/// submodule content, `effect.toolchains`'s neighboring retention rule); +/// [`Error::NotUtf8`] for a non-UTF-8 filename; [`Error::Missing`] or +/// [`Error::Decode`] for an unreadable object; [`Error::Io`] for a host +/// filesystem failure. A symlink entry is written as a real symlink +/// (`std::os::unix::fs::symlink`) pointing at its recorded target text. +pub fn checkout(objects: &impl Find, tree: ObjectId, dest: &Path) -> Result<()> { + let mut buf = Vec::new(); + let data = objects + .try_find(&tree, &mut buf) + .map_err(|source| Error::Decode { + oid: tree, + detail: source.to_string(), + })? + .ok_or(Error::Missing { oid: tree })?; + if data.kind != Kind::Tree { + return Err(Error::Decode { + oid: tree, + detail: "expected a tree".to_owned(), + }); + } + let entries: Vec<(String, EntryKind, ObjectId)> = TreeRef::from_bytes(data.data, tree.kind()) + .map_err(|e| Error::Decode { + oid: tree, + detail: e.to_string(), + })? + .entries + .iter() + .map(|entry| { + let name = entry + .filename + .to_str() + .map_err(|_not_utf8| Error::NotUtf8(dest.join(entry.filename.to_string())))? + .to_owned(); + Ok((name, entry.mode.kind(), entry.oid.to_owned())) + }) + .collect::<Result<Vec<_>>>()?; + + for (name, kind, oid) in entries { + let path = dest.join(&name); + match kind { + EntryKind::Tree => { + make_dir(&path)?; + checkout(objects, oid, &path)?; + } + EntryKind::Commit => { + return Err(Error::Submodule { path: name }); + } + EntryKind::Link => { + let mut buf = Vec::new(); + let data = objects + .try_find(&oid, &mut buf) + .map_err(|source| Error::Decode { + oid, + detail: source.to_string(), + })? + .ok_or(Error::Missing { oid })?; + let target = std::str::from_utf8(data.data) + .map_err(|_not_utf8| Error::NotUtf8(path.clone()))?; + symlink(target, &path)?; + } + EntryKind::Blob | EntryKind::BlobExecutable => { + let mut buf = Vec::new(); + let data = objects + .try_find(&oid, &mut buf) + .map_err(|source| Error::Decode { + oid, + detail: source.to_string(), + })? + .ok_or(Error::Missing { oid })?; + std::fs::write(&path, data.data).map_err(|source| Error::Io { + path: path.clone(), + source, + })?; + if kind == EntryKind::BlobExecutable { + set_executable(&path)?; + } + } + } + } + Ok(()) +} + +fn make_dir(path: &Path) -> Result<()> { + std::fs::create_dir_all(path).map_err(|source| Error::Io { + path: path.to_owned(), + source, + }) +} + +#[cfg(unix)] +fn set_executable(path: &Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt as _; + let mut perms = std::fs::metadata(path) + .map_err(|source| Error::Io { + path: path.to_owned(), + source, + })? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(path, perms).map_err(|source| Error::Io { + path: path.to_owned(), + source, + }) +} + +#[cfg(not(unix))] +fn set_executable(_path: &Path) -> Result<()> { + Ok(()) +} + +#[cfg(unix)] +fn symlink(target: &str, path: &Path) -> Result<()> { + std::os::unix::fs::symlink(target, path).map_err(|source| Error::Io { + path: path.to_owned(), + source, + }) +} + +#[cfg(not(unix))] +fn symlink(target: &str, path: &Path) -> Result<()> { + std::fs::write(path, target).map_err(|source| Error::Io { + path: path.to_owned(), + source, + }) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use ents_testutil::ObjectStore; + use gix_object::tree::{Entry, EntryMode}; + use gix_object::{Kind, Tree, Write as _}; + + use super::*; + + #[test] + // @relation(effect.toolchains, effect.execution, scope=function, role=Verifies) + fn checkout_writes_blobs_and_sets_the_executable_bit() { + let objects = ObjectStore::default(); + let script = objects + .write_buf(Kind::Blob, b"#!/bin/sh\necho hi\n") + .expect("write"); + let readme = objects.write_buf(Kind::Blob, b"hello\n").expect("write"); + let tree = Tree { + entries: vec![ + Entry { + mode: EntryMode::from(EntryKind::Blob), + filename: "README".into(), + oid: readme, + }, + Entry { + mode: EntryMode::from(EntryKind::BlobExecutable), + filename: "run.sh".into(), + oid: script, + }, + ], + }; + let tree_oid = objects.write(&tree).expect("write tree"); + + let dir = tempfile::tempdir().expect("tempdir"); + checkout(&objects, tree_oid, dir.path()).expect("checkout"); + + let script_path = dir.path().join("run.sh"); + assert_eq!( + std::fs::read_to_string(&script_path).expect("read"), + "#!/bin/sh\necho hi\n" + ); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + let mode = std::fs::metadata(&script_path) + .expect("stat") + .permissions() + .mode(); + assert_eq!(mode & 0o111, 0o111, "run.sh must be executable"); + } + assert_eq!( + std::fs::read_to_string(dir.path().join("README")).expect("read"), + "hello\n" + ); + } + + #[test] + // @relation(effect.toolchains, scope=function, role=Verifies) + fn checkout_recurses_into_subdirectories() { + let objects = ObjectStore::default(); + let leaf = objects.write_buf(Kind::Blob, b"leaf\n").expect("write"); + let inner = Tree { + entries: vec![Entry { + mode: EntryMode::from(EntryKind::Blob), + filename: "leaf.txt".into(), + oid: leaf, + }], + }; + let inner_oid = objects.write(&inner).expect("write inner tree"); + let outer = Tree { + entries: vec![Entry { + mode: EntryMode::from(EntryKind::Tree), + filename: "sub".into(), + oid: inner_oid, + }], + }; + let outer_oid = objects.write(&outer).expect("write outer tree"); + + let dir = tempfile::tempdir().expect("tempdir"); + checkout(&objects, outer_oid, dir.path()).expect("checkout"); + + assert_eq!( + std::fs::read_to_string(dir.path().join("sub").join("leaf.txt")).expect("read"), + "leaf\n" + ); + } + + #[test] + // @relation(effect.toolchains, scope=function, role=Verifies) + fn checkout_refuses_a_submodule_entry() { + let objects = ObjectStore::default(); + let tree = Tree { + entries: vec![Entry { + mode: EntryMode::from(EntryKind::Commit), + filename: "vendor".into(), + oid: ObjectId::null(gix_hash::Kind::Sha1), + }], + }; + let tree_oid = objects.write(&tree).expect("write tree"); + + let dir = tempfile::tempdir().expect("tempdir"); + let err = checkout(&objects, tree_oid, dir.path()).expect_err("must refuse a gitlink"); + assert!(matches!(err, Error::Submodule { .. })); + } +}

crates/ents-effect/src/results.rs @@ -1,0 +1,242 @@ +//! Writing a run's outcome back to the repository (`effect.results-writeback`, +//! `effect.identity`): an ordinary [`ents_receive::receive`] client, never a +//! privileged write outside the gate. +//! +//! [`write_result`] builds the [`ents_model::Status`] typed tree, seals it +//! into a signed commit exactly the way [`ents_sync::resolve::merge_heads`] +//! seals a merge tip (`sign` is a caller-injected closure, so this crate +//! never holds key material — the composition root injects the worker's own +//! member key, `effect.identity`: "its result commit MUST be signed with +//! its own member key"), and hands the result to `receive` like any other +//! frontend. + +use ents_model::Status; +use ents_model::trailer::Trailers; +use ents_receive::{EventSink, Mode, Outcome, Proposal, RefTransition}; +use gix::refs::FullName; +use gix_object::{Commit, Find, Kind, Write, WriteTo as _}; +use gix_ref_store::RefStore; + +use crate::error::{Error, Result}; + +/// Build a signed commit recording `status` on `results_ref`, and push it +/// through [`ents_receive::receive`] — the sole path an effect's outcome +/// may re-enter the repository (`effect.results-writeback`). +/// +/// `results_ref` is the caller's choice: the canonical +/// `refs/meta/results/<effect>/<short-oid>` for a designated worker, or the +/// self-run `refs/meta/self/<member>/<effect>/<short-oid>` for any other +/// member running the same effect on their own account +/// (`effect.self-run`). Which one is "official" is a refname authorization +/// rule the gate enforces (`effect.official`), not a decision this +/// function makes. +/// +/// # Errors +/// +/// [`Error::Facet`] if `status` cannot be serialized; [`Error::Refs`] if +/// reading `results_ref`'s current tip fails; [`Error::Receive`] if +/// `receive` itself could not reach an outcome. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::write_result; +/// use ents_model::Status; +/// use ents_receive::{Mode, NullEventSink}; +/// use ents_testutil::{Keypair, MemRefStore, ObjectStore}; +/// +/// let refs = MemRefStore::default(); +/// let objects = ObjectStore::default(); +/// let key = Keypair::from_seed(1); +/// let author = gix::actor::Signature { +/// name: "worker".into(), +/// email: "worker@ents.test".into(), +/// time: gix::date::Time { seconds: 1_000, offset: 0 }, +/// }; +/// +/// let name: gix::refs::FullName = +/// "refs/meta/results/unit/abc123456789".try_into().expect("valid"); +/// let outcome = write_result( +/// &refs, &objects, &NullEventSink, name, Status::Pass, &author, +/// |payload| key.sign(payload), Mode::Advisory, +/// ).expect("evaluates"); +/// assert_eq!(outcome.result, ents_receive::TxResult::Applied); +/// ``` +// @relation(effect.results-writeback, effect.identity, effect.result-taxonomy, effect.self-run, scope=function) +#[expect( + clippy::too_many_arguments, + reason = "one input per commit-building step, mirrors ents_sync::resolve::merge_heads's shape" +)] +pub fn write_result( + refs: &dyn RefStore, + objects: &(impl Find + Write), + events: &dyn EventSink, + results_ref: FullName, + status: Status, + author: &gix::actor::Signature, + sign: impl FnOnce(&[u8]) -> String, + mode: Mode, +) -> Result<Outcome> { + let tree = facet_git_tree::serialize_into(&status, objects)?; + let old = refs.get(results_ref.as_ref())?; + let parents: Vec<_> = old.into_iter().collect(); + + let trailers = Trailers { + ents_ref: Some(results_ref.clone()), + schema_version: None, + }; + let summary = match status { + Status::Pass => "Record pass", + Status::Fail => "Record fail", + Status::Error => "Record error", + }; + let message = format!("{summary}\n\n{}", trailers.render()); + let mut commit = Commit { + tree, + parents: parents.clone().into(), + author: author.clone(), + committer: author.clone(), + encoding: None, + message: message.into(), + extra_headers: Vec::new(), + }; + + let mut payload = Vec::new(); + commit.write_to(&mut payload).map_err(|e| Error::Decode { + oid: tree, + detail: format!("serializing result commit failed: {e}"), + })?; + let pem = sign(&payload); + commit + .extra_headers + .push(("gpgsig".into(), pem.trim_end().into())); + + let mut raw = Vec::new(); + commit.write_to(&mut raw).map_err(|e| Error::Decode { + oid: tree, + detail: format!("serializing signed result commit failed: {e}"), + })?; + let tip = objects + .write_buf(Kind::Commit, &raw) + .map_err(|e| Error::Decode { + oid: tree, + detail: e.to_string(), + })?; + + let proposal = Proposal { + transitions: vec![RefTransition { + name: results_ref, + old, + new: Some(tip), + }], + objects: vec![tip], + auth: None, + }; + Ok(ents_receive::receive( + refs, objects, events, &proposal, mode, + )?) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use ents_model::{Provenance, namespace}; + use ents_receive::{NullEventSink, TxResult}; + use ents_testutil::{Keypair, MemRefStore, ObjectStore, enroll_member}; + use gix_ref_store::RefStoreRead as _; + use rstest::rstest; + + use super::*; + + fn author() -> gix::actor::Signature { + gix::actor::Signature { + name: "worker".into(), + email: "worker@ents.test".into(), + time: gix::date::Time { + seconds: 1_000, + offset: 0, + }, + } + } + + #[rstest] + #[case::pass(Status::Pass)] + #[case::fail(Status::Fail)] + #[case::error(Status::Error)] + // @relation(effect.results-writeback, effect.identity, effect.result-taxonomy, scope=function, role=Verifies) + fn write_result_lands_a_signed_commit_on_the_canonical_results_ref(#[case] status: Status) { + let refs = MemRefStore::default(); + let objects = ObjectStore::default(); + let worker = Keypair::from_seed(1); + enroll_member( + &refs, + &objects, + "worker", + &worker, + Provenance::AdminRegistered, + 100, + ); + + let name = namespace::result_ref("unit", "deadbeefcafe").expect("valid"); + let outcome = write_result( + &refs, + &objects, + &NullEventSink, + name.clone(), + status, + &author(), + |payload| worker.sign(payload), + Mode::Advisory, + ) + .expect("evaluates"); + assert_eq!(outcome.result, TxResult::Applied); + // The gate validated the commit's signature against the worker's + // own member key — `effect.identity`: "its result commit MUST be + // signed with its own member key". + let (_, verdict) = outcome.verdicts.first().expect("one transition proposed"); + assert!(verdict.is_pass()); + + let tip = refs.get(name.as_ref()).expect("readable").expect("landed"); + let mut buf = Vec::new(); + let data = gix_object::Find::try_find(&objects, &tip, &mut buf) + .expect("readable") + .expect("present"); + let commit = gix_object::CommitRef::from_bytes(data.data, tip.kind()).expect("decodes"); + let landed: Status = + facet_git_tree::deserialize(&commit.tree(), &objects).expect("deserializes"); + assert_eq!(landed, status); + } + + #[rstest] + // @relation(effect.self-run, effect.results-writeback, scope=function, role=Verifies) + fn write_result_can_target_the_self_run_namespace() { + let refs = MemRefStore::default(); + let objects = ObjectStore::default(); + let member = Keypair::from_seed(2); + enroll_member( + &refs, + &objects, + "bob", + &member, + Provenance::AdminRegistered, + 100, + ); + + let name = namespace::self_result_ref(&ents_model::MemberId::new("bob"), "unit", "abc") + .expect("valid"); + let outcome = write_result( + &refs, + &objects, + &NullEventSink, + name.clone(), + Status::Fail, + &author(), + |payload| member.sign(payload), + Mode::Advisory, + ) + .expect("evaluates"); + assert_eq!(outcome.result, TxResult::Applied); + assert!(refs.get(name.as_ref()).expect("readable").is_some()); + } +}

crates/ents-effect/src/run.rs @@ -1,0 +1,464 @@ +//! The run loop (`effect.execution`, `effect.local-run`): materialize an +//! effect's declared toolchains and the tested commit's tree, hand both to +//! an [`Executor`], and write the outcome back through +//! [`crate::write_result`]. +//! +//! [`run_one`] is the one code path a hosted worker and `git effect run` +//! both call (`effect.local-run`: "the identical code path a hosted worker +//! uses"); only what surrounds it differs — a durable queue feeding a +//! worker's loop of [`run_one`] calls, versus [`run_effect`] deriving the +//! same obligations directly from [`ents_query::Evaluator::outstanding`] +//! and calling [`run_one`] once per commit, with no queue at all +//! (`effect.local-run`: "only the durable queue MUST be skipped"). + +use std::path::Path; + +use ents_model::Effect; +use ents_query::{Evaluator, Query}; +use ents_receive::{EventSink, Mode, Outcome}; +use gix::refs::FullName; +use gix_hash::ObjectId; +use gix_object::{CommitRef, Find, Kind, Write}; +use gix_ref_store::RefStore; + +use crate::error::{Error, Result}; +use crate::executor::{Executor, RunStatus, SandboxInputs}; +use crate::results::write_result; +use crate::toolchain; + +/// The tree of the commit at `oid`. +fn commit_tree(objects: &impl Find, oid: ObjectId) -> Result<ObjectId> { + let mut buf = Vec::new(); + let data = objects + .try_find(&oid, &mut buf) + .map_err(|source| Error::Decode { + oid, + detail: source.to_string(), + })? + .ok_or(Error::Missing { oid })?; + if data.kind != Kind::Commit { + return Err(Error::Decode { + oid, + detail: "expected a commit".to_owned(), + }); + } + let commit = CommitRef::from_bytes(data.data, oid.kind()).map_err(|e| Error::Decode { + oid, + detail: e.to_string(), + })?; + Ok(commit.tree()) +} + +/// The short-oid segment convention every results refname uses: +/// `refs/meta/results/<effect>/<short-oid>` (`effect.results-writeback`) — +/// the first 12 hex characters, long enough to stay unambiguous within one +/// effect's results namespace while keeping refnames short. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::run::short_oid; +/// +/// let oid = gix_hash::ObjectId::null(gix_hash::Kind::Sha1); +/// assert_eq!(short_oid(oid), "000000000000"); +/// ``` +#[must_use] +pub fn short_oid(oid: ObjectId) -> String { + let hex = oid.to_string(); + hex.get(..12).unwrap_or(&hex).to_owned() +} + +/// Materialize `effect`'s declared toolchains against `toolchain_cache`, +/// returning each name paired with its host `bin/` directory, in the +/// effect's declared order (`crate::executor::activate`'s PATH-collision +/// tiebreak depends on this order surviving). +/// +/// # Errors +/// +/// [`Error::UnknownToolchain`] or [`Error::InvalidRecipe`] for a name the +/// effect declares but that does not resolve; see +/// [`crate::toolchain::materialize`] for extraction failures. +fn resolve_toolchains( + refs: &dyn gix_ref_store::RefStoreRead, + objects: &impl Find, + effect: &Effect, + toolchain_cache: &Path, +) -> Result<Vec<(String, std::path::PathBuf)>> { + let mut out = Vec::with_capacity(effect.toolchains.len()); + for name in &effect.toolchains { + let (_, recipe) = toolchain::resolve(refs, objects, name)?; + let bin = toolchain::materialize(&recipe, objects, toolchain_cache)?; + out.push((name.clone(), bin)); + } + Ok(out) +} + +/// Run `effect` against the single commit `oid`: materialize its +/// toolchains and `oid`'s tree, execute via `executor`, and write the +/// outcome to `results_ref` — the one code path `effect.local-run` names. +/// +/// `results_ref` is the caller's choice (`effect.self-run`, +/// `effect.official`): the canonical results ref for a designated worker, +/// or a self-run mirror for any other member. `scratch` holds the +/// per-run, never-cached tree checkout (a Docker container is thrown away +/// per run; a Sprite's `sync_dir` re-syncs it every time +/// too, so nothing here needs it to survive); `toolchain_cache` holds the +/// extract-once toolchain cache [`crate::toolchain::materialize`] shares +/// across runs. +/// +/// # Errors +/// +/// Any [`Error`] from resolving or materializing a toolchain, checking out +/// `oid`'s tree, the executor itself, or [`crate::write_result`]. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::run::run_one; +/// use ents_effect::{Executor, RunOutput, RunStatus, SandboxInputs}; +/// use ents_model::{Effect, Provenance, namespace}; +/// use ents_receive::{Mode, NullEventSink}; +/// use ents_testutil::{Keypair, MemRefStore, ObjectStore, advance_ref, enroll_member}; +/// +/// struct AlwaysPass; +/// impl Executor for AlwaysPass { +/// fn run(&self, _inputs: &SandboxInputs<'_>) -> ents_effect::Result<RunOutput> { +/// Ok(RunOutput { status: RunStatus::Pass, log: String::new() }) +/// } +/// } +/// +/// let refs = MemRefStore::default(); +/// let objects = ObjectStore::default(); +/// let worker = Keypair::from_seed(1); +/// enroll_member(&refs, &objects, "worker", &worker, Provenance::AdminRegistered, 100); +/// let commits = advance_ref(&refs, &objects, "refs/heads/main", 1, 200); +/// +/// let effect = Effect { trigger: "rev(refs/heads/main)".into(), toolchains: vec![], run: "true".into() }; +/// let results_ref = namespace::result_ref("unit", "abcabcabcabc").expect("valid"); +/// let author = gix::actor::Signature { +/// name: "worker".into(), email: "worker@ents.test".into(), +/// time: gix::date::Time { seconds: 300, offset: 0 }, +/// }; +/// let scratch = tempfile::tempdir().expect("tempdir"); +/// let cache = tempfile::tempdir().expect("tempdir"); +/// +/// let outcome = run_one( +/// &refs, &objects, &NullEventSink, &AlwaysPass, scratch.path(), cache.path(), +/// commits[0], &effect, results_ref, &author, |p| worker.sign(p), Mode::Advisory, +/// ).expect("runs"); +/// assert_eq!(outcome.result, ents_receive::TxResult::Applied); +/// ``` +// @relation(effect.execution, effect.local-run, effect.toolchains, scope=function) +#[expect( + clippy::too_many_arguments, + reason = "one input per materialization step, mirrors pre-redo's engine::run shape" +)] +pub fn run_one( + refs: &dyn RefStore, + objects: &(impl Find + Write), + events: &dyn EventSink, + executor: &dyn Executor, + scratch: &Path, + toolchain_cache: &Path, + oid: ObjectId, + effect: &Effect, + results_ref: FullName, + author: &gix::actor::Signature, + sign: impl FnOnce(&[u8]) -> String, + mode: Mode, +) -> Result<Outcome> { + let toolchains = resolve_toolchains(refs, objects, effect, toolchain_cache)?; + + let workdir = scratch.join(oid.to_string()); + std::fs::create_dir_all(&workdir).map_err(|source| Error::Io { + path: workdir.clone(), + source, + })?; + let tree = commit_tree(objects, oid)?; + crate::materialize::checkout(objects, tree, &workdir)?; + + let inputs = SandboxInputs { + workdir: &workdir, + toolchains: &toolchains, + command: &effect.run, + }; + let output = executor.run(&inputs)?; + let status = match output.status { + RunStatus::Pass => ents_model::Status::Pass, + RunStatus::Fail => ents_model::Status::Fail, + }; + + write_result( + refs, + objects, + events, + results_ref, + status, + author, + sign, + mode, + ) +} + +/// Run `effect` against every commit currently owed a result +/// (`ents_query::Evaluator::outstanding`, `query.workset`), or against the +/// single commit `at` when given — the boot-time/on-demand form +/// [`run_one`]'s doc names, and the shape `git effect run [--at <commit>]` +/// (a future frontend) calls. +/// +/// `results_ref` builds each run's target refname from its short oid +/// (`crate::run::short_oid`) — pass `ents_model::namespace::result_ref` for +/// a canonical worker or `ents_model::namespace::self_result_ref` curried +/// to one member for a self-run (`effect.self-run`); this function makes +/// no canonical-vs-self decision itself. +/// +/// # Errors +/// +/// [`Error::Eval`] if the work set cannot be computed; otherwise anything +/// [`run_one`] can fail with, for the first commit that fails — later +/// commits in the set are not attempted once one fails, since a caller +/// wrapping this in its own retry policy (`effect.deployment-property`) +/// needs to know exactly which commit stopped the batch. +// @relation(effect.local-run, query.workset, scope=function) +#[expect( + clippy::too_many_arguments, + reason = "one input per materialization step plus the target-ref builder" +)] +pub fn run_effect( + refs: &dyn RefStore, + objects: &(impl Find + Write), + events: &dyn EventSink, + executor: &dyn Executor, + scratch: &Path, + toolchain_cache: &Path, + effect_name: &str, + effect: &Effect, + at: Option<ObjectId>, + results_ref: impl Fn(&str) -> Result<FullName>, + author: &gix::actor::Signature, + sign: &impl Fn(&[u8]) -> String, + mode: Mode, +) -> Result<Vec<(ObjectId, Outcome)>> { + let trigger: Query = effect.trigger.parse()?; + let oids: Vec<ObjectId> = match at { + Some(oid) => vec![oid], + None => { + // `query.workset`'s dedup marker is always the effect's own + // *canonical* results namespace, regardless of which ref this + // particular run's outcome ends up targeting + // (`results_ref`) — a self-run mirror never discharges the + // canonical obligation, by construction. + let evaluator = Evaluator::new(refs, objects); + evaluator + .outstanding(effect_name, &trigger)? + .into_iter() + .collect() + } + }; + + let mut outcomes = Vec::with_capacity(oids.len()); + for oid in oids { + let target = results_ref(&short_oid(oid))?; + let outcome = run_one( + refs, + objects, + events, + executor, + scratch, + toolchain_cache, + oid, + effect, + target, + author, + sign, + mode, + )?; + outcomes.push((oid, outcome)); + } + Ok(outcomes) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use ents_model::{Provenance, namespace}; + use ents_receive::{NullEventSink, TxResult}; + use ents_testutil::{Keypair, MemRefStore, ObjectStore, advance_ref, enroll_member}; + use gix_ref_store::RefStoreRead as _; + use rstest::rstest; + + use super::*; + use crate::executor::{RunOutput, RunStatus, SandboxInputs}; + + struct AlwaysPass; + impl Executor for AlwaysPass { + fn run(&self, _inputs: &SandboxInputs<'_>) -> Result<RunOutput> { + Ok(RunOutput { + status: RunStatus::Pass, + log: String::new(), + }) + } + } + + fn author() -> gix::actor::Signature { + gix::actor::Signature { + name: "worker".into(), + email: "worker@ents.test".into(), + time: gix::date::Time { + seconds: 500, + offset: 0, + }, + } + } + + #[rstest] + // @relation(effect.local-run, query.workset, scope=function, role=Verifies) + fn run_effect_derives_the_full_outstanding_set_with_no_queue_at_all() { + let refs = MemRefStore::default(); + let objects = ObjectStore::default(); + let worker = Keypair::from_seed(1); + enroll_member( + &refs, + &objects, + "worker", + &worker, + Provenance::AdminRegistered, + 100, + ); + let commits = advance_ref(&refs, &objects, "refs/heads/main", 2, 200); + + let effect = Effect { + trigger: "rev(refs/heads/main)".into(), + toolchains: vec![], + run: "true".into(), + }; + let scratch = tempfile::tempdir().expect("tempdir"); + let cache = tempfile::tempdir().expect("tempdir"); + + // `NullEventSink`: the only component `effect.local-run` says this + // path skips is the durable queue, and this run derives its work + // set directly from `query.workset` instead of draining one. + let outcomes = run_effect( + &refs, + &objects, + &NullEventSink, + &AlwaysPass, + scratch.path(), + cache.path(), + "unit", + &effect, + None, + |short| Ok(namespace::result_ref("unit", short).expect("valid")), + &author(), + &|payload| worker.sign(payload), + Mode::Advisory, + ) + .expect("runs"); + + assert_eq!(outcomes.len(), 2); + let mut ran: Vec<_> = outcomes.iter().map(|(oid, _)| *oid).collect(); + ran.sort(); + let mut expected = commits.clone(); + expected.sort(); + assert_eq!(ran, expected); + for (_, outcome) in &outcomes { + assert_eq!(outcome.result, TxResult::Applied); + } + } + + #[rstest] + // @relation(effect.local-run, scope=function, role=Verifies) + fn run_effect_at_a_single_commit_skips_the_work_set_scan() { + let refs = MemRefStore::default(); + let objects = ObjectStore::default(); + let worker = Keypair::from_seed(1); + enroll_member( + &refs, + &objects, + "worker", + &worker, + Provenance::AdminRegistered, + 100, + ); + let commits = advance_ref(&refs, &objects, "refs/heads/main", 3, 200); + + let effect = Effect { + trigger: "rev(refs/heads/main)".into(), + toolchains: vec![], + run: "true".into(), + }; + let scratch = tempfile::tempdir().expect("tempdir"); + let cache = tempfile::tempdir().expect("tempdir"); + + let first = *commits.first().expect("advance_ref produced a commit"); + let outcomes = run_effect( + &refs, + &objects, + &NullEventSink, + &AlwaysPass, + scratch.path(), + cache.path(), + "unit", + &effect, + Some(first), + |short| Ok(namespace::result_ref("unit", short).expect("valid")), + &author(), + &|payload| worker.sign(payload), + Mode::Advisory, + ) + .expect("runs"); + + assert_eq!(outcomes.len(), 1); + let (oid, _) = outcomes.first().expect("one outcome"); + assert_eq!(*oid, first); + } + + #[rstest] + // @relation(effect.self-run, effect.local-run, scope=function, role=Verifies) + fn run_effect_can_target_the_self_run_namespace_via_its_results_ref_closure() { + let refs = MemRefStore::default(); + let objects = ObjectStore::default(); + let bob = Keypair::from_seed(2); + enroll_member( + &refs, + &objects, + "bob", + &bob, + Provenance::AdminRegistered, + 100, + ); + let commits = advance_ref(&refs, &objects, "refs/heads/main", 1, 200); + + let effect = Effect { + trigger: "rev(refs/heads/main)".into(), + toolchains: vec![], + run: "true".into(), + }; + let scratch = tempfile::tempdir().expect("tempdir"); + let cache = tempfile::tempdir().expect("tempdir"); + let member = ents_model::MemberId::new("bob"); + + let outcomes = run_effect( + &refs, + &objects, + &NullEventSink, + &AlwaysPass, + scratch.path(), + cache.path(), + "unit", + &effect, + None, + |short| Ok(namespace::self_result_ref(&member, "unit", short).expect("valid")), + &author(), + &|payload| bob.sign(payload), + Mode::Advisory, + ) + .expect("runs"); + + assert_eq!(outcomes.len(), 1); + let first = *commits.first().expect("advance_ref produced a commit"); + let name = namespace::self_result_ref(&member, "unit", &short_oid(first)).expect("valid"); + assert!(refs.get(name.as_ref()).expect("readable").is_some()); + } +}

crates/ents-effect/src/sprite.rs @@ -1,0 +1,345 @@ +//! The Fly.io Sprite [`Executor`] backend (`effect.execution`, +//! `roots.hosted`): a persistent, hardware-isolated sandbox driven through +//! the `sprite` CLI, one Sprite kept per deployment so a toolchain's +//! extracted bytes survive between runs (`crate::toolchain::materialize`'s +//! host-side cache has a Sprite-side mirror, `sync_dir`'s extract-once +//! check). +//! +//! Ported from `pre-redo`'s `git-effect::engine` Sprite half: the CLI +//! authentication quirk ([`ensure_auth`]), the idempotent-create quirk +//! ([`ensure_sprite`]), and the orphan-process-kill quirk in +//! `unpack_script` all carry over verbatim — these are exactly the +//! "environment is the risk" gotchas the development plan calls out. +//! Rewritten against this phase's design: no `git archive` (this crate +//! never assumes an on-disk `.git`, `arch.no-object-store-trait`) — the +//! workdir and each toolchain are materialized to a host directory first +//! (the same `crate::materialize::checkout` and +//! [`crate::toolchain::materialize`] every backend shares), then `tar`'d +//! from that host directory into the Sprite over `sprite exec`'s stdin; +//! and no PTY/asciicast live-streaming (`effect.adoc` names no such +//! requirement for this phase — deferred to `ents-web`, which owns any +//! live view). + +use std::path::Path; +use std::process::{Command, Stdio}; + +use crate::error::{Error, Result}; +use crate::executor::{Executor, RunOutput, RunStatus, SandboxInputs, activate}; + +/// Where the workdir is unpacked inside the Sprite. +pub const WORKDIR: &str = "/work"; + +/// Where a toolchain's `bin/` is extracted inside the Sprite, one directory +/// per content key (`{TOOLCHAINS_DIR}/<key>/bin`) — never cleared: the +/// Sprite's persistent filesystem is the cache. +pub const TOOLCHAINS_DIR: &str = "/toolchains"; + +/// The env var the hosted worker passes the `sprite` CLI's auth token +/// through, per [`ensure_auth`]. +pub const SPRITES_TOKEN_VAR: &str = "SPRITES_TOKEN"; + +/// A Sprite name derived from `seed`, kept to the `[a-z0-9-]` a Sprite name +/// allows so the same seed (a deployment id, a repository path) always +/// reuses the same sandbox. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::sprite::sprite_name; +/// +/// assert_eq!(sprite_name("git-ents.cloud"), "ents-effect-git-ents-cloud"); +/// assert_eq!(sprite_name(""), "ents-effect-sprite"); +/// ``` +#[must_use] +pub fn sprite_name(seed: &str) -> String { + let sanitized: String = seed + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() { + c.to_ascii_lowercase() + } else { + '-' + } + }) + .collect(); + let trimmed = sanitized.trim_matches('-'); + format!( + "ents-effect-{}", + if trimmed.is_empty() { + "sprite" + } else { + trimmed + } + ) +} + +/// Configure the `sprite` CLI from [`SPRITES_TOKEN_VAR`]. The CLI persists +/// its credentials to a config file rather than reading the token per +/// call, so without this it reports "no organizations configured" even +/// with the token in the environment. `auth setup` is idempotent, so a +/// caller may run this before every batch of runs to keep the steady state +/// self-healing. +/// +/// # Errors +/// +/// [`Error::Process`] if [`SPRITES_TOKEN_VAR`] is unset, or the CLI ran and +/// refused it; [`Error::Spawn`] if the CLI could not be started. +pub fn ensure_auth() -> Result<()> { + let token = std::env::var(SPRITES_TOKEN_VAR).map_err(|_unset| Error::Process { + program: "sprite".to_owned(), + detail: format!("{SPRITES_TOKEN_VAR} is not set in the worker's environment"), + })?; + let output = Command::new("sprite") + .args(["auth", "setup", "--token", &token]) + .output() + .map_err(|e| Error::Spawn { + program: "sprite".to_owned(), + detail: e.to_string(), + })?; + if output.status.success() { + Ok(()) + } else { + Err(Error::Process { + program: "sprite".to_owned(), + detail: format!( + "auth setup failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + ), + }) + } +} + +/// Create the Sprite named `name` if it does not already exist. +/// `sprite create` fails when the Sprite is already there — the steady +/// state once the first run has happened — so its failure is tolerated +/// here and surfaces only later if the Sprite turns out unreachable. +/// +/// # Errors +/// +/// [`Error::Spawn`] if the CLI could not be started. +pub fn ensure_sprite(name: &str) -> Result<()> { + let _existing = Command::new("sprite") + .args(["create", "--skip-console", name]) + .output() + .map_err(|e| Error::Spawn { + program: "sprite".to_owned(), + detail: e.to_string(), + })?; + Ok(()) +} + +/// The in-Sprite script [`sync_dir`] runs to replace `dest`'s contents with +/// the tar streamed over stdin, first killing any process still working +/// under `dest`: a worker killed mid-run (a deploy, a restart) leaves its +/// in-Sprite build processes alive, since `sprite exec` only tethers the +/// local CLI process — an orphaned build still writing under `dest` races +/// the wipe, failing `rm -rf` with "Directory not empty". +fn unpack_script(dest: &str) -> String { + format!( + "for cwd in /proc/[0-9]*/cwd; do\n\ + case \"$(readlink \"$cwd\" 2>/dev/null)\" in\n\ + {dest}|{dest}/*) kill -9 \"$(basename \"${{cwd%/cwd}}\")\" 2>/dev/null || true ;;\n\ + esac\n\ + done\n\ + rm -rf {dest} && mkdir -p {dest} && tar -x -C {dest}" + ) +} + +/// Stream `host_dir`'s contents into the Sprite `name` at `dest`, replacing +/// whatever was there — used both for the workdir (always re-synced: a +/// fresh checkout per run) and, via [`sync_toolchain`], for a toolchain not +/// already cached in-Sprite. +/// +/// # Errors +/// +/// [`Error::Spawn`] if `tar` or `sprite` could not be started; +/// [`Error::Process`] if either exited nonzero. +fn sync_dir(host_dir: &Path, name: &str, dest: &str) -> Result<()> { + let mut archive = Command::new("tar") + .args(["-c", "-C"]) + .arg(host_dir) + .arg(".") + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| Error::Spawn { + program: "tar".to_owned(), + detail: e.to_string(), + })?; + let tar_stdout = archive.stdout.take().ok_or_else(|| Error::Process { + program: "tar".to_owned(), + detail: "no stdout".to_owned(), + })?; + + let unpack = Command::new("sprite") + .args(["exec", "-s", name, "--", "sh", "-c", &unpack_script(dest)]) + .stdin(Stdio::from(tar_stdout)) + .output() + .map_err(|e| Error::Spawn { + program: "sprite".to_owned(), + detail: e.to_string(), + })?; + + let tar_status = archive.wait().map_err(|e| Error::Process { + program: "tar".to_owned(), + detail: e.to_string(), + })?; + if !tar_status.success() { + return Err(Error::Process { + program: "tar".to_owned(), + detail: format!("could not archive {}", host_dir.display()), + }); + } + if !unpack.status.success() { + return Err(Error::Process { + program: "sprite".to_owned(), + detail: format!( + "could not sync into {dest}: {}", + String::from_utf8_lossy(&unpack.stderr).trim() + ), + }); + } + Ok(()) +} + +/// Extract-once sync of one toolchain's `bin/` directory into the Sprite +/// `name`, at `{TOOLCHAINS_DIR}/<key>/bin` — a directory already present +/// from an earlier run is left alone rather than re-extracted, since the +/// Sprite's persistent filesystem is the cache. `key` is the same content +/// key [`crate::toolchain::materialize`] cached `host_bin`'s parent +/// directory under, so two runs of the same toolchain content sync it at +/// most once. +/// +/// # Errors +/// +/// See [`sync_dir`]. +fn sync_toolchain(host_bin: &Path, name: &str, key: &str) -> Result<String> { + let sandbox_dir = format!("{TOOLCHAINS_DIR}/{key}/bin"); + let cached = Command::new("sprite") + .args([ + "exec", + "-s", + name, + "--", + "sh", + "-c", + &format!("[ -d {sandbox_dir} ]"), + ]) + .status() + .map_err(|e| Error::Spawn { + program: "sprite".to_owned(), + detail: e.to_string(), + })?; + if !cached.success() { + sync_dir(host_bin, name, &sandbox_dir)?; + } + Ok(sandbox_dir) +} + +/// The content key [`crate::toolchain::materialize`] cached `host_bin` +/// under — `host_bin`'s parent directory name, since `materialize` always +/// returns `<cache_root>/<key>/bin`. +fn content_key(host_bin: &Path) -> Result<String> { + host_bin + .parent() + .and_then(Path::file_name) + .and_then(std::ffi::OsStr::to_str) + .map(str::to_owned) + .ok_or_else(|| Error::Process { + program: "sprite".to_owned(), + detail: format!( + "{} is not a materialize()-shaped toolchain directory", + host_bin.display() + ), + }) +} + +/// [`Executor`] running each effect in a persistent, hardware-isolated Fly +/// Sprite (`roots.hosted`). +#[derive(Debug, Clone)] +pub struct SpriteExecutor { + /// The Sprite's name, from [`sprite_name`] or chosen by the + /// composition root. + pub name: String, +} + +impl SpriteExecutor { + /// A Sprite executor targeting the Sprite named `name`. + #[must_use] + pub fn new(name: impl Into<String>) -> Self { + Self { name: name.into() } + } +} + +impl Executor for SpriteExecutor { + fn run(&self, inputs: &SandboxInputs<'_>) -> Result<RunOutput> { + ensure_auth()?; + ensure_sprite(&self.name)?; + sync_dir(inputs.workdir, &self.name, WORKDIR)?; + + let mut sandbox_dirs: Vec<(String, String)> = Vec::with_capacity(inputs.toolchains.len()); + for (toolchain_name, host_bin) in inputs.toolchains { + let key = content_key(host_bin)?; + let sandbox_dir = sync_toolchain(host_bin, &self.name, &key)?; + sandbox_dirs.push((toolchain_name.clone(), sandbox_dir)); + } + + let script = format!( + "cd {WORKDIR} && {} 2>&1", + activate(inputs.command, &sandbox_dirs) + ); + let output = Command::new("sprite") + .args(["exec", "-s", &self.name, "--", "sh", "-c", &script]) + .output() + .map_err(|e| Error::Spawn { + program: "sprite".to_owned(), + detail: e.to_string(), + })?; + let log = String::from_utf8_lossy(&output.stdout).into_owned(); + let status = if output.status.success() { + RunStatus::Pass + } else { + RunStatus::Fail + }; + Ok(RunOutput { status, log }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::normal("git-ents.cloud", "ents-effect-git-ents-cloud")] + #[case::empty("", "ents-effect-sprite")] + #[case::only_punctuation("///", "ents-effect-sprite")] + #[case::mixed_case("Repo_Name", "ents-effect-repo-name")] + // @relation(effect.execution, scope=function, role=Verifies) + fn sprite_name_sanitizes_to_a_valid_shape(#[case] seed: &str, #[case] expected: &str) { + assert_eq!(sprite_name(seed), expected); + } + + #[rstest] + // @relation(effect.execution, scope=function, role=Verifies) + fn unpack_script_kills_orphans_before_wiping_the_destination() { + let script = unpack_script("/work"); + assert!(script.contains("kill -9")); + assert!(script.contains("rm -rf /work && mkdir -p /work && tar -x -C /work")); + } + + #[rstest] + // @relation(effect.toolchains, scope=function, role=Verifies) + fn content_key_reads_the_materialize_cache_layout() { + let bin = Path::new("/cache/deadbeef/bin"); + assert_eq!(content_key(bin).expect("valid shape"), "deadbeef"); + } + + #[rstest] + // @relation(effect.toolchains, scope=function, role=Verifies) + fn content_key_rejects_a_path_with_no_parent() { + content_key(Path::new("/")).expect_err("no parent"); + } +}

crates/ents-effect/src/toolchain.rs @@ -1,0 +1,551 @@ +//! Toolchain resolution and materialization (`effect.toolchains`, +//! `model.toolchain`). +//! +//! [`ents_model::Toolchain::recipe`] is deliberately an opaque `String` — +//! `model.toolchain`'s own doc names this crate as the one that gives it +//! structure. [`Recipe`] is that structure: a toolchain's `bin` is either +//! [`Recipe::Embedded`] (a tree already in the object database, captured +//! whole by whatever wrote the toolchain) or [`Recipe::Downloaded`] (a set +//! of externally-hosted, sha256-pinned archives), ported from `pre-redo`'s +//! `git_toolchain::Bin` — the design pre-redo settled on and this phase +//! carries forward, not a fresh design. [`Recipe::render`]/[`Recipe::parse`] +//! round-trip it through the plain-text `recipe` field. +//! +//! [`materialize`] resolves a toolchain to a host directory containing its +//! activated `bin/`, extract-once cached under a content key (a tree oid +//! for `Embedded`, a hash of each component's pin for `Downloaded`) so a +//! backend that runs the same toolchain repeatedly (a Sprite's persistent +//! filesystem, a developer's local cache) never re-extracts unchanged +//! bytes. Fetching a [`Recipe::Downloaded`] component shells to `curl`, +//! `tar`, and `sha256sum`/`shasum`, the same pattern `pre-redo` used and +//! this phase ports rather than adding an HTTP or hashing dependency. + +use std::io::Read as _; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use ents_model::{Toolchain, namespace}; +use gix_hash::ObjectId; +use gix_object::{CommitRef, Find, Kind}; +use gix_ref_store::RefStoreRead; + +use crate::error::{Error, Result}; + +/// How a toolchain's `bin` is provisioned — the structure inside +/// [`ents_model::Toolchain::recipe`] (`effect.toolchains`). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Recipe { + /// `bin`'s directory tree, already in the object database — the whole + /// tree's entries become the toolchain's activated `bin/` contents. + Embedded { + /// The tree object id. + tree: ObjectId, + }, + /// A set of archives fetched, sha256-verified, and merged onto disk at + /// materialization time. + Downloaded { + /// Each archive making up the toolchain. + components: Vec<Component>, + }, +} + +/// One archive making up a [`Recipe::Downloaded`] toolchain: fetched from +/// `url` and checked against `sha256` before being extracted per +/// `strip`/`dest` — ported verbatim from `pre-redo`'s +/// `git_toolchain::Component`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Component { + /// Where to fetch the archive from. + pub url: String, + /// The archive's expected sha256, hex-encoded. + pub sha256: String, + /// Leading path segments `tar` strips at extraction. + pub strip: u8, + /// Subdirectory under the toolchain's `bin/` extraction root to extract + /// into: empty for an archive that already carries its own `bin/` top + /// level, `bin` for a flat archive whose payload should itself land on + /// `PATH`. + pub dest: String, +} + +const EMBEDDED: &str = "embedded"; +const DOWNLOADED: &str = "downloaded"; + +impl Recipe { + /// Parse a [`Recipe`] out of a [`ents_model::Toolchain::recipe`] string. + /// + /// The format is deliberately small rather than a general one (no new + /// dependency for two variants and four fields): one line naming the + /// kind, then either the embedded tree's hex oid, or one + /// `url sha256 strip dest` line per component (`dest` last, so it may + /// be empty without ambiguity). + /// + /// # Errors + /// + /// [`Error::InvalidRecipe`] if the text does not match this shape. + /// + /// # Examples + /// + /// ``` + /// use ents_effect::Recipe; + /// + /// let text = "embedded 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n"; + /// let recipe = Recipe::parse(text).expect("parses"); + /// assert!(matches!(recipe, Recipe::Embedded { .. })); + /// ``` + pub fn parse(text: &str) -> Result<Self> { + let mut lines = text.lines().filter(|line| !line.trim().is_empty()); + let Some(first) = lines.next() else { + return Err(invalid("empty recipe")); + }; + let mut words = first.split_whitespace(); + match words.next() { + Some(EMBEDDED) => { + let hex = words + .next() + .ok_or_else(|| invalid("embedded recipe missing a tree oid"))?; + let tree = ObjectId::from_hex(hex.as_bytes()) + .map_err(|e| invalid(format!("invalid tree oid {hex:?}: {e}")))?; + Ok(Self::Embedded { tree }) + } + Some(DOWNLOADED) => { + let mut components = Vec::new(); + for line in lines { + let mut fields = line.split_whitespace(); + let url = fields + .next() + .ok_or_else(|| invalid("component line missing a url"))? + .to_owned(); + let sha256 = fields + .next() + .ok_or_else(|| invalid("component line missing a sha256"))? + .to_owned(); + let strip = fields + .next() + .ok_or_else(|| invalid("component line missing a strip count"))? + .parse::<u8>() + .map_err(|e| invalid(format!("invalid strip count: {e}")))?; + let dest = fields.next().unwrap_or("").to_owned(); + components.push(Component { + url, + sha256, + strip, + dest, + }); + } + if components.is_empty() { + return Err(invalid( + "a downloaded toolchain must list at least one component", + )); + } + Ok(Self::Downloaded { components }) + } + Some(other) => Err(invalid(format!("unknown recipe kind {other:?}"))), + None => Err(invalid("empty recipe")), + } + } + + /// Render this [`Recipe`] back into the text stored in + /// [`ents_model::Toolchain::recipe`]. + /// + /// # Examples + /// + /// ``` + /// use ents_effect::Recipe; + /// + /// let recipe = Recipe::Embedded { + /// tree: gix_hash::ObjectId::null(gix_hash::Kind::Sha1), + /// }; + /// let text = recipe.render(); + /// assert_eq!(Recipe::parse(&text).expect("round-trips"), recipe); + /// ``` + #[must_use] + pub fn render(&self) -> String { + match self { + Self::Embedded { tree } => format!("{EMBEDDED} {tree}\n"), + Self::Downloaded { components } => { + let mut out = format!("{DOWNLOADED}\n"); + for c in components { + out.push_str(&format!("{} {} {} {}\n", c.url, c.sha256, c.strip, c.dest)); + } + out + } + } + } +} + +/// Read the [`Toolchain`] entity named `name` from +/// `refs/meta/toolchains/<name>`, and parse its [`Toolchain::recipe`] as a +/// [`Recipe`]. +/// +/// # Errors +/// +/// [`Error::UnknownToolchain`] when the ref does not exist or does not +/// resolve to a commit tree; [`Error::InvalidRecipe`] when its `recipe` +/// field does not parse. +/// +/// # Examples +/// +/// ``` +/// use ents_effect::toolchain::resolve; +/// use ents_model::Toolchain; +/// use ents_testutil::{MemRefStore, ObjectStore, write_meta_entity}; +/// +/// let refs = MemRefStore::default(); +/// let objects = ObjectStore::default(); +/// let toolchain = Toolchain { +/// name: "rust-stable".into(), +/// recipe: "embedded 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n".into(), +/// }; +/// let name: gix::refs::FullName = "refs/meta/toolchains/rust-stable".try_into().expect("valid"); +/// write_meta_entity(&refs, &objects, name, &toolchain, None, 100); +/// +/// let (entity, recipe) = resolve(&refs, &objects, "rust-stable").expect("resolves"); +/// assert_eq!(entity.name, "rust-stable"); +/// assert!(matches!(recipe, ents_effect::Recipe::Embedded { .. })); +/// ``` +pub fn resolve( + refs: &dyn RefStoreRead, + objects: &impl Find, + name: &str, +) -> Result<(Toolchain, Recipe)> { + let refname = namespace::toolchain_ref(name) + .map_err(|e| Error::UnknownToolchain(format!("{name}: {e}")))?; + let Some(tip) = refs.get(refname.as_ref())? else { + return Err(Error::UnknownToolchain(name.to_owned())); + }; + let mut buf = Vec::new(); + let data = objects + .try_find(&tip, &mut buf) + .map_err(|source| Error::Decode { + oid: tip, + detail: source.to_string(), + })? + .ok_or(Error::Missing { oid: tip })?; + if data.kind != Kind::Commit { + return Err(Error::Decode { + oid: tip, + detail: "toolchain ref does not point at a commit".to_owned(), + }); + } + let commit = CommitRef::from_bytes(data.data, tip.kind()).map_err(|e| Error::Decode { + oid: tip, + detail: e.to_string(), + })?; + let toolchain: Toolchain = facet_git_tree::deserialize(&commit.tree(), objects)?; + let recipe = Recipe::parse(&toolchain.recipe).map_err(|e| match e { + Error::InvalidRecipe { detail, .. } => Error::InvalidRecipe { + name: name.to_owned(), + detail, + }, + other => other, + })?; + Ok((toolchain, recipe)) +} + +fn invalid(detail: impl Into<String>) -> Error { + Error::InvalidRecipe { + name: String::new(), + detail: detail.into(), + } +} + +/// A stable, filesystem-safe cache key for a [`Recipe`]: the embedded +/// tree's own hex oid, or each downloaded component's sha256 joined in +/// extraction order — the same bytes extracted differently (a different +/// `strip`/`dest`) are a different toolchain on disk, so those fields join +/// the key too. +#[must_use] +pub fn cache_key(recipe: &Recipe) -> String { + match recipe { + Recipe::Embedded { tree } => tree.to_string(), + Recipe::Downloaded { components } => components + .iter() + .map(|c| format!("{}.{}.{}", c.sha256, c.strip, c.dest)) + .collect::<Vec<_>>() + .join("-"), + } +} + +/// Resolve `recipe` to a host directory containing the toolchain's +/// activated `bin/`, extracted once under `cache_root` and reused on every +/// later call with the same recipe (`effect.toolchains`: "resolved during +/// effect execution"). +/// +/// # Errors +/// +/// [`Error::Submodule`] or [`Error::NotUtf8`] for a tree this crate cannot +/// materialize; [`Error::Spawn`]/[`Error::Process`]/[`Error::HashMismatch`] +/// for a downloaded component that could not be fetched, verified, or +/// extracted; [`Error::Io`] for a host filesystem failure. +/// +/// # Examples +/// +/// An embedded recipe over the empty tree materializes an empty `bin/`. +/// +/// ``` +/// use ents_effect::Recipe; +/// use ents_effect::toolchain::materialize; +/// use ents_testutil::ObjectStore; +/// use gix_object::{Kind, Write as _}; +/// +/// let objects = ObjectStore::default(); +/// let empty = objects.write_buf(Kind::Tree, b"").expect("write"); +/// let recipe = Recipe::Embedded { tree: empty }; +/// +/// let dir = tempfile::tempdir().expect("tempdir"); +/// let bin = materialize(&recipe, &objects, dir.path()).expect("materializes"); +/// assert!(bin.ends_with("bin")); +/// assert!(bin.is_dir()); +/// ``` +pub fn materialize(recipe: &Recipe, objects: &impl Find, cache_root: &Path) -> Result<PathBuf> { + let root = cache_root.join(cache_key(recipe)); + let bin = root.join("bin"); + if bin.is_dir() { + return Ok(bin); + } + let tmp = cache_root.join(format!("{}.tmp", cache_key(recipe))); + if tmp.exists() { + remove_dir(&tmp)?; + } + make_dir(&tmp)?; + + match recipe { + Recipe::Embedded { tree } => { + let bin_tmp = tmp.join("bin"); + make_dir(&bin_tmp)?; + crate::materialize::checkout(objects, *tree, &bin_tmp)?; + } + Recipe::Downloaded { components } => { + make_dir(&tmp.join("bin"))?; + for component in components { + fetch_component(component, &tmp)?; + } + } + } + + // Land atomically: a transient failure partway through must never leave + // `bin` existing-but-incomplete, or the next call would trust a half + // extraction forever. + std::fs::rename(&tmp, &root).map_err(|source| Error::Io { + path: root.clone(), + source, + })?; + Ok(bin) +} + +fn fetch_component(component: &Component, root: &Path) -> Result<()> { + if component.dest.contains('/') || component.dest.contains("..") { + return Err(Error::InvalidComponent(format!( + "unsafe dest {:?}", + component.dest + ))); + } + let dest = if component.dest.is_empty() { + root.join("bin") + } else { + root.join("bin").join(&component.dest) + }; + make_dir(&dest)?; + + let bytes = fetch(&component.url)?; + let actual = sha256_hex(&bytes)?; + if !actual.eq_ignore_ascii_case(&component.sha256) { + return Err(Error::HashMismatch { + url: component.url.clone(), + expected: component.sha256.clone(), + actual, + }); + } + + let mut child = Command::new("tar") + .args([ + "-x", + "-C", + dest.to_str().ok_or_else(|| Error::NotUtf8(dest.clone()))?, + &format!("--strip-components={}", component.strip), + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| Error::Spawn { + program: "tar".to_owned(), + detail: e.to_string(), + })?; + { + use std::io::Write as _; + let mut stdin = child.stdin.take().ok_or_else(|| Error::Process { + program: "tar".to_owned(), + detail: "no stdin".to_owned(), + })?; + stdin.write_all(&bytes).map_err(|e| Error::Process { + program: "tar".to_owned(), + detail: e.to_string(), + })?; + } + let output = child.wait_with_output().map_err(|e| Error::Process { + program: "tar".to_owned(), + detail: e.to_string(), + })?; + if !output.status.success() { + return Err(Error::Process { + program: "tar".to_owned(), + detail: String::from_utf8_lossy(&output.stderr).trim().to_owned(), + }); + } + Ok(()) +} + +/// `GET url`, via the system `curl` — shells out rather than adding an HTTP +/// dependency, the same rationale `pre-redo` used. +fn fetch(url: &str) -> Result<Vec<u8>> { + let output = Command::new("curl") + .args(["-sSL", "--fail", url]) + .output() + .map_err(|e| Error::Spawn { + program: "curl".to_owned(), + detail: e.to_string(), + })?; + if !output.status.success() { + return Err(Error::Process { + program: "curl".to_owned(), + detail: format!("could not fetch {url}"), + }); + } + Ok(output.stdout) +} + +/// Hex-encoded sha256 of `bytes`, via the system `shasum` (macOS) or +/// `sha256sum` (Linux) — shells out rather than adding a hashing +/// dependency, the same rationale `pre-redo` used. +fn sha256_hex(bytes: &[u8]) -> Result<String> { + let (program, args): (&str, &[&str]) = if Command::new("sha256sum") + .arg("--version") + .output() + .is_ok_and(|o| o.status.success()) + { + ("sha256sum", &[]) + } else { + ("shasum", &["-a", "256"]) + }; + let mut child = Command::new(program) + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .map_err(|e| Error::Spawn { + program: program.to_owned(), + detail: e.to_string(), + })?; + { + use std::io::Write as _; + let mut stdin = child.stdin.take().ok_or_else(|| Error::Process { + program: program.to_owned(), + detail: "no stdin".to_owned(), + })?; + stdin.write_all(bytes).map_err(|e| Error::Process { + program: program.to_owned(), + detail: e.to_string(), + })?; + } + let mut out = String::new(); + child + .stdout + .take() + .ok_or_else(|| Error::Process { + program: program.to_owned(), + detail: "no stdout".to_owned(), + })? + .read_to_string(&mut out) + .map_err(|e| Error::Process { + program: program.to_owned(), + detail: e.to_string(), + })?; + let status = child.wait().map_err(|e| Error::Process { + program: program.to_owned(), + detail: e.to_string(), + })?; + if !status.success() { + return Err(Error::Process { + program: program.to_owned(), + detail: "hashing failed".to_owned(), + }); + } + out.split_whitespace() + .next() + .map(str::to_owned) + .ok_or_else(|| Error::Process { + program: program.to_owned(), + detail: "no hash in output".to_owned(), + }) +} + +fn make_dir(path: &Path) -> Result<()> { + std::fs::create_dir_all(path).map_err(|source| Error::Io { + path: path.to_owned(), + source, + }) +} + +fn remove_dir(path: &Path) -> Result<()> { + std::fs::remove_dir_all(path).map_err(|source| Error::Io { + path: path.to_owned(), + source, + }) +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::embedded(Recipe::Embedded { tree: ObjectId::null(gix_hash::Kind::Sha1) })] + #[case::downloaded(Recipe::Downloaded { + components: vec![ + Component { url: "https://example.test/a.tar.gz".into(), sha256: "a".repeat(64), strip: 2, dest: String::new() }, + Component { url: "https://example.test/b.tar.gz".into(), sha256: "b".repeat(64), strip: 1, dest: "bin".into() }, + ], + })] + // @relation(effect.toolchains, model.toolchain, scope=function, role=Verifies) + fn recipe_round_trips_through_text(#[case] recipe: Recipe) { + let text = recipe.render(); + assert_eq!(Recipe::parse(&text).expect("parses"), recipe); + } + + #[rstest] + #[case::empty("")] + #[case::unknown_kind("frobnicated\n")] + #[case::embedded_missing_oid("embedded\n")] + #[case::downloaded_no_components("downloaded\n")] + // @relation(effect.toolchains, scope=function, role=Verifies) + fn parse_rejects_malformed_text(#[case] text: &str) { + Recipe::parse(text).expect_err("malformed"); + } + + #[rstest] + // @relation(effect.toolchains, scope=function, role=Verifies) + fn cache_key_differs_by_extraction_shape() { + let a = Recipe::Downloaded { + components: vec![Component { + url: "u".into(), + sha256: "s".repeat(64), + strip: 1, + dest: String::new(), + }], + }; + let b = Recipe::Downloaded { + components: vec![Component { + url: "u".into(), + sha256: "s".repeat(64), + strip: 2, + dest: String::new(), + }], + }; + assert_ne!(cache_key(&a), cache_key(&b)); + } +}

crates/ents-effect/src/unsandboxed.rs @@ -1,0 +1,104 @@ +//! Host-direct execution, with no sandbox at all (`effect.execution`: +//! "Host-direct execution... MUST require an explicit `--unsandboxed` flag +//! and MUST be available only locally, never on canonical hosted +//! infrastructure"). +//! +//! This module only provides the [`Executor`] implementation; enforcing +//! that it is reachable solely behind an explicit flag, and never wired at +//! a hosted composition root, is `roots.local`'s and the future CLI's job +//! (`roots.config-isolation`: selection happens at the root, never inside +//! a library). + +use std::process::Command; + +use crate::error::{Error, Result}; +use crate::executor::{Executor, RunOutput, RunStatus, SandboxInputs, activate}; + +/// [`Executor`] running a command directly on the host, with the tested +/// tree's checkout as its working directory and every declared toolchain's +/// `bin/` activated on `PATH` — no isolation whatsoever. +#[derive(Debug, Clone, Copy, Default)] +pub struct UnsandboxedExecutor; + +impl Executor for UnsandboxedExecutor { + fn run(&self, inputs: &SandboxInputs<'_>) -> Result<RunOutput> { + let dirs: Vec<(String, String)> = inputs + .toolchains + .iter() + .map(|(name, dir)| (name.clone(), dir.display().to_string())) + .collect(); + let output = Command::new("sh") + .arg("-c") + .arg(activate(inputs.command, &dirs)) + .current_dir(inputs.workdir) + .output() + .map_err(|e| Error::Spawn { + program: "sh".to_owned(), + detail: e.to_string(), + })?; + let mut log = String::from_utf8_lossy(&output.stdout).into_owned(); + log.push_str(&String::from_utf8_lossy(&output.stderr)); + let status = if output.status.success() { + RunStatus::Pass + } else { + RunStatus::Fail + }; + Ok(RunOutput { status, log }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::expect_used, reason = "unit test")] + + use std::path::PathBuf; + + use rstest::rstest; + + use super::*; + + #[rstest] + // @relation(effect.execution, scope=function, role=Verifies) + fn unsandboxed_reports_pass_on_exit_zero() { + let dir = tempfile::tempdir().expect("tempdir"); + let inputs = SandboxInputs { + workdir: dir.path(), + toolchains: &[], + command: "true", + }; + let output = UnsandboxedExecutor.run(&inputs).expect("runs"); + assert_eq!(output.status, RunStatus::Pass); + } + + #[rstest] + // @relation(effect.execution, scope=function, role=Verifies) + fn unsandboxed_reports_fail_on_nonzero_exit_never_as_an_error() { + let dir = tempfile::tempdir().expect("tempdir"); + let inputs = SandboxInputs { + workdir: dir.path(), + toolchains: &[], + command: "false", + }; + let output = UnsandboxedExecutor + .run(&inputs) + .expect("a completed run is never Err"); + assert_eq!(output.status, RunStatus::Fail); + } + + #[rstest] + // @relation(effect.execution, effect.toolchains, scope=function, role=Verifies) + fn unsandboxed_activates_declared_toolchains_on_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let bin = dir.path().join("bin"); + std::fs::create_dir_all(&bin).expect("mkdir"); + let toolchains = vec![("t".to_owned(), bin)]; + let inputs = SandboxInputs { + workdir: dir.path(), + toolchains: &toolchains, + command: "echo $PATH", + }; + let output = UnsandboxedExecutor.run(&inputs).expect("runs"); + assert!(output.log.contains("bin")); + let _: Vec<(String, PathBuf)> = toolchains; + } +}

effect: add ents-effect, the Executor trait and Docker/Sprite run loop

Reviews

Start a review