git ents: chore: merge Docker sandbox backend

crates/git-effect/src/cache.rs @@ -197,3 +197,97 @@ .store_tree_replace(&cache_ref(name), tree, "Update cache") .map_err(|e| format!("could not store cache {name}: {e}")) } + +/// [`restore`]'s local-backend equivalent: `dest` is already a host directory +/// (a [`crate::local::Sandbox`] bind-mounted straight into a Docker +/// container, or used as-is for host-direct execution), so restoring is just +/// extracting the snapshot tree onto it — no sandbox CLI transport needed. +/// +/// ## Requirements +/// +/// @relation(checks.cache) +pub fn restore_local(repo: &Path, dest: &Path, name: &str) -> Result<(), String> { + std::fs::create_dir_all(dest).map_err(|e| format!("could not create cache directory: {e}"))?; + + let store = git_store::Store::open(repo).map_err(|e| format!("could not open store: {e}"))?; + let Ok(tree) = store.ref_tree(&cache_ref(name)) else { + return Ok(()); + }; + + let archive = Command::new("git") + .arg("-C") + .arg(repo) + .args(["archive", "--format=tar", &tree.to_string()]) + .output() + .map_err(|e| format!("could not run git archive: {e}"))?; + if !archive.status.success() { + return Err(format!("git archive failed for cache {name}")); + } + + let mut child = Command::new("tar") + .args(["-x", "-C"]) + .arg(dest) + .stdin(Stdio::piped()) + .spawn() + .map_err(|e| format!("could not run tar: {e}"))?; + child + .stdin + .take() + .ok_or("tar did not accept stdin")? + .write_all(&archive.stdout) + .map_err(|e| format!("could not extract cache {name}: {e}"))?; + let status = child + .wait() + .map_err(|e| format!("tar did not complete: {e}"))?; + if status.success() { + Ok(()) + } else { + Err(format!("could not restore cache {name}")) + } +} + +/// [`snapshot`]'s local-backend equivalent: `src` is already a host +/// directory, so snapshotting is building a tree from it directly, without +/// first archiving it out of a sandbox. Uses a scratch index alongside `src` +/// (never inside it) for the same reason [`snapshot`] does: a `.git-index` +/// left inside `src` would poison the next run's restore/snapshot cycle. +/// +/// ## Requirements +/// +/// @relation(checks.cache) +pub fn snapshot_local(repo: &Path, src: &Path, name: &str) -> Result<(), String> { + let index_dir = + tempfile::tempdir().map_err(|e| format!("could not create scratch dir: {e}"))?; + let index = index_dir.path().join(".git-index"); + + let add = Command::new("git") + .arg("-C") + .arg(repo) + .env("GIT_INDEX_FILE", &index) + .env("GIT_WORK_TREE", src) + .args(["add", "-A", "."]) + .status() + .map_err(|e| format!("could not stage cache {name}'s tree: {e}"))?; + if !add.success() { + return Err(format!("could not stage cache {name}'s tree")); + } + let write_tree = Command::new("git") + .arg("-C") + .arg(repo) + .env("GIT_INDEX_FILE", &index) + .env("GIT_WORK_TREE", src) + .args(["write-tree"]) + .output() + .map_err(|e| format!("could not write cache {name}'s tree: {e}"))?; + if !write_tree.status.success() { + return Err(format!("could not write cache {name}'s tree")); + } + let tree = String::from_utf8_lossy(&write_tree.stdout); + let tree = ObjectId::from_hex(tree.trim().as_bytes()) + .map_err(|e| format!("git write-tree returned an invalid tree oid: {e}"))?; + + let store = git_store::Store::open(repo).map_err(|e| format!("could not open store: {e}"))?; + store + .store_tree_replace(&cache_ref(name), tree, "Update cache") + .map_err(|e| format!("could not store cache {name}: {e}")) +}

crates/git-effect/src/engine.rs @@ -36,6 +36,8 @@ use crate::cache; use crate::definition::{self, Effect}; +use crate::docker; +use crate::local; use crate::results::{self, RunOutcome, Status}; /// Where the pushed tree is unpacked inside the Sprite. @@ -190,8 +192,8 @@ /// /// ## Requirements /// -/// @relation(checks.worker, nonfunctional.concurrency) -pub async fn worker(queue: PathBuf, live: LiveRegistry) { +/// @relation(checks.worker, nonfunctional.concurrency, checks.sandbox) +pub async fn worker(queue: PathBuf, live: LiveRegistry, kind: BackendKind) { if let Err(e) = std::fs::create_dir_all(&queue) { eprintln!("effects: could not create queue directory {queue:?}: {e}"); return; @@ -209,7 +211,7 @@ } let inflight = Arc::clone(&inflight); let live = live.clone(); - let handle = tokio::task::spawn_blocking(move || drain_repo(&jobs, &live)); + let handle = tokio::task::spawn_blocking(move || drain_repo(&jobs, &live, kind)); tokio::spawn(async move { let _done = handle.await; inflight.lock().await.remove(&repo); @@ -218,6 +220,55 @@ } } +/// Which sandbox a job's effects run in. +/// +/// [`Sprite`](BackendKind::Sprite) is the hosted backend, driven through the +/// `sprite` CLI. [`Docker`](BackendKind::Docker) is the local default (`git +/// effect run`, and `git ents serve`'s own worker): a throwaway container per +/// effect, with toolchains materialized on the host and bind-mounted in — see +/// [`crate::local`] and [`crate::docker`]. [`Host`](BackendKind::Host) is +/// `--unsandboxed`: the command runs directly on the machine running the +/// worker, no isolation at all. +/// +/// ## Requirements +/// +/// @relation(checks.sandbox) +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BackendKind { + /// The hosted Fly.io Sprite backend. + Sprite, + /// The local Docker backend. + Docker, + /// Host-direct execution (`--unsandboxed`), no sandbox at all. + Host, +} + +/// The backend `git ents serve`/`git-ents-server` fall back to when not told +/// otherwise: [`BackendKind::Sprite`] when `SPRITES_TOKEN` is set in the +/// environment — the hosted deployment's own signal that a Sprite is +/// configured (see [`ensure_auth`]) — [`BackendKind::Docker`] otherwise. This +/// is exactly the Deployment table's split: hosted mode always carries +/// `SPRITES_TOKEN`, local `git ents serve` never does. +/// +/// ## Requirements +/// +/// @relation(checks.sandbox) +#[must_use] +pub fn default_backend() -> BackendKind { + backend_for_token(std::env::var("SPRITES_TOKEN").ok().as_deref()) +} + +/// [`default_backend`]'s pure decision, taking `SPRITES_TOKEN`'s value +/// directly rather than reading the environment — the part worth unit +/// testing without mutating process-global state. +fn backend_for_token(sprites_token: Option<&str>) -> BackendKind { + if sprites_token.is_some() { + BackendKind::Sprite + } else { + BackendKind::Docker + } +} + /// The pending jobs in the queue directory grouped by repository, so each /// repository can be drained independently. A malformed job file is dropped here /// rather than grouped — a poison job is never retried. @@ -246,34 +297,72 @@ /// Drain one repository's queued jobs in order, deleting each job file after it /// is handled (whether it ran cleanly or failed) so it is never retried. -fn drain_repo(jobs: &[(PathBuf, Job)], live: &LiveRegistry) { +fn drain_repo(jobs: &[(PathBuf, Job)], live: &LiveRegistry, kind: BackendKind) { for (path, job) in jobs { - if let Err(e) = process_job(job, live) { + if let Err(e) = process_job(job, live, kind) { eprintln!("effects: {e}"); } let _removed = std::fs::remove_file(path); } } -/// Run the effects for one queued push in its repository's Sprite, advancing -/// the recorded run as it goes: `running` while the Sprite is prepared, then -/// each effect flipped to its result as it finishes. Effects settle in the -/// dependency order `definition::order` fixed at write time: an effect whose -/// dependency did not pass is recorded `skipped` without touching the Sprite, -/// and a composite (no command) derives its status from its dependencies -/// alone. An infra failure (an unreachable Sprite, a tree that will not sync, -/// an effect set that fails re-validation) finalizes the run as `error` rather -/// than leaving it stuck at `running`, then returns `Err`. Returns `Ok` even -/// when an effect fails — a failing effect is a recorded result, not an error. +/// Run one queued job's effects in `kind`'s backend, discarding the outcomes +/// (already recorded — see [`run_all`]) since nothing else needs them here. /// /// ## Requirements /// /// @relation(checks.worker) -fn process_job(job: &Job, live: &LiveRegistry) -> Result<(), String> { +fn process_job(job: &Job, live: &LiveRegistry, kind: BackendKind) -> Result<(), String> { + run_all(&job.repo, job.new, &job.ref_name, kind, live)?; + Ok(()) +} + +/// Run every configured effect in `repo` against `at` outside the queue — +/// `git effect run`'s local execution path. Identical toolchain +/// materialization and sandbox path to a push-triggered run (see +/// [`run_all`]); only the queue is skipped, exactly as the porcelain +/// promises. `at` is a full hex commit id, already resolved by the caller. +/// +/// ## Requirements +/// +/// @relation(checks.worker, cli.account-checks) +pub fn run_effect_at( + repo: &Path, + at: &str, + kind: BackendKind, + live: &LiveRegistry, +) -> Result<Vec<RunOutcome>, String> { + let oid = ObjectId::from_hex(at.trim().as_bytes()) + .map_err(|e| format!("{at:?} is not a valid commit id: {e}"))?; + run_all(repo, oid, "<local run>", kind, live) +} + +/// Run every effect for `new` in `repo`'s given backend, advancing the +/// recorded run as it goes: `running` while the sandbox is prepared, then +/// each effect flipped to its result as it finishes. Effects settle in the +/// dependency order `definition::order` fixed at write time: an effect whose +/// dependency did not pass is recorded `skipped` without touching the +/// sandbox, and a composite (no command) derives its status from its +/// dependencies alone. An infra failure (an unreachable sandbox, a tree that +/// will not sync, an effect set that fails re-validation) finalizes the run +/// as `error` rather than leaving it stuck at `running`, then returns `Err`. +/// Returns the settled outcomes on success — even one that includes a +/// failing effect, which is a recorded result, not an error. +/// +/// ## Requirements +/// +/// @relation(checks.worker, checks.sandbox) +fn run_all( + repo: &Path, + new: ObjectId, + ref_name: &str, + kind: BackendKind, + live: &LiveRegistry, +) -> Result<Vec<RunOutcome>, String> { let runnable = - definition::load_all(&job.repo).map_err(|e| format!("could not read effects: {e}"))?; + definition::load_all(repo).map_err(|e| format!("could not read effects: {e}"))?; if runnable.is_empty() { - return Ok(()); + return Ok(Vec::new()); } let mut outcomes = statuses(&runnable, Status::Running); @@ -286,31 +375,38 @@ .filter_map(|effect| runnable.iter().position(|c| c.name == effect.name)) .collect(), Err(e) => { - finalize_error(&job.repo, job.new, &mut outcomes); + finalize_error(repo, new, &mut outcomes); return Err(format!("invalid effect set: {e}")); } }; - let sprite = sprite_name(&job.repo); - if let Err(e) = ensure_auth().and_then(|()| ensure_sprite(&sprite)) { - finalize_error(&job.repo, job.new, &mut outcomes); + + let backend = match Backend::new(kind, repo) { + Ok(backend) => backend, + Err(e) => { + finalize_error(repo, new, &mut outcomes); + return Err(e); + } + }; + if let Err(e) = backend.ensure() { + finalize_error(repo, new, &mut outcomes); return Err(e); } eprintln!( "effects: running {} effect(s) on {}", runnable.len(), - job.ref_name + ref_name ); - advance(&job.repo, job.new, &outcomes); - if let Err(e) = sync_tree(&job.repo, &sprite, job.new) { - finalize_error(&job.repo, job.new, &mut outcomes); + advance(repo, new, &outcomes); + if let Err(e) = backend.sync_tree(repo, new) { + finalize_error(repo, new, &mut outcomes); return Err(e); } - let toolchain_dirs = match resolve_toolchains(&job.repo, &sprite, &runnable) { + let toolchain_dirs = match backend.resolve_toolchains(repo, &runnable) { Ok(dirs) => dirs, Err(e) => { - finalize_error(&job.repo, job.new, &mut outcomes); + finalize_error(repo, new, &mut outcomes); return Err(e); } }; @@ -322,8 +418,8 @@ cache_names.sort_unstable(); cache_names.dedup(); for name in cache_names { - if let Err(e) = cache::restore(&job.repo, &sprite, name) { - finalize_error(&job.repo, job.new, &mut outcomes); + if let Err(e) = backend.restore_cache(repo, name) { + finalize_error(repo, new, &mut outcomes); return Err(e); } } @@ -347,13 +443,17 @@ match &effect.command { Some(command) if all_pass => { let command = activate(command, &effect.toolchains, &toolchain_dirs); - let command = with_cache_env(&command, effect.cache.as_deref()); - let key: LiveKey = (job.repo.clone(), job.new, effect.name.clone()); + let cache_dir = effect + .cache + .as_deref() + .map(|name| backend.cache_dir_for(name)); + let command = with_cache_env(&command, cache_dir.as_deref()); + let key: LiveKey = (repo.to_path_buf(), new, effect.name.clone()); let buffer = live_start(live, key.clone()); - let result = run_one(&sprite, &effect.name, &command, &buffer); + let result = backend.run_one(&effect.name, &command, &buffer); live_finish(live, &key); if let Some(name) = &effect.cache - && let Err(e) = cache::snapshot(&job.repo, &sprite, name) + && let Err(e) = backend.snapshot_cache(repo, name) { eprintln!("effects: could not snapshot cache {name}: {e}"); } @@ -382,9 +482,116 @@ } } } - advance(&job.repo, job.new, &outcomes); + advance(repo, new, &outcomes); + } + Ok(outcomes) +} + +/// One ready-to-use sandbox backend: the Sprite's name, or a fresh +/// [`local::Sandbox`] materialized on the host for the Docker or host-direct +/// backends. Constructing it is the one place a backend-specific setup +/// failure (no `docker` on `PATH`, no scratch directory) surfaces before any +/// sandbox work starts. +/// +/// ## Requirements +/// +/// @relation(checks.sandbox) +enum Backend { + Sprite(String), + Docker(local::Sandbox), + Host(local::Sandbox), +} + +impl Backend { + fn new(kind: BackendKind, repo: &Path) -> Result<Self, String> { + match kind { + BackendKind::Sprite => Ok(Backend::Sprite(sprite_name(repo))), + BackendKind::Docker => { + docker::ensure_docker()?; + Ok(Backend::Docker(local::Sandbox::new()?)) + } + BackendKind::Host => Ok(Backend::Host(local::Sandbox::new()?)), + } + } + + /// Sprite-only setup (auth, create-if-absent); the local backends need + /// none, since [`Backend::new`] already prepared their sandbox. + fn ensure(&self) -> Result<(), String> { + match self { + Backend::Sprite(name) => ensure_auth().and_then(|()| ensure_sprite(name)), + Backend::Docker(_) | Backend::Host(_) => Ok(()), + } + } + + fn sync_tree(&self, repo: &Path, new: ObjectId) -> Result<(), String> { + match self { + Backend::Sprite(name) => sync_tree(repo, name, new), + Backend::Docker(sandbox) | Backend::Host(sandbox) => { + local::sync_tree(repo, sandbox, new) + } + } + } + + /// A `name -> PATH entry` map: an in-Sprite/in-container path for the + /// Sprite and Docker backends, the real host path for host-direct + /// execution, since it runs with no container to bind-mount into. + fn resolve_toolchains( + &self, + repo: &Path, + runnable: &[Effect], + ) -> Result<HashMap<String, String>, String> { + match self { + Backend::Sprite(name) => resolve_toolchains(repo, name, runnable), + Backend::Docker(sandbox) => { + let names = local::resolve_toolchains(repo, sandbox, runnable)?; + Ok(names + .into_iter() + .map(|name| { + let dir = format!("{}/{name}/bin", docker::TOOLCHAINS_DIR); + (name, dir) + }) + .collect()) + } + Backend::Host(sandbox) => { + let names = local::resolve_toolchains(repo, sandbox, runnable)?; + Ok(local::host_toolchain_dirs(sandbox, &names)) + } + } + } + + fn restore_cache(&self, repo: &Path, name: &str) -> Result<(), String> { + match self { + Backend::Sprite(sprite) => cache::restore(repo, sprite, name), + Backend::Docker(sandbox) | Backend::Host(sandbox) => { + cache::restore_local(repo, &sandbox.cache_dir(name), name) + } + } + } + + fn snapshot_cache(&self, repo: &Path, name: &str) -> Result<(), String> { + match self { + Backend::Sprite(sprite) => cache::snapshot(repo, sprite, name), + Backend::Docker(sandbox) | Backend::Host(sandbox) => { + cache::snapshot_local(repo, &sandbox.cache_dir(name), name) + } + } + } + + fn cache_dir_for(&self, name: &str) -> String { + match self { + Backend::Sprite(_) => cache::cache_dir(name), + Backend::Docker(_) => format!("{}/{name}", docker::CACHE_DIR), + Backend::Host(sandbox) => sandbox.cache_dir(name).display().to_string(), + } + } + + fn run_one(&self, name: &str, command: &str, live: &Arc<StdMutex<String>>) -> RunResult { + match self { + Backend::Sprite(sprite) => run_one(sprite, name, command, live), + Backend::Docker(sandbox) => run_one_docker(sandbox, name, command, live), + Backend::Host(sandbox) => run_one_host(sandbox, name, command, live), + } } - Ok(()) } /// A composite effect's status, derived from its dependencies' settled @@ -749,20 +956,18 @@ format!("export PATH={path}:$PATH; {command}") } -/// Prefix `command` with an `EFFECT_CACHE_DIR` export pointing at `cache`'s -/// restored directory (see [`cache::restore`]), so the command can point a +/// Prefix `command` with an `EFFECT_CACHE_DIR` export pointing at +/// `cache_dir` (the cache's restored directory in whichever backend is +/// running — see [`Backend::cache_dir_for`]), so the command can point a /// tool (`sccache`, ...) at it; an effect with no cache is returned /// unchanged. /// /// ## Requirements /// /// @relation(checks.cache) -fn with_cache_env(command: &str, cache: Option<&str>) -> String { - match cache { - Some(name) => format!( - "export EFFECT_CACHE_DIR={}; {command}", - cache::cache_dir(name) - ), +fn with_cache_env(command: &str, cache_dir: Option<&str>) -> String { + match cache_dir { + Some(dir) => format!("export EFFECT_CACHE_DIR={dir}; {command}"), None => command.to_owned(), } } @@ -998,46 +1203,13 @@ drop(pair.slave); let master = pair.master; - let Ok(mut reader) = master.try_clone_reader() else { + let Ok(reader) = master.try_clone_reader() else { eprintln!("effects: ERROR {name} (could not read the pty)"); let _killed = child.kill(); return finish(Status::Error, start, None, live); }; - // The pty's `Read` is blocking, so it gets its own thread; the main thread - // times the whole run out against [`CHECK_TIMEOUT`] by bounding how long it - // waits on the channel rather than the read itself. - let (tx, rx) = std::sync::mpsc::channel::<Vec<u8>>(); - std::thread::spawn(move || { - let mut buf = [0u8; 4096]; - loop { - match reader.read(&mut buf) { - Ok(0) | Err(_) => break, - Ok(n) => { - let Some(chunk) = buf.get(..n) else { break }; - if tx.send(chunk.to_vec()).is_err() { - break; - } - } - } - } - }); - - let deadline = start.checked_add(CHECK_TIMEOUT).unwrap_or(start); - let timed_out = loop { - let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { - break true; - }; - match rx.recv_timeout(remaining) { - Ok(chunk) => { - let elapsed = start.elapsed().as_secs_f64(); - let data = String::from_utf8_lossy(&chunk); - push_event(&mut lock(live), elapsed, &data); - } - Err(RecvTimeoutError::Timeout) => break true, - Err(RecvTimeoutError::Disconnected) => break false, - } - }; + let timed_out = drain(reader, start, live); drop(master); if timed_out { @@ -1064,6 +1236,154 @@ } } +/// Run one effect in the Docker backend's throwaway `--rm` container, per +/// [`docker::run_args`]. Otherwise identical to [`run_one`]: same timeout, +/// same asciicast recording, same `live` buffer — just a plain pipe instead +/// of a pty, since nothing here needs an interactive terminal, only a +/// captured one. +/// +/// ## Requirements +/// +/// @relation(checks.sandbox) +fn run_one_docker( + sandbox: &local::Sandbox, + name: &str, + command: &str, + live: &Arc<StdMutex<String>>, +) -> RunResult { + let start = Instant::now(); + lock(live).push_str(&asciicast_header()); + + let args = docker::run_args( + &sandbox.work_dir(), + &sandbox.toolchains_dir(), + &sandbox.cache_root(), + command, + ); + let mut cmd = Command::new("docker"); + cmd.args(&args); + run_captured(&mut cmd, name, command, start, live) +} + +/// Run one effect directly on the host (`--unsandboxed`), in the sandbox's +/// materialized work directory — no container, no isolation. Otherwise +/// identical to [`run_one_docker`]. +/// +/// ## Requirements +/// +/// @relation(checks.sandbox) +fn run_one_host( + sandbox: &local::Sandbox, + name: &str, + command: &str, + live: &Arc<StdMutex<String>>, +) -> RunResult { + let start = Instant::now(); + lock(live).push_str(&asciicast_header()); + + let mut cmd = Command::new("sh"); + cmd.arg("-c") + .arg(format!("{command} 2>&1")) + .current_dir(sandbox.work_dir()); + run_captured(&mut cmd, name, command, start, live) +} + +/// Spawn `cmd` (already built, stdout not yet configured), capture its +/// combined output into `live` via [`drain`], and assemble the [`RunResult`] +/// — the part [`run_one_docker`] and [`run_one_host`] share. +fn run_captured( + cmd: &mut Command, + name: &str, + command: &str, + start: Instant, + live: &Arc<StdMutex<String>>, +) -> RunResult { + let mut child = match cmd.stdin(Stdio::null()).stdout(Stdio::piped()).spawn() { + Ok(child) => child, + Err(e) => { + eprintln!("effects: ERROR {name} (could not run: {e})"); + return finish(Status::Error, start, None, live); + } + }; + let Some(stdout) = child.stdout.take() else { + eprintln!("effects: ERROR {name} (could not capture output)"); + let _killed = child.kill(); + return finish(Status::Error, start, None, live); + }; + + let timed_out = drain(stdout, start, live); + if timed_out { + eprintln!("effects: ERROR {name} (timed out after {CHECK_TIMEOUT:?})"); + let _killed = child.kill(); + return finish(Status::Error, start, None, live); + } + + let status = match child.wait() { + Ok(status) => status, + Err(e) => { + eprintln!("effects: ERROR {name} (could not wait: {e})"); + return finish(Status::Error, start, None, live); + } + }; + + let exit_code = status.code(); + if status.success() { + eprintln!("effects: PASS {name}"); + finish(Status::Pass, start, exit_code, live) + } else { + eprintln!("effects: FAIL {name} ({command})"); + finish(Status::Fail, start, exit_code, live) + } +} + +/// Read `reader` until EOF or [`CHECK_TIMEOUT`] elapses since `start`, +/// appending each chunk to `live` as an asciicast v2 output event, exactly +/// the format [`run_one`]'s pty capture already produces. Shared by every +/// backend so the Checks tab's live/final recording looks the same +/// regardless of which one ran: the Sprite backend feeds this a pty's +/// reader, the Docker/host backends a plain child pipe. `reader`'s own +/// (blocking) read runs on a dedicated thread; the caller's thread only +/// waits on a channel, so it can time out the whole run without depending on +/// the read itself returning promptly. Returns whether the timeout (rather +/// than EOF) ended the read. +fn drain( + mut reader: impl Read + Send + 'static, + start: Instant, + live: &Arc<StdMutex<String>>, +) -> bool { + let (tx, rx) = std::sync::mpsc::channel::<Vec<u8>>(); + std::thread::spawn(move || { + let mut buf = [0u8; 4096]; + loop { + match reader.read(&mut buf) { + Ok(0) | Err(_) => break, + Ok(n) => { + let Some(chunk) = buf.get(..n) else { break }; + if tx.send(chunk.to_vec()).is_err() { + break; + } + } + } + } + }); + + let deadline = start.checked_add(CHECK_TIMEOUT).unwrap_or(start); + loop { + let Some(remaining) = deadline.checked_duration_since(Instant::now()) else { + return true; + }; + match rx.recv_timeout(remaining) { + Ok(chunk) => { + let elapsed = start.elapsed().as_secs_f64(); + let data = String::from_utf8_lossy(&chunk); + push_event(&mut lock(live), elapsed, &data); + } + Err(RecvTimeoutError::Timeout) => return true, + Err(RecvTimeoutError::Disconnected) => return false, + } + } +} + /// Assemble a [`RunResult`] from `live`'s accumulated recording — used on /// every exit path, including the failure ones, so an effect that errors out /// still keeps whatever terminal output it produced before that happened. @@ -1276,7 +1596,7 @@ #[test] fn with_cache_env_exports_the_restored_directory() { assert_eq!( - with_cache_env("cargo build", Some("sccache")), + with_cache_env("cargo build", Some("/cache/sccache")), "export EFFECT_CACHE_DIR=/cache/sccache; cargo build" ); } @@ -1333,4 +1653,58 @@ assert!(!queue.path().join("d.job").exists()); assert!(queue.path().join("ignored.tmp").exists()); } + + // @relation(checks.sandbox, role=Verifies) + #[test] + fn backend_for_token_is_docker_without_a_token() { + assert_eq!(backend_for_token(None), BackendKind::Docker); + } + + // @relation(checks.sandbox, role=Verifies) + #[test] + fn backend_for_token_is_sprite_with_a_token() { + assert_eq!(backend_for_token(Some("test-token")), BackendKind::Sprite); + } + + // @relation(checks.sandbox, role=Verifies) + #[test] + fn docker_backend_runs_a_trivial_effect() { + if docker::ensure_docker().is_err() { + eprintln!("skipping docker_backend_runs_a_trivial_effect: docker is not available"); + return; + } + + let repo = crate::testutil::unique_repo("docker-run"); + crate::testutil::write_effect_doc(&repo, "hello", "echo hi-from-docker"); + let status = Command::new("git") + .arg("-C") + .arg(&repo) + .args(["commit", "--allow-empty", "-q", "-m", "seed"]) + .status() + .unwrap(); + assert!(status.success()); + let head = Command::new("git") + .arg("-C") + .arg(&repo) + .args(["rev-parse", "HEAD"]) + .output() + .unwrap(); + assert!(head.status.success()); + let head = String::from_utf8(head.stdout).unwrap(); + + let live = new_live_registry(); + let outcomes = run_effect_at(&repo, head.trim(), BackendKind::Docker, &live).unwrap(); + let outcome = outcomes + .iter() + .find(|outcome| outcome.name == "hello") + .unwrap(); + assert_eq!(outcome.status, Status::Pass); + assert!( + outcome + .recording + .as_deref() + .unwrap_or_default() + .contains("hi-from-docker") + ); + } }

crates/git-effect/src/lib.rs @@ -15,7 +15,9 @@ pub mod cache; pub mod definition; +pub mod docker; pub mod engine; +pub mod local; pub mod results; #[cfg(test)] mod testutil;

crates/git-ents-server/src/lib.rs @@ -179,10 +179,14 @@ live_runs: git_effect::engine::new_live_registry(), }; - // Drain queued pushes and run their effects for the life of the server. + // Drain queued pushes and run their effects for the life of the server: + // the Sprite backend when `SPRITES_TOKEN` says this is the hosted + // deployment, the local Docker backend otherwise — see + // `git_effect::engine::default_backend`. tokio::spawn(git_effect::engine::worker( state.checks_queue.clone(), state.live_runs.clone(), + git_effect::engine::default_backend(), )); // @relation(protocol.routing, deploy.health)

crates/git-ents/src/main.rs @@ -242,6 +242,23 @@ /// Show recorded effect runs (queued/running/pass/fail/error) from /// `refs/meta/results/*` on a remote, newest first. Log, + /// Run this repository's effects locally against `at`, identical + /// toolchain materialization and sandbox path to a push-triggered run — + /// the queue is skipped, nothing else differs. Runs in the local Docker + /// sandbox by default; `--unsandboxed` runs directly on the host + /// instead. + Run { + /// Name (`effects/<name>`) whose result to report. + #[facet(args::positional)] + name: String, + /// Commit-ish to check (defaults to `HEAD`). + #[facet(args::named)] + at: Option<String>, + /// Run directly on the host instead of in the Docker sandbox — no + /// isolation; this used to be local execution's only mode. + #[facet(args::named, default)] + unsandboxed: bool, + }, } /// ## Requirements @@ -504,6 +521,46 @@ EffectAction::Remove { name } => effect_remove(&name, remote), EffectAction::Debug => effect_debug(remote), EffectAction::Log => effect_log(remote), + EffectAction::Run { + name, + at, + unsandboxed, + } => effect_run(&name, at.as_deref(), unsandboxed), + } +} + +/// Run this repository's effects locally against `at` (default `HEAD`), +/// printing `name`'s settled outcome — the local execution path: identical +/// toolchain materialization and sandbox as a push, minus the queue. Runs in +/// the local Docker sandbox by default; `unsandboxed` runs directly on the +/// host instead. +/// +/// ## Requirements +/// +/// @relation(cli.account-checks, checks.sandbox) +fn effect_run(name: &str, at: Option<&str>, unsandboxed: bool) -> Result<(), String> { + let repo = repo()?; + let rev = at.unwrap_or("HEAD"); + let commit = git_capture(&["-C", &repo.to_string_lossy(), "rev-parse", "--verify", rev])?; + + let kind = if unsandboxed { + git_effect::engine::BackendKind::Host + } else { + git_effect::engine::BackendKind::Docker + }; + let live = git_effect::engine::new_live_registry(); + let outcomes = git_effect::engine::run_effect_at(&repo, commit.trim(), kind, &live) + .map_err(|e| format!("effects: {e}"))?; + let outcome = outcomes + .iter() + .find(|outcome| outcome.name == name) + .ok_or_else(|| format!("no effect named {name} is configured"))?; + println!("{}: {}", outcome.name, outcome.status); + match outcome.status { + git_effect::Status::Fail | git_effect::Status::Error => { + Err(format!("effect {name} did not pass")) + } + _ => Ok(()), } }

crates/git-effect/src/docker.rs @@ -1,0 +1,116 @@ +//! Docker sandbox backend for local effect execution: shells out to the +//! `docker` CLI via `std::process` (no docker API crate — see +//! [`crate::engine`] for why the Sprite backend does the same with `sprite`), +//! running each effect in a throwaway `--rm` container with the +//! [`crate::local::Sandbox`] materialized on the host bind-mounted in. Unlike +//! the Sprite backend's persistent per-repository sandbox, a container never +//! outlives its run, so nothing here needs an extract-once cache: toolchains +//! are re-materialized on the host per run (cheap — it is a local `git +//! archive`/tree walk, not a network fetch) rather than kept warm across +//! runs. +//! +//! `git effect run` uses this backend by default; `--unsandboxed` skips it +//! for host-direct execution instead (see [`crate::local`]). + +use std::path::Path; + +/// The minimal base image every effect runs in — no toolchain of its own; +/// everything the command needs comes from the bind-mounted, host-exported +/// toolchains. +pub const IMAGE: &str = "debian:stable-slim"; + +/// Where the sandbox's work directory is bind-mounted in the container. +pub const WORKDIR: &str = "/work"; + +/// Where the sandbox's toolchains directory is bind-mounted in the +/// container, read-only — toolchains are extract-once-per-run and never +/// written to by the command. +pub const TOOLCHAINS_DIR: &str = "/toolchains"; + +/// Where the sandbox's cache directory is bind-mounted in the container, +/// read-write. +pub const CACHE_DIR: &str = "/cache"; + +/// Confirm `docker` is on `PATH` and the daemon answers, with a clean error +/// (rather than a raw "os error 2") when it is not — the one place this +/// backend can fail before anything else runs. +pub fn ensure_docker() -> Result<(), String> { + let status = std::process::Command::new("docker") + .arg("version") + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .status() + .map_err(|e| format!("docker is not installed or not on PATH: {e}"))?; + if status.success() { + Ok(()) + } else { + Err("docker is installed but the daemon did not respond (`docker version` failed); is it running?".to_owned()) + } +} + +/// Assemble `docker run`'s argv for one effect's `command` against the +/// sandbox's host directories — pure, so the exact invocation is unit tested +/// without a daemon. `command` runs under `sh -c`, stderr folded into stdout +/// so the captured recording is one interleaved stream, matching what the +/// Sprite backend's pty capture already gives a developer. +#[must_use] +pub fn run_args(work: &Path, toolchains: &Path, cache: &Path, command: &str) -> Vec<String> { + vec![ + "run".to_owned(), + "--rm".to_owned(), + "-v".to_owned(), + format!("{}:{WORKDIR}", work.display()), + "-v".to_owned(), + format!("{}:{TOOLCHAINS_DIR}:ro", toolchains.display()), + "-v".to_owned(), + format!("{}:{CACHE_DIR}", cache.display()), + "-w".to_owned(), + WORKDIR.to_owned(), + IMAGE.to_owned(), + "sh".to_owned(), + "-c".to_owned(), + format!("{command} 2>&1"), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + // @relation(checks.sandbox, role=Verifies) + #[test] + fn run_args_binds_work_toolchains_and_cache() { + let args = run_args( + Path::new("/tmp/s/work"), + Path::new("/tmp/s/toolchains"), + Path::new("/tmp/s/cache"), + "cargo test", + ); + assert_eq!( + args, + vec![ + "run", + "--rm", + "-v", + "/tmp/s/work:/work", + "-v", + "/tmp/s/toolchains:/toolchains:ro", + "-v", + "/tmp/s/cache:/cache", + "-w", + "/work", + IMAGE, + "sh", + "-c", + "cargo test 2>&1", + ] + ); + } + + // @relation(checks.sandbox, role=Verifies) + #[test] + fn run_args_uses_the_minimal_base_image() { + let args = run_args(Path::new("/w"), Path::new("/t"), Path::new("/c"), "true"); + assert_eq!(args.get(args.len() - 4).map(String::as_str), Some(IMAGE)); + } +}

crates/git-effect/src/local.rs @@ -1,0 +1,188 @@ +//! Host-side materialization shared by every *local* effect backend (Docker, +//! and host-direct/`--unsandboxed`) — the Fly.io Sprite backend +//! ([`crate::engine`]) instead streams bytes into the Sprite's own +//! filesystem, since there is no host directory to bind-mount there. +//! +//! A [`Sandbox`] is one effect run's scratch area: a fresh temp directory +//! holding the checked-out tree (`work`), every declared toolchain's +//! extracted `bin` (`toolchains/<name>`), and every declared cache +//! (`cache/<name>`) — laid out on the *host* filesystem so the Docker backend +//! can bind-mount it straight into the container, and host-direct execution +//! can just point `PATH`/`$PWD` at it. Toolchain extraction goes through +//! [`git_toolchain::export`], the same function the Sprite path's own doc +//! comments call out as its local/hosted parity anchor, so a toolchain's +//! materialized bytes are identical no matter which backend runs it. + +use std::collections::HashMap; +use std::io::Write as _; +use std::path::{Path, PathBuf}; +use std::process::Stdio; + +use gix_hash::ObjectId; +use std::process::Command; + +use crate::definition::Effect; + +/// One effect run's host-side scratch area, torn down when dropped. +pub struct Sandbox { + root: tempfile::TempDir, +} + +impl Sandbox { + /// A fresh sandbox with empty `work`/`toolchains`/`cache` directories. + pub fn new() -> Result<Self, String> { + let root = tempfile::tempdir().map_err(|e| format!("could not create scratch dir: {e}"))?; + for name in ["work", "toolchains", "cache"] { + std::fs::create_dir_all(root.path().join(name)) + .map_err(|e| format!("could not create {name} dir: {e}"))?; + } + Ok(Self { root }) + } + + /// The checked-out tree's directory. + #[must_use] + pub fn work_dir(&self) -> PathBuf { + self.root.path().join("work") + } + + /// The parent of every extracted toolchain's `<name>` directory. + #[must_use] + pub fn toolchains_dir(&self) -> PathBuf { + self.root.path().join("toolchains") + } + + /// The parent of every restored cache's `<name>` directory. + #[must_use] + pub fn cache_root(&self) -> PathBuf { + self.root.path().join("cache") + } + + /// Where cache `name` is restored, created even absent a prior snapshot + /// so a tool populating it fresh always finds it there. + #[must_use] + pub fn cache_dir(&self, name: &str) -> PathBuf { + self.cache_root().join(name) + } +} + +/// Replace the sandbox's [`Sandbox::work_dir`] with the tree at `new`, via +/// `git archive | tar -x` straight onto the host filesystem — no sandbox CLI +/// involved, unlike the Sprite path's streamed unpack. +pub fn sync_tree(repo: &Path, sandbox: &Sandbox, new: ObjectId) -> Result<(), String> { + let archive = Command::new("git") + .arg("-C") + .arg(repo) + .args(["archive", "--format=tar", &new.to_string()]) + .output() + .map_err(|e| format!("could not run git archive: {e}"))?; + if !archive.status.success() { + return Err(format!("git archive failed for {new}")); + } + let work = sandbox.work_dir(); + let mut child = Command::new("tar") + .args(["-x", "-C"]) + .arg(&work) + .stdin(Stdio::piped()) + .spawn() + .map_err(|e| format!("could not run tar: {e}"))?; + child + .stdin + .take() + .ok_or("tar did not accept stdin")? + .write_all(&archive.stdout) + .map_err(|e| format!("could not extract the tree: {e}"))?; + let status = child + .wait() + .map_err(|e| format!("tar did not complete: {e}"))?; + if status.success() { + Ok(()) + } else { + Err(format!("could not unpack the tree at {new}")) + } +} + +/// Resolve and extract every distinct toolchain named across `runnable` into +/// `sandbox.toolchains_dir()/<name>/bin` via [`git_toolchain::export`], +/// returning the resolved (deduplicated) names — the exported bytes are +/// identical regardless of whether `bin` is [`git_toolchain::Bin::Embedded`] +/// or [`git_toolchain::Bin::Downloaded`], since `export` normalizes both to +/// the same `<dest>/bin/…` shape. +pub fn resolve_toolchains( + repo: &Path, + sandbox: &Sandbox, + runnable: &[Effect], +) -> Result<Vec<String>, String> { + let mut names: Vec<&str> = runnable + .iter() + .flat_map(|effect| effect.toolchains.iter().map(String::as_str)) + .collect(); + names.sort_unstable(); + names.dedup(); + + for name in &names { + let dest = sandbox.toolchains_dir().join(name); + if dest.exists() { + continue; + } + git_toolchain::export(repo, name, &dest) + .map_err(|e| format!("could not resolve toolchain {name}: {e}"))?; + } + Ok(names.into_iter().map(str::to_owned).collect()) +} + +/// The container/host-relative path a toolchain named `name` was exported to +/// (see [`resolve_toolchains`]), for building an `activate()` `PATH`. +#[must_use] +pub fn toolchain_bin_dir(sandbox: &Sandbox, name: &str) -> PathBuf { + sandbox.toolchains_dir().join(name).join("bin") +} + +/// A `name -> bin dir` map from `names`, each pointing at its host path under +/// `sandbox` — used by host-direct execution, which runs outside any +/// container and so needs the real host path rather than a bind-mounted +/// in-container one. +#[must_use] +pub fn host_toolchain_dirs(sandbox: &Sandbox, names: &[String]) -> HashMap<String, String> { + names + .iter() + .map(|name| { + ( + name.clone(), + toolchain_bin_dir(sandbox, name).display().to_string(), + ) + }) + .collect() +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used, clippy::indexing_slicing, reason = "unit test")] + + use super::*; + + // @relation(checks.sandbox, role=Verifies) + #[test] + fn sandbox_starts_with_empty_work_toolchains_cache() { + let sandbox = Sandbox::new().unwrap(); + assert!(sandbox.work_dir().is_dir()); + assert!(sandbox.toolchains_dir().is_dir()); + assert!(sandbox.cache_root().is_dir()); + } + + // @relation(checks.sandbox, role=Verifies) + #[test] + fn host_toolchain_dirs_map_to_the_sandbox_bin_directory() { + let sandbox = Sandbox::new().unwrap(); + let names = vec!["gcc".to_owned()]; + let dirs = host_toolchain_dirs(&sandbox, &names); + assert_eq!( + dirs["gcc"], + sandbox + .toolchains_dir() + .join("gcc") + .join("bin") + .display() + .to_string() + ); + } +}

chore: merge Docker sandbox backend

Reviews

Start a review