git-ents.gitmain
⌘K
foforge

crates/kernel/ents-effect/src/toolchain.rs

toolchain.rs634 lines · 22.5 KB · rusthistorycomment on this file
1//! Toolchain resolution and materialization (`effect.toolchains`,
2//! `model.toolchain`).
3//!
4//! [`ents_model::Toolchain::recipe`] is deliberately an opaque `String` —
5//! `model.toolchain`'s own doc names this crate as the one that gives it
6//! structure. [`Recipe`] is that structure: a toolchain's `bin` is either
7//! [`Recipe::Embedded`] (a tree already in the object database, captured
8//! whole by whatever wrote the toolchain) or [`Recipe::Downloaded`] (a set
9//! of externally-hosted, sha256-pinned archives), ported from `pre-redo`'s
10//! `git_toolchain::Bin` — the design pre-redo settled on and this phase
11//! carries forward, not a fresh design. [`Recipe::render`]/[`Recipe::parse`]
12//! round-trip it through the plain-text `recipe` field.
13//!
14//! [`materialize`] resolves a toolchain to a host directory containing its
15//! activated `bin/`, extract-once cached under a content key (a tree oid
16//! for `Embedded`, a hash of each component's pin for `Downloaded`) so a
17//! backend that runs the same toolchain repeatedly (a Sprite's persistent
18//! filesystem, a developer's local cache) never re-extracts unchanged
19//! bytes. Fetching a [`Recipe::Downloaded`] component shells to `curl`,
20//! `tar`, and `sha256sum`/`shasum`, the same pattern `pre-redo` used and
21//! this phase ports rather than adding an HTTP or hashing dependency.
22
23use std::io::Read as _;
24use std::path::{Path, PathBuf};
25use std::process::{Command, Stdio};
26
27use ents_model::{Toolchain, namespace};
28use gix_hash::ObjectId;
29use gix_object::{CommitRef, Find, Kind};
30use gix_ref_store::RefStoreRead;
31
32use crate::error::{Error, Result};
33
34/// How a toolchain's `bin` is provisioned — the structure inside
35/// [`ents_model::Toolchain::recipe`] (`effect.toolchains`).
36#[derive(Debug, Clone, PartialEq, Eq)]
37pub enum Recipe {
38 /// `bin`'s directory tree, already in the object database — the whole
39 /// tree's entries become the toolchain's activated `bin/` contents.
40 Embedded {
41 /// The tree object id.
42 tree: ObjectId,
43 },
44 /// A set of archives fetched, sha256-verified, and merged onto disk at
45 /// materialization time.
46 Downloaded {
47 /// Each archive making up the toolchain.
48 components: Vec<Component>,
49 },
50}
51
52/// One archive making up a [`Recipe::Downloaded`] toolchain: fetched from
53/// `url` and checked against `sha256` before being extracted per
54/// `strip`/`dest` — ported verbatim from `pre-redo`'s
55/// `git_toolchain::Component`.
56#[derive(Debug, Clone, PartialEq, Eq)]
57pub struct Component {
58 /// Where to fetch the archive from.
59 pub url: String,
60 /// The archive's expected sha256, hex-encoded.
61 pub sha256: String,
62 /// Leading path segments `tar` strips at extraction.
63 pub strip: u8,
64 /// Subdirectory under the toolchain's `bin/` extraction root to extract
65 /// into: empty for an archive that already carries its own `bin/` top
66 /// level, `bin` for a flat archive whose payload should itself land on
67 /// `PATH`.
68 pub dest: String,
69}
70
71const EMBEDDED: &str = "embedded";
72const DOWNLOADED: &str = "downloaded";
73
74impl Recipe {
75 /// Parse a [`Recipe`] out of a [`ents_model::Toolchain::recipe`] string.
76 ///
77 /// The format is deliberately small rather than a general one (no new
78 /// dependency for two variants and four fields): one line naming the
79 /// kind, then either the embedded tree's hex oid, or one
80 /// `url sha256 strip dest` line per component (`dest` last, so it may
81 /// be empty without ambiguity).
82 ///
83 /// # Errors
84 ///
85 /// [`Error::InvalidRecipe`] if the text does not match this shape.
86 ///
87 /// # Examples
88 ///
89 /// ```
90 /// use ents_effect::Recipe;
91 ///
92 /// let text = "embedded 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n";
93 /// let recipe = Recipe::parse(text).expect("parses");
94 /// assert!(matches!(recipe, Recipe::Embedded { .. }));
95 /// ```
96 pub fn parse(text: &str) -> Result<Self> {
97 let mut lines = text.lines().filter(|line| !line.trim().is_empty());
98 let Some(first) = lines.next() else {
99 return Err(invalid("empty recipe"));
100 };
101 let mut words = first.split_whitespace();
102 match words.next() {
103 Some(EMBEDDED) => {
104 let hex = words
105 .next()
106 .ok_or_else(|| invalid("embedded recipe missing a tree oid"))?;
107 let tree = ObjectId::from_hex(hex.as_bytes())
108 .map_err(|e| invalid(format!("invalid tree oid {hex:?}: {e}")))?;
109 Ok(Self::Embedded { tree })
110 }
111 Some(DOWNLOADED) => {
112 let mut components = Vec::new();
113 for line in lines {
114 let mut fields = line.split_whitespace();
115 let url = fields
116 .next()
117 .ok_or_else(|| invalid("component line missing a url"))?
118 .to_owned();
119 let sha256 = fields
120 .next()
121 .ok_or_else(|| invalid("component line missing a sha256"))?
122 .to_owned();
123 let strip = fields
124 .next()
125 .ok_or_else(|| invalid("component line missing a strip count"))?
126 .parse::<u8>()
127 .map_err(|e| invalid(format!("invalid strip count: {e}")))?;
128 let dest = fields.next().unwrap_or("").to_owned();
129 let component = Component {
130 url,
131 sha256,
132 strip,
133 dest,
134 };
135 validate_component(&component)?;
136 components.push(component);
137 }
138 if components.is_empty() {
139 return Err(invalid(
140 "a downloaded toolchain must list at least one component",
141 ));
142 }
143 Ok(Self::Downloaded { components })
144 }
145 Some(other) => Err(invalid(format!("unknown recipe kind {other:?}"))),
146 None => Err(invalid("empty recipe")),
147 }
148 }
149
150 /// Render this [`Recipe`] back into the text stored in
151 /// [`ents_model::Toolchain::recipe`].
152 ///
153 /// # Examples
154 ///
155 /// ```
156 /// use ents_effect::Recipe;
157 ///
158 /// let recipe = Recipe::Embedded {
159 /// tree: gix_hash::ObjectId::null(gix_hash::Kind::Sha1),
160 /// };
161 /// let text = recipe.render();
162 /// assert_eq!(Recipe::parse(&text).expect("round-trips"), recipe);
163 /// ```
164 #[must_use]
165 pub fn render(&self) -> String {
166 match self {
167 Self::Embedded { tree } => format!("{EMBEDDED} {tree}\n"),
168 Self::Downloaded { components } => {
169 let mut out = format!("{DOWNLOADED}\n");
170 for c in components {
171 out.push_str(&format!("{} {} {} {}\n", c.url, c.sha256, c.strip, c.dest));
172 }
173 out
174 }
175 }
176 }
177}
178
179/// Read the [`Toolchain`] entity named `name` from
180/// `refs/meta/toolchains/<name>`, and parse its [`Toolchain::recipe`] as a
181/// [`Recipe`].
182///
183/// # Errors
184///
185/// [`Error::UnknownToolchain`] when the ref does not exist or does not
186/// resolve to a commit tree; [`Error::InvalidRecipe`] when its `recipe`
187/// field does not parse.
188///
189/// # Examples
190///
191/// ```
192/// use ents_effect::toolchain::resolve;
193/// use ents_model::Toolchain;
194/// use ents_testutil::{MemRefStore, ObjectStore, write_meta_entity};
195///
196/// let refs = MemRefStore::default();
197/// let objects = ObjectStore::default();
198/// let toolchain = Toolchain {
199/// name: "rust-stable".into(),
200/// recipe: "embedded 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n".into(),
201/// };
202/// let name: gix::refs::FullName = "refs/meta/toolchains/rust-stable".try_into().expect("valid");
203/// write_meta_entity(&refs, &objects, name, &toolchain, None, 100);
204///
205/// let (entity, recipe) = resolve(&refs, &objects, "rust-stable").expect("resolves");
206/// assert_eq!(entity.name, "rust-stable");
207/// assert!(matches!(recipe, ents_effect::Recipe::Embedded { .. }));
208/// ```
209pub fn resolve(
210 refs: &dyn RefStoreRead,
211 objects: &impl Find,
212 name: &str,
213) -> Result<(Toolchain, Recipe)> {
214 let refname = namespace::toolchain_ref(name)
215 .map_err(|e| Error::UnknownToolchain(format!("{name}: {e}")))?;
216 let Some(tip) = refs.get(refname.as_ref())? else {
217 return Err(Error::UnknownToolchain(name.to_owned()));
218 };
219 let mut buf = Vec::new();
220 let data = objects
221 .try_find(&tip, &mut buf)
222 .map_err(|source| Error::Decode {
223 oid: tip,
224 detail: source.to_string(),
225 })?
226 .ok_or(Error::Missing { oid: tip })?;
227 if data.kind != Kind::Commit {
228 return Err(Error::Decode {
229 oid: tip,
230 detail: "toolchain ref does not point at a commit".to_owned(),
231 });
232 }
233 let commit = CommitRef::from_bytes(data.data, tip.kind()).map_err(|e| Error::Decode {
234 oid: tip,
235 detail: e.to_string(),
236 })?;
237 let toolchain: Toolchain = facet_git_tree::deserialize(&commit.tree(), objects)?;
238 let recipe = Recipe::parse(&toolchain.recipe).map_err(|e| match e {
239 Error::InvalidRecipe { detail, .. } => Error::InvalidRecipe {
240 name: name.to_owned(),
241 detail,
242 },
243 other => other,
244 })?;
245 Ok((toolchain, recipe))
246}
247
248fn invalid(detail: impl Into<String>) -> Error {
249 Error::InvalidRecipe {
250 name: String::new(),
251 detail: detail.into(),
252 }
253}
254
255/// Refuse a [`Component`] whose fields are unsafe downstream: `sha256` and
256/// `dest` become filesystem path segments ([`cache_key`]) and, via the
257/// Sprite backend's extract-once check, part of an in-sandbox `sh -c`
258/// string; `url` is handed to `curl` and quoted contexts. Checked at
259/// [`Recipe::parse`] so a hostile recipe never reaches those sites, and
260/// re-checked before any fetch for a [`Recipe`] constructed directly in
261/// code.
262fn validate_component(component: &Component) -> Result<()> {
263 if component.sha256.len() != 64 || !component.sha256.bytes().all(|b| b.is_ascii_hexdigit()) {
264 return Err(Error::InvalidComponent(format!(
265 "sha256 must be 64 hex characters, got {:?}",
266 component.sha256
267 )));
268 }
269 if component.url.is_empty()
270 || component.url.contains('\'')
271 || component.url.chars().any(char::is_whitespace)
272 {
273 return Err(Error::InvalidComponent(format!(
274 "unsafe url {:?}",
275 component.url
276 )));
277 }
278 let dest_ok = component.dest.is_empty()
279 || (component.dest != "."
280 && component.dest != ".."
281 && component
282 .dest
283 .bytes()
284 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')));
285 if !dest_ok {
286 return Err(Error::InvalidComponent(format!(
287 "unsafe dest {:?}",
288 component.dest
289 )));
290 }
291 Ok(())
292}
293
294/// A stable, filesystem-safe cache key for a [`Recipe`]: the embedded
295/// tree's own hex oid, or each downloaded component's sha256 joined in
296/// extraction order — the same bytes extracted differently (a different
297/// `strip`/`dest`) are a different toolchain on disk, so those fields join
298/// the key too.
299#[must_use]
300pub fn cache_key(recipe: &Recipe) -> String {
301 match recipe {
302 Recipe::Embedded { tree } => tree.to_string(),
303 Recipe::Downloaded { components } => components
304 .iter()
305 .map(|c| format!("{}.{}.{}", c.sha256, c.strip, c.dest))
306 .collect::<Vec<_>>()
307 .join("-"),
308 }
309}
310
311/// Resolve `recipe` to a host directory containing the toolchain's
312/// activated `bin/`, extracted once under `cache_root` and reused on every
313/// later call with the same recipe (`effect.toolchains`: "resolved during
314/// effect execution").
315///
316/// # Errors
317///
318/// [`Error::Submodule`] or [`Error::NotUtf8`] for a tree this crate cannot
319/// materialize; [`Error::Spawn`]/[`Error::Process`]/[`Error::HashMismatch`]
320/// for a downloaded component that could not be fetched, verified, or
321/// extracted; [`Error::Io`] for a host filesystem failure.
322///
323/// # Examples
324///
325/// An embedded recipe over the empty tree materializes an empty `bin/`.
326///
327/// ```
328/// use ents_effect::Recipe;
329/// use ents_effect::toolchain::materialize;
330/// use ents_testutil::ObjectStore;
331/// use gix_object::{Kind, Write as _};
332///
333/// let objects = ObjectStore::default();
334/// let empty = objects.write_buf(Kind::Tree, b"").expect("write");
335/// let recipe = Recipe::Embedded { tree: empty };
336///
337/// let dir = tempfile::tempdir().expect("tempdir");
338/// let bin = materialize(&recipe, &objects, dir.path()).expect("materializes");
339/// assert!(bin.ends_with("bin"));
340/// assert!(bin.is_dir());
341/// ```
342pub fn materialize(recipe: &Recipe, objects: &impl Find, cache_root: &Path) -> Result<PathBuf> {
343 // Validate before `cache_key` embeds any component field in a path:
344 // `Recipe::parse` already refused a hostile recipe, but a `Recipe`
345 // built directly in code never went through parse.
346 if let Recipe::Downloaded { components } = recipe {
347 for component in components {
348 validate_component(component)?;
349 }
350 }
351 let root = cache_root.join(cache_key(recipe));
352 let bin = root.join("bin");
353 if bin.is_dir() {
354 return Ok(bin);
355 }
356 let tmp = cache_root.join(format!("{}.tmp", cache_key(recipe)));
357 if tmp.exists() {
358 remove_dir(&tmp)?;
359 }
360 make_dir(&tmp)?;
361
362 match recipe {
363 Recipe::Embedded { tree } => {
364 let bin_tmp = tmp.join("bin");
365 make_dir(&bin_tmp)?;
366 crate::materialize::checkout(objects, *tree, &bin_tmp)?;
367 }
368 Recipe::Downloaded { components } => {
369 make_dir(&tmp.join("bin"))?;
370 for component in components {
371 fetch_component(component, &tmp)?;
372 }
373 }
374 }
375
376 // Land atomically: a transient failure partway through must never leave
377 // `bin` existing-but-incomplete, or the next call would trust a half
378 // extraction forever.
379 std::fs::rename(&tmp, &root).map_err(|source| Error::Io {
380 path: root.clone(),
381 source,
382 })?;
383 Ok(bin)
384}
385
386fn fetch_component(component: &Component, root: &Path) -> Result<()> {
387 validate_component(component)?;
388 let dest = if component.dest.is_empty() {
389 root.join("bin")
390 } else {
391 root.join("bin").join(&component.dest)
392 };
393 make_dir(&dest)?;
394
395 let bytes = fetch(&component.url)?;
396 let actual = sha256_hex(&bytes)?;
397 if !actual.eq_ignore_ascii_case(&component.sha256) {
398 return Err(Error::HashMismatch {
399 url: component.url.clone(),
400 expected: component.sha256.clone(),
401 actual,
402 });
403 }
404
405 let mut child = Command::new("tar")
406 .args([
407 "-x",
408 "-C",
409 dest.to_str().ok_or_else(|| Error::NotUtf8(dest.clone()))?,
410 &format!("--strip-components={}", component.strip),
411 ])
412 .stdin(Stdio::piped())
413 .stdout(Stdio::null())
414 .stderr(Stdio::piped())
415 .spawn()
416 .map_err(|e| Error::Spawn {
417 program: "tar".to_owned(),
418 detail: e.to_string(),
419 })?;
420 {
421 use std::io::Write as _;
422 let mut stdin = child.stdin.take().ok_or_else(|| Error::Process {
423 program: "tar".to_owned(),
424 detail: "no stdin".to_owned(),
425 })?;
426 stdin.write_all(&bytes).map_err(|e| Error::Process {
427 program: "tar".to_owned(),
428 detail: e.to_string(),
429 })?;
430 }
431 let output = child.wait_with_output().map_err(|e| Error::Process {
432 program: "tar".to_owned(),
433 detail: e.to_string(),
434 })?;
435 if !output.status.success() {
436 return Err(Error::Process {
437 program: "tar".to_owned(),
438 detail: String::from_utf8_lossy(&output.stderr).trim().to_owned(),
439 });
440 }
441 Ok(())
442}
443
444/// `GET url`, via the system `curl` — shells out rather than adding an HTTP
445/// dependency, the same rationale `pre-redo` used.
446fn fetch(url: &str) -> Result<Vec<u8>> {
447 let output = Command::new("curl")
448 .args(["-sSL", "--fail", url])
449 .output()
450 .map_err(|e| Error::Spawn {
451 program: "curl".to_owned(),
452 detail: e.to_string(),
453 })?;
454 if !output.status.success() {
455 return Err(Error::Process {
456 program: "curl".to_owned(),
457 detail: format!("could not fetch {url}"),
458 });
459 }
460 Ok(output.stdout)
461}
462
463/// Hex-encoded sha256 of `bytes`, via the system `shasum` (macOS) or
464/// `sha256sum` (Linux) — shells out rather than adding a hashing
465/// dependency, the same rationale `pre-redo` used.
466fn sha256_hex(bytes: &[u8]) -> Result<String> {
467 let (program, args): (&str, &[&str]) = if Command::new("sha256sum")
468 .arg("--version")
469 .output()
470 .is_ok_and(|o| o.status.success())
471 {
472 ("sha256sum", &[])
473 } else {
474 ("shasum", &["-a", "256"])
475 };
476 let mut child = Command::new(program)
477 .args(args)
478 .stdin(Stdio::piped())
479 .stdout(Stdio::piped())
480 .spawn()
481 .map_err(|e| Error::Spawn {
482 program: program.to_owned(),
483 detail: e.to_string(),
484 })?;
485 {
486 use std::io::Write as _;
487 let mut stdin = child.stdin.take().ok_or_else(|| Error::Process {
488 program: program.to_owned(),
489 detail: "no stdin".to_owned(),
490 })?;
491 stdin.write_all(bytes).map_err(|e| Error::Process {
492 program: program.to_owned(),
493 detail: e.to_string(),
494 })?;
495 }
496 let mut out = String::new();
497 child
498 .stdout
499 .take()
500 .ok_or_else(|| Error::Process {
501 program: program.to_owned(),
502 detail: "no stdout".to_owned(),
503 })?
504 .read_to_string(&mut out)
505 .map_err(|e| Error::Process {
506 program: program.to_owned(),
507 detail: e.to_string(),
508 })?;
509 let status = child.wait().map_err(|e| Error::Process {
510 program: program.to_owned(),
511 detail: e.to_string(),
512 })?;
513 if !status.success() {
514 return Err(Error::Process {
515 program: program.to_owned(),
516 detail: "hashing failed".to_owned(),
517 });
518 }
519 out.split_whitespace()
520 .next()
521 .map(str::to_owned)
522 .ok_or_else(|| Error::Process {
523 program: program.to_owned(),
524 detail: "no hash in output".to_owned(),
525 })
526}
527
528fn make_dir(path: &Path) -> Result<()> {
529 std::fs::create_dir_all(path).map_err(|source| Error::Io {
530 path: path.to_owned(),
531 source,
532 })
533}
534
535fn remove_dir(path: &Path) -> Result<()> {
536 std::fs::remove_dir_all(path).map_err(|source| Error::Io {
537 path: path.to_owned(),
538 source,
539 })
540}
541
542#[cfg(test)]
543mod tests {
544 #![allow(clippy::expect_used, reason = "unit test")]
545
546 use ents_testutil::ObjectStore;
547 use rstest::rstest;
548
549 use super::*;
550
551 #[rstest]
552 #[case::embedded(Recipe::Embedded { tree: ObjectId::null(gix_hash::Kind::Sha1) })]
553 #[case::downloaded(Recipe::Downloaded {
554 components: vec![
555 Component { url: "https://example.test/a.tar.gz".into(), sha256: "a".repeat(64), strip: 2, dest: String::new() },
556 Component { url: "https://example.test/b.tar.gz".into(), sha256: "b".repeat(64), strip: 1, dest: "bin".into() },
557 ],
558 })]
559 // @relation(effect.toolchains, model.toolchain, scope=function, role=Verifies)
560 fn recipe_round_trips_through_text(#[case] recipe: Recipe) {
561 let text = recipe.render();
562 assert_eq!(Recipe::parse(&text).expect("parses"), recipe);
563 }
564
565 #[rstest]
566 #[case::empty("")]
567 #[case::unknown_kind("frobnicated\n")]
568 #[case::embedded_missing_oid("embedded\n")]
569 #[case::downloaded_no_components("downloaded\n")]
570 // @relation(effect.toolchains, scope=function, role=Verifies)
571 fn parse_rejects_malformed_text(#[case] text: &str) {
572 Recipe::parse(text).expect_err("malformed");
573 }
574
575 #[rstest]
576 // @relation(effect.toolchains, scope=function, role=Verifies)
577 fn cache_key_differs_by_extraction_shape() {
578 let a = Recipe::Downloaded {
579 components: vec![Component {
580 url: "u".into(),
581 sha256: "a".repeat(64),
582 strip: 1,
583 dest: String::new(),
584 }],
585 };
586 let b = Recipe::Downloaded {
587 components: vec![Component {
588 url: "u".into(),
589 sha256: "a".repeat(64),
590 strip: 2,
591 dest: String::new(),
592 }],
593 };
594 assert_ne!(cache_key(&a), cache_key(&b));
595 }
596
597 #[rstest]
598 #[case::sha256_too_short(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 \n", "a".repeat(40)))]
599 #[case::sha256_not_hex(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 \n", "z".repeat(64)))]
600 #[case::url_with_quote(&format!("downloaded\nhttps://x'y.test/a.tar.gz {} 1 \n", "a".repeat(64)))]
601 #[case::dest_parent_dir(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 ..\n", "a".repeat(64)))]
602 #[case::dest_with_unsafe_bytes(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 a$b\n", "a".repeat(64)))]
603 // @relation(effect.toolchains, scope=function, role=Verifies)
604 fn parse_rejects_a_component_unsafe_for_paths_or_shell(#[case] text: &str) {
605 let err = Recipe::parse(text).expect_err("unsafe component");
606 assert!(matches!(err, Error::InvalidComponent(_)), "got {err:?}");
607 }
608
609 #[rstest]
610 // @relation(effect.toolchains, scope=function, role=Verifies)
611 fn materialize_validates_a_directly_constructed_component() {
612 // A Recipe built in code never went through parse, but its sha256
613 // and dest still become filesystem path segments (cache_key) and,
614 // on the Sprite backend, part of an in-sandbox shell string — so
615 // materialize re-checks before touching either.
616 let objects = ObjectStore::default();
617 let recipe = Recipe::Downloaded {
618 components: vec![Component {
619 url: "https://x.test/a.tar.gz".into(),
620 sha256: "../escape".into(),
621 strip: 1,
622 dest: String::new(),
623 }],
624 };
625 let dir = tempfile::tempdir().expect("tempdir");
626 let err = materialize(&recipe, &objects, dir.path()).expect_err("unsafe component");
627 assert!(matches!(err, Error::InvalidComponent(_)), "got {err:?}");
628 assert_eq!(
629 std::fs::read_dir(dir.path()).expect("readable").count(),
630 0,
631 "the refusal must come before any cache path is created"
632 );
633 }
634}