git-ents.gitmain
⌘K
foforge

crates/kiln/ents-kiln/src/toolchain/recipe.rs

recipe.rs641 lines · 22.8 KB · rusthistorycomment on this file
1//! Toolchain resolution and materialization (`effect.toolchains`,
2//! `model.toolchain`).
3//!
4//! [`super::Toolchain::recipe`] is deliberately an opaque `String` —
5//! `model.toolchain`'s own doc names this crate as the one that gives it
6//! structure. [`Recipe`] is that structure: a toolchain's `bin` is either
7//! [`Recipe::Embedded`] (a tree already in the object database, captured
8//! whole by whatever wrote the toolchain) or [`Recipe::Downloaded`] (a set
9//! of externally-hosted, sha256-pinned archives), ported from `pre-redo`'s
10//! `git_toolchain::Bin` — the design pre-redo settled on and this phase
11//! carries forward, not a fresh design. [`Recipe::render`]/[`Recipe::parse`]
12//! round-trip it through the plain-text `recipe` field.
13//!
14//! [`materialize`] resolves a toolchain to a host directory containing its
15//! activated `bin/`, extract-once cached under a content key (a tree oid
16//! for `Embedded`, a hash of each component's pin for `Downloaded`) so a
17//! backend that runs the same toolchain repeatedly (a Sprite's persistent
18//! filesystem, a developer's local cache) never re-extracts unchanged
19//! bytes. Fetching a [`Recipe::Downloaded`] component shells to `curl`,
20//! `tar`, and `sha256sum`/`shasum`, the same pattern `pre-redo` used and
21//! this phase ports rather than adding an HTTP or hashing dependency.
22#![expect(
23 clippy::result_large_err,
24 reason = "every function here returns ents_effect::Result — that crate's own Error type, \
25 reused as-is rather than wrapped, since the toolchain domain's errors already live \
26 naturally in its enum; not this crate's to box"
27)]
28
29use std::io::Read as _;
30use std::path::{Path, PathBuf};
31use std::process::{Command, Stdio};
32
33use ents_effect::{Error, Result};
34use ents_model::namespace;
35use gix_hash::ObjectId;
36use gix_object::{CommitRef, Find, Kind};
37use gix_ref_store::RefStoreRead;
38
39use crate::Toolchain;
40
41/// How a toolchain's `bin` is provisioned — the structure inside
42/// [`Toolchain::recipe`] (`effect.toolchains`).
43#[derive(Debug, Clone, PartialEq, Eq)]
44pub enum Recipe {
45 /// `bin`'s directory tree, already in the object database — the whole
46 /// tree's entries become the toolchain's activated `bin/` contents.
47 Embedded {
48 /// The tree object id.
49 tree: ObjectId,
50 },
51 /// A set of archives fetched, sha256-verified, and merged onto disk at
52 /// materialization time.
53 Downloaded {
54 /// Each archive making up the toolchain.
55 components: Vec<Component>,
56 },
57}
58
59/// One archive making up a [`Recipe::Downloaded`] toolchain: fetched from
60/// `url` and checked against `sha256` before being extracted per
61/// `strip`/`dest` — ported verbatim from `pre-redo`'s
62/// `git_toolchain::Component`.
63#[derive(Debug, Clone, PartialEq, Eq)]
64pub struct Component {
65 /// Where to fetch the archive from.
66 pub url: String,
67 /// The archive's expected sha256, hex-encoded.
68 pub sha256: String,
69 /// Leading path segments `tar` strips at extraction.
70 pub strip: u8,
71 /// Subdirectory under the toolchain's `bin/` extraction root to extract
72 /// into: empty for an archive that already carries its own `bin/` top
73 /// level, `bin` for a flat archive whose payload should itself land on
74 /// `PATH`.
75 pub dest: String,
76}
77
78const EMBEDDED: &str = "embedded";
79const DOWNLOADED: &str = "downloaded";
80
81impl Recipe {
82 /// Parse a [`Recipe`] out of a [`Toolchain::recipe`] string.
83 ///
84 /// The format is deliberately small rather than a general one (no new
85 /// dependency for two variants and four fields): one line naming the
86 /// kind, then either the embedded tree's hex oid, or one
87 /// `url sha256 strip dest` line per component (`dest` last, so it may
88 /// be empty without ambiguity).
89 ///
90 /// # Errors
91 ///
92 /// [`Error::InvalidRecipe`] if the text does not match this shape.
93 ///
94 /// # Examples
95 ///
96 /// ```
97 /// use ents_kiln::Recipe;
98 ///
99 /// let text = "embedded 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n";
100 /// let recipe = Recipe::parse(text).expect("parses");
101 /// assert!(matches!(recipe, Recipe::Embedded { .. }));
102 /// ```
103 pub fn parse(text: &str) -> Result<Self> {
104 let mut lines = text.lines().filter(|line| !line.trim().is_empty());
105 let Some(first) = lines.next() else {
106 return Err(invalid("empty recipe"));
107 };
108 let mut words = first.split_whitespace();
109 match words.next() {
110 Some(EMBEDDED) => {
111 let hex = words
112 .next()
113 .ok_or_else(|| invalid("embedded recipe missing a tree oid"))?;
114 let tree = ObjectId::from_hex(hex.as_bytes())
115 .map_err(|e| invalid(format!("invalid tree oid {hex:?}: {e}")))?;
116 Ok(Self::Embedded { tree })
117 }
118 Some(DOWNLOADED) => {
119 let mut components = Vec::new();
120 for line in lines {
121 let mut fields = line.split_whitespace();
122 let url = fields
123 .next()
124 .ok_or_else(|| invalid("component line missing a url"))?
125 .to_owned();
126 let sha256 = fields
127 .next()
128 .ok_or_else(|| invalid("component line missing a sha256"))?
129 .to_owned();
130 let strip = fields
131 .next()
132 .ok_or_else(|| invalid("component line missing a strip count"))?
133 .parse::<u8>()
134 .map_err(|e| invalid(format!("invalid strip count: {e}")))?;
135 let dest = fields.next().unwrap_or("").to_owned();
136 let component = Component {
137 url,
138 sha256,
139 strip,
140 dest,
141 };
142 validate_component(&component)?;
143 components.push(component);
144 }
145 if components.is_empty() {
146 return Err(invalid(
147 "a downloaded toolchain must list at least one component",
148 ));
149 }
150 Ok(Self::Downloaded { components })
151 }
152 Some(other) => Err(invalid(format!("unknown recipe kind {other:?}"))),
153 None => Err(invalid("empty recipe")),
154 }
155 }
156
157 /// Render this [`Recipe`] back into the text stored in
158 /// [`Toolchain::recipe`].
159 ///
160 /// # Examples
161 ///
162 /// ```
163 /// use ents_kiln::Recipe;
164 ///
165 /// let recipe = Recipe::Embedded {
166 /// tree: gix_hash::ObjectId::null(gix_hash::Kind::Sha1),
167 /// };
168 /// let text = recipe.render();
169 /// assert_eq!(Recipe::parse(&text).expect("round-trips"), recipe);
170 /// ```
171 #[must_use]
172 pub fn render(&self) -> String {
173 match self {
174 Self::Embedded { tree } => format!("{EMBEDDED} {tree}\n"),
175 Self::Downloaded { components } => {
176 let mut out = format!("{DOWNLOADED}\n");
177 for c in components {
178 out.push_str(&format!("{} {} {} {}\n", c.url, c.sha256, c.strip, c.dest));
179 }
180 out
181 }
182 }
183 }
184}
185
186/// Read the [`Toolchain`] entity named `name` from
187/// `refs/meta/toolchains/<name>`, and parse its [`Toolchain::recipe`] as a
188/// [`Recipe`].
189///
190/// # Errors
191///
192/// [`Error::UnknownToolchain`] when the ref does not exist or does not
193/// resolve to a commit tree; [`Error::InvalidRecipe`] when its `recipe`
194/// field does not parse.
195///
196/// # Examples
197///
198/// ```
199/// use ents_kiln::Toolchain;
200/// use ents_kiln::toolchain::resolve;
201/// use ents_testutil::{MemRefStore, ObjectStore, write_meta_entity};
202///
203/// let refs = MemRefStore::default();
204/// let objects = ObjectStore::default();
205/// let toolchain = Toolchain {
206/// name: "rust-stable".into(),
207/// recipe: "embedded 4b825dc642cb6eb9a060e54bf8d69288fbee4904\n".into(),
208/// };
209/// let name: gix::refs::FullName = "refs/meta/toolchains/rust-stable".try_into().expect("valid");
210/// write_meta_entity(&refs, &objects, name, &toolchain, None, 100);
211///
212/// let (entity, recipe) = resolve(&refs, &objects, "rust-stable").expect("resolves");
213/// assert_eq!(entity.name, "rust-stable");
214/// assert!(matches!(recipe, ents_kiln::Recipe::Embedded { .. }));
215/// ```
216pub fn resolve(
217 refs: &dyn RefStoreRead,
218 objects: &impl Find,
219 name: &str,
220) -> Result<(Toolchain, Recipe)> {
221 let refname = namespace::toolchain_ref(name)
222 .map_err(|e| Error::UnknownToolchain(format!("{name}: {e}")))?;
223 let Some(tip) = refs.get(refname.as_ref())? else {
224 return Err(Error::UnknownToolchain(name.to_owned()));
225 };
226 let mut buf = Vec::new();
227 let data = objects
228 .try_find(&tip, &mut buf)
229 .map_err(|source| Error::Decode {
230 oid: tip,
231 detail: source.to_string(),
232 })?
233 .ok_or(Error::Missing { oid: tip })?;
234 if data.kind != Kind::Commit {
235 return Err(Error::Decode {
236 oid: tip,
237 detail: "toolchain ref does not point at a commit".to_owned(),
238 });
239 }
240 let commit = CommitRef::from_bytes(data.data, tip.kind()).map_err(|e| Error::Decode {
241 oid: tip,
242 detail: e.to_string(),
243 })?;
244 let toolchain: Toolchain = facet_git_tree::deserialize(&commit.tree(), objects)?;
245 let recipe = Recipe::parse(&toolchain.recipe).map_err(|e| match e {
246 Error::InvalidRecipe { detail, .. } => Error::InvalidRecipe {
247 name: name.to_owned(),
248 detail,
249 },
250 other => other,
251 })?;
252 Ok((toolchain, recipe))
253}
254
255fn invalid(detail: impl Into<String>) -> Error {
256 Error::InvalidRecipe {
257 name: String::new(),
258 detail: detail.into(),
259 }
260}
261
262/// Refuse a [`Component`] whose fields are unsafe downstream: `sha256` and
263/// `dest` become filesystem path segments ([`cache_key`]) and, via the
264/// Sprite backend's extract-once check, part of an in-sandbox `sh -c`
265/// string; `url` is handed to `curl` and quoted contexts. Checked at
266/// [`Recipe::parse`] so a hostile recipe never reaches those sites, and
267/// re-checked before any fetch for a [`Recipe`] constructed directly in
268/// code.
269fn validate_component(component: &Component) -> Result<()> {
270 if component.sha256.len() != 64 || !component.sha256.bytes().all(|b| b.is_ascii_hexdigit()) {
271 return Err(Error::InvalidComponent(format!(
272 "sha256 must be 64 hex characters, got {:?}",
273 component.sha256
274 )));
275 }
276 if component.url.is_empty()
277 || component.url.contains('\'')
278 || component.url.chars().any(char::is_whitespace)
279 {
280 return Err(Error::InvalidComponent(format!(
281 "unsafe url {:?}",
282 component.url
283 )));
284 }
285 let dest_ok = component.dest.is_empty()
286 || (component.dest != "."
287 && component.dest != ".."
288 && component
289 .dest
290 .bytes()
291 .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'.' | b'_' | b'-')));
292 if !dest_ok {
293 return Err(Error::InvalidComponent(format!(
294 "unsafe dest {:?}",
295 component.dest
296 )));
297 }
298 Ok(())
299}
300
301/// A stable, filesystem-safe cache key for a [`Recipe`]: the embedded
302/// tree's own hex oid, or each downloaded component's sha256 joined in
303/// extraction order — the same bytes extracted differently (a different
304/// `strip`/`dest`) are a different toolchain on disk, so those fields join
305/// the key too.
306#[must_use]
307pub fn cache_key(recipe: &Recipe) -> String {
308 match recipe {
309 Recipe::Embedded { tree } => tree.to_string(),
310 Recipe::Downloaded { components } => components
311 .iter()
312 .map(|c| format!("{}.{}.{}", c.sha256, c.strip, c.dest))
313 .collect::<Vec<_>>()
314 .join("-"),
315 }
316}
317
318/// Resolve `recipe` to a host directory containing the toolchain's
319/// activated `bin/`, extracted once under `cache_root` and reused on every
320/// later call with the same recipe (`effect.toolchains`: "resolved during
321/// effect execution").
322///
323/// # Errors
324///
325/// [`Error::Submodule`] or [`Error::NotUtf8`] for a tree this crate cannot
326/// materialize; [`Error::Spawn`]/[`Error::Process`]/[`Error::HashMismatch`]
327/// for a downloaded component that could not be fetched, verified, or
328/// extracted; [`Error::Io`] for a host filesystem failure.
329///
330/// # Examples
331///
332/// An embedded recipe over the empty tree materializes an empty `bin/`.
333///
334/// ```
335/// use ents_kiln::Recipe;
336/// use ents_kiln::toolchain::materialize;
337/// use ents_testutil::ObjectStore;
338/// use gix_object::{Kind, Write as _};
339///
340/// let objects = ObjectStore::default();
341/// let empty = objects.write_buf(Kind::Tree, b"").expect("write");
342/// let recipe = Recipe::Embedded { tree: empty };
343///
344/// let dir = tempfile::tempdir().expect("tempdir");
345/// let bin = materialize(&recipe, &objects, dir.path()).expect("materializes");
346/// assert!(bin.ends_with("bin"));
347/// assert!(bin.is_dir());
348/// ```
349pub fn materialize(recipe: &Recipe, objects: &impl Find, cache_root: &Path) -> Result<PathBuf> {
350 // Validate before `cache_key` embeds any component field in a path:
351 // `Recipe::parse` already refused a hostile recipe, but a `Recipe`
352 // built directly in code never went through parse.
353 if let Recipe::Downloaded { components } = recipe {
354 for component in components {
355 validate_component(component)?;
356 }
357 }
358 let root = cache_root.join(cache_key(recipe));
359 let bin = root.join("bin");
360 if bin.is_dir() {
361 return Ok(bin);
362 }
363 let tmp = cache_root.join(format!("{}.tmp", cache_key(recipe)));
364 if tmp.exists() {
365 remove_dir(&tmp)?;
366 }
367 make_dir(&tmp)?;
368
369 match recipe {
370 Recipe::Embedded { tree } => {
371 let bin_tmp = tmp.join("bin");
372 make_dir(&bin_tmp)?;
373 ents_effect::materialize::checkout(objects, *tree, &bin_tmp)?;
374 }
375 Recipe::Downloaded { components } => {
376 make_dir(&tmp.join("bin"))?;
377 for component in components {
378 fetch_component(component, &tmp)?;
379 }
380 }
381 }
382
383 // Land atomically: a transient failure partway through must never leave
384 // `bin` existing-but-incomplete, or the next call would trust a half
385 // extraction forever.
386 std::fs::rename(&tmp, &root).map_err(|source| Error::Io {
387 path: root.clone(),
388 source,
389 })?;
390 Ok(bin)
391}
392
393fn fetch_component(component: &Component, root: &Path) -> Result<()> {
394 validate_component(component)?;
395 let dest = if component.dest.is_empty() {
396 root.join("bin")
397 } else {
398 root.join("bin").join(&component.dest)
399 };
400 make_dir(&dest)?;
401
402 let bytes = fetch(&component.url)?;
403 let actual = sha256_hex(&bytes)?;
404 if !actual.eq_ignore_ascii_case(&component.sha256) {
405 return Err(Error::HashMismatch {
406 url: component.url.clone(),
407 expected: component.sha256.clone(),
408 actual,
409 });
410 }
411
412 let mut child = Command::new("tar")
413 .args([
414 "-x",
415 "-C",
416 dest.to_str().ok_or_else(|| Error::NotUtf8(dest.clone()))?,
417 &format!("--strip-components={}", component.strip),
418 ])
419 .stdin(Stdio::piped())
420 .stdout(Stdio::null())
421 .stderr(Stdio::piped())
422 .spawn()
423 .map_err(|e| Error::Spawn {
424 program: "tar".to_owned(),
425 detail: e.to_string(),
426 })?;
427 {
428 use std::io::Write as _;
429 let mut stdin = child.stdin.take().ok_or_else(|| Error::Process {
430 program: "tar".to_owned(),
431 detail: "no stdin".to_owned(),
432 })?;
433 stdin.write_all(&bytes).map_err(|e| Error::Process {
434 program: "tar".to_owned(),
435 detail: e.to_string(),
436 })?;
437 }
438 let output = child.wait_with_output().map_err(|e| Error::Process {
439 program: "tar".to_owned(),
440 detail: e.to_string(),
441 })?;
442 if !output.status.success() {
443 return Err(Error::Process {
444 program: "tar".to_owned(),
445 detail: String::from_utf8_lossy(&output.stderr).trim().to_owned(),
446 });
447 }
448 Ok(())
449}
450
451/// `GET url`, via the system `curl` — shells out rather than adding an HTTP
452/// dependency, the same rationale `pre-redo` used.
453fn fetch(url: &str) -> Result<Vec<u8>> {
454 let output = Command::new("curl")
455 .args(["-sSL", "--fail", url])
456 .output()
457 .map_err(|e| Error::Spawn {
458 program: "curl".to_owned(),
459 detail: e.to_string(),
460 })?;
461 if !output.status.success() {
462 return Err(Error::Process {
463 program: "curl".to_owned(),
464 detail: format!("could not fetch {url}"),
465 });
466 }
467 Ok(output.stdout)
468}
469
470/// Hex-encoded sha256 of `bytes`, via the system `shasum` (macOS) or
471/// `sha256sum` (Linux) — shells out rather than adding a hashing
472/// dependency, the same rationale `pre-redo` used.
473fn sha256_hex(bytes: &[u8]) -> Result<String> {
474 let (program, args): (&str, &[&str]) = if Command::new("sha256sum")
475 .arg("--version")
476 .output()
477 .is_ok_and(|o| o.status.success())
478 {
479 ("sha256sum", &[])
480 } else {
481 ("shasum", &["-a", "256"])
482 };
483 let mut child = Command::new(program)
484 .args(args)
485 .stdin(Stdio::piped())
486 .stdout(Stdio::piped())
487 .spawn()
488 .map_err(|e| Error::Spawn {
489 program: program.to_owned(),
490 detail: e.to_string(),
491 })?;
492 {
493 use std::io::Write as _;
494 let mut stdin = child.stdin.take().ok_or_else(|| Error::Process {
495 program: program.to_owned(),
496 detail: "no stdin".to_owned(),
497 })?;
498 stdin.write_all(bytes).map_err(|e| Error::Process {
499 program: program.to_owned(),
500 detail: e.to_string(),
501 })?;
502 }
503 let mut out = String::new();
504 child
505 .stdout
506 .take()
507 .ok_or_else(|| Error::Process {
508 program: program.to_owned(),
509 detail: "no stdout".to_owned(),
510 })?
511 .read_to_string(&mut out)
512 .map_err(|e| Error::Process {
513 program: program.to_owned(),
514 detail: e.to_string(),
515 })?;
516 let status = child.wait().map_err(|e| Error::Process {
517 program: program.to_owned(),
518 detail: e.to_string(),
519 })?;
520 if !status.success() {
521 return Err(Error::Process {
522 program: program.to_owned(),
523 detail: "hashing failed".to_owned(),
524 });
525 }
526 out.split_whitespace()
527 .next()
528 .map(str::to_owned)
529 .ok_or_else(|| Error::Process {
530 program: program.to_owned(),
531 detail: "no hash in output".to_owned(),
532 })
533}
534
535fn make_dir(path: &Path) -> Result<()> {
536 std::fs::create_dir_all(path).map_err(|source| Error::Io {
537 path: path.to_owned(),
538 source,
539 })
540}
541
542fn remove_dir(path: &Path) -> Result<()> {
543 std::fs::remove_dir_all(path).map_err(|source| Error::Io {
544 path: path.to_owned(),
545 source,
546 })
547}
548
549#[cfg(test)]
550mod tests {
551 #![allow(clippy::expect_used, reason = "unit test")]
552
553 use ents_testutil::ObjectStore;
554 use rstest::rstest;
555
556 use super::*;
557
558 #[rstest]
559 #[case::embedded(Recipe::Embedded { tree: ObjectId::null(gix_hash::Kind::Sha1) })]
560 #[case::downloaded(Recipe::Downloaded {
561 components: vec![
562 Component { url: "https://example.test/a.tar.gz".into(), sha256: "a".repeat(64), strip: 2, dest: String::new() },
563 Component { url: "https://example.test/b.tar.gz".into(), sha256: "b".repeat(64), strip: 1, dest: "bin".into() },
564 ],
565 })]
566 // @relation(effect.toolchains, model.toolchain, scope=function, role=Verifies)
567 fn recipe_round_trips_through_text(#[case] recipe: Recipe) {
568 let text = recipe.render();
569 assert_eq!(Recipe::parse(&text).expect("parses"), recipe);
570 }
571
572 #[rstest]
573 #[case::empty("")]
574 #[case::unknown_kind("frobnicated\n")]
575 #[case::embedded_missing_oid("embedded\n")]
576 #[case::downloaded_no_components("downloaded\n")]
577 // @relation(effect.toolchains, scope=function, role=Verifies)
578 fn parse_rejects_malformed_text(#[case] text: &str) {
579 Recipe::parse(text).expect_err("malformed");
580 }
581
582 #[rstest]
583 // @relation(effect.toolchains, scope=function, role=Verifies)
584 fn cache_key_differs_by_extraction_shape() {
585 let a = Recipe::Downloaded {
586 components: vec![Component {
587 url: "u".into(),
588 sha256: "a".repeat(64),
589 strip: 1,
590 dest: String::new(),
591 }],
592 };
593 let b = Recipe::Downloaded {
594 components: vec![Component {
595 url: "u".into(),
596 sha256: "a".repeat(64),
597 strip: 2,
598 dest: String::new(),
599 }],
600 };
601 assert_ne!(cache_key(&a), cache_key(&b));
602 }
603
604 #[rstest]
605 #[case::sha256_too_short(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 \n", "a".repeat(40)))]
606 #[case::sha256_not_hex(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 \n", "z".repeat(64)))]
607 #[case::url_with_quote(&format!("downloaded\nhttps://x'y.test/a.tar.gz {} 1 \n", "a".repeat(64)))]
608 #[case::dest_parent_dir(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 ..\n", "a".repeat(64)))]
609 #[case::dest_with_unsafe_bytes(&format!("downloaded\nhttps://x.test/a.tar.gz {} 1 a$b\n", "a".repeat(64)))]
610 // @relation(effect.toolchains, scope=function, role=Verifies)
611 fn parse_rejects_a_component_unsafe_for_paths_or_shell(#[case] text: &str) {
612 let err = Recipe::parse(text).expect_err("unsafe component");
613 assert!(matches!(err, Error::InvalidComponent(_)), "got {err:?}");
614 }
615
616 #[rstest]
617 // @relation(effect.toolchains, scope=function, role=Verifies)
618 fn materialize_validates_a_directly_constructed_component() {
619 // A Recipe built in code never went through parse, but its sha256
620 // and dest still become filesystem path segments (cache_key) and,
621 // on the Sprite backend, part of an in-sandbox shell string — so
622 // materialize re-checks before touching either.
623 let objects = ObjectStore::default();
624 let recipe = Recipe::Downloaded {
625 components: vec![Component {
626 url: "https://x.test/a.tar.gz".into(),
627 sha256: "../escape".into(),
628 strip: 1,
629 dest: String::new(),
630 }],
631 };
632 let dir = tempfile::tempdir().expect("tempdir");
633 let err = materialize(&recipe, &objects, dir.path()).expect_err("unsafe component");
634 assert!(matches!(err, Error::InvalidComponent(_)), "got {err:?}");
635 assert_eq!(
636 std::fs::read_dir(dir.path()).expect("readable").count(),
637 0,
638 "the refusal must come before any cache path is created"
639 );
640 }
641}