fix: stream TOFU archive hashing instead of buffering the whole file
commit
e075b78fix: stream TOFU archive hashing instead of buffering the whole file
http_get_bytes caps the response body at 10MB, which a real hosted toolchain archive (zig’s release is 53MB) exceeds well before the pin can even be computed. A trust-on-first-use pin only ever needs the digest, so stream the response straight into the hashing subprocess instead of buffering it in memory first, removing the cap entirely.
feat: add sha256_hex_reader for streaming hash computation feat: add http_get_sha256 for TOFU pins Assisted-by: Claude:claude-sonnet-5
Reviews
No reviews of this commit yet — record a verdict below.
Start a review
crates/git-ents/src/main.rs
@@ -1716,6 +1716,24 @@
.map_err(|error| format!("could not read the response: {error}"))
}
+/// GET `url` and return its sha256, streamed straight into the hash rather
+/// than buffered — [`http_get_bytes`]'s counterpart for a trust-on-first-use
+/// pin, which only ever needs the digest and otherwise discards the archive.
+fn http_get_sha256(url: &str) -> Result<String, String> {
+ let mut response = ureq::get(url)
+ .config()
+ .http_status_as_error(false)
+ .build()
+ .call()
+ .map_err(|error| format!("GET {url} failed: {error}"))?;
+ let status = response.status();
+ if !status.is_success() {
+ return Err(format!("GET {url} returned {status}"));
+ }
+ git_toolchain::sha256_hex_reader(response.body_mut().as_reader())
+ .map_err(|error| format!("could not hash: {error}"))
+}
+
/// POST an `application/x-www-form-urlencoded` `body` to `url`, returning the
/// response body, or its body text as the error on a non-2xx status.
fn http_post_form(url: &str, body: &str) -> Result<String, String> {
crates/git-ents/src/registry.rs
@@ -284,11 +284,9 @@
let url = format!(
"https://github.com/mozilla/sccache/releases/download/{tag}/sccache-{tag}-{target}.tar.gz"
);
- let bytes = crate::http_get_bytes(&url)?;
if platform.is_some() {
- let sha256 = git_toolchain::sha256_hex(&bytes)
- .map_err(|error| format!("could not hash: {error}"))?;
+ let sha256 = crate::http_get_sha256(&url)?;
return Ok(Resolved {
bin: Bin::Components(vec![Component {
url,
@@ -304,6 +302,7 @@
});
}
+ let bytes = crate::http_get_bytes(&url)?;
let staging = tempfile::tempdir()
.map_err(|error| format!("could not create a staging directory: {error}"))?;
stage_sccache(&bytes, &tag, &target, staging.path())?;
@@ -334,9 +333,7 @@
if spec.is_empty() {
return Err("the url recipe needs --spec <archive-url>".to_owned());
}
- let bytes = crate::http_get_bytes(spec)?;
- let sha256 =
- git_toolchain::sha256_hex(&bytes).map_err(|error| format!("could not hash: {error}"))?;
+ let sha256 = crate::http_get_sha256(spec)?;
Ok(Resolved {
bin: Bin::Components(vec![Component {
url: spec.to_owned(),
crates/git-toolchain/src/lib.rs
@@ -793,6 +793,14 @@
/// to this crate. Public so a recipe pinning a hosted archive (trust on
/// first use) computes its hash the same way every later verification does.
pub fn sha256_hex(bytes: &[u8]) -> Result<String, Error> {
+ sha256_hex_reader(bytes)
+}
+
+/// Hex-encoded sha256 of everything `reader` yields, streamed straight into
+/// the hashing subprocess rather than buffered — so a recipe computing a
+/// trust-on-first-use pin over a multi-hundred-MB archive doesn't have to
+/// hold the whole thing in memory just to hash it once and discard it.
+pub fn sha256_hex_reader(mut reader: impl std::io::Read) -> Result<String, Error> {
let (program, args): (&str, &[&str]) = match std::env::consts::OS {
"macos" => ("shasum", &["-a", "256"]),
_ => ("sha256sum", &[]),
@@ -803,12 +811,13 @@
.stdout(Stdio::piped())
.spawn()
.map_err(|error| Error::Fetch(program.to_owned(), error.to_string()))?;
- child
+ let mut stdin = child
.stdin
.take()
- .ok_or_else(|| Error::Fetch(program.to_owned(), "no stdin".to_owned()))?
- .write_all(bytes)
+ .ok_or_else(|| Error::Fetch(program.to_owned(), "no stdin".to_owned()))?;
+ std::io::copy(&mut reader, &mut stdin)
.map_err(|error| Error::Fetch(program.to_owned(), error.to_string()))?;
+ drop(stdin);
let output = child
.wait_with_output()
.map_err(|error| Error::Fetch(program.to_owned(), error.to_string()))?;