feat: add git-hydrate crate for the WS0 interim hydration backend
commit
300703ffeat: add git-hydrate crate for the WS0 interim hydration backend
Turns the existing stock-git CGI path into docs/scale-out.adoc’s WS0 hydration backend: ephemeral local disk hydrated from Postgres refs and Tigris packs, so cloud deployment stays additive configuration rather than a second code path. Hydration is optional (GIT_ENTS_HYDRATE_POSTGRES_URL plus GIT_ENTS_HYDRATE_BLOB_ROOT or the S3 equivalents); unset, a deployment keeps the current direct-disk behavior.
feat: add git_hydrate::hydrate::ensure_hydrated, idempotently topping up a local bare repo objects/pack/ from the pack registry feat: add git_hydrate::packed_refs::regenerate, rewriting packed-refs from Postgres on every service request feat: add git_hydrate::pre_receive::run, applying a push through git_protocol::native::NativeBackend::receive against Postgres/Tigris and logging its corpus entry feat: wire hydration into git-ents-server backend() and Command::PreReceive dispatch test: add a docker-gated end-to-end test pushing through the hydration path and replaying its corpus against the files backends Assisted-by: Claude:claude-sonnet-5
Reviews
No reviews of this commit yet — record a verdict below.
Start a review
Cargo.lock
@@ -1673,6 +1673,7 @@
"arborium",
"askama",
"axum",
+ "backend-conformance",
"facet",
"figue",
"form_urlencoded",
@@ -1682,6 +1683,7 @@
"git-comment",
"git-effect",
"git-ents-core",
+ "git-hydrate",
"git-member",
"git-protocol",
"git-reachability",
@@ -1694,15 +1696,37 @@
"gix-object",
"maud",
"odb-files",
+ "odb-tigris",
"portable-pty",
"pulldown-cmark",
"refstore-files",
+ "refstore-postgres",
"rstest",
"tempfile",
"tokio",
"uuid",
]
+[[package]]
+name = "git-hydrate"
+version = "0.0.0"
+dependencies = [
+ "backend-conformance",
+ "git-backend",
+ "git-ents-core",
+ "git-member",
+ "git-protocol",
+ "git-reachability",
+ "gix-hash",
+ "gix-object",
+ "odb-files",
+ "odb-tigris",
+ "refstore-files",
+ "refstore-postgres",
+ "tempfile",
+ "uuid",
+]
+
[[package]]
name = "git-member"
version = "0.0.0"
Cargo.toml
@@ -9,6 +9,7 @@
"crates/git-ents",
"crates/git-ents-core",
"crates/git-ents-server",
+ "crates/git-hydrate",
"crates/git-member",
"crates/git-protocol",
"crates/git-reachability",
@@ -70,6 +71,7 @@
git-effect = { path = "crates/git-effect" }
git-ents-core = { path = "crates/git-ents-core" }
git-ents-server = { path = "crates/git-ents-server" }
+git-hydrate = { path = "crates/git-hydrate" }
git-member = { path = "crates/git-member" }
git-protocol = { path = "crates/git-protocol" }
git-reachability = { path = "crates/git-reachability" }
crates/git-ents-server/Cargo.toml
@@ -22,6 +22,7 @@
git-backend = { workspace = true }
git-comment = { workspace = true }
git-effect = { workspace = true }
+git-hydrate = { workspace = true }
git-member = { workspace = true }
git-protocol = { workspace = true }
git-reachability = { workspace = true }
@@ -34,14 +35,17 @@
gix-object = { workspace = true }
maud = { workspace = true }
odb-files = { workspace = true }
+odb-tigris = { workspace = true }
pulldown-cmark = { workspace = true }
refstore-files = { workspace = true }
+refstore-postgres = { workspace = true }
tempfile = { workspace = true }
tokio = { workspace = true }
uuid = { workspace = true }
portable-pty = "0.9.0"
[dev-dependencies]
+backend-conformance = { workspace = true }
rstest = { workspace = true }
[lints]
crates/git-ents-server/src/http.rs
@@ -136,6 +136,23 @@
return response;
}
+ // WS0 hydration (`docs/scale-out.adoc`): before handing this request to
+ // `git http-backend`, top up the ephemeral local repo from the durable
+ // stores and, for anything answering a ref advertisement, regenerate
+ // `packed-refs` from Postgres — bounding advertisement staleness to
+ // this one request. A no-op (and `state.hydrate` is `None`) for a
+ // local-only deployment.
+ if let Some(hydrate) = &state.hydrate
+ && is_service_request(path_info, query_string)
+ && let Some(relative) = repo_path(path_info)
+ {
+ let repo_path = state.data_dir.join(&relative);
+ let repo_id = repo_id_string(&relative);
+ if let Err(response) = hydrate_repo(hydrate, &repo_path, &repo_id).await {
+ return response;
+ }
+ }
+
let content_type = header_value(headers, "Content-Type");
let content_length = header_value(headers, "Content-Length");
// `Content-Type`/`Content-Length` are CGI-special-cased env vars with no
@@ -172,6 +189,38 @@
if let Some(value) = &content_encoding {
cmd.env("HTTP_CONTENT_ENCODING", value);
}
+ // Hand the hydration-mode `pre-receive` (`git_hydrate::pre_receive`) the
+ // durable-store config it needs: these env vars propagate through
+ // `http-backend` -> `receive-pack` -> the hook exactly as
+ // `git_effect::engine::QUEUE_ENV` above already does, and
+ // `git_hydrate::HydrateConfig::from_env` reads them back.
+ if let Some(hydrate) = &state.hydrate {
+ // The op record's signer: `git_hydrate::pre_receive` needs the same
+ // key `AppState::web_signing_key` already holds in this process,
+ // but the hook is a separate process with no access to it.
+ if let Some(key) = &state.web_signing_key {
+ cmd.env("GIT_ENTS_WEB_SIGNING_KEY", key);
+ }
+ cmd.env("GIT_ENTS_HYDRATE_POSTGRES_URL", &hydrate.postgres_conninfo);
+ match &hydrate.blob {
+ git_hydrate::config::BlobStore::Fs(root) => {
+ cmd.env("GIT_ENTS_HYDRATE_BLOB_ROOT", root);
+ }
+ git_hydrate::config::BlobStore::S3(s3) => {
+ cmd.env("GIT_ENTS_HYDRATE_S3_BUCKET", &s3.bucket)
+ .env("GIT_ENTS_HYDRATE_S3_REGION", &s3.region)
+ .env("GIT_ENTS_HYDRATE_S3_ENDPOINT", &s3.endpoint)
+ .env("GIT_ENTS_HYDRATE_S3_ACCESS_KEY_ID", &s3.access_key_id)
+ .env(
+ "GIT_ENTS_HYDRATE_S3_SECRET_ACCESS_KEY",
+ &s3.secret_access_key,
+ );
+ if s3.allow_http {
+ cmd.env("GIT_ENTS_HYDRATE_S3_ALLOW_HTTP", "1");
+ }
+ }
+ }
+ }
// Push these through `GIT_CONFIG_*` rather than `git -c` so they reach the
// `receive-pack` and `pre-receive` processes http-backend spawns, where the
@@ -405,18 +454,21 @@
}
// @relation(namespace.path)
-/// The target repository of a push, as a validated path relative to the data
-/// directory, or `None` if the request does not name an acceptable repository.
+/// The target repository of a smart-HTTP service request (push or fetch),
+/// as a validated path relative to the data directory, or `None` if the
+/// request does not name an acceptable repository.
///
-/// The repository is everything before git's service suffix (`/info/refs` or
-/// `/git-receive-pack`), limited to [`MAX_REPO_DEPTH`] segments each drawn from
-/// a conservative character set. Validating the segments here is what keeps a
-/// push from escaping the data directory or fabricating arbitrary paths on
-/// disk: every returned component is a plain, dot-free, separator-free name, so
-/// the join below can only ever descend into `data_dir`.
+/// The repository is everything before git's service suffix (`/info/refs`,
+/// `/git-receive-pack`, or `/git-upload-pack`), limited to
+/// [`MAX_REPO_DEPTH`] segments each drawn from a conservative character
+/// set. Validating the segments here is what keeps a request from escaping
+/// the data directory or fabricating arbitrary paths on disk: every
+/// returned component is a plain, dot-free, separator-free name, so the
+/// join below can only ever descend into `data_dir`.
fn repo_path(path_info: &str) -> Option<PathBuf> {
let repo = path_info
.strip_suffix("/git-receive-pack")
+ .or_else(|| path_info.strip_suffix("/git-upload-pack"))
.or_else(|| path_info.strip_suffix("/info/refs"))?;
let segments: Vec<&str> = repo.split('/').filter(|s| !s.is_empty()).collect();
if segments.is_empty() || segments.len() > MAX_REPO_DEPTH {
@@ -524,6 +576,69 @@
.await;
}
+/// `relative`'s repository id, the way every hydration-mode component
+/// (this module, `git_hydrate::pre_receive`, `native_git`'s own resolver)
+/// names one: its data-dir-relative path with forward slashes, regardless
+/// of host path-separator conventions.
+fn repo_id_string(relative: &Path) -> String {
+ relative.to_string_lossy().replace('\\', "/")
+}
+
+// @relation(protocol.git, storage.bare)
+/// WS0's read-path hydration step for one request: top up `repo_path`'s
+/// local packs from `hydrate`'s durable stores (idempotent — a no-op past
+/// the first pack a given ephemeral instance has already fetched) and
+/// regenerate its `packed-refs` from Postgres, bounding advertisement
+/// staleness to this one request (`docs/scale-out.adoc`, "WS0").
+///
+/// Runs on a blocking task: [`refstore_postgres::PostgresRefStore`] and
+/// [`odb_tigris::OdbTigris`] are synchronous (each owns its own dedicated
+/// runtime for the async clients underneath), so driving them straight from
+/// this async handler would block the executor thread they're called from.
+async fn hydrate_repo(
+ hydrate: &git_hydrate::HydrateConfig,
+ repo_path: &Path,
+ repo_id: &str,
+) -> Result<(), Response> {
+ let hydrate = hydrate.clone();
+ let repo_path = repo_path.to_path_buf();
+ let repo_id = repo_id.to_owned();
+ let result = tokio::task::spawn_blocking(move || -> git_backend::Result<()> {
+ let registry = refstore_postgres::PostgresRefStore::connect(
+ &hydrate.postgres_conninfo,
+ repo_id.clone(),
+ )?;
+ match &hydrate.blob {
+ git_hydrate::config::BlobStore::Fs(root) => {
+ let transport = odb_tigris::transport::fs::FsTransport::open(root)?;
+ git_hydrate::hydrate::ensure_hydrated(&repo_path, &repo_id, &transport, ®istry)?;
+ }
+ git_hydrate::config::BlobStore::S3(s3) => {
+ let transport = odb_tigris::transport::s3::S3Transport::connect(s3)?;
+ git_hydrate::hydrate::ensure_hydrated(&repo_path, &repo_id, &transport, ®istry)?;
+ }
+ }
+ let refs =
+ refstore_postgres::PostgresRefStore::connect(&hydrate.postgres_conninfo, repo_id)?;
+ git_hydrate::packed_refs::regenerate(&repo_path, &refs)?;
+ Ok(())
+ })
+ .await;
+ match result {
+ Ok(Ok(())) => Ok(()),
+ Ok(Err(error)) => Err((
+ StatusCode::INTERNAL_SERVER_ERROR,
+ format!("hydration failed: {error}"),
+ )
+ .into_response()),
+ Err(join_error) => Err((
+ StatusCode::INTERNAL_SERVER_ERROR,
+ format!("hydration task panicked: {join_error}"),
+ )
+ .into_response()),
+ }
+}
+
fn header_value(headers: &HeaderMap, field: &str) -> Option<String> {
headers
.get(field)
@@ -602,6 +717,7 @@
challenges: crate::web::new_challenges(),
web_signing_key: None,
live_runs: git_effect::engine::new_live_registry(),
+ hydrate: None,
}
}
crates/git-ents-server/src/lib.rs
@@ -105,6 +105,13 @@
/// Live output for checks the worker currently has running, polled by the
/// Checks tab's live view.
pub(crate) live_runs: git_effect::engine::LiveRegistry,
+ /// WS0 hydration config (`docs/scale-out.adoc`, "WS0 — Interim
+ /// hydration backend"), read once at startup from the environment
+ /// (see [`git_hydrate::HydrateConfig::from_env`]). `None` keeps this
+ /// deployment on the current direct-disk behavior; `Some` hydrates
+ /// every served repo from Postgres/blob-store durable state on every
+ /// request, per `crate::http`'s wiring.
+ pub(crate) hydrate: Option<git_hydrate::HydrateConfig>,
}
/// The non-empty value of the environment variable `key`, or `None`.
@@ -118,6 +125,23 @@
/// wins over the hardcoded default.
pub fn run(args: Args) -> ExitCode {
if let Some(Command::PreReceive) = args.command {
+ // Hydration mode (`docs/scale-out.adoc`, WS0): `crate::http`'s
+ // backend invocation injects `GIT_ENTS_HYDRATE_*` env vars onto
+ // every `git http-backend` it spawns whenever `AppState::hydrate`
+ // is configured, and they propagate down to this hook exactly as
+ // `GIT_ENTS_HOOKS_DIR`/`GIT_ENTS_CHECKS_QUEUE` already do. Their
+ // presence here is this (separate) hook process's only way to
+ // learn hydration is on; there is no shared `AppState` to consult.
+ if let Some(hydrate) = git_hydrate::HydrateConfig::from_env() {
+ let signing_key = env_var("GIT_ENTS_WEB_SIGNING_KEY").map(PathBuf::from);
+ return match git_hydrate::pre_receive::run(&hydrate, signing_key.as_deref()) {
+ Ok(()) => ExitCode::SUCCESS,
+ Err(reason) => {
+ eprintln!("error: {reason}");
+ ExitCode::FAILURE
+ }
+ };
+ }
return match git_signed_push::pre_receive() {
Ok(()) => ExitCode::SUCCESS,
Err(reason) => {
@@ -167,6 +191,12 @@
let web_signing_key = args
.web_signing_key
.or_else(|| env_var("GIT_ENTS_WEB_SIGNING_KEY").map(PathBuf::from));
+ // @relation(protocol.routing)
+ // Read once at startup, exactly as `git_hydrate::pre_receive`'s hook
+ // process reads it again for itself (see `Command::PreReceive` above)
+ // — the one config both this process and every hook subprocess it
+ // spawns must agree on.
+ let hydrate = git_hydrate::HydrateConfig::from_env();
let state = AppState {
data_dir,
@@ -178,6 +208,7 @@
challenges: web::new_challenges(),
web_signing_key,
live_runs: git_effect::engine::new_live_registry(),
+ hydrate,
};
// Drain queued pushes and run their effects for the life of the server:
crates/git-ents-server/tests/hydrate.rs
@@ -1,0 +1,336 @@
+#![allow(
+ missing_docs,
+ clippy::unwrap_used,
+ clippy::panic,
+ clippy::arithmetic_side_effects,
+ reason = "integration test binary"
+)]
+
+//! End-to-end coverage for the WS0 hydration backend (`docs/scale-out.adoc`'s
+//! "WS0 Interim hydration backend" section).
+//!
+//! Two real `git push` invocations run over HTTP against a server
+//! configured with `GIT_ENTS_HYDRATE_POSTGRES_URL` and
+//! `GIT_ENTS_HYDRATE_BLOB_ROOT`, so every request goes through
+//! `git_hydrate`'s read/write paths rather than direct disk. The test then
+//! replays the corpus the write path logged, using
+//! `backend_conformance::replay_corpus`, against fresh
+//! `refstore-files`/`odb-files` backends, and asserts the replayed backend
+//! ends up with identical content refs and an identical reachable object
+//! set to the original Postgres/Tigris-backed repository: the conformance
+//! seed corpus `docs/scale-out.adoc` asks WS0 to produce.
+//!
+//! Gated on a reachable Postgres (`GIT_ENTS_TEST_POSTGRES_URL`, or a
+//! throwaway docker container), matching `refstore-postgres`'s own tests.
+//! See that crate's `tests/conformance.rs` module doc for the priority
+//! order and the visible-skip rationale.
+
+use std::collections::BTreeSet;
+use std::net::{TcpListener, TcpStream};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::time::Duration;
+
+use git_backend::{RefName, RefStore as _};
+use gix_hash::ObjectId;
+use odb_tigris::OdbTigris;
+use odb_tigris::transport::fs::FsTransport;
+use refstore_postgres::PostgresRefStore;
+
+const BIN: &str = env!("CARGO_BIN_EXE_git-ents-server");
+
+enum TestPostgres {
+ External(String),
+ Docker { container_id: String, url: String },
+}
+
+impl TestPostgres {
+ fn url(&self) -> &str {
+ match self {
+ Self::External(url) | Self::Docker { url, .. } => url,
+ }
+ }
+}
+
+impl Drop for TestPostgres {
+ fn drop(&mut self) {
+ if let Self::Docker { container_id, .. } = self {
+ let _ignored = Command::new("docker")
+ .args(["rm", "-f", container_id])
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status();
+ }
+ }
+}
+
+fn test_postgres() -> Option<TestPostgres> {
+ if let Ok(url) = std::env::var("GIT_ENTS_TEST_POSTGRES_URL") {
+ return Some(TestPostgres::External(url));
+ }
+ if !docker_available() {
+ return None;
+ }
+ start_docker_postgres()
+}
+
+fn docker_available() -> bool {
+ Command::new("docker")
+ .arg("version")
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()
+ .map(|status| status.success())
+ .unwrap_or(false)
+}
+
+fn start_docker_postgres() -> Option<TestPostgres> {
+ let output = Command::new("docker")
+ .args([
+ "run",
+ "-d",
+ "--rm",
+ "-e",
+ "POSTGRES_PASSWORD=postgres",
+ "-p",
+ "127.0.0.1::5432",
+ "postgres:16-alpine",
+ ])
+ .output()
+ .ok()?;
+ if !output.status.success() {
+ eprintln!(
+ "git-ents-server hydrate test: docker run failed: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+ return None;
+ }
+ let container_id = String::from_utf8_lossy(&output.stdout).trim().to_owned();
+
+ for _ in 0..120 {
+ let ready = Command::new("docker")
+ .args(["exec", &container_id, "pg_isready", "-U", "postgres"])
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()
+ .map(|status| status.success())
+ .unwrap_or(false);
+ if ready {
+ break;
+ }
+ std::thread::sleep(Duration::from_millis(250));
+ }
+
+ let port_output = Command::new("docker")
+ .args(["port", &container_id, "5432"])
+ .output()
+ .ok()?;
+ let mapping = String::from_utf8_lossy(&port_output.stdout);
+ let port = mapping
+ .lines()
+ .next()?
+ .rsplit(':')
+ .next()?
+ .trim()
+ .to_owned();
+
+ Some(TestPostgres::Docker {
+ url: format!("host=127.0.0.1 port={port} user=postgres password=postgres dbname=postgres"),
+ container_id,
+ })
+}
+
+macro_rules! require_postgres {
+ ($name:literal) => {
+ match test_postgres() {
+ Some(pg) => pg,
+ None => {
+ eprintln!(concat!(
+ "skipping ",
+ $name,
+ ": set GIT_ENTS_TEST_POSTGRES_URL, or make docker available"
+ ));
+ return;
+ }
+ }
+ };
+}
+
+// @relation(protocol.git, storage.bare, role=Verifies)
+#[test]
+fn pushes_through_hydration_replay_identically_against_the_files_backends() {
+ let pg =
+ require_postgres!("pushes_through_hydration_replay_identically_against_the_files_backends");
+ let repo_id = format!("hydrate-{}.git", uuid::Uuid::new_v4());
+
+ let scratch = tempfile::tempdir().unwrap();
+ let data = tempfile::tempdir().unwrap();
+ let blob_root = tempfile::tempdir().unwrap();
+ let hooks = tempfile::tempdir().unwrap();
+
+ let hook = hooks.path().join("pre-receive");
+ std::fs::write(&hook, format!("#!/bin/sh\nexec \"{BIN}\" pre-receive\n")).unwrap();
+ #[cfg(unix)]
+ {
+ use std::os::unix::fs::PermissionsExt;
+ std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap();
+ }
+
+ let signing_key = keygen(scratch.path(), "op-signer");
+
+ let port = free_port();
+ let mut child = Command::new(BIN)
+ .args(["--port", &port.to_string()])
+ .arg("--data-dir")
+ .arg(data.path())
+ .arg("--hooks-dir")
+ .arg(hooks.path())
+ .arg("--web-signing-key")
+ .arg(&signing_key)
+ .arg("--checks-queue")
+ .arg(scratch.path().join("checks-queue"))
+ .env("GIT_ENTS_HYDRATE_POSTGRES_URL", pg.url())
+ .env("GIT_ENTS_HYDRATE_BLOB_ROOT", blob_root.path())
+ .spawn()
+ .unwrap();
+ wait_for_port(port);
+
+ let url = format!("http://127.0.0.1:{port}/{repo_id}");
+ let work = scratch.path().join("work");
+ std::fs::create_dir_all(&work).unwrap();
+ run(&work, "git", &["init", "-q", "-b", "main"]);
+ std::fs::write(work.join("file.txt"), "one\n").unwrap();
+ run(&work, "git", &["add", "."]);
+ run(
+ &work,
+ "git",
+ &["-c", "commit.gpgsign=false", "commit", "-q", "-m", "first"],
+ );
+ run(&work, "git", &["push", "-q", &url, "main"]);
+
+ // A second push, so the corpus carries more than one entry and the
+ // second's pack excludes the first's already-known objects.
+ std::fs::write(work.join("file.txt"), "two\n").unwrap();
+ run(&work, "git", &["add", "."]);
+ run(
+ &work,
+ "git",
+ &["-c", "commit.gpgsign=false", "commit", "-q", "-m", "second"],
+ );
+ run(&work, "git", &["push", "-q", &url, "main"]);
+
+ child.kill().unwrap();
+ let _wait = child.wait();
+
+ // The source of truth: Postgres refs, Tigris (here, `FsTransport`)
+ // objects, and the corpus this repository's pushes logged.
+ let source_refs = PostgresRefStore::connect(pg.url(), repo_id.clone()).unwrap();
+ let source_registry = PostgresRefStore::connect(pg.url(), repo_id.clone()).unwrap();
+ let source_transport = FsTransport::open(blob_root.path()).unwrap();
+ let source_objects = OdbTigris::new(source_transport, source_registry, repo_id.clone());
+
+ let entries = source_refs.corpus_log().unwrap();
+ assert_eq!(
+ entries.len(),
+ 2,
+ "both pushes should have logged a corpus entry"
+ );
+
+ let main = RefName::new("refs/heads/main");
+ let source_main = source_refs.get(&main).unwrap();
+ assert!(
+ source_main.is_some(),
+ "the pushed branch must exist in Postgres"
+ );
+
+ // Replay the corpus against fresh `refstore-files`/`odb-files` — the
+ // conformance seed corpus (`docs/scale-out.adoc`, WS2) this backend
+ // feeds.
+ let target_dir = tempfile::tempdir().unwrap();
+ run(target_dir.path(), "git", &["init", "-q", "--bare"]);
+ let target_refs = refstore_files::FilesRefStore::open(target_dir.path()).unwrap();
+ let target_objects = odb_files::OdbFiles::open(target_dir.path()).unwrap();
+ backend_conformance::replay_corpus(&entries, &target_refs, &target_objects).unwrap();
+
+ let target_main = target_refs.get(&main).unwrap();
+ assert_eq!(
+ target_main, source_main,
+ "replaying the corpus must reproduce the same final ref"
+ );
+
+ let source_reachable = reachable_from_heads(&source_refs, &source_objects);
+ let target_reachable = reachable_from_heads(&target_refs, &target_objects);
+ assert_eq!(
+ source_reachable, target_reachable,
+ "replaying the corpus must reproduce the same reachable object set"
+ );
+}
+
+/// The set of objects reachable from every `refs/heads/*` tip — deliberately
+/// narrower than `backend_conformance::reachable_object_set` (which walks
+/// every ref, including `refs/meta/ops/log`): the corpus intentionally does
+/// not carry the op-record chain (it re-signs on every replay, so it never
+/// hash-matches — see `git_protocol::corpus`'s module doc), so comparing the
+/// content refs' reachable closure is the correct scope for this assertion.
+fn reachable_from_heads(
+ refs: &dyn git_backend::RefStore,
+ objects: &dyn git_backend::ObjectStore,
+) -> BTreeSet<ObjectId> {
+ let roots: Vec<ObjectId> = refs
+ .iter_prefix(&RefName::new("refs/heads/"))
+ .unwrap()
+ .map(|entry| entry.unwrap().1)
+ .collect();
+ let source = git_reachability::walk::StoreSource::new(objects);
+ git_reachability::walk::reachable(roots, &source, |_id| false, false).unwrap()
+}
+
+fn keygen(base: &Path, name: &str) -> PathBuf {
+ let key = base.join(name);
+ let status = Command::new("ssh-keygen")
+ .args(["-q", "-t", "ed25519", "-N", "", "-C", name, "-f"])
+ .arg(&key)
+ .status()
+ .unwrap();
+ assert!(status.success(), "ssh-keygen failed");
+ key
+}
+
+fn free_port() -> u16 {
+ let probe = TcpListener::bind("127.0.0.1:0").unwrap();
+ let port = probe.local_addr().unwrap().port();
+ drop(probe);
+ port
+}
+
+fn wait_for_port(port: u16) {
+ let deadline = std::time::Instant::now() + Duration::from_secs(5);
+ loop {
+ match TcpStream::connect(format!("127.0.0.1:{port}")) {
+ Ok(_) => return,
+ Err(_) if std::time::Instant::now() < deadline => {
+ std::thread::sleep(Duration::from_millis(10));
+ }
+ Err(error) => panic!("server never accepted connections: {error}"),
+ }
+ }
+}
+
+fn run(dir: &Path, program: &str, args: &[&str]) {
+ let output = Command::new(program)
+ .current_dir(dir)
+ .args(args)
+ .env("GIT_CONFIG_GLOBAL", "/dev/null")
+ .env("GIT_CONFIG_SYSTEM", "/dev/null")
+ .env("GIT_AUTHOR_NAME", "T")
+ .env("GIT_AUTHOR_EMAIL", "t@e")
+ .env("GIT_COMMITTER_NAME", "T")
+ .env("GIT_COMMITTER_EMAIL", "t@e")
+ .output()
+ .unwrap();
+ assert!(
+ output.status.success(),
+ "{program} {args:?} failed: {}",
+ String::from_utf8_lossy(&output.stderr)
+ );
+}
crates/git-hydrate/Cargo.toml
@@ -1,0 +1,27 @@
+[package]
+name = "git-hydrate"
+version = "0.0.0"
+edition.workspace = true
+publish.workspace = true
+license.workspace = true
+
+[dependencies]
+git-backend = { workspace = true }
+git-ents-core = { workspace = true }
+git-member = { workspace = true }
+git-protocol = { workspace = true }
+git-reachability = { workspace = true }
+gix-hash = { workspace = true }
+gix-object = { workspace = true }
+odb-tigris = { workspace = true }
+refstore-postgres = { workspace = true }
+tempfile = { workspace = true }
+
+[dev-dependencies]
+backend-conformance = { workspace = true }
+odb-files = { workspace = true }
+refstore-files = { workspace = true }
+uuid = { workspace = true }
+
+[lints]
+workspace = true
crates/git-hydrate/src/config.rs
@@ -1,0 +1,86 @@
+//! [`HydrateConfig`]: how to reach the durable stores hydration reads from
+//! and writes through. Present (`Some`) enables hydration mode; absent
+//! keeps a deployment on the current direct-disk behavior, per
+//! `docs/scale-out.adoc`'s thesis that cloud deployment is additive
+//! configuration, not a different code path.
+
+use std::path::PathBuf;
+
+use odb_tigris::transport::s3::S3Config;
+
+/// Which [`odb_tigris::transport::BlobTransport`] hydration reads packs
+/// from and writes them to.
+#[derive(Debug, Clone)]
+pub enum BlobStore {
+ /// A local directory, standing in for the bucket — used by tests and
+ /// small/self-hosted deployments that don't need S3.
+ Fs(PathBuf),
+ /// A real S3-compatible bucket (Tigris in production).
+ S3(S3Config),
+}
+
+/// Everything hydration needs to reach the durable stores: a Postgres
+/// connection string (`refstore-postgres`'s ref store, reflog, pack
+/// registry, and op-replay corpus log) and a blob store (`odb-tigris`'s
+/// packs).
+#[derive(Debug, Clone)]
+pub struct HydrateConfig {
+ /// Libpq connection string for the Postgres ref store / pack registry /
+ /// corpus log.
+ pub postgres_conninfo: String,
+ /// Where packs live.
+ pub blob: BlobStore,
+}
+
+impl HydrateConfig {
+ /// Build a config over a local directory blob store — the common case
+ /// for tests and small deployments.
+ #[must_use]
+ pub fn with_fs_blob(postgres_conninfo: impl Into<String>, root: impl Into<PathBuf>) -> Self {
+ Self {
+ postgres_conninfo: postgres_conninfo.into(),
+ blob: BlobStore::Fs(root.into()),
+ }
+ }
+
+ /// Read a config from the environment, or `None` if hydration is not
+ /// configured (the caller should then keep the current direct-disk
+ /// behavior). Recognizes:
+ ///
+ /// - `GIT_ENTS_HYDRATE_POSTGRES_URL` (required to enable hydration).
+ /// - `GIT_ENTS_HYDRATE_BLOB_ROOT` — a local directory blob store, or
+ /// - `GIT_ENTS_HYDRATE_S3_BUCKET`/`_REGION`/`_ENDPOINT`/
+ /// `_ACCESS_KEY_ID`/`_SECRET_ACCESS_KEY`/`_ALLOW_HTTP` — an
+ /// S3-compatible bucket. The `Fs` root wins if both are set.
+ #[must_use]
+ pub fn from_env() -> Option<Self> {
+ let postgres_conninfo = env_var("GIT_ENTS_HYDRATE_POSTGRES_URL")?;
+ if let Some(root) = env_var("GIT_ENTS_HYDRATE_BLOB_ROOT") {
+ return Some(Self {
+ postgres_conninfo,
+ blob: BlobStore::Fs(PathBuf::from(root)),
+ });
+ }
+ let bucket = env_var("GIT_ENTS_HYDRATE_S3_BUCKET")?;
+ let region = env_var("GIT_ENTS_HYDRATE_S3_REGION").unwrap_or_else(|| "auto".to_owned());
+ let endpoint = env_var("GIT_ENTS_HYDRATE_S3_ENDPOINT")?;
+ let access_key_id = env_var("GIT_ENTS_HYDRATE_S3_ACCESS_KEY_ID")?;
+ let secret_access_key = env_var("GIT_ENTS_HYDRATE_S3_SECRET_ACCESS_KEY")?;
+ let allow_http = env_var("GIT_ENTS_HYDRATE_S3_ALLOW_HTTP").is_some_and(|v| v == "1");
+ Some(Self {
+ postgres_conninfo,
+ blob: BlobStore::S3(S3Config {
+ bucket,
+ region,
+ endpoint,
+ access_key_id,
+ secret_access_key,
+ allow_http,
+ }),
+ })
+ }
+}
+
+fn env_var(key: &str) -> Option<String> {
+ std::env::var(key).ok().filter(|value| !value.is_empty())
+}
crates/git-hydrate/src/hydrate.rs
@@ -1,0 +1,101 @@
+//! The read-path hydration step (`docs/scale-out.adoc`, WS0's read path):
+//! copy a repository's registered packs into a local bare repository's
+//! `objects/pack/`, idempotently.
+
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+use git_backend::Result;
+use odb_tigris::registry::PackRegistry;
+use odb_tigris::transport::BlobTransport;
+
+/// Ensure `repo_path` is a bare repository on local (ephemeral) disk
+/// carrying every pack `registry` has registered for `repo_id`, fetched
+/// from `transport`.
+///
+/// Idempotent and cheap to call on every request: a pack already present
+/// locally (named after its [`odb_tigris::registry::PackId`], so presence
+/// is a plain file check) is never re-fetched. Nothing here is
+/// correctness-bearing — ephemeral disk death just means the next call
+/// starts from an empty `objects/pack/` and re-copies everything
+/// (`docs/scale-out.adoc`: "Ephemeral disk death -> re-hydrate. Nothing
+/// correctness-bearing on disk").
+///
+/// # Errors
+///
+/// Returns an error if the bare repository cannot be initialized, the
+/// registry cannot be listed, or a pack/idx cannot be fetched or written.
+pub fn ensure_hydrated<T, R>(
+ repo_path: &Path,
+ repo_id: &str,
+ transport: &T,
+ registry: &R,
+) -> Result<()>
+where
+ T: BlobTransport,
+ R: PackRegistry,
+{
+ if !is_bare_repo(repo_path) {
+ init_bare_repo(repo_path)?;
+ }
+ let pack_dir = repo_path.join("objects").join("pack");
+ std::fs::create_dir_all(&pack_dir)?;
+
+ for record in registry.list(repo_id)? {
+ let pack_path = pack_dir.join(format!("pack-{}.pack", record.id.as_str()));
+ let idx_path = pack_dir.join(format!("pack-{}.idx", record.id.as_str()));
+ if pack_path.is_file() && idx_path.is_file() {
+ // Already hydrated from a previous request/instance — the
+ // whole point of naming local files after the registry's own
+ // pack id.
+ continue;
+ }
+ let pack_bytes = transport.get(&record.pack_key)?;
+ let idx_bytes = transport.get(&record.idx_key)?;
+ atomic_write(&pack_path, &pack_bytes)?;
+ atomic_write(&idx_path, &idx_bytes)?;
+ }
+ Ok(())
+}
+
+/// Whether `path` is the root of a bare git repository.
+fn is_bare_repo(path: &Path) -> bool {
+ path.join("HEAD").is_file() && path.join("objects").is_dir()
+}
+
+/// Create an empty bare repository at `repo_path`, creating parent
+/// directories as needed.
+fn init_bare_repo(repo_path: &Path) -> Result<()> {
+ if let Some(parent) = repo_path.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+ let status = Command::new("git")
+ .arg("init")
+ .arg("--bare")
+ .arg("-q")
+ .arg(repo_path)
+ .stdout(Stdio::null())
+ .stderr(Stdio::null())
+ .status()?;
+ if !status.success() {
+ return Err(git_backend::Error::ObjectStore(
+ "git init --bare failed while hydrating a repository".to_owned(),
+ ));
+ }
+ Ok(())
+}
+
+/// Write `bytes` to `path` via a same-directory temp file and rename, so a
+/// reader never observes a partially-written pack or idx.
+fn atomic_write(path: &Path, bytes: &[u8]) -> Result<()> {
+ let tmp_path = tmp_path_for(path);
+ std::fs::write(&tmp_path, bytes)?;
+ std::fs::rename(&tmp_path, path)?;
+ Ok(())
+}
+
+fn tmp_path_for(path: &Path) -> PathBuf {
+ let mut tmp = path.as_os_str().to_owned();
+ tmp.push(".tmp");
+ PathBuf::from(tmp)
+}
crates/git-hydrate/src/lib.rs
@@ -1,0 +1,65 @@
+//! WS0 — the interim hydration backend (`docs/scale-out.adoc`, "WS0 —
+//! Interim hydration backend"): stock `git http-backend` over ephemeral
+//! disk, hydrated from the durable stores (`refstore-postgres` for refs,
+//! `odb-tigris` for packs). Not a hack outside the architecture — this
+//! crate *is* the stock-git-wrapped backend the protocol traits permit,
+//! built first, exactly as the doc's decision record on invariant
+//! stratification describes.
+//!
+//! # Read path
+//!
+//! [`hydrate::ensure_hydrated`] copies a repository's registered packs
+//! (`.pack` + `.idx`, from [`odb_tigris::registry::PackRegistry`]) into
+//! `objects/pack/` of a local bare repository, skipping any pack already
+//! present by its own content-addressed filename — idempotent, and cheap on
+//! every call after the first: ephemeral disk death means nothing more than
+//! re-copying everything again next time (`docs/scale-out.adoc`: "nothing
+//! correctness-bearing on ephemeral disk"). [`packed_refs::regenerate`]
+//! rewrites `packed-refs` from one `RefStore::iter_prefix("refs/")` scan,
+//! atomically, to bound advertisement staleness — call it on every
+//! `info/refs` request, not just the first.
+//!
+//! # Write path
+//!
+//! [`pre_receive::run`] is the `pre-receive` hook body for a repository
+//! configured with a [`config::HydrateConfig`]: it authenticates and applies
+//! the push through [`git_protocol::native::NativeBackend::receive`]
+//! (`IngestPack`) against a [`resolver::PostgresResolver`] — Postgres as the
+//! ref store, Tigris (or a local directory in tests) as the object store —
+//! exactly the "IngestPack via receive-pack against a scratch repo with
+//! Postgres as the commit point" shape `docs/scale-out.adoc`'s "Protocol
+//! traits" section names as a conforming implementation. `receive-pack`'s
+//! own tmp objdir plays no special role here (unlike a hand-rolled
+//! quarantine): staging, the atomic ref transaction, and promotion are all
+//! `NativeBackend::receive`'s existing, already-tested ordering, so causal
+//! collection safety holds by construction, not by convention. Local disk
+//! is a demoted cache: git's own post-hook ref update reconciles it to
+//! match Postgres automatically, since our applied edits are the exact ones
+//! `receive-pack` was asked to make. The one ref this doesn't reconcile
+//! locally — `refs/meta/ops/log`, added to the same atomic transaction
+//! internally — self-heals on the next `info/refs` (packed-refs
+//! regeneration reads every ref back from Postgres, this one included).
+//!
+//! Every accepted push through this path also logs a
+//! [`git_protocol::CorpusEntry`] (see [`refstore_postgres::PostgresRefStore::log_corpus_entry`]):
+//! the seed corpus `backend_conformance::replay_corpus` replays against the
+//! local files backends (WS2).
+//!
+//! # Known limits (short-term, accepted)
+//!
+//! - Whole-pack hydration makes first-touch read latency scale with repo
+//! size; ranged reads (WS5) are the fix, not this crate's job.
+//! - Concurrent pushes to one repo from multiple serve machines are safe
+//! under Postgres's compare-and-swap (no split-brain ref state is ever
+//! possible), but a machine whose local disk cache is stale relative to
+//! another machine's last-accepted push will advertise a stale `old` and
+//! see spurious rejections until its next `info/refs` re-hydration. Pin
+//! writes for one repository to one machine, or accept client retries.
+
+pub mod config;
+pub mod hydrate;
+pub mod packed_refs;
+pub mod pre_receive;
+pub mod resolver;
+
+pub use config::HydrateConfig;
crates/git-hydrate/src/packed_refs.rs
@@ -1,0 +1,44 @@
+//! The read-path's other half (`docs/scale-out.adoc`, WS0's read path):
+//! regenerate `packed-refs` from the ref store on every `info/refs`
+//! request, bounding advertisement staleness to one request's worth.
+
+use std::path::Path;
+
+use git_backend::{RefName, RefStore, Result};
+
+/// Rewrite `repo_path`'s `packed-refs` from one
+/// [`RefStore::iter_prefix`]`("refs/")` scan over `refs`, atomically (a
+/// temp file, then a rename) so a concurrent `git` reader never observes a
+/// half-written file.
+///
+/// No peeled (`^{}`) entries are emitted for annotated tags — this rewrite
+/// intentionally does not claim the `fully-peeled` trait git's
+/// `packed-refs` format supports, so a reader that needs a tag's peeled
+/// target still resolves it correctly by opening the tag object itself,
+/// just without the fast path a fully-peeled file would offer. Correctness
+/// over an optimization this backend does not need yet.
+///
+/// # Errors
+///
+/// Returns an error if `refs` cannot be scanned or the file cannot be
+/// written.
+pub fn regenerate(repo_path: &Path, refs: &dyn RefStore) -> Result<()> {
+ let mut entries: Vec<(String, String)> = refs
+ .iter_prefix(&RefName::new("refs/"))?
+ .map(|entry| entry.map(|(name, oid)| (name.as_str().to_owned(), oid.to_hex().to_string())))
+ .collect::<Result<_>>()?;
+ entries.sort();
+
+ let mut body = String::from("# pack-refs with: sorted\n");
+ for (name, oid) in entries {
+ body.push_str(&oid);
+ body.push(' ');
+ body.push_str(&name);
+ body.push('\n');
+ }
+
+ let tmp_path = repo_path.join("packed-refs.tmp");
+ std::fs::write(&tmp_path, body)?;
+ std::fs::rename(&tmp_path, repo_path.join("packed-refs"))?;
+ Ok(())
+}
crates/git-hydrate/src/pre_receive.rs
@@ -1,0 +1,249 @@
+//! The `pre-receive` hook body for a hydration-configured repository
+//! (`docs/scale-out.adoc`, WS0's write path): apply the push through
+//! [`git_protocol::native::NativeBackend::receive`] against
+//! [`crate::resolver::PostgresResolver`], then log the accepted push's
+//! [`git_protocol::CorpusEntry`] for later replay (WS2's seed corpus).
+
+use std::io::{Cursor, Read as _};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::Arc;
+
+use git_backend::{Expected, PackStream, RefEdit, RefName};
+use git_protocol::attestation::{OpSigner, SshOpSigner};
+use git_protocol::native::NativeBackend;
+use git_protocol::{
+ CorpusEntry, IngestPack as _, PushCertificate, PushOutcome, PushRequest, RepoId,
+};
+use gix_hash::ObjectId;
+
+use crate::config::HydrateConfig;
+use crate::resolver::PostgresResolver;
+
+/// One ref update as git hands it to `pre-receive` on stdin.
+struct RefUpdate {
+ name: RefName,
+ old: Option<ObjectId>,
+ new: Option<ObjectId>,
+}
+
+impl RefUpdate {
+ fn to_ref_edit(&self) -> RefEdit {
+ RefEdit {
+ name: self.name.clone(),
+ expected: match self.old {
+ Some(oid) => Expected::MustExistAndMatch(oid),
+ None => Expected::MustNotExist,
+ },
+ new: self.new,
+ }
+ }
+}
+
+/// Run the hook: read the push git is about to apply, commit it through
+/// the durable stores, and log its corpus entry.
+///
+/// `op_signing_key` signs the accepted push's server op record; `None`
+/// rejects every push closed (mirrors `git_ents_server::native_git`'s own
+/// rule: no signing key configured, no accepted push — reads are
+/// unaffected). `config` names the Postgres/blob-store pair this
+/// repository hydrates from and writes through.
+///
+/// # Errors
+///
+/// Returns `Err(reason)` — the caller prints `reason` to stderr and exits
+/// non-zero, rejecting the whole push — if the ref updates or push
+/// certificate cannot be read, the incoming pack cannot be built, or the
+/// push itself is rejected (failed attestation, failed connectivity, or a
+/// failed compare-and-swap against Postgres).
+pub fn run(config: &HydrateConfig, op_signing_key: Option<&Path>) -> Result<(), String> {
+ let repo_path =
+ std::env::current_dir().map_err(|error| format!("cannot resolve repository: {error}"))?;
+ let repo_id = repo_id_for(&repo_path);
+
+ let updates = read_ref_updates()?;
+ if updates.is_empty() {
+ return Ok(());
+ }
+
+ let roots: Vec<ObjectId> = updates.iter().filter_map(|update| update.new).collect();
+ let pack_bytes = build_pack(&repo_path, &roots)?;
+
+ let cert_text = read_push_cert(&repo_path)?;
+ let cert_bytes = cert_text
+ .as_deref()
+ .map(str::as_bytes)
+ .unwrap_or_default()
+ .to_vec();
+ let cert_oid =
+ gix_object::compute_hash(gix_hash::Kind::Sha1, gix_object::Kind::Blob, &cert_bytes)
+ .map_err(|error| format!("could not hash push certificate: {error}"))?;
+
+ let signer: Arc<dyn OpSigner> = match op_signing_key {
+ Some(key) => Arc::new(SshOpSigner::new(key.to_path_buf())),
+ // No signing key: op records fail to sign, so every otherwise
+ // acceptable push is rejected — fail-closed, since an accepted
+ // push without its op record breaks the "universal server op
+ // record" rule.
+ None => Arc::new(SshOpSigner::new(PathBuf::from("/dev/null"))),
+ };
+ let resolver = PostgresResolver::new(config.clone(), repo_path.clone());
+ let backend = NativeBackend::new(resolver, signer);
+
+ let ref_edits: Vec<RefEdit> = updates.iter().map(RefUpdate::to_ref_edit).collect();
+ let push = PushRequest {
+ repo: RepoId::new(repo_id.clone()),
+ ref_edits,
+ pack: PackStream::new(Cursor::new(pack_bytes.clone())),
+ push_cert: cert_text.clone().map(PushCertificate::new),
+ };
+
+ match backend.receive(push).map_err(|error| error.to_string())? {
+ PushOutcome::Accepted { applied, .. } => {
+ let entry =
+ CorpusEntry::new(cert_text.is_some().then_some(cert_oid), applied, pack_bytes);
+ log_corpus_entry(config, &repo_id, &entry);
+ Ok(())
+ }
+ PushOutcome::Rejected { reason } => Err(reason),
+ }
+}
+
+/// This repository's id, relative to `$GIT_PROJECT_ROOT` when set (the
+/// same env var `git-ents-server`'s CGI gateway hands every backend
+/// invocation, inherited down through `receive-pack` to this hook) —
+/// otherwise the repository's own directory name, so the hook still runs
+/// (against a single-repo id) outside that server.
+fn repo_id_for(repo_path: &Path) -> String {
+ if let Ok(root) = std::env::var("GIT_PROJECT_ROOT")
+ && let Ok(relative) = repo_path.strip_prefix(root)
+ && !relative.as_os_str().is_empty()
+ {
+ return relative.to_string_lossy().replace('\\', "/");
+ }
+ repo_path
+ .file_name()
+ .map(|name| name.to_string_lossy().into_owned())
+ .unwrap_or_else(|| repo_path.to_string_lossy().into_owned())
+}
+
+/// Read the ref updates git hands `pre-receive` on stdin: `<old> <new>
+/// <refname>` per line.
+fn read_ref_updates() -> Result<Vec<RefUpdate>, String> {
+ let mut input = String::new();
+ std::io::stdin()
+ .read_to_string(&mut input)
+ .map_err(|error| format!("could not read ref updates: {error}"))?;
+
+ let null = ObjectId::null(gix_hash::Kind::Sha1);
+ let mut updates = Vec::new();
+ for line in input.lines() {
+ let mut parts = line.split_whitespace();
+ let (Some(old_hex), Some(new_hex), Some(name)) = (parts.next(), parts.next(), parts.next())
+ else {
+ continue;
+ };
+ let old = ObjectId::from_hex(old_hex.as_bytes())
+ .map_err(|error| format!("bad old oid {old_hex:?}: {error}"))?;
+ let new = ObjectId::from_hex(new_hex.as_bytes())
+ .map_err(|error| format!("bad new oid {new_hex:?}: {error}"))?;
+ updates.push(RefUpdate {
+ name: RefName::new(name),
+ old: (old != null).then_some(old),
+ new: (new != null).then_some(new),
+ });
+ }
+ Ok(updates)
+}
+
+/// The push certificate git verified the nonce of, read from the blob
+/// `$GIT_PUSH_CERT` names — `None` when the push carried no certificate at
+/// all (only acceptable during the bootstrap window, which
+/// `NativeBackend::receive`'s attestation check enforces).
+///
+/// # Errors
+///
+/// Returns `Err` if a certificate was sent but its anti-replay nonce did
+/// not validate, or its blob cannot be read.
+fn read_push_cert(repo: &Path) -> Result<Option<String>, String> {
+ let Some(oid) = std::env::var("GIT_PUSH_CERT")
+ .ok()
+ .filter(|value| !value.is_empty())
+ else {
+ return Ok(None);
+ };
+ if std::env::var("GIT_PUSH_CERT_NONCE_STATUS").as_deref() != Ok("OK") {
+ return Err("push certificate nonce was missing or stale".to_owned());
+ }
+ let output = Command::new("git")
+ .arg("-C")
+ .arg(repo)
+ .args(["cat-file", "blob", &oid])
+ .output()
+ .map_err(|error| format!("could not read push certificate: {error}"))?;
+ if !output.status.success() {
+ return Err("could not read the push certificate from the object store".to_owned());
+ }
+ String::from_utf8(output.stdout)
+ .map(Some)
+ .map_err(|_invalid| "push certificate is not valid UTF-8".to_owned())
+}
+
+/// Build the pack introducing every object reachable from `roots` that
+/// `repo` (plus its inherited quarantine — `$GIT_OBJECT_DIRECTORY`/
+/// `$GIT_ALTERNATE_OBJECT_DIRECTORIES`, set by `receive-pack` for this
+/// very hook) doesn't already have, by shelling out to `git rev-list`/`git
+/// pack-objects` exactly as a real client push transmits one. An empty
+/// `roots` (a batch of pure ref deletions) still needs a valid, empty pack.
+fn build_pack(repo: &Path, roots: &[ObjectId]) -> Result<Vec<u8>, String> {
+ if roots.is_empty() {
+ return git_protocol::pack::build_pack(&[]).map_err(|error| error.to_string());
+ }
+ let mut rev_list = Command::new("git")
+ .arg("-C")
+ .arg(repo)
+ .args(["rev-list", "--objects"])
+ .args(roots.iter().map(|oid| oid.to_hex().to_string()))
+ .args(["--not", "--all"])
+ .stdout(Stdio::piped())
+ .spawn()
+ .map_err(|error| format!("could not spawn git rev-list: {error}"))?;
+ let rev_list_stdout = rev_list
+ .stdout
+ .take()
+ .ok_or_else(|| "git rev-list produced no stdout".to_owned())?;
+ let pack_objects = Command::new("git")
+ .arg("-C")
+ .arg(repo)
+ .args(["pack-objects", "--stdout", "-q"])
+ .stdin(rev_list_stdout)
+ .stdout(Stdio::piped())
+ .spawn()
+ .map_err(|error| format!("could not spawn git pack-objects: {error}"))?;
+ let output = pack_objects
+ .wait_with_output()
+ .map_err(|error| format!("git pack-objects failed: {error}"))?;
+ let rev_list_status = rev_list
+ .wait()
+ .map_err(|error| format!("git rev-list failed: {error}"))?;
+ if !rev_list_status.success() {
+ return Err("git rev-list failed while building the push's pack".to_owned());
+ }
+ if !output.status.success() {
+ return Err("git pack-objects failed while building the push's pack".to_owned());
+ }
+ Ok(output.stdout)
+}
+
+/// Best-effort: the push already committed to Postgres by the time this
+/// runs, so a corpus-logging failure must not undo (or even report as
+/// failing) an otherwise-accepted push — the same "a failure here cannot
+/// undo the push" stance `git_effect::engine::post_receive` takes.
+fn log_corpus_entry(config: &HydrateConfig, repo_id: &str, entry: &CorpusEntry) {
+ let Ok(store) =
+ refstore_postgres::PostgresRefStore::connect(&config.postgres_conninfo, repo_id.to_owned())
+ else {
+ return;
+ };
+ let _ignored = store.log_corpus_entry(entry);
+}
crates/git-hydrate/src/resolver.rs
@@ -1,0 +1,89 @@
+//! [`PostgresResolver`]: a [`git_protocol::native::BackendResolver`] over
+//! the durable stores — Postgres for refs (and, doubling as the pack
+//! registry, for `odb-tigris`'s bookkeeping), the configured blob store for
+//! packs. Feeding this resolver to
+//! [`git_protocol::native::NativeBackend`] is what makes
+//! [`crate::pre_receive::run`] the "IngestPack via receive-pack against a
+//! scratch repo with Postgres as the commit point" backend
+//! `docs/scale-out.adoc`'s "Protocol traits" section names.
+
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use git_backend::ObjectStore;
+use git_protocol::native::{BackendResolver, RepoBackends};
+use git_protocol::types::RepoId;
+use odb_tigris::OdbTigris;
+use odb_tigris::transport::fs::FsTransport;
+use odb_tigris::transport::s3::S3Transport;
+use refstore_postgres::PostgresRefStore;
+
+use crate::config::{BlobStore, HydrateConfig};
+
+/// Resolves a [`RepoId`] to `refstore-postgres`/`odb-tigris` backends, and
+/// to the repository's currently enrolled members/config — read from the
+/// local hydrated disk cache at `repo_path`, exactly as
+/// `git_ents_server::native_git::DiskResolver` reads them for the WS3
+/// native path, and as `git-signed-push`'s own `pre-receive` verifier
+/// always has: a fresh disk read per call, which is close enough to
+/// Postgres truth by the time a push reaches this resolver (`packed-refs`
+/// was just regenerated from Postgres on the preceding `info/refs`).
+pub struct PostgresResolver {
+ config: HydrateConfig,
+ repo_path: PathBuf,
+}
+
+impl PostgresResolver {
+ /// Resolve repositories through `config`'s durable stores, reading
+ /// members/config from the local hydrated cache at `repo_path`.
+ #[must_use]
+ pub fn new(config: HydrateConfig, repo_path: impl Into<PathBuf>) -> Self {
+ Self {
+ config,
+ repo_path: repo_path.into(),
+ }
+ }
+}
+
+impl BackendResolver for PostgresResolver {
+ fn resolve(&self, repo: &RepoId) -> git_protocol::Result<RepoBackends> {
+ let refs = PostgresRefStore::connect(&self.config.postgres_conninfo, repo.as_str())?;
+ let registry = PostgresRefStore::connect(&self.config.postgres_conninfo, repo.as_str())?;
+ let objects: Arc<dyn ObjectStore> = match &self.config.blob {
+ BlobStore::Fs(root) => {
+ let transport = FsTransport::open(root)?;
+ Arc::new(OdbTigris::new(
+ transport,
+ registry,
+ repo.as_str().to_owned(),
+ ))
+ }
+ BlobStore::S3(s3_config) => {
+ let transport = S3Transport::connect(s3_config)?;
+ Arc::new(OdbTigris::new(
+ transport,
+ registry,
+ repo.as_str().to_owned(),
+ ))
+ }
+ };
+
+ let members = git_member::members::load_all(&self.repo_path)
+ .map_err(|error| git_protocol::Error::UnknownRepo(error.to_string()))?;
+ let revoked = git_member::revocations::fingerprints(&self.repo_path)
+ .map_err(|error| git_protocol::Error::UnknownRepo(error.to_string()))?;
+ let config = git_ents_core::config::load(&self.repo_path)
+ .map_err(|error| git_protocol::Error::UnknownRepo(error.to_string()))?;
+
+ Ok(RepoBackends {
+ refs: Arc::new(refs),
+ objects,
+ authorized_members: git_member::members::without_revoked(members, &revoked),
+ config,
+ // No reachability artifacts wired for this resolver yet (same
+ // gap `DiskResolver` documents): negotiation/ingest degrade to
+ // the plain walk, never a wrong answer.
+ reachability: git_reachability::ArtifactBundle::empty(),
+ })
+ }
+}