feat: add odb-tigris, an ObjectStore over ranged S3-compatible reads
commit
a05d5c6feat: add odb-tigris, an ObjectStore over ranged S3-compatible reads
Implements WS5’s Tigris object store: packs and .idx live in an
S3-compatible bucket under a per-repo prefix, OID lookup resolves via
cached-in-memory pack indexes, and object reads are ranged GETs of just
the needed slice rather than whole-pack downloads. OFS/REF delta chains
resolve over further ranged reads with bounded recursion, since
gix-pack’s own delta decoder and applier are crate-private.
feat: add a BlobTransport seam (filesystem + object_store-backed S3) feat: add a PackRegistry trait plus an in-memory implementation feat: add a pack writer partitioning objects by pack-lifetime class test: add a hand-built OfsDelta pack fixture exercising ranged decode test: instantiate the backend-conformance ObjectStore suite for odb-tigris Assisted-by: Claude:claude-sonnet-5
Reviews
No reviews of this commit yet — record a verdict below.
Start a review
Cargo.lock
@@ -488,6 +488,29 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
+[[package]]
+name = "aws-lc-rs"
+version = "1.17.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
+dependencies = [
+ "aws-lc-sys",
+ "zeroize",
+]
+
+[[package]]
+name = "aws-lc-sys"
+version = "0.42.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
+dependencies = [
+ "cc",
+ "cmake",
+ "dunce",
+ "fs_extra",
+ "pkg-config",
+]
+
[[package]]
name = "axum"
version = "0.8.9"
@@ -656,6 +679,8 @@
checksum = "e228eec9be7c17ccb640b59b36a5cd805ea2a564a4c5e162c2f659fea30d3b96"
dependencies = [
"find-msvc-tools",
+ "jobserver",
+ "libc",
"shlex",
]
@@ -697,6 +722,7 @@
"iana-time-zone",
"js-sys",
"num-traits",
+ "serde",
"wasm-bindgen",
"windows-link",
]
@@ -710,12 +736,31 @@
"hashbrown 0.16.1",
]
+[[package]]
+name = "cmake"
+version = "0.1.58"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "cmov"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
+[[package]]
+name = "combine"
+version = "4.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+dependencies = [
+ "bytes",
+ "memchr",
+]
+
[[package]]
name = "comfy-table"
version = "7.2.2"
@@ -768,6 +813,16 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "980c2afde4af43d6a05c5be738f9eae595cff86dce1f38f88b95058a98c027f3"
+[[package]]
+name = "core-foundation"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "core-foundation-sys"
version = "0.8.7"
@@ -792,6 +847,16 @@
"libc",
]
+[[package]]
+name = "crc-fast"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5"
+dependencies = [
+ "digest 0.10.7",
+ "spin",
+]
+
[[package]]
name = "crc32fast"
version = "1.5.0"
@@ -1391,6 +1456,12 @@
"percent-encoding",
]
+[[package]]
+name = "fs_extra"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
+
[[package]]
name = "futures-channel"
version = "0.3.32"
@@ -1407,6 +1478,12 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
+[[package]]
+name = "futures-io"
+version = "0.3.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
+
[[package]]
name = "futures-macro"
version = "0.3.32"
@@ -1443,9 +1520,11 @@
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
dependencies = [
"futures-core",
+ "futures-io",
"futures-macro",
"futures-sink",
"futures-task",
+ "memchr",
"pin-project-lite",
"slab",
]
@@ -1467,8 +1546,10 @@
checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"wasi 0.11.1+wasi-snapshot-preview1",
+ "wasm-bindgen",
]
[[package]]
@@ -1490,9 +1571,11 @@
checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099"
dependencies = [
"cfg-if",
+ "js-sys",
"libc",
"r-efi 6.0.0",
"rand_core 0.10.1",
+ "wasm-bindgen",
]
[[package]]
@@ -2592,6 +2675,25 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
+[[package]]
+name = "h2"
+version = "0.4.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cb093c84e8bd9b188d4c4a8cb6579fc016968d14c99882163cd3ff402a4f155"
+dependencies = [
+ "atomic-waker",
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "http",
+ "indexmap",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
[[package]]
name = "hash32"
version = "0.3.1"
@@ -2711,6 +2813,12 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eaec953f16e5bcf6b8a3cb3aa959b17e5577dbd2693e94554c462c08be22624b"
+[[package]]
+name = "humantime"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
+
[[package]]
name = "hybrid-array"
version = "0.4.13"
@@ -2730,6 +2838,7 @@
"bytes",
"futures-channel",
"futures-core",
+ "h2",
"http",
"http-body",
"httparse",
@@ -2738,6 +2847,22 @@
"pin-project-lite",
"smallvec 1.15.2",
"tokio",
+ "want",
+]
+
+[[package]]
+name = "hyper-rustls"
+version = "0.27.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f"
+dependencies = [
+ "http",
+ "hyper",
+ "hyper-util",
+ "rustls",
+ "tokio",
+ "tokio-rustls",
+ "tower-service",
]
[[package]]
@@ -2746,13 +2871,21 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0"
dependencies = [
+ "base64",
"bytes",
+ "futures-channel",
+ "futures-util",
"http",
"http-body",
"hyper",
+ "ipnet",
+ "libc",
+ "percent-encoding",
"pin-project-lite",
+ "socket2",
"tokio",
"tower-service",
+ "tracing",
]
[[package]]
@@ -2945,6 +3078,12 @@
"winapi",
]
+[[package]]
+name = "ipnet"
+version = "2.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
+
[[package]]
name = "is-terminal"
version = "0.4.17"
@@ -2962,6 +3101,15 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7655c9839580ee829dfacba1d1278c2b7883e50a277ff7541299489d6bdfdc45"
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
[[package]]
name = "itoa"
version = "1.0.18"
@@ -3010,6 +3158,65 @@
"jiff-tzdb",
]
+[[package]]
+name = "jni"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
+dependencies = [
+ "cfg-if",
+ "combine",
+ "jni-macros",
+ "jni-sys",
+ "log",
+ "simd_cesu8",
+ "thiserror 2.0.18",
+ "walkdir",
+ "windows-link",
+]
+
+[[package]]
+name = "jni-macros"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "simd_cesu8",
+ "syn",
+]
+
+[[package]]
+name = "jni-sys"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2"
+dependencies = [
+ "jni-sys-macros",
+]
+
+[[package]]
+name = "jni-sys-macros"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264"
+dependencies = [
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "jobserver"
+version = "0.1.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3"
+dependencies = [
+ "getrandom 0.4.3",
+ "libc",
+]
+
[[package]]
name = "js-sys"
version = "0.3.103"
@@ -3107,6 +3314,12 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad"
+[[package]]
+name = "lru-slab"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
+
[[package]]
name = "matchit"
version = "0.8.4"
@@ -3286,6 +3499,45 @@
"objc2-core-foundation",
]
+[[package]]
+name = "object_store"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "765784b4390c6bcf80316e5a22f4e3661b639c9d8c83246856643c27d8ce9dbe"
+dependencies = [
+ "async-trait",
+ "aws-lc-rs",
+ "base64",
+ "bytes",
+ "chrono",
+ "crc-fast",
+ "form_urlencoded",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body-util",
+ "humantime",
+ "hyper",
+ "itertools",
+ "md-5",
+ "parking_lot",
+ "percent-encoding",
+ "quick-xml 0.40.1",
+ "rand 0.10.2",
+ "reqwest",
+ "rustls-pki-types",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "url",
+ "wasm-bindgen-futures",
+ "web-time",
+]
+
[[package]]
name = "odb-files"
version = "0.0.0"
@@ -3300,6 +3552,22 @@
"uuid",
]
+[[package]]
+name = "odb-tigris"
+version = "0.0.0"
+dependencies = [
+ "backend-conformance",
+ "git-backend",
+ "gix-features",
+ "gix-hash",
+ "gix-object",
+ "gix-pack",
+ "object_store",
+ "tempfile",
+ "tokio",
+ "uuid",
+]
+
[[package]]
name = "once_cell"
version = "1.21.4"
@@ -3328,6 +3596,12 @@
"pkg-config",
]
+[[package]]
+name = "openssl-probe"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
+
[[package]]
name = "owo-colors"
version = "4.3.0"
@@ -3433,7 +3707,7 @@
dependencies = [
"base64",
"indexmap",
- "quick-xml",
+ "quick-xml 0.41.0",
"serde",
"time",
]
@@ -3586,6 +3860,16 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae"
+[[package]]
+name = "quick-xml"
+version = "0.40.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
+dependencies = [
+ "memchr",
+ "serde",
+]
+
[[package]]
name = "quick-xml"
version = "0.41.0"
@@ -3595,6 +3879,63 @@
"memchr",
]
+[[package]]
+name = "quinn"
+version = "0.11.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8"
+dependencies = [
+ "bytes",
+ "cfg_aliases 0.2.1",
+ "pin-project-lite",
+ "quinn-proto",
+ "quinn-udp",
+ "rustc-hash",
+ "rustls",
+ "socket2",
+ "thiserror 2.0.18",
+ "tokio",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-proto"
+version = "0.11.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
+dependencies = [
+ "aws-lc-rs",
+ "bytes",
+ "getrandom 0.4.3",
+ "lru-slab",
+ "rand 0.10.2",
+ "rand_pcg",
+ "ring",
+ "rustc-hash",
+ "rustls",
+ "rustls-pki-types",
+ "slab",
+ "thiserror 2.0.18",
+ "tinyvec",
+ "tracing",
+ "web-time",
+]
+
+[[package]]
+name = "quinn-udp"
+version = "0.5.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694"
+dependencies = [
+ "cfg_aliases 0.2.1",
+ "libc",
+ "once_cell",
+ "socket2",
+ "tracing",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "quote"
version = "1.0.46"
@@ -3662,6 +4003,15 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+[[package]]
+name = "rand_pcg"
+version = "0.10.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a"
+dependencies = [
+ "rand_core 0.10.1",
+]
+
[[package]]
name = "rayon"
version = "1.12.0"
@@ -3749,6 +4099,45 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+[[package]]
+name = "reqwest"
+version = "0.13.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
+dependencies = [
+ "base64",
+ "bytes",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "http-body-util",
+ "hyper",
+ "hyper-rustls",
+ "hyper-util",
+ "js-sys",
+ "log",
+ "percent-encoding",
+ "pin-project-lite",
+ "quinn",
+ "rustls",
+ "rustls-pki-types",
+ "rustls-platform-verifier",
+ "sync_wrapper",
+ "tokio",
+ "tokio-rustls",
+ "tokio-util",
+ "tower",
+ "tower-http",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-streams",
+ "web-sys",
+]
+
[[package]]
name = "rgb"
version = "0.8.53"
@@ -3832,6 +4221,7 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
dependencies = [
+ "aws-lc-rs",
"log",
"once_cell",
"ring",
@@ -3841,21 +4231,62 @@
"zeroize",
]
+[[package]]
+name = "rustls-native-certs"
+version = "0.8.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d"
+dependencies = [
+ "openssl-probe",
+ "rustls-pki-types",
+ "schannel",
+ "security-framework",
+]
+
[[package]]
name = "rustls-pki-types"
version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
dependencies = [
+ "web-time",
"zeroize",
]
+[[package]]
+name = "rustls-platform-verifier"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
+dependencies = [
+ "core-foundation",
+ "core-foundation-sys",
+ "jni",
+ "log",
+ "once_cell",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-platform-verifier-android",
+ "rustls-webpki",
+ "security-framework",
+ "security-framework-sys",
+ "webpki-root-certs",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls-platform-verifier-android"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
+
[[package]]
name = "rustls-webpki"
version = "0.103.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
dependencies = [
+ "aws-lc-rs",
"ring",
"rustls-pki-types",
"untrusted",
@@ -3888,12 +4319,44 @@
"winapi-util",
]
+[[package]]
+name = "schannel"
+version = "0.1.29"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
+dependencies = [
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "scopeguard"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+[[package]]
+name = "security-framework"
+version = "3.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
+dependencies = [
+ "bitflags 2.13.0",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
[[package]]
name = "self_cell"
version = "1.2.2"
@@ -4084,6 +4547,22 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
+[[package]]
+name = "simd_cesu8"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
+dependencies = [
+ "rustc_version",
+ "simdutf8",
+]
+
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
[[package]]
name = "siphasher"
version = "1.0.3"
@@ -4127,6 +4606,12 @@
"smallvec 1.15.2",
]
+[[package]]
+name = "spin"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d5fe4ccb98d9c292d56fec89a5e07da7fc4cf0dc11e156b41793132775d3e591"
+
[[package]]
name = "stable_deref_trait"
version = "1.2.1"
@@ -4212,6 +4697,9 @@
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
+dependencies = [
+ "futures-core",
+]
[[package]]
name = "synstructure"
@@ -4509,6 +4997,24 @@
"tracing",
]
+[[package]]
+name = "tower-http"
+version = "0.6.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840"
+dependencies = [
+ "bitflags 2.13.0",
+ "bytes",
+ "futures-util",
+ "http",
+ "http-body",
+ "pin-project-lite",
+ "tower",
+ "tower-layer",
+ "tower-service",
+ "url",
+]
+
[[package]]
name = "tower-layer"
version = "0.3.3"
@@ -4559,6 +5065,12 @@
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
[[package]]
name = "tungstenite"
version = "0.29.0"
@@ -4750,6 +5262,15 @@
"winapi-util",
]
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
[[package]]
name = "wasi"
version = "0.11.1+wasi-snapshot-preview1"
@@ -4796,6 +5317,16 @@
"wasm-bindgen-shared",
]
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.76"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.126"
@@ -4828,6 +5359,19 @@
"unicode-ident",
]
+[[package]]
+name = "wasm-streams"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
[[package]]
name = "weavy"
version = "0.2.2"
@@ -4844,6 +5388,25 @@
"wasm-bindgen",
]
+[[package]]
+name = "web-time"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "webpki-root-certs"
+version = "1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0d46a5a140e6f7afeccd8eae97eff335163939eac8b929834875168b29b3d267"
+dependencies = [
+ "rustls-pki-types",
+]
+
[[package]]
name = "webpki-roots"
version = "0.26.11"
Cargo.toml
@@ -15,6 +15,7 @@
"crates/git-store",
"crates/git-toolchain",
"crates/odb-files",
+ "crates/odb-tigris",
"crates/refstore-files",
"crates/refstore-postgres",
]
@@ -73,6 +74,7 @@
git-store = { path = "crates/git-store" }
git-toolchain = { path = "crates/git-toolchain" }
odb-files = { path = "crates/odb-files" }
+odb-tigris = { path = "crates/odb-tigris" }
refstore-files = { path = "crates/refstore-files" }
refstore-postgres = { path = "crates/refstore-postgres" }
gix = "0.84"
@@ -84,6 +86,9 @@
gix-odb = "0.81"
gix-pack = "0.71"
iddqd = "0.4"
+object_store = { version = "0.14", default-features = false, features = [
+ "aws",
+] }
maud = { version = "0.27", features = ["axum"] }
pulldown-cmark = { version = "0.13", default-features = false, features = ["html"] }
rayon = "1"
crates/odb-tigris/Cargo.toml
@@ -1,0 +1,23 @@
+[package]
+name = "odb-tigris"
+version = "0.0.0"
+edition.workspace = true
+publish.workspace = true
+license.workspace = true
+
+[dependencies]
+git-backend = { workspace = true }
+gix-features = { workspace = true, features = ["progress"] }
+gix-hash = { workspace = true }
+gix-object = { workspace = true }
+gix-pack = { workspace = true }
+object_store = { workspace = true }
+tempfile = { workspace = true }
+tokio = { workspace = true }
+uuid = { workspace = true }
+
+[dev-dependencies]
+backend-conformance = { workspace = true }
+
+[lints]
+workspace = true
crates/odb-tigris/src/decode.rs
@@ -1,0 +1,376 @@
+//! Resolving one object out of a pack over ranged reads (`docs/scale-out.adoc`,
+//! WS5 / Q4).
+//!
+//! # Survey: what gix-pack offers
+//!
+//! `gix_pack::index::File::from_data` parses a `.idx` from any
+//! `Deref<Target = [u8]>` (a plain `Vec<u8>` qualifies), so a fetched index
+//! can be parsed straight out of memory — see [`crate::index_cache`].
+//! `gix_pack::data::Entry::from_bytes` decodes a pack entry's header
+//! (kind, decompressed size, and — for deltas — the base reference) from an
+//! arbitrary byte slice plus the absolute pack offset it came from, and
+//! `Entry::checked_base_pack_offset` turns an `OfsDelta`'s distance into a
+//! bounds-checked absolute base offset. Both are exactly what's needed to
+//! decode one entry from a ranged read rather than a full pack mmap.
+//!
+//! What gix-pack does *not* expose publicly is the rest of the decode path:
+//! `gix_pack::data::File::decode_entry` (delta chain resolution) and
+//! `gix_pack::data::delta::apply` (the actual copy/insert interpreter) both
+//! assume a fully-mapped `data::File` addressed by absolute offset, and the
+//! delta-apply primitives (`data::delta::apply`,
+//! `data::delta::decode_header_size`) are `pub(crate)` — not reachable from
+//! outside gix-pack at all. So this module reimplements the (small, stable,
+//! documented) pack delta format itself: [`apply_delta`] is the same
+//! copy/insert interpreter gix-pack's private `data::delta::apply` performs,
+//! and [`decode_varint_size`] the same header-size varint it decodes:
+//! neither has changed shape across pack format versions.
+//!
+//! # Ranged reads and growth
+//!
+//! Every fetch here is a bounded window, never the whole pack: entry headers
+//! are probed with a small initial window that grows (geometrically, capped)
+//! only if the header turns out to need more bytes than guessed, and
+//! compressed entry data is fetched as `decompressed_size` plus a slack
+//! margin, regrown the same way if zlib reports it needs more input. Real
+//! packs' zlib streams essentially never need the regrowth path since
+//! compressed size is bounded by decompressed size plus a small constant
+//! overhead; the loop exists so a pathological pack degrades to a few extra
+//! round trips instead of a wrong answer.
+//!
+//! # Delta chains
+//!
+//! [`resolve`] recurses on `OfsDelta`/`RefDelta` bases, bounded by
+//! `max_depth` (matching git's own default `--depth` limit of 50) as a
+//! defense against corrupt or adversarial packs with absurdly long chains —
+//! independently of the fact that `OfsDelta` offsets are already
+//! strictly-decreasing (a base is always earlier in the pack than the entry
+//! that deltas against it) and so terminate on their own in any well-formed
+//! pack. Binaries are stored un-delta'd at write time
+//! (`docs/scale-out.adoc`, rule 5, and `crate::pack_writer`) precisely so
+//! this recursion stays shallow in practice: only trees/commits/manifests
+//! are ever candidates for a delta chain at all under this store's own
+//! writer, and this store's `stage_pack` only ever indexes self-contained
+//! packs (no thin-pack bases outside the pack — see `crate::OdbTigris`), so
+//! `RefDelta` bases always resolve to an offset within the same pack.
+
+use gix_object::Kind;
+use gix_pack::data::Entry as PackEntry;
+use gix_pack::data::entry::Header;
+
+use crate::transport::BlobTransport;
+
+/// git's own delta-depth ceiling, reused here as a recursion bound rather
+/// than invented fresh.
+const MAX_DELTA_DEPTH: u32 = 50;
+
+/// Initial header probe window, generous enough for the overwhelming
+/// majority of entries (type/size varint plus, for a ref-delta, a 20-byte
+/// SHA-1) without ever growing.
+const HEADER_PROBE_BYTES: u64 = 64;
+
+/// Slack added to `decompressed_size` when fetching an entry's compressed
+/// bytes, before any regrowth.
+const DECOMPRESS_SLACK_BYTES: u64 = 32;
+
+/// How many times a fetch window is allowed to double before giving up and
+/// reporting corruption.
+const MAX_FETCH_GROWTHS: u32 = 8;
+
+fn corrupt(message: impl Into<String>) -> git_backend::Error {
+ git_backend::Error::ObjectStore(format!("corrupt pack entry: {}", message.into()))
+}
+
+/// Look up the base of a `RefDelta` within the same pack, returning its
+/// pack offset. Implemented by [`crate::OdbTigris`] over the pack's cached
+/// index — see this module's doc comment for why a ref-delta base is always
+/// in-pack for stores built by this crate.
+pub trait RefDeltaResolver {
+ /// Resolve `base_id` to a pack offset, or `None` if not found in this
+ /// pack.
+ fn resolve(&self, base_id: &gix_hash::oid) -> Option<u64>;
+}
+
+/// Fully resolve the object at `offset` in the pack at `pack_key`,
+/// returning its kind and undeltified bytes.
+///
+/// # Errors
+///
+/// Returns an error if a ranged read fails, an entry header or delta stream
+/// is corrupt, or the delta chain exceeds [`MAX_DELTA_DEPTH`].
+pub fn resolve(
+ transport: &dyn BlobTransport,
+ pack_key: &str,
+ offset: u64,
+ hash_len: usize,
+ ref_deltas: &dyn RefDeltaResolver,
+) -> git_backend::Result<(Kind, Vec<u8>)> {
+ resolve_at_depth(transport, pack_key, offset, hash_len, ref_deltas, 0)
+}
+
+fn resolve_at_depth(
+ transport: &dyn BlobTransport,
+ pack_key: &str,
+ offset: u64,
+ hash_len: usize,
+ ref_deltas: &dyn RefDeltaResolver,
+ depth: u32,
+) -> git_backend::Result<(Kind, Vec<u8>)> {
+ if depth > MAX_DELTA_DEPTH {
+ return Err(corrupt("delta chain exceeds the maximum supported depth"));
+ }
+
+ let entry = fetch_entry_header(transport, pack_key, offset, hash_len)?;
+ match entry.header {
+ Header::Commit | Header::Tree | Header::Blob | Header::Tag => {
+ let kind = entry
+ .header
+ .as_kind()
+ .ok_or_else(|| corrupt("a non-delta header failed to convert to an object kind"))?;
+ let data = decompress_entry(transport, pack_key, &entry)?;
+ Ok((kind, data))
+ }
+ Header::OfsDelta { base_distance } => {
+ let base_offset = entry
+ .checked_base_pack_offset(base_distance)
+ .ok_or_else(|| corrupt("ofs-delta base distance out of range"))?;
+ let (kind, base_data) = resolve_at_depth(
+ transport,
+ pack_key,
+ base_offset,
+ hash_len,
+ ref_deltas,
+ depth.saturating_add(1),
+ )?;
+ let delta_data = decompress_entry(transport, pack_key, &entry)?;
+ let target = apply_delta(&base_data, &delta_data)?;
+ Ok((kind, target))
+ }
+ Header::RefDelta { base_id } => {
+ let base_offset = ref_deltas
+ .resolve(base_id.as_ref())
+ .ok_or_else(|| corrupt("ref-delta base not found in this pack"))?;
+ let (kind, base_data) = resolve_at_depth(
+ transport,
+ pack_key,
+ base_offset,
+ hash_len,
+ ref_deltas,
+ depth.saturating_add(1),
+ )?;
+ let delta_data = decompress_entry(transport, pack_key, &entry)?;
+ let target = apply_delta(&base_data, &delta_data)?;
+ Ok((kind, target))
+ }
+ }
+}
+
+/// Ranged-fetch and parse the entry header at `offset`, growing the fetch
+/// window if the header (e.g. a long size varint, or a ref-delta's base id)
+/// doesn't fit in the initial probe.
+fn fetch_entry_header(
+ transport: &dyn BlobTransport,
+ pack_key: &str,
+ offset: u64,
+ hash_len: usize,
+) -> git_backend::Result<PackEntry> {
+ let mut window = HEADER_PROBE_BYTES;
+ for _ in 0..MAX_FETCH_GROWTHS {
+ let bytes = transport.get_range(pack_key, offset..offset.saturating_add(window))?;
+ match PackEntry::from_bytes(&bytes, offset, hash_len) {
+ Ok(entry) => return Ok(entry),
+ Err(_) if (bytes.len() as u64) < window => {
+ // The transport handed back less than we asked for, which
+ // only happens at the tail of the pack: growing further
+ // would never produce more bytes, so this is a real parse
+ // failure, not a too-small window.
+ return Err(corrupt("truncated entry header at end of pack"));
+ }
+ Err(_) => window = window.saturating_mul(2),
+ }
+ }
+ Err(corrupt(
+ "entry header did not fit within the fetch growth budget",
+ ))
+}
+
+/// Ranged-fetch and decompress `entry`'s compressed data, growing the fetch
+/// window if the zlib stream needs more input than the initial guess.
+fn decompress_entry(
+ transport: &dyn BlobTransport,
+ pack_key: &str,
+ entry: &PackEntry,
+) -> git_backend::Result<Vec<u8>> {
+ let out_len = usize::try_from(entry.decompressed_size)
+ .map_err(|_size_error| corrupt("decompressed size does not fit in memory"))?;
+ let mut window = entry
+ .decompressed_size
+ .saturating_add(DECOMPRESS_SLACK_BYTES);
+ for _ in 0..MAX_FETCH_GROWTHS {
+ let input = transport.get_range(
+ pack_key,
+ entry.data_offset..entry.data_offset.saturating_add(window),
+ )?;
+ let grew_to_end = (input.len() as u64) < window;
+ let mut inflate = gix_features::zlib::Inflate::default();
+ let mut out = vec![0u8; out_len];
+ match inflate.once(&input, &mut out) {
+ Ok((gix_features::zlib::Status::StreamEnd, _consumed_in, consumed_out))
+ if consumed_out == out.len() =>
+ {
+ return Ok(out);
+ }
+ _ if grew_to_end => {
+ return Err(corrupt("zlib stream did not end within the pack's bounds"));
+ }
+ _ => window = window.saturating_mul(2),
+ }
+ }
+ Err(corrupt(
+ "entry data did not decompress within the fetch growth budget",
+ ))
+}
+
+/// Decode a delta header size varint (used for both the base-object-size
+/// and result-object-size fields at the start of a delta stream). Same
+/// encoding as gix-pack's private `data::delta::decode_header_size`.
+fn decode_varint_size(d: &[u8]) -> git_backend::Result<(u64, usize)> {
+ let mut shift: u32 = 0;
+ let mut size: u64 = 0;
+ for (consumed, &byte) in d.iter().enumerate() {
+ if shift >= u64::BITS {
+ return Err(corrupt("delta header size uses more bits than fit in u64"));
+ }
+ size |= (u64::from(byte) & 0x7f) << shift;
+ shift = shift.saturating_add(7);
+ if byte & 0x80 == 0 {
+ return Ok((size, consumed.saturating_add(1)));
+ }
+ }
+ Err(corrupt("delta header size is truncated"))
+}
+
+/// Apply a pack delta: `base` plus `delta`'s copy/insert instructions
+/// produce the target object's bytes. Same instruction format as
+/// gix-pack's private `data::delta::apply`; reimplemented here because that
+/// function is `pub(crate)` in gix-pack (see this module's doc comment).
+fn apply_delta(base: &[u8], delta: &[u8]) -> git_backend::Result<Vec<u8>> {
+ let (base_size, consumed) = decode_varint_size(delta)?;
+ if usize::try_from(base_size).ok() != Some(base.len()) {
+ return Err(corrupt(
+ "delta base size does not match resolved base object",
+ ));
+ }
+ let rest = delta
+ .get(consumed..)
+ .ok_or_else(|| corrupt("delta is truncated after base size"))?;
+ let (target_size, consumed2) = decode_varint_size(rest)?;
+ let target_size = usize::try_from(target_size)
+ .map_err(|_size_error| corrupt("delta target size does not fit in memory"))?;
+
+ let mut out = Vec::with_capacity(target_size);
+ let mut i = consumed.saturating_add(consumed2);
+ while let Some(&cmd) = delta.get(i) {
+ i = i.saturating_add(1);
+ if cmd & 0b1000_0000 != 0 {
+ let (ofs, size) = read_copy_operands(delta, &mut i, cmd)?;
+ let end = ofs
+ .checked_add(size)
+ .ok_or_else(|| corrupt("delta copy range overflows"))?;
+ out.extend_from_slice(
+ base.get(ofs..end)
+ .ok_or_else(|| corrupt("delta copy range exceeds base object"))?,
+ );
+ } else if cmd == 0 {
+ return Err(corrupt("delta command 0 is reserved and invalid"));
+ } else {
+ let size = usize::from(cmd);
+ let end = i
+ .checked_add(size)
+ .ok_or_else(|| corrupt("delta insert range overflows"))?;
+ out.extend_from_slice(
+ delta
+ .get(i..end)
+ .ok_or_else(|| corrupt("delta insert data is truncated"))?,
+ );
+ i = end;
+ }
+ }
+ if out.len() != target_size {
+ return Err(corrupt(
+ "delta instructions produced a different size than promised",
+ ));
+ }
+ Ok(out)
+}
+
+/// Decode a copy instruction's offset/size operand bytes, per the pack
+/// delta format's variable-length little-endian encoding selected by the
+/// command byte's low 7 bits. Accumulates in `usize` throughout: every term
+/// is a `u8` widened losslessly, so no truncating cast is ever needed.
+fn read_copy_operands(delta: &[u8], i: &mut usize, cmd: u8) -> git_backend::Result<(usize, usize)> {
+ let mut ofs: usize = 0;
+ let mut size: usize = 0;
+ let mut next = || -> git_backend::Result<usize> {
+ let byte = *delta
+ .get(*i)
+ .ok_or_else(|| corrupt("delta copy instruction is truncated"))?;
+ *i = i.saturating_add(1);
+ Ok(usize::from(byte))
+ };
+ if cmd & 0x01 != 0 {
+ ofs |= next()?;
+ }
+ if cmd & 0x02 != 0 {
+ ofs |= next()? << 8;
+ }
+ if cmd & 0x04 != 0 {
+ ofs |= next()? << 16;
+ }
+ if cmd & 0x08 != 0 {
+ ofs |= next()? << 24;
+ }
+ if cmd & 0x10 != 0 {
+ size |= next()?;
+ }
+ if cmd & 0x20 != 0 {
+ size |= next()? << 8;
+ }
+ if cmd & 0x40 != 0 {
+ size |= next()? << 16;
+ }
+ if size == 0 {
+ size = 0x10000;
+ }
+ Ok((ofs, size))
+}
+
+/// Not part of this module's public surface, but exercised via
+/// [`crate::OdbTigris`]'s own conformance and unit tests, since a
+/// meaningful test here needs a real pack fixture — see
+/// `crates/odb-tigris/tests/conformance.rs`.
+#[cfg(test)]
+mod tests {
+ #![allow(
+ clippy::expect_used,
+ clippy::assertions_on_result_states,
+ reason = "unit test"
+ )]
+
+ use super::*;
+
+ #[test]
+ fn decode_varint_size_round_trips_small_values() {
+ // 200 encodes as two leb128-like bytes (continuation bit set on the
+ // first): 0xC8 -> low 7 bits 0x48 with continuation, then 0x01.
+ let (size, consumed) = decode_varint_size(&[0xC8, 0x01]).expect("decode");
+ assert_eq!(size, 200);
+ assert_eq!(consumed, 2);
+ }
+
+ #[test]
+ fn apply_delta_rejects_a_base_size_mismatch() {
+ // base_size varint says 5, but the supplied base is empty.
+ let delta = [5u8, 0u8];
+ assert!(apply_delta(&[], &delta).is_err());
+ }
+}
crates/odb-tigris/src/index_cache.rs
@@ -1,0 +1,65 @@
+//! In-memory cache of parsed `.idx` bytes, one per [`crate::OdbTigris`]
+//! instance (`docs/scale-out.adoc`, WS5: "cache fetched `.idx` bytes in
+//! memory per store"). Indexes are small (a few percent of pack size) and
+//! reused across every `read`/`contains` call, so fetching one once per
+//! pack per store lifetime — rather than per object lookup — is the whole
+//! point of this module.
+
+use std::collections::HashMap;
+use std::sync::{Arc, Mutex, MutexGuard, PoisonError};
+
+use git_backend::Result;
+use gix_pack::index::File as IndexFile;
+
+use crate::transport::BlobTransport;
+
+/// A cache of parsed pack indexes, keyed by their bucket key.
+#[derive(Default)]
+pub struct IndexCache {
+ parsed: Mutex<HashMap<String, Arc<IndexFile<Vec<u8>>>>>,
+}
+
+impl IndexCache {
+ /// An empty cache.
+ #[must_use]
+ pub fn new() -> Self {
+ Self::default()
+ }
+
+ /// Return the parsed index for `idx_key`, fetching and parsing it via
+ /// `transport` on a cache miss.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the fetch or parse fails.
+ pub fn get(
+ &self,
+ transport: &dyn BlobTransport,
+ idx_key: &str,
+ object_hash: gix_hash::Kind,
+ ) -> Result<Arc<IndexFile<Vec<u8>>>> {
+ if let Some(cached) = lock(&self.parsed).get(idx_key) {
+ return Ok(Arc::clone(cached));
+ }
+ let bytes = transport.get(idx_key)?;
+ let parsed = IndexFile::from_data(bytes, std::path::PathBuf::from(idx_key), object_hash)
+ .map_err(|error| {
+ git_backend::Error::ObjectStore(format!("parsing index {idx_key}: {error}"))
+ })?;
+ let parsed = Arc::new(parsed);
+ lock(&self.parsed).insert(idx_key.to_owned(), Arc::clone(&parsed));
+ Ok(parsed)
+ }
+
+ /// Drop a cached index, e.g. because its pack was deleted from the
+ /// registry. Not currently called on any path in this crate (GC is out
+ /// of scope for WS5), but present so a future maintenance path doesn't
+ /// need to add cache invalidation from scratch.
+ pub fn invalidate(&self, idx_key: &str) {
+ lock(&self.parsed).remove(idx_key);
+ }
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().unwrap_or_else(PoisonError::into_inner)
+}
crates/odb-tigris/src/lib.rs
@@ -1,0 +1,279 @@
+//! [`ObjectStore`] over an S3-compatible bucket (Tigris in production),
+//! designed around ranged reads rather than whole-pack hydration
+//! (`docs/scale-out.adoc`, "ObjectStore" / WS5).
+//!
+//! # Layout
+//!
+//! Packs and their `.idx` live under a per-repo prefix (rule 7: "namespace
+//! per repo" — no cross-tenant dedup, so every key this crate writes or
+//! reads is scoped under `{repo_id}/...`):
+//!
+//! - `{repo_id}/quarantine/{id}/pack.pack` + `.../pack.idx` — staged, not
+//! yet visible (written by [`OdbTigris::stage_pack`]).
+//! - `{repo_id}/live/{id}.pack` + `{id}.idx` — promoted, registered, visible
+//! to `read`/`contains` (written by [`OdbTigris::promote`]).
+//!
+//! midx (multi-pack-index) support is not implemented: nothing here
+//! prevents adding one alongside the per-pack indexes later
+//! (`docs/scale-out.adoc`'s ObjectStore row mentions it as "when
+//! available"), but with no multi-index yet, `read`/`contains` scan every
+//! registered pack's own `.idx` — correct, and cheap enough for the pack
+//! counts this store is expected to carry before WS6 lands.
+//!
+//! # Never a bucket listing
+//!
+//! [`OdbTigris`] never calls anything resembling a bucket "list objects"
+//! operation — the [`transport::BlobTransport`] trait doesn't even expose
+//! one. Every key this store touches comes from either a
+//! [`registry::PackRegistry`] record or a quarantine id it minted itself
+//! (`docs/scale-out.adoc`, "Reachability": "nothing may traverse Tigris
+//! object-by-object").
+//!
+//! # Q2: promotion visibility
+//!
+//! See [`transport`]'s module doc: `promote` assumes the bucket offers
+//! read-after-write consistency for a key it just copied.
+
+pub mod decode;
+pub mod index_cache;
+pub mod pack_writer;
+pub mod registry;
+pub mod transport;
+
+use std::collections::HashMap;
+use std::sync::{Mutex, MutexGuard, PoisonError};
+
+pub use git_backend::{Error, Result};
+use git_backend::{Object, ObjectStore, PackStream, QuarantineId};
+use gix_hash::ObjectId;
+use gix_pack::index::File as IndexFile;
+
+use crate::decode::RefDeltaResolver;
+use crate::index_cache::IndexCache;
+use crate::registry::{PackId, PackRecord, PackRegistry};
+use crate::transport::BlobTransport;
+
+/// A registered pack paired with its parsed index, as gathered by
+/// [`OdbTigris::indexes`].
+type RecordAndIndex = (PackRecord, std::sync::Arc<IndexFile<Vec<u8>>>);
+
+/// A pack awaiting promotion: the quarantine keys [`OdbTigris::stage_pack`]
+/// uploaded to, kept around so [`OdbTigris::promote`] knows what to copy.
+struct Quarantine {
+ pack_key: String,
+ idx_key: String,
+ object_count: u64,
+}
+
+/// [`ObjectStore`] over an S3-compatible bucket, generic over its blob
+/// transport and pack registry so tests can run with
+/// [`transport::fs::FsTransport`] + [`registry::memory::InMemoryRegistry`]
+/// and production wires up [`transport::s3::S3Transport`] plus a
+/// Postgres-backed registry (see `refstore-postgres`).
+pub struct OdbTigris<T, R> {
+ transport: T,
+ registry: R,
+ repo_id: String,
+ hash_kind: gix_hash::Kind,
+ index_cache: IndexCache,
+ quarantines: Mutex<HashMap<QuarantineId, Quarantine>>,
+}
+
+impl<T, R> OdbTigris<T, R>
+where
+ T: BlobTransport,
+ R: PackRegistry,
+{
+ /// Open a store scoped to `repo_id`, over `transport` and `registry`.
+ /// Object hashes are always SHA-1, matching every other backend in this
+ /// workspace.
+ pub fn new(transport: T, registry: R, repo_id: impl Into<String>) -> Self {
+ Self {
+ transport,
+ registry,
+ repo_id: repo_id.into(),
+ hash_kind: gix_hash::Kind::Sha1,
+ index_cache: IndexCache::new(),
+ quarantines: Mutex::new(HashMap::new()),
+ }
+ }
+
+ fn quarantine_pack_key(&self, id: &str) -> String {
+ format!("{}/quarantine/{id}/pack.pack", self.repo_id)
+ }
+
+ fn quarantine_idx_key(&self, id: &str) -> String {
+ format!("{}/quarantine/{id}/pack.idx", self.repo_id)
+ }
+
+ fn live_pack_key(&self, id: &str) -> String {
+ format!("{}/live/{id}.pack", self.repo_id)
+ }
+
+ fn live_idx_key(&self, id: &str) -> String {
+ format!("{}/live/{id}.idx", self.repo_id)
+ }
+
+ /// Every registered pack's parsed index, fetched (and cached) on
+ /// demand. Iterated in full on every `read`/`contains` since there is
+ /// no multi-pack-index yet — see this crate's module doc.
+ fn indexes(&self) -> Result<Vec<RecordAndIndex>> {
+ self.registry
+ .list(&self.repo_id)?
+ .into_iter()
+ .map(|record| {
+ let idx = self
+ .index_cache
+ .get(&self.transport, &record.idx_key, self.hash_kind)?;
+ Ok((record, idx))
+ })
+ .collect()
+ }
+}
+
+impl<T, R> ObjectStore for OdbTigris<T, R>
+where
+ T: BlobTransport,
+ R: PackRegistry,
+{
+ fn read(&self, id: ObjectId) -> Result<Object> {
+ for (record, idx) in self.indexes()? {
+ let Some(entry_index) = idx.lookup(id) else {
+ continue;
+ };
+ let offset = idx.pack_offset_at_index(entry_index);
+ let resolver = IndexRefDeltaResolver { idx: &idx };
+ let (kind, data) = decode::resolve(
+ &self.transport,
+ &record.pack_key,
+ offset,
+ self.hash_kind.len_in_bytes(),
+ &resolver,
+ )?;
+ return Ok(Object { kind, data });
+ }
+ Err(Error::ObjectStore(format!(
+ "object {id} not found in any registered pack for repo {}",
+ self.repo_id
+ )))
+ }
+
+ fn contains(&self, id: ObjectId) -> Result<bool> {
+ for (_record, idx) in self.indexes()? {
+ if idx.lookup(id).is_some() {
+ return Ok(true);
+ }
+ }
+ Ok(false)
+ }
+
+ fn stage_pack(&self, pack: PackStream) -> Result<QuarantineId> {
+ let id = uuid::Uuid::new_v4().to_string();
+ let scratch = tempfile::tempdir()?;
+
+ let mut reader = std::io::BufReader::new(pack);
+ let outcome = gix_pack::Bundle::write_to_directory(
+ &mut reader,
+ Some(scratch.path()),
+ &mut gix_features::progress::Discard,
+ &std::sync::atomic::AtomicBool::new(false),
+ None::<NoThinBaseLookup>,
+ gix_pack::bundle::write::Options {
+ object_hash: self.hash_kind,
+ ..Default::default()
+ },
+ )
+ .map_err(|error| Error::ObjectStore(error.to_string()))?;
+
+ let data_path = outcome
+ .data_path
+ .ok_or_else(|| Error::ObjectStore("pack write produced no data file".to_owned()))?;
+ let index_path = outcome
+ .index_path
+ .ok_or_else(|| Error::ObjectStore("pack write produced no index file".to_owned()))?;
+ let pack_bytes = std::fs::read(&data_path)?;
+ let idx_bytes = std::fs::read(&index_path)?;
+
+ self.transport
+ .put(&self.quarantine_pack_key(&id), pack_bytes)?;
+ self.transport
+ .put(&self.quarantine_idx_key(&id), idx_bytes)?;
+
+ lock(&self.quarantines).insert(
+ QuarantineId::new(id.clone()),
+ Quarantine {
+ pack_key: self.quarantine_pack_key(&id),
+ idx_key: self.quarantine_idx_key(&id),
+ object_count: u64::from(outcome.index.num_objects),
+ },
+ );
+ Ok(QuarantineId::new(id))
+ }
+
+ fn promote(&self, q: QuarantineId) -> Result<()> {
+ let quarantine = lock(&self.quarantines)
+ .remove(&q)
+ .ok_or_else(|| Error::ObjectStore(format!("unknown quarantine {q}")))?;
+
+ let live_pack_key = self.live_pack_key(q.as_str());
+ let live_idx_key = self.live_idx_key(q.as_str());
+
+ // Q2: the copy below must be durably visible to a subsequent `get`/
+ // `get_range` against `live_pack_key`/`live_idx_key` before we
+ // return — see `transport`'s module doc. This is a fact about the
+ // bucket to verify, not something this code can enforce.
+ self.transport.copy(&quarantine.pack_key, &live_pack_key)?;
+ self.transport.copy(&quarantine.idx_key, &live_idx_key)?;
+
+ self.registry.record(PackRecord {
+ id: PackId::new(q.as_str()),
+ repo_id: self.repo_id.clone(),
+ pack_key: live_pack_key,
+ idx_key: live_idx_key,
+ object_count: Some(quarantine.object_count),
+ })?;
+
+ // Best-effort cleanup: the registry record above is the actual
+ // commit point (rule 2). Leaving these behind would waste space,
+ // not correctness, so a failure here is not propagated.
+ let _ignored = self.transport.delete(&quarantine.pack_key);
+ let _ignored = self.transport.delete(&quarantine.idx_key);
+ Ok(())
+ }
+}
+
+/// Resolves `RefDelta` bases against one pack's already-fetched index — see
+/// `decode`'s module doc for why a ref-delta base is always in the same
+/// pack for stores built by this crate.
+struct IndexRefDeltaResolver<'a> {
+ idx: &'a IndexFile<Vec<u8>>,
+}
+
+impl RefDeltaResolver for IndexRefDeltaResolver<'_> {
+ fn resolve(&self, base_id: &gix_hash::oid) -> Option<u64> {
+ let entry_index = self.idx.lookup(base_id)?;
+ Some(self.idx.pack_offset_at_index(entry_index))
+ }
+}
+
+/// A `gix_object::Find` that never finds anything, satisfying
+/// `Bundle::write_to_directory`'s thin-pack-base-lookup parameter without
+/// pulling in a full `gix`/`gix-odb` dependency. Correct because this
+/// store's `stage_pack` never receives a thin pack: every pack it indexes
+/// is expected to be self-contained (the same assumption `odb-files` makes
+/// by passing `None` there too).
+struct NoThinBaseLookup;
+
+impl gix_object::Find for NoThinBaseLookup {
+ fn try_find<'a>(
+ &self,
+ _id: &gix_hash::oid,
+ _buffer: &'a mut Vec<u8>,
+ ) -> std::result::Result<Option<gix_object::Data<'a>>, gix_object::find::Error> {
+ Ok(None)
+ }
+}
+
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().unwrap_or_else(PoisonError::into_inner)
+}
crates/odb-tigris/src/pack_writer.rs
@@ -1,0 +1,213 @@
+//! Partitioning objects into packs that obey the pack-lifetime rule
+//! (`docs/scale-out.adoc`, rule 5): "Objects with different lifetimes never
+//! share a pack. Cache-namespace objects get their own packs, so eviction is
+//! a registry delete, never repack surgery. Within a lifetime class, delta
+//! policy is per content class: trees/manifests delta'd, binaries stored
+//! raw."
+//!
+//! # Decision point: whole-object encoding only
+//!
+//! `gix_pack::data::output::Entry::from_data` — the constructor this module
+//! (and `git-protocol`'s `pack::build_pack`, the precedent this mirrors)
+//! uses to turn a materialized object into a pack entry — only ever
+//! produces [`gix_pack::data::output::entry::Kind::Base`], i.e. a full
+//! object. The `DeltaRef`/`DeltaOid` variants of that enum exist, but the
+//! only public constructor that produces them,
+//! `output::Entry::from_pack_entry`, *reuses* a delta already present in a
+//! source pack being repacked — gix-pack exposes no API to diff two fresh
+//! objects and encode a new delta from scratch. That's a real gap, not a
+//! missed method: encoding a new delta requires an actual diff algorithm,
+//! which is out of scope to bolt on here (`docs/scale-out.adoc`, Q6, calls
+//! this out as its own risk-budgeted item).
+//!
+//! So [`ContentClass`] is honestly a *policy* label, not yet a behavior:
+//! this module partitions by [`LifetimeClass`] (rule 5's actual correctness
+//! requirement — cache and durable objects never share a pack) and records
+//! each object's [`ContentClass`] purely as the documented decision point
+//! for when gix-pack (or a future dependency) gains real delta-encoding.
+//! Every object, regardless of class, is written whole today. Do not read
+//! `ContentClass::Structural` as "this gets delta-compressed" — it does
+//! not, yet.
+
+use gix_hash::ObjectId;
+use gix_object::Kind;
+use gix_pack::data::Version;
+use gix_pack::data::output::{Count, Entry, bytes::FromEntriesIter};
+
+use crate::Result;
+
+/// Which lifetime an object belongs to (`docs/scale-out.adoc`, rule 5).
+/// Determines which of the two output packs an object lands in; never mixed
+/// within one pack.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum LifetimeClass {
+ /// Ordinary repository objects: reachable from durable refs, never
+ /// evicted by a cache TTL.
+ Durable,
+ /// Objects reachable only from `refs/cache/*` / `refs/meta/cache/*`
+ /// (`docs/scale-out.adoc`, rule 4): evictable, reconstructible, exempt
+ /// from provenance.
+ Cache,
+}
+
+/// The content-class delta policy this object is a candidate for, per rule
+/// 5's "within a lifetime class, delta policy is per content class" —
+/// currently a documented decision point rather than an applied behavior;
+/// see this module's doc comment.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum ContentClass {
+ /// Trees, commits, and typed manifests: eligible for delta compression
+ /// once gix-pack (or a replacement) can encode one.
+ Structural,
+ /// Blobs at or above the raw-storage threshold: stored whole
+ /// deliberately (rule 5, and `crate::decode`'s doc comment on why short
+ /// delta chains matter for ranged reads).
+ Binary,
+}
+
+/// Size, in bytes, at or above which a blob is classified [`ContentClass::Binary`]
+/// rather than [`ContentClass::Structural`]. Chosen as a plausible default,
+/// not measured; `docs/scale-out.adoc`'s Q5 applies to the tiered store's
+/// small-object threshold specifically, but the same "measure, don't guess"
+/// caution applies here.
+pub const BINARY_THRESHOLD_BYTES: usize = 16 * 1024;
+
+/// One object to be written into a pack by [`partition_and_pack`].
+pub struct ClassifiedObject {
+ /// The object's id.
+ pub id: ObjectId,
+ /// The object's kind.
+ pub kind: Kind,
+ /// The object's raw, undeltified content.
+ pub data: Vec<u8>,
+ /// Which lifetime this object belongs to.
+ pub lifetime: LifetimeClass,
+}
+
+impl ClassifiedObject {
+ /// This object's content class, derived from its kind and size against
+ /// [`BINARY_THRESHOLD_BYTES`] (see [`ContentClass`]).
+ #[must_use]
+ pub fn content_class(&self) -> ContentClass {
+ match self.kind {
+ Kind::Tree | Kind::Commit | Kind::Tag => ContentClass::Structural,
+ Kind::Blob if self.data.len() < BINARY_THRESHOLD_BYTES => ContentClass::Structural,
+ Kind::Blob => ContentClass::Binary,
+ }
+ }
+}
+
+/// The result of [`partition_and_pack`]: up to two whole-object version-2
+/// packs, one per [`LifetimeClass`] actually present in the input. A class
+/// with no objects produces no pack at all — rule 5 forbids an empty
+/// cache-namespace pack sharing anything with the durable one, but there's
+/// no reason to write one when there's nothing to put in it.
+#[derive(Debug, Clone, Default, PartialEq, Eq)]
+pub struct PartitionedPacks {
+ /// The durable-class pack, if any durable objects were supplied.
+ pub durable: Option<Vec<u8>>,
+ /// The cache-class pack, if any cache-namespace objects were supplied.
+ pub cache: Option<Vec<u8>>,
+}
+
+/// Partition `objects` by [`LifetimeClass`] and encode each non-empty
+/// partition as its own version-2 pack (`docs/scale-out.adoc`, rule 5).
+///
+/// # Errors
+///
+/// Returns an error if pack encoding fails (e.g. zlib deflate failure).
+pub fn partition_and_pack(objects: Vec<ClassifiedObject>) -> Result<PartitionedPacks> {
+ let (durable, cache): (Vec<_>, Vec<_>) = objects
+ .into_iter()
+ .partition(|object| object.lifetime == LifetimeClass::Durable);
+ Ok(PartitionedPacks {
+ durable: pack_whole_objects(&durable)?,
+ cache: pack_whole_objects(&cache)?,
+ })
+}
+
+/// Encode `objects` as a version-2 pack, every entry a full base object
+/// (mirrors `git-protocol::pack::build_pack`, this crate's precedent for
+/// "gix-pack's writer only does whole objects" — see this module's doc
+/// comment for why). Returns `None` for an empty slice rather than an
+/// empty-but-valid pack: callers only stage a pack that has at least one
+/// object in it.
+fn pack_whole_objects(objects: &[ClassifiedObject]) -> Result<Option<Vec<u8>>> {
+ if objects.is_empty() {
+ return Ok(None);
+ }
+ let entries: Vec<Entry> = objects
+ .iter()
+ .map(|object| {
+ let count = Count::from_data(object.id, None);
+ let data = gix_object::Data::new(&object.data, object.kind, gix_hash::Kind::Sha1);
+ Entry::from_data(&count, &data)
+ .map_err(|error| crate::Error::ObjectStore(error.to_string()))
+ })
+ .collect::<Result<_>>()?;
+ let num_entries = u32::try_from(entries.len()).map_err(|_too_many| {
+ crate::Error::ObjectStore("too many objects for one pack".to_owned())
+ })?;
+ let input = std::iter::once(Ok::<_, std::convert::Infallible>(entries));
+ let mut writer = FromEntriesIter::new(
+ input,
+ Vec::new(),
+ num_entries,
+ Version::V2,
+ gix_hash::Kind::Sha1,
+ );
+ for step in &mut writer {
+ step.map_err(|error| crate::Error::ObjectStore(error.to_string()))?;
+ }
+ Ok(Some(writer.into_write()))
+}
+
+#[cfg(test)]
+mod tests {
+ #![allow(clippy::expect_used, reason = "unit test")]
+
+ use gix_hash::ObjectId;
+
+ use super::*;
+
+ fn blob(id_seed: u8, data: Vec<u8>, lifetime: LifetimeClass) -> ClassifiedObject {
+ let mut bytes = [0u8; 20];
+ bytes[0] = id_seed;
+ ClassifiedObject {
+ id: ObjectId::from(bytes),
+ kind: Kind::Blob,
+ data,
+ lifetime,
+ }
+ }
+
+ #[test]
+ fn empty_partitions_produce_no_pack() {
+ let result = partition_and_pack(Vec::new()).expect("partition");
+ assert_eq!(result, PartitionedPacks::default());
+ }
+
+ #[test]
+ fn durable_and_cache_objects_land_in_separate_packs() {
+ let objects = vec![
+ blob(1, b"durable content".to_vec(), LifetimeClass::Durable),
+ blob(2, b"cache content".to_vec(), LifetimeClass::Cache),
+ ];
+ let result = partition_and_pack(objects).expect("partition");
+ assert!(result.durable.is_some());
+ assert!(result.cache.is_some());
+ assert_ne!(result.durable, result.cache);
+ }
+
+ #[test]
+ fn content_class_splits_blobs_by_size_threshold() {
+ let small = blob(1, vec![0u8; 4], LifetimeClass::Durable);
+ let large = blob(
+ 2,
+ vec![0u8; BINARY_THRESHOLD_BYTES + 1],
+ LifetimeClass::Durable,
+ );
+ assert_eq!(small.content_class(), ContentClass::Structural);
+ assert_eq!(large.content_class(), ContentClass::Binary);
+ }
+}
crates/odb-tigris/src/registry.rs
@@ -1,0 +1,91 @@
+//! [`PackRegistry`]: the only source of truth `odb-tigris` consults for
+//! which packs exist and are live (`docs/scale-out.adoc`, "Reachability":
+//! "nothing may traverse Tigris object-by-object"). `read`/`contains` walk
+//! [`PackRegistry::list`]'s result and nothing else — never a bucket listing
+//! call, which the [`crate::transport::BlobTransport`] trait doesn't even
+//! expose.
+//!
+//! [`memory::InMemoryRegistry`] is the in-process stand-in for tests and
+//! conformance; the Postgres-backed implementation lives in
+//! `refstore-postgres` (extending its `git_ents_pack_registry` table) rather
+//! than here, so this crate never needs a `tokio-postgres` dependency of its
+//! own — it depends only on the trait.
+
+pub mod memory;
+
+use git_backend::Result;
+
+/// Opaque identifier for one registered pack, unique within a repo. Chosen
+/// by whoever calls [`PackRegistry::record`] (in practice, the same id
+/// [`crate::OdbTigris::stage_pack`] used for its quarantine key prefix).
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct PackId(String);
+
+impl PackId {
+ /// Wrap a backend-chosen opaque token as a `PackId`.
+ #[must_use]
+ pub fn new(id: impl Into<String>) -> Self {
+ Self(id.into())
+ }
+
+ /// The id as a `&str`.
+ #[must_use]
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl std::fmt::Display for PackId {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(&self.0)
+ }
+}
+
+/// One promoted pack, as recorded in the registry: enough to fetch its
+/// index and data from the bucket, scoped to the repo it belongs to
+/// (`docs/scale-out.adoc`, rule 7: "namespace per repo").
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct PackRecord {
+ /// This pack's id.
+ pub id: PackId,
+ /// The repo this pack belongs to.
+ pub repo_id: String,
+ /// The bucket key holding the pack's object data.
+ pub pack_key: String,
+ /// The bucket key holding the pack's `.idx`.
+ pub idx_key: String,
+ /// The number of objects in the pack, if known. Optional: informational
+ /// only, never relied on for correctness (a stale or absent count never
+ /// changes what `read`/`contains` return, since those consult the idx
+ /// itself).
+ pub object_count: Option<u64>,
+}
+
+/// Registry of promoted packs: the commit point [`crate::OdbTigris::promote`]
+/// writes to, and the only thing [`crate::OdbTigris::read`] and
+/// [`crate::OdbTigris::contains`] consult to learn which packs exist.
+pub trait PackRegistry: Send + Sync {
+ /// Record `record` as promoted and live. Called once per pack, after its
+ /// bytes are durably in the bucket at the live keys `record` names.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the record cannot be durably written.
+ fn record(&self, record: PackRecord) -> Result<()>;
+
+ /// All packs currently registered for `repo_id`, in no particular order.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the registry cannot be read.
+ fn list(&self, repo_id: &str) -> Result<Vec<PackRecord>>;
+
+ /// Remove a pack from the registry (maintenance/GC use only — no code
+ /// in this crate calls it on the read path). Not an error if `id` is
+ /// already absent.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the registry cannot be written.
+ fn delete(&self, repo_id: &str, id: &PackId) -> Result<()>;
+}
crates/odb-tigris/src/registry/memory.rs
@@ -1,0 +1,49 @@
+//! [`InMemoryRegistry`]: an in-process [`PackRegistry`], used by tests and
+//! by the conformance instantiation
+//! (`crates/odb-tigris/tests/conformance.rs`).
+
+use std::sync::{Mutex, MutexGuard, PoisonError};
+
+use git_backend::Result;
+
+use super::{PackId, PackRecord, PackRegistry};
+
+/// A [`PackRegistry`] held entirely in memory, scoped to one process.
+#[derive(Default)]
+pub struct InMemoryRegistry {
+ records: Mutex<Vec<PackRecord>>,
+}
+
+impl InMemoryRegistry {
+ /// An empty registry.
+ #[must_use]
+ pub fn new() -> Self {
+ Self::default()
+ }
+}
+
+impl PackRegistry for InMemoryRegistry {
+ fn record(&self, record: PackRecord) -> Result<()> {
+ lock(&self.records).push(record);
+ Ok(())
+ }
+
+ fn list(&self, repo_id: &str) -> Result<Vec<PackRecord>> {
+ Ok(lock(&self.records)
+ .iter()
+ .filter(|record| record.repo_id == repo_id)
+ .cloned()
+ .collect())
+ }
+
+ fn delete(&self, repo_id: &str, id: &PackId) -> Result<()> {
+ lock(&self.records).retain(|record| !(record.repo_id == repo_id && &record.id == id));
+ Ok(())
+ }
+}
+
+/// Lock `mutex`, recovering the guard from a poisoned lock rather than
+/// panicking, mirroring `odb-files`'s own quarantine-map lock helper.
+fn lock<T>(mutex: &Mutex<T>) -> MutexGuard<'_, T> {
+ mutex.lock().unwrap_or_else(PoisonError::into_inner)
+}
crates/odb-tigris/src/transport.rs
@@ -1,0 +1,66 @@
+//! [`BlobTransport`]: the seam between `odb-tigris`'s object-store logic and
+//! however bytes actually move to and from the bucket (`docs/scale-out.adoc`,
+//! "ObjectStore" / WS5). Every method takes plain string keys so the rest of
+//! the crate never has to know whether it's talking to S3 or a local
+//! directory — [`fs::FsTransport`] is a no-network stand-in used by tests and
+//! conformance, [`s3::S3Transport`] is the real one.
+//!
+//! # Q2: read-after-write visibility
+//!
+//! [`ObjectStore::promote`](git_backend::ObjectStore::promote) assumes that a
+//! `put` (or `copy`) completed here is immediately visible to a subsequent
+//! `get`/`get_range` against the same key — i.e. the bucket offers
+//! read-after-write consistency for the keys this crate writes. Tigris (and
+//! S3 today) document this, but it is a fact to verify against the real
+//! service, not an assumption to bake in silently
+//! (`docs/scale-out.adoc`, Q2).
+
+pub mod fs;
+pub mod s3;
+
+use std::ops::Range;
+
+use git_backend::{Error, Result};
+
+/// One blob store, addressed by opaque string keys. Deliberately narrower
+/// than a filesystem: no directories, no rename other than [`copy`], no
+/// listing — the object-store layer above resolves everything it needs
+/// (OID → pack, pack → key) through the [`crate::registry::PackRegistry`]
+/// and cached pack indexes, never by asking the transport what keys exist
+/// (`docs/scale-out.adoc`, "Reachability": never traverse the bucket
+/// object-by-object).
+pub trait BlobTransport: Send + Sync {
+ /// Durably write `bytes` to `key`, replacing any prior content.
+ fn put(&self, key: &str, bytes: Vec<u8>) -> Result<()>;
+
+ /// Read the entirety of `key`.
+ fn get(&self, key: &str) -> Result<Vec<u8>>;
+
+ /// Read the byte range `range` of `key` — the operation the whole point
+ /// of this crate exists to make cheap: an HTTP Range-GET of just the
+ /// slice a caller needs, never the whole object. Implementations clamp
+ /// `range.end` to the object's actual length rather than erroring, so
+ /// callers can probe with a generous, possibly-too-large window (see
+ /// `decode`'s growth loop) without special-casing the tail of a key.
+ fn get_range(&self, key: &str, range: Range<u64>) -> Result<Vec<u8>>;
+
+ /// Whether `key` exists.
+ fn exists(&self, key: &str) -> Result<bool>;
+
+ /// Remove `key`. Not an error if it does not exist.
+ fn delete(&self, key: &str) -> Result<()>;
+
+ /// Copy `from` to `to` without a round trip through the caller — used by
+ /// [`crate::OdbTigris::promote`] to move a quarantined pack to its live
+ /// key. All CAS (compare-and-swap) stays in Postgres via the pack
+ /// registry; this is a plain durable copy, not a conditional write
+ /// (`docs/scale-out.adoc`, WS5: "Tigris needs only durable PUT/GET").
+ fn copy(&self, from: &str, to: &str) -> Result<()>;
+}
+
+/// Map any transport-level failure into [`git_backend::Error::ObjectStore`],
+/// prefixed with `context` so failures are traceable to the operation that
+/// caused them.
+pub(crate) fn transport_err(context: &str, error: impl std::fmt::Display) -> Error {
+ Error::ObjectStore(format!("{context}: {error}"))
+}
crates/odb-tigris/src/transport/fs.rs
@@ -1,0 +1,98 @@
+//! [`FsTransport`]: a [`BlobTransport`] over a local directory, standing in
+//! for the bucket in tests and conformance — no network, so the suite
+//! (`crates/odb-tigris/tests/conformance.rs`) runs anywhere
+//! (`docs/scale-out.adoc`, WS5, "Conformance").
+
+use std::ops::Range;
+use std::path::PathBuf;
+
+use git_backend::Result;
+
+use super::{BlobTransport, transport_err};
+
+/// A [`BlobTransport`] backed by plain files under a root directory. Keys
+/// (e.g. `"repo/live/abc.pack"`) map onto `root/repo/live/abc.pack`,
+/// creating parent directories as needed on `put`.
+pub struct FsTransport {
+ root: PathBuf,
+}
+
+impl FsTransport {
+ /// Store blobs under `root`, creating it if it does not exist.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if `root` cannot be created.
+ pub fn open(root: impl Into<PathBuf>) -> Result<Self> {
+ let root = root.into();
+ std::fs::create_dir_all(&root)?;
+ Ok(Self { root })
+ }
+
+ fn path_for(&self, key: &str) -> PathBuf {
+ self.root.join(key)
+ }
+}
+
+impl BlobTransport for FsTransport {
+ fn put(&self, key: &str, bytes: Vec<u8>) -> Result<()> {
+ let path = self.path_for(key);
+ if let Some(parent) = path.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+ std::fs::write(path, bytes)?;
+ Ok(())
+ }
+
+ fn get(&self, key: &str) -> Result<Vec<u8>> {
+ Ok(std::fs::read(self.path_for(key))?)
+ }
+
+ fn get_range(&self, key: &str, range: Range<u64>) -> Result<Vec<u8>> {
+ // No partial-file read syscall is worth the complexity here: this
+ // transport exists for tests, not for latency. It still honors the
+ // clamping contract so growth-loop callers see the same behavior a
+ // real ranged GET would.
+ let data = self.get(key)?;
+ let len = data.len() as u64;
+ let start = range.start.min(len);
+ let end = range.end.min(len);
+ Ok(if start >= end {
+ Vec::new()
+ } else {
+ data.get(usize_of(start)..usize_of(end))
+ .map(<[u8]>::to_vec)
+ .unwrap_or_default()
+ })
+ }
+
+ fn exists(&self, key: &str) -> Result<bool> {
+ Ok(self.path_for(key).is_file())
+ }
+
+ fn delete(&self, key: &str) -> Result<()> {
+ let path = self.path_for(key);
+ match std::fs::remove_file(path) {
+ Ok(()) => Ok(()),
+ Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()),
+ Err(error) => Err(error.into()),
+ }
+ }
+
+ fn copy(&self, from: &str, to: &str) -> Result<()> {
+ let to_path = self.path_for(to);
+ if let Some(parent) = to_path.parent() {
+ std::fs::create_dir_all(parent)?;
+ }
+ std::fs::copy(self.path_for(from), &to_path)
+ .map_err(|error| transport_err(&format!("copy {from} -> {to}"), error))?;
+ Ok(())
+ }
+}
+
+/// Fallible-looking but total for the lengths this module deals with:
+/// callers only ever pass values already clamped to a file's actual byte
+/// length, which never approaches `usize::MAX`.
+fn usize_of(n: u64) -> usize {
+ usize::try_from(n).unwrap_or(usize::MAX)
+}
crates/odb-tigris/src/transport/s3.rs
@@ -1,0 +1,138 @@
+//! [`S3Transport`]: the real [`BlobTransport`], over any S3-compatible
+//! bucket (Tigris included) via the `object_store` crate's `aws` feature
+//! (`docs/scale-out.adoc`, WS5). Dependency policy: `object_store` only,
+//! no `aws-sdk-*`, no `reqwest`/`hyper` pulled in directly.
+//!
+//! `object_store`'s trait is `async`; [`BlobTransport`] is not, so this type
+//! owns one dedicated [`tokio::runtime::Runtime`] and `block_on`s every
+//! call, mirroring `refstore-postgres::PostgresRefStore`'s own bridge from a
+//! sync trait to an async client.
+
+use std::ops::Range;
+
+use git_backend::Result;
+use object_store::aws::AmazonS3Builder;
+use object_store::path::Path as ObjectPath;
+use object_store::{ObjectStoreExt as _, PutPayload};
+
+use super::{BlobTransport, transport_err};
+
+/// Connection parameters for an S3-compatible bucket. Kept minimal and
+/// explicit rather than reading environment variables implicitly, so a
+/// caller embedding this crate controls exactly what credentials and
+/// endpoint it targets.
+#[derive(Debug, Clone)]
+pub struct S3Config {
+ /// The bucket name.
+ pub bucket: String,
+ /// The region to sign requests for (Tigris and most S3-compatibles
+ /// accept an arbitrary non-empty value here).
+ pub region: String,
+ /// The S3-compatible endpoint URL (e.g. `https://fly.storage.tigris.dev`).
+ pub endpoint: String,
+ /// Access key id.
+ pub access_key_id: String,
+ /// Secret access key.
+ pub secret_access_key: String,
+ /// Whether to allow plain HTTP (only ever `true` in tests against a
+ /// local S3-compatible stand-in).
+ pub allow_http: bool,
+}
+
+/// A [`BlobTransport`] over an S3-compatible bucket.
+pub struct S3Transport {
+ runtime: tokio::runtime::Runtime,
+ store: object_store::aws::AmazonS3,
+}
+
+impl S3Transport {
+ /// Connect to the bucket described by `config`.
+ ///
+ /// # Errors
+ ///
+ /// Returns an error if the dedicated runtime cannot be created or the
+ /// `object_store` client fails to build (e.g. malformed config).
+ pub fn connect(config: &S3Config) -> Result<Self> {
+ let runtime = tokio::runtime::Runtime::new()
+ .map_err(|error| transport_err("creating S3 transport runtime", error))?;
+ let store = AmazonS3Builder::new()
+ .with_bucket_name(&config.bucket)
+ .with_region(&config.region)
+ .with_endpoint(&config.endpoint)
+ .with_access_key_id(&config.access_key_id)
+ .with_secret_access_key(&config.secret_access_key)
+ .with_allow_http(config.allow_http)
+ .build()
+ .map_err(|error| transport_err("building S3 client", error))?;
+ Ok(Self { runtime, store })
+ }
+}
+
+impl BlobTransport for S3Transport {
+ fn put(&self, key: &str, bytes: Vec<u8>) -> Result<()> {
+ let path = ObjectPath::from(key);
+ self.runtime
+ .block_on(async { self.store.put(&path, PutPayload::from(bytes)).await })
+ .map_err(|error| transport_err(&format!("put {key}"), error))?;
+ Ok(())
+ }
+
+ fn get(&self, key: &str) -> Result<Vec<u8>> {
+ let path = ObjectPath::from(key);
+ self.runtime
+ .block_on(async {
+ let bytes = self.store.get(&path).await?.bytes().await?;
+ Ok::<_, object_store::Error>(bytes.to_vec())
+ })
+ .map_err(|error| transport_err(&format!("get {key}"), error))
+ }
+
+ fn get_range(&self, key: &str, range: Range<u64>) -> Result<Vec<u8>> {
+ let path = ObjectPath::from(key);
+ // Clamp to the object's actual length so growth-loop callers (see
+ // `crate::decode`) can probe with an intentionally generous window
+ // without the last object in a pack erroring on out-of-bounds ends.
+ self.runtime
+ .block_on(async {
+ let meta = self.store.head(&path).await?;
+ let len = meta.size;
+ let start = range.start.min(len);
+ let end = range.end.min(len);
+ if start >= end {
+ return Ok::<_, object_store::Error>(Vec::new());
+ }
+ let bytes = self.store.get_range(&path, start..end).await?;
+ Ok(bytes.to_vec())
+ })
+ .map_err(|error| transport_err(&format!("get_range {key}"), error))
+ }
+
+ fn exists(&self, key: &str) -> Result<bool> {
+ let path = ObjectPath::from(key);
+ self.runtime.block_on(async {
+ match self.store.head(&path).await {
+ Ok(_meta) => Ok(true),
+ Err(object_store::Error::NotFound { .. }) => Ok(false),
+ Err(error) => Err(transport_err(&format!("exists {key}"), error)),
+ }
+ })
+ }
+
+ fn delete(&self, key: &str) -> Result<()> {
+ let path = ObjectPath::from(key);
+ self.runtime.block_on(async {
+ match self.store.delete(&path).await {
+ Ok(()) | Err(object_store::Error::NotFound { .. }) => Ok(()),
+ Err(error) => Err(transport_err(&format!("delete {key}"), error)),
+ }
+ })
+ }
+
+ fn copy(&self, from: &str, to: &str) -> Result<()> {
+ let from_path = ObjectPath::from(from);
+ let to_path = ObjectPath::from(to);
+ self.runtime
+ .block_on(async { self.store.copy(&from_path, &to_path).await })
+ .map_err(|error| transport_err(&format!("copy {from} -> {to}"), error))
+ }
+}
crates/odb-tigris/tests/conformance.rs
@@ -1,0 +1,54 @@
+//! This crate's instantiation of the shared backend conformance suite
+//! (`docs/scale-out.adoc`, WS2): every `ObjectStore` property run against
+//! `OdbTigris` over its no-network stand-ins
+//! ([`FsTransport`](odb_tigris::transport::fs::FsTransport) +
+//! [`InMemoryRegistry`](odb_tigris::registry::memory::InMemoryRegistry)), so
+//! this test needs no bucket and no Postgres to run.
+
+#![allow(clippy::expect_used, reason = "test harness, not application code")]
+
+use backend_conformance::NoopCollector;
+use git_backend::{Object, ObjectStore, PackStream, QuarantineId, Result};
+use gix_hash::ObjectId;
+use odb_tigris::OdbTigris;
+use odb_tigris::registry::memory::InMemoryRegistry;
+use odb_tigris::transport::fs::FsTransport;
+
+/// Bundles an `OdbTigris` over `FsTransport` with the tempdir its bucket
+/// root lives under, so the directory outlives the store.
+struct WithBucketDir {
+ store: OdbTigris<FsTransport, InMemoryRegistry>,
+ _dir: tempfile::TempDir,
+}
+
+impl WithBucketDir {
+ fn new() -> Self {
+ let dir = tempfile::tempdir().expect("tempdir");
+ let transport = FsTransport::open(dir.path().join("bucket")).expect("open transport");
+ let store = OdbTigris::new(transport, InMemoryRegistry::new(), "conformance-repo");
+ Self { store, _dir: dir }
+ }
+}
+
+impl ObjectStore for WithBucketDir {
+ fn read(&self, id: ObjectId) -> Result<Object> {
+ self.store.read(id)
+ }
+
+ fn contains(&self, id: ObjectId) -> Result<bool> {
+ self.store.contains(id)
+ }
+
+ fn stage_pack(&self, pack: PackStream) -> Result<QuarantineId> {
+ self.store.stage_pack(pack)
+ }
+
+ fn promote(&self, q: QuarantineId) -> Result<()> {
+ self.store.promote(q)
+ }
+}
+
+#[test]
+fn conforms_to_object_store_properties() {
+ backend_conformance::object_store_properties(WithBucketDir::new, &NoopCollector);
+}
crates/odb-tigris/tests/delta_chain.rs
@@ -1,0 +1,174 @@
+//! Exercises `odb-tigris`'s ranged-read `OfsDelta` resolution
+//! (`crates/odb-tigris/src/decode.rs`) against a pack this test hand-builds
+//! specifically to contain a delta entry.
+//!
+//! The conformance suite's fixture pack (one small commit) is realistic but
+//! too small for `git pack-objects` to bother delta-compressing anything,
+//! so it never exercises `decode::resolve`'s delta branch. Relying on git's
+//! own (unspecified, version-dependent) heuristics to *maybe* produce a
+//! delta would make this test flaky, so instead this file constructs a
+//! minimal, deliberately-deltified two-object pack by hand: a full blob,
+//! and an `OfsDelta` entry against it, each zlib-wrapped with a trivial
+//! *stored* (uncompressed) deflate block — valid per the DEFLATE spec and
+//! decodable by any conforming inflate implementation, without pulling in a
+//! compression crate. `gix_pack::Bundle::write_to_directory` (invoked via
+//! `OdbTigris::stage_pack`) indexes this pack exactly like a real one, and
+//! `OdbTigris::read` then has to reconstruct the delta target purely from
+//! ranged reads to get the right answer.
+
+#![allow(
+ clippy::expect_used,
+ clippy::indexing_slicing,
+ clippy::arithmetic_side_effects,
+ reason = "test fixture hand-building a pack from fixed, known-small lengths, not application code"
+)]
+
+use std::io::Cursor;
+
+use git_backend::{ObjectStore as _, PackStream};
+use gix_hash::{Kind as HashKind, ObjectId};
+use gix_object::Kind;
+use gix_pack::data::entry::Header;
+use odb_tigris::OdbTigris;
+use odb_tigris::registry::memory::InMemoryRegistry;
+use odb_tigris::transport::fs::FsTransport;
+
+/// The git blob object id for `data`: `sha1("blob {len}\0" + data)`, the
+/// same content address `gix_pack`'s own indexer computes for our
+/// hand-built pack's objects — used here only to know what to `read()`
+/// back, not something the store is told.
+fn blob_oid(data: &[u8]) -> ObjectId {
+ let mut hasher = gix_hash::hasher(HashKind::Sha1);
+ hasher.update(format!("blob {}\0", data.len()).as_bytes());
+ hasher.update(data);
+ hasher.try_finalize().expect("hash blob")
+}
+
+/// Adler-32, the checksum zlib appends after its deflate stream.
+fn adler32(data: &[u8]) -> u32 {
+ const MOD_ADLER: u32 = 65521;
+ let mut a: u32 = 1;
+ let mut b: u32 = 0;
+ for &byte in data {
+ a = (a + u32::from(byte)) % MOD_ADLER;
+ b = (b + a) % MOD_ADLER;
+ }
+ (b << 16) | a
+}
+
+/// Wrap `data` in a valid zlib stream using a single uncompressed ("stored")
+/// deflate block — no real compression, just the minimal envelope zlib
+/// requires, which `gix_features::zlib::Inflate` (backed by `zlib-rs`, a
+/// spec-conforming implementation) must decode like any other zlib stream.
+fn zlib_store(data: &[u8]) -> Vec<u8> {
+ let len = u16::try_from(data.len()).expect("test fixture data fits in one stored block");
+ let mut out = vec![0x78, 0x01]; // valid zlib header (CM=8/CINFO=7, FLEVEL=fastest)
+ out.push(0x01); // final stored block: BFINAL=1, BTYPE=00
+ out.extend_from_slice(&len.to_le_bytes());
+ out.extend_from_slice(&(!len).to_le_bytes());
+ out.extend_from_slice(data);
+ out.extend_from_slice(&adler32(data).to_be_bytes());
+ out
+}
+
+/// A copy instruction copying `size` bytes starting at `ofs` from the base
+/// object, encoded per the pack delta format (mirrors
+/// `odb_tigris`'s own `apply_delta` decoder, used here in reverse).
+fn copy_op(ofs: u8, size: u8) -> Vec<u8> {
+ // Only the low byte of each of ofs/size is ever needed for this test's
+ // small fixture, so only bits 0x01 (ofs low byte) and 0x10 (size low
+ // byte) of the command byte are ever set.
+ vec![0x80 | 0x01 | 0x10, ofs, size]
+}
+
+/// An insert instruction embedding `bytes` literally (length must be 1..=127).
+fn insert_op(bytes: &[u8]) -> Vec<u8> {
+ let len = u8::try_from(bytes.len()).expect("test fixture insert fits one instruction");
+ assert!(len > 0 && len < 128, "insert length must fit the opcode");
+ let mut out = vec![len];
+ out.extend_from_slice(bytes);
+ out
+}
+
+/// Hand-encode a delta transforming `base` into `target`, where `target` is
+/// `base`'s first `prefix_len` bytes, then `middle`, then `base`'s last
+/// `suffix_len` bytes — exactly the shape this test's fixture uses.
+fn build_delta(base: &[u8], prefix_len: u8, middle: &[u8], suffix_len: u8) -> Vec<u8> {
+ let mut out = Vec::new();
+ out.push(u8::try_from(base.len()).expect("test base fits one varint byte"));
+ let target_len = usize::from(prefix_len) + middle.len() + usize::from(suffix_len);
+ out.push(u8::try_from(target_len).expect("test target fits one varint byte"));
+ out.extend(copy_op(0, prefix_len));
+ out.extend(insert_op(middle));
+ let suffix_ofs = u8::try_from(base.len()).expect("fits u8") - suffix_len;
+ out.extend(copy_op(suffix_ofs, suffix_len));
+ out
+}
+
+/// Build a minimal, valid version-2 pack containing exactly two objects:
+/// `base` as a full blob, then an `OfsDelta` entry against it decoding to
+/// `target`.
+fn build_pack(base: &[u8], target_prefix: u8, target_middle: &[u8], target_suffix: u8) -> Vec<u8> {
+ let mut pack = Vec::new();
+ pack.extend_from_slice(b"PACK");
+ pack.extend_from_slice(&2u32.to_be_bytes()); // pack version
+ pack.extend_from_slice(&2u32.to_be_bytes()); // object count
+
+ let entry0_offset = u64::try_from(pack.len()).expect("fits u64");
+ let mut header0 = Vec::new();
+ Header::Blob
+ .write_to(base.len() as u64, &mut header0)
+ .expect("write blob header");
+ pack.extend_from_slice(&header0);
+ pack.extend_from_slice(&zlib_store(base));
+
+ let entry1_offset = u64::try_from(pack.len()).expect("fits u64");
+ let base_distance = entry1_offset - entry0_offset;
+ let delta = build_delta(base, target_prefix, target_middle, target_suffix);
+ let mut header1 = Vec::new();
+ Header::OfsDelta { base_distance }
+ .write_to(delta.len() as u64, &mut header1)
+ .expect("write ofs-delta header");
+ pack.extend_from_slice(&header1);
+ pack.extend_from_slice(&zlib_store(&delta));
+
+ let mut hasher = gix_hash::hasher(HashKind::Sha1);
+ hasher.update(&pack);
+ let trailer = hasher.try_finalize().expect("hash pack");
+ pack.extend_from_slice(trailer.as_slice());
+ pack
+}
+
+#[test]
+fn resolves_an_ofs_delta_entry_via_ranged_reads() {
+ let base = b"01234567890123456789".to_vec(); // 20 bytes
+ let middle = b"DELTA-INSERTED-MIDDLE".to_vec();
+ let (prefix_len, suffix_len) = (5u8, 5u8);
+ let pack_bytes = build_pack(&base, prefix_len, &middle, suffix_len);
+
+ let mut target = base[..usize::from(prefix_len)].to_vec();
+ target.extend_from_slice(&middle);
+ target.extend_from_slice(&base[base.len() - usize::from(suffix_len)..]);
+
+ let dir = tempfile::tempdir().expect("tempdir");
+ let transport = FsTransport::open(dir.path().join("bucket")).expect("open transport");
+ let store = OdbTigris::new(transport, InMemoryRegistry::new(), "delta-test-repo");
+
+ let quarantine = store
+ .stage_pack(PackStream::new(Cursor::new(pack_bytes)))
+ .expect("stage_pack indexes the hand-built pack");
+ store.promote(quarantine).expect("promote");
+
+ let base_object = store.read(blob_oid(&base)).expect("read base object");
+ assert_eq!(base_object.kind, Kind::Blob);
+ assert_eq!(base_object.data, base);
+
+ let target_object = store
+ .read(blob_oid(&target))
+ .expect("read delta-reconstructed object");
+ assert_eq!(target_object.kind, Kind::Blob);
+ assert_eq!(
+ target_object.data, target,
+ "OfsDelta resolution over ranged reads must reproduce the exact target bytes"
+ );
+}