refactor(cache): redesign cache system with structured keys and improved performance
- Add repo_path parameter to cached_response and cached_vec_response functions - Implement structured cache key format with namespace, repo_path, and request proto - Replace global cache with Moka in-memory cache using weight-based eviction - Set 256MB memory cap with 10-minute TTL and 2-minute TTI policy - Add metrics collection for cache operations and evictions - Implement efficient repo-scoped invalidation using key structure - Add detailed documentation comments explaining cache architecture - Remove outdated dependencies and update dependency versions - Add error handling for encoding failures in cache operations - Optimize Vec responses with length-delimited encoding and pre-allocation
This commit is contained in:
zhenyi
Generated
+2
-445
@@ -61,28 +61,6 @@ version = "1.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-rs"
|
||||
version = "1.17.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
|
||||
dependencies = [
|
||||
"aws-lc-sys",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-lc-sys"
|
||||
version = "0.41.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"cmake",
|
||||
"dunce",
|
||||
"fs_extra",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "axum"
|
||||
version = "0.8.9"
|
||||
@@ -159,29 +137,6 @@ dependencies = [
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bon"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "97493a391b4b18ee918675fb8663e53646fd09321c58b46afa04e8ce2499c869"
|
||||
dependencies = [
|
||||
"bon-macros",
|
||||
"rustversion",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bon-macros"
|
||||
version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2a2af3eac944c12cdf4423eab70d310da0a8e5851a18ffb192c0a5e3f7ae1663"
|
||||
dependencies = [
|
||||
"darling",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bstr"
|
||||
version = "1.12.1"
|
||||
@@ -218,8 +173,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
|
||||
dependencies = [
|
||||
"find-msvc-tools",
|
||||
"jobserver",
|
||||
"libc",
|
||||
"shlex",
|
||||
]
|
||||
|
||||
@@ -238,15 +191,6 @@ dependencies = [
|
||||
"hashbrown 0.16.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "cmake"
|
||||
version = "0.1.58"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678"
|
||||
dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "const-oid"
|
||||
version = "0.10.2"
|
||||
@@ -323,41 +267,6 @@ dependencies = [
|
||||
"hybrid-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"darling_macro",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_core"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e"
|
||||
dependencies = [
|
||||
"fnv",
|
||||
"ident_case",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"strsim",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "darling_macro"
|
||||
version = "0.20.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead"
|
||||
dependencies = [
|
||||
"darling_core",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dashmap"
|
||||
version = "6.2.1"
|
||||
@@ -550,27 +459,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
|
||||
|
||||
[[package]]
|
||||
name = "fs_extra"
|
||||
version = "1.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c"
|
||||
|
||||
[[package]]
|
||||
name = "futures"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8b147ee9d1f6d097cef9ce628cd2ee62288d963e16fb287bd9286455b241382d"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-executor",
|
||||
"futures-io",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-channel"
|
||||
version = "0.3.32"
|
||||
@@ -578,7 +466,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-sink",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -587,34 +474,6 @@ version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d"
|
||||
|
||||
[[package]]
|
||||
name = "futures-executor"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"futures-task",
|
||||
"futures-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-io"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718"
|
||||
|
||||
[[package]]
|
||||
name = "futures-macro"
|
||||
version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "futures-sink"
|
||||
version = "0.3.32"
|
||||
@@ -633,13 +492,8 @@ version = "0.3.32"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"futures-io",
|
||||
"futures-macro",
|
||||
"futures-sink",
|
||||
"futures-task",
|
||||
"memchr",
|
||||
"pin-project-lite",
|
||||
"slab",
|
||||
]
|
||||
@@ -665,18 +519,6 @@ dependencies = [
|
||||
"wasi",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi 5.3.0",
|
||||
"wasip2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "getrandom"
|
||||
version = "0.4.2"
|
||||
@@ -685,7 +527,7 @@ checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"r-efi 6.0.0",
|
||||
"r-efi",
|
||||
"wasip2",
|
||||
"wasip3",
|
||||
]
|
||||
@@ -694,7 +536,6 @@ dependencies = [
|
||||
name = "gitks"
|
||||
version = "1.0.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
"dashmap",
|
||||
@@ -709,8 +550,6 @@ dependencies = [
|
||||
"moka",
|
||||
"prost",
|
||||
"prost-types",
|
||||
"ractor",
|
||||
"ractor_cluster",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_yml",
|
||||
@@ -727,6 +566,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-appender",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1786,12 +1626,6 @@ version = "2.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954"
|
||||
|
||||
[[package]]
|
||||
name = "ident_case"
|
||||
version = "1.0.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "2.14.0"
|
||||
@@ -1860,16 +1694,6 @@ dependencies = [
|
||||
"jiff-tzdb",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "jobserver"
|
||||
version = "0.1.34"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
|
||||
dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"libc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "js-sys"
|
||||
version = "0.3.100"
|
||||
@@ -2164,15 +1988,6 @@ version = "0.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
|
||||
|
||||
[[package]]
|
||||
name = "ppv-lite86"
|
||||
version = "0.2.21"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
|
||||
dependencies = [
|
||||
"zerocopy",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "prettyplease"
|
||||
version = "0.2.37"
|
||||
@@ -2254,70 +2069,6 @@ dependencies = [
|
||||
"prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d1c381df33c98266b5f08186583660090a4ffa0889e76c7e9a5e175f645a67fa"
|
||||
dependencies = [
|
||||
"protoc-bin-vendored-linux-aarch_64",
|
||||
"protoc-bin-vendored-linux-ppcle_64",
|
||||
"protoc-bin-vendored-linux-s390_64",
|
||||
"protoc-bin-vendored-linux-x86_32",
|
||||
"protoc-bin-vendored-linux-x86_64",
|
||||
"protoc-bin-vendored-macos-aarch_64",
|
||||
"protoc-bin-vendored-macos-x86_64",
|
||||
"protoc-bin-vendored-win32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-aarch_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c350df4d49b5b9e3ca79f7e646fde2377b199e13cfa87320308397e1f37e1a4c"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-ppcle_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a55a63e6c7244f19b5c6393f025017eb5d793fd5467823a099740a7a4222440c"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-s390_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1dba5565db4288e935d5330a07c264a4ee8e4a5b4a4e6f4e83fad824cc32f3b0"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-x86_32"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8854774b24ee28b7868cd71dccaae8e02a2365e67a4a87a6cd11ee6cdbdf9cf5"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-linux-x86_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b38b07546580df720fa464ce124c4b03630a6fb83e05c336fea2a241df7e5d78"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-macos-aarch_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "89278a9926ce312e51f1d999fee8825d324d603213344a9a706daa009f1d8092"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-macos-x86_64"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "81745feda7ccfb9471d7a4de888f0652e806d5795b61480605d4943176299756"
|
||||
|
||||
[[package]]
|
||||
name = "protoc-bin-vendored-win32"
|
||||
version = "3.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "95067976aca6421a523e491fce939a3e65249bac4b977adee0ee9771568e8aa3"
|
||||
|
||||
[[package]]
|
||||
name = "pulldown-cmark"
|
||||
version = "0.13.4"
|
||||
@@ -2347,102 +2098,12 @@ dependencies = [
|
||||
"proc-macro2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "5.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
|
||||
|
||||
[[package]]
|
||||
name = "r-efi"
|
||||
version = "6.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf"
|
||||
|
||||
[[package]]
|
||||
name = "ractor"
|
||||
version = "0.15.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f12c86deb2af198b10a04c4fb3fba73baf3bb300df765a29272f0e5583da7510"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bon",
|
||||
"dashmap",
|
||||
"futures",
|
||||
"js-sys",
|
||||
"once_cell",
|
||||
"strum",
|
||||
"tokio",
|
||||
"tokio_with_wasm",
|
||||
"tracing",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ractor_cluster"
|
||||
version = "0.15.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0fc5566dade327a8b4fa8dc046653cec2b889f00b6ddbbc8df6b020472dc62f5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"prost",
|
||||
"prost-build",
|
||||
"prost-types",
|
||||
"protoc-bin-vendored",
|
||||
"ractor",
|
||||
"ractor_cluster_derive",
|
||||
"rand",
|
||||
"sha2",
|
||||
"socket2",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ractor_cluster_derive"
|
||||
version = "0.15.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ed9db7ea11020d50ad74f9ed3eb25ba43c61ab1d8c24986ad967e80d092c01b"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand"
|
||||
version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5ca0ecfa931c29007047d1bc58e623ab12e5590e8c7cc53200d5202b69266d8a"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"rand_chacha",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_chacha"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
|
||||
dependencies = [
|
||||
"ppv-lite86",
|
||||
"rand_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rand_core"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
|
||||
dependencies = [
|
||||
"getrandom 0.2.17",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rawzip"
|
||||
version = "0.4.4"
|
||||
@@ -2520,7 +2181,6 @@ version = "0.23.40"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"log",
|
||||
"once_cell",
|
||||
"ring",
|
||||
@@ -2545,7 +2205,6 @@ version = "0.103.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"untrusted",
|
||||
@@ -2786,33 +2445,6 @@ version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
|
||||
|
||||
[[package]]
|
||||
name = "strsim"
|
||||
version = "0.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
|
||||
|
||||
[[package]]
|
||||
name = "strum"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd"
|
||||
dependencies = [
|
||||
"strum_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "strum_macros"
|
||||
version = "0.28.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "subtle"
|
||||
version = "2.6.1"
|
||||
@@ -2960,7 +2592,6 @@ dependencies = [
|
||||
"signal-hook-registry",
|
||||
"socket2",
|
||||
"tokio-macros",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
@@ -3010,30 +2641,6 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio_with_wasm"
|
||||
version = "0.8.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34e40fbbbd95441133fe9483f522db15dbfd26dc636164ebd8f2dd28759a6aa6"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"tokio",
|
||||
"tokio_with_wasm_proc",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tokio_with_wasm_proc"
|
||||
version = "0.8.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d01145a2c788d6aae4cd653afec1e8332534d7d783d01897cefcafe4428de992"
|
||||
dependencies = [
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
version = "0.14.6"
|
||||
@@ -3374,16 +2981,6 @@ dependencies = [
|
||||
"wasm-bindgen-shared",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-futures"
|
||||
version = "0.4.73"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wasm-bindgen-macro"
|
||||
version = "0.2.123"
|
||||
@@ -3450,26 +3047,6 @@ dependencies = [
|
||||
"semver",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-sys"
|
||||
version = "0.3.100"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-time"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winapi-util"
|
||||
version = "0.1.11"
|
||||
@@ -3745,26 +3322,6 @@ dependencies = [
|
||||
"rustix",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b065d4f0e55f82fae73202e189638116a87c55ab6b8e6c2721e13dd9d854ad1"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.50"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0b631b19d36a892ab55420c92dbc83ccd79274f25be714855d3074aa71cab639"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zeroize"
|
||||
version = "1.8.2"
|
||||
|
||||
+3
-5
@@ -20,6 +20,7 @@ moka = { version = "0.12", default-features = false, features = ["sync"] }
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
sha2 = "0.11"
|
||||
uuid = { version = "1", features = ["v7"] }
|
||||
gix = { version = "0.84", default-features = false, features = ["serde", "blame", "sha256", "sha1", "tracing", "merge", "max-performance-safe", "revision"] }
|
||||
gix-archive = { version = "0.33", features = ["sha256","sha1","document-features"] }
|
||||
duct = { version = "1", features = [] }
|
||||
@@ -33,14 +34,11 @@ thiserror = { version = "2", features = [] }
|
||||
prost = "0.14"
|
||||
prost-types = "0.14"
|
||||
tonic = { version = "0.14", features = ["transport"] }
|
||||
tonic-health = "0.14.6"
|
||||
tonic-health = "0.14"
|
||||
tonic-prost = "0.14"
|
||||
tempfile = "3"
|
||||
dotenvy = "0.15"
|
||||
ractor = { version = "0.15.13", features = ["cluster","tokio_runtime","monitors","message_span_propogation","async-trait"]}
|
||||
ractor_cluster = { version = "0.15.13", features = ["async-trait"] }
|
||||
async-trait = "0.1.89"
|
||||
etcd-client = { version = "0.18.0", features = ["tls"] }
|
||||
etcd-client = { version = "0.18", features = ["tls"] }
|
||||
dashmap = "6"
|
||||
hyper = { version = "1", features = ["server", "http1"] }
|
||||
hyper-util = { version = "0.1", features = ["tokio"] }
|
||||
|
||||
@@ -22,9 +22,11 @@ RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends git ca-certificates && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
COPY --from=builder /app/target/release/gitks /usr/local/bin/gitks
|
||||
|
||||
ENV GITKS_HOST=0.0.0.0
|
||||
ENV GITKS_PORT=50051
|
||||
ENV REPO_PREFIX_PATH=/data/repos
|
||||
|
||||
RUN mkdir -p /data/repos
|
||||
EXPOSE 50051
|
||||
ENTRYPOINT ["gitks"]
|
||||
-1141
File diff suppressed because it is too large
Load Diff
@@ -1,631 +0,0 @@
|
||||
use crate::pb::RepositoryHeader;
|
||||
use ractor::RpcReplyPort;
|
||||
use ractor_cluster::BytesConvertable;
|
||||
use ractor_cluster::RactorClusterMessage;
|
||||
|
||||
use super::raft_log::{Command as RaftCmd, LogEntry as RaftEntry};
|
||||
|
||||
/// Protocol version for Raft messages (forward/backward compatibility).
|
||||
pub const RAFT_MSG_VERSION: u32 = 1;
|
||||
|
||||
impl BytesConvertable for RepositoryHeader {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
prost::Message::encode_to_vec(&self)
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
prost::Message::decode(bytes.as_slice()).unwrap_or_default()
|
||||
}
|
||||
}
|
||||
|
||||
pub const ROLE_PRIMARY: &str = "primary";
|
||||
pub const ROLE_REPLICA: &str = "replica";
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RouteDecision {
|
||||
pub found: bool,
|
||||
pub storage_name: String,
|
||||
pub relative_path: String,
|
||||
pub actor_name: String,
|
||||
pub grpc_addr: String,
|
||||
pub role: String,
|
||||
}
|
||||
|
||||
impl BytesConvertable for RouteDecision {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
encode_strings(&[
|
||||
if self.found { "1" } else { "0" }.to_string(),
|
||||
self.storage_name,
|
||||
self.relative_path,
|
||||
self.actor_name,
|
||||
self.grpc_addr,
|
||||
self.role,
|
||||
])
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let values = decode_strings(bytes);
|
||||
Self {
|
||||
found: values.first().is_some_and(|v| v == "1"),
|
||||
storage_name: values.get(1).cloned().unwrap_or_default(),
|
||||
relative_path: values.get(2).cloned().unwrap_or_default(),
|
||||
actor_name: values.get(3).cloned().unwrap_or_default(),
|
||||
grpc_addr: values.get(4).cloned().unwrap_or_default(),
|
||||
role: values.get(5).cloned().unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct NodeHealth {
|
||||
pub storage_name: String,
|
||||
pub repo_count: u64,
|
||||
pub healthy: bool,
|
||||
pub version: String,
|
||||
}
|
||||
|
||||
impl BytesConvertable for NodeHealth {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
encode_strings(&[
|
||||
self.storage_name,
|
||||
self.repo_count.to_string(),
|
||||
if self.healthy { "1" } else { "0" }.to_string(),
|
||||
self.version,
|
||||
])
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let values = decode_strings(bytes);
|
||||
Self {
|
||||
storage_name: values.first().cloned().unwrap_or_default(),
|
||||
repo_count: values
|
||||
.get(1)
|
||||
.and_then(|v| v.parse().ok())
|
||||
.unwrap_or_default(),
|
||||
healthy: values.get(2).is_some_and(|v| v == "1"),
|
||||
version: values.get(3).cloned().unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RefUpdateEvent {
|
||||
pub relative_path: String,
|
||||
pub ref_name: String,
|
||||
pub old_oid: String,
|
||||
pub new_oid: String,
|
||||
pub primary_grpc_addr: String,
|
||||
pub primary_storage_name: String,
|
||||
}
|
||||
|
||||
impl BytesConvertable for RefUpdateEvent {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
encode_strings(&[
|
||||
self.relative_path,
|
||||
self.ref_name,
|
||||
self.old_oid,
|
||||
self.new_oid,
|
||||
self.primary_grpc_addr,
|
||||
self.primary_storage_name,
|
||||
])
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let values = decode_strings(bytes);
|
||||
Self {
|
||||
relative_path: values.first().cloned().unwrap_or_default(),
|
||||
ref_name: values.get(1).cloned().unwrap_or_default(),
|
||||
old_oid: values.get(2).cloned().unwrap_or_default(),
|
||||
new_oid: values.get(3).cloned().unwrap_or_default(),
|
||||
primary_grpc_addr: values.get(4).cloned().unwrap_or_default(),
|
||||
primary_storage_name: values.get(5).cloned().unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(RactorClusterMessage)]
|
||||
pub enum GitNodeMessage {
|
||||
ScanAndRegister,
|
||||
|
||||
RegisterRepository(RepositoryHeader),
|
||||
|
||||
RemoveRepository(RepositoryHeader),
|
||||
|
||||
RefUpdated(RefUpdateEvent),
|
||||
|
||||
#[rpc]
|
||||
FindPrimary(RepositoryHeader, RpcReplyPort<RouteDecision>),
|
||||
|
||||
#[rpc]
|
||||
FindReplica(RepositoryHeader, RpcReplyPort<RouteDecision>),
|
||||
|
||||
#[rpc]
|
||||
ListRepositoryPaths(RpcReplyPort<String>),
|
||||
|
||||
#[rpc]
|
||||
RepositoryExists(RepositoryHeader, RpcReplyPort<bool>),
|
||||
|
||||
#[rpc]
|
||||
GetNodeHealth(RpcReplyPort<NodeHealth>),
|
||||
|
||||
/// Election: vote for a candidate to become PRIMARY.
|
||||
#[rpc]
|
||||
ElectPrimary(ElectionRequest, RpcReplyPort<ElectionResult>),
|
||||
|
||||
/// A role change has occurred in the cluster.
|
||||
RoleChanged(RoleChangedEvent),
|
||||
|
||||
/// Health checker detected primary failure, trigger election.
|
||||
TriggerElection,
|
||||
|
||||
// ── Raft consensus messages ──────────────────────────────
|
||||
/// AppendEntries RPC: Leader → Follower log replication.
|
||||
#[rpc]
|
||||
AppendEntries(AppendEntriesRequest, RpcReplyPort<AppendEntriesResponse>),
|
||||
|
||||
/// ReadIndex RPC: confirm Leader is still valid for read operations.
|
||||
#[rpc]
|
||||
ReadIndex(ReadIndexRequest, RpcReplyPort<ReadIndexResponse>),
|
||||
|
||||
/// Raft write command: submit a command through Raft consensus.
|
||||
/// Returns true if consensus achieved, false otherwise.
|
||||
#[rpc]
|
||||
RaftWrite(crate::actor::raft_log::Command, RpcReplyPort<bool>),
|
||||
}
|
||||
|
||||
#[derive(ractor_cluster::RactorMessage)]
|
||||
pub enum RepoActorMessage {
|
||||
UpdateMetadata(RepositoryHeader),
|
||||
}
|
||||
|
||||
/// Request for a node to vote in a PRIMARY election.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ElectionRequest {
|
||||
pub candidate_storage_name: String,
|
||||
pub candidate_grpc_addr: String,
|
||||
pub candidate_actor_name: String,
|
||||
pub term: u64,
|
||||
pub reason: String, // "primary_failed" etc.
|
||||
/// Raft: candidate's last log index (for log consistency check).
|
||||
pub last_log_index: u64,
|
||||
/// Raft: candidate's last log term (for log consistency check).
|
||||
pub last_log_term: u64,
|
||||
}
|
||||
|
||||
impl BytesConvertable for ElectionRequest {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
encode_strings(&[
|
||||
self.candidate_storage_name,
|
||||
self.candidate_grpc_addr,
|
||||
self.candidate_actor_name,
|
||||
self.term.to_string(),
|
||||
self.reason,
|
||||
self.last_log_index.to_string(),
|
||||
self.last_log_term.to_string(),
|
||||
])
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let values = decode_strings(bytes);
|
||||
let term = values.get(3).and_then(|v| v.parse::<u64>().ok());
|
||||
if term.is_none() {
|
||||
tracing::warn!("ElectionRequest.from_bytes: failed to parse term field");
|
||||
}
|
||||
Self {
|
||||
candidate_storage_name: values.first().cloned().unwrap_or_default(),
|
||||
candidate_grpc_addr: values.get(1).cloned().unwrap_or_default(),
|
||||
candidate_actor_name: values.get(2).cloned().unwrap_or_default(),
|
||||
term: term.unwrap_or(0),
|
||||
reason: values.get(4).cloned().unwrap_or_default(),
|
||||
last_log_index: values.get(5).and_then(|v| v.parse().ok()).unwrap_or(0),
|
||||
last_log_term: values.get(6).and_then(|v| v.parse().ok()).unwrap_or(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of an election vote.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ElectionResult {
|
||||
pub accepted: bool,
|
||||
pub current_term: u64,
|
||||
pub voter_storage_name: String,
|
||||
pub voter_role: String,
|
||||
}
|
||||
|
||||
impl BytesConvertable for ElectionResult {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
encode_strings(&[
|
||||
if self.accepted { "1" } else { "0" }.to_string(),
|
||||
self.current_term.to_string(),
|
||||
self.voter_storage_name,
|
||||
self.voter_role,
|
||||
])
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let values = decode_strings(bytes);
|
||||
let current_term = values.get(1).and_then(|v| v.parse::<u64>().ok());
|
||||
if current_term.is_none() {
|
||||
tracing::warn!("ElectionResult.from_bytes: failed to parse current_term field");
|
||||
}
|
||||
Self {
|
||||
accepted: values.first().is_some_and(|v| v == "1"),
|
||||
current_term: current_term.unwrap_or(0),
|
||||
voter_storage_name: values.get(2).cloned().unwrap_or_default(),
|
||||
voter_role: values.get(3).cloned().unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Event broadcast when a node's role changes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RoleChangedEvent {
|
||||
pub storage_name: String,
|
||||
pub grpc_addr: String,
|
||||
pub new_role: String, // "primary" or "replica"
|
||||
pub term: u64,
|
||||
pub relative_paths: Vec<String>, // repos that changed role
|
||||
}
|
||||
|
||||
impl BytesConvertable for RoleChangedEvent {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
let mut strings = vec![
|
||||
self.storage_name,
|
||||
self.grpc_addr,
|
||||
self.new_role,
|
||||
self.term.to_string(),
|
||||
];
|
||||
strings.extend(self.relative_paths);
|
||||
encode_strings(&strings)
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let values = decode_strings(bytes);
|
||||
Self {
|
||||
storage_name: values.first().cloned().unwrap_or_default(),
|
||||
grpc_addr: values.get(1).cloned().unwrap_or_default(),
|
||||
new_role: values.get(2).cloned().unwrap_or_default(),
|
||||
term: {
|
||||
let t = values.get(3).and_then(|v| v.parse::<u64>().ok());
|
||||
if t.is_none() {
|
||||
tracing::warn!("RoleChangedEvent.from_bytes: failed to parse term field");
|
||||
}
|
||||
t.unwrap_or(0)
|
||||
},
|
||||
relative_paths: values.iter().skip(4).cloned().collect(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Raft consensus messages ──────────────────────────────────
|
||||
|
||||
/// Serialized Raft log entry for cross-node transfer.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct SerializedRaftEntry {
|
||||
pub term: u64,
|
||||
pub index: u64,
|
||||
pub command_bytes: Vec<u8>,
|
||||
pub checksum: u32,
|
||||
}
|
||||
|
||||
impl SerializedRaftEntry {
|
||||
pub fn from_entry(entry: &RaftEntry) -> Self {
|
||||
Self {
|
||||
term: entry.term,
|
||||
index: entry.index,
|
||||
command_bytes: entry.command.encode(),
|
||||
checksum: entry.checksum,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn to_entry(&self) -> Option<RaftEntry> {
|
||||
let command = RaftCmd::decode(&self.command_bytes)?;
|
||||
Some(RaftEntry {
|
||||
term: self.term,
|
||||
index: self.index,
|
||||
command,
|
||||
checksum: self.checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// AppendEntries RPC: Leader → Follower replication.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AppendEntriesRequest {
|
||||
/// Protocol version for forward/backward compatibility.
|
||||
pub version: u32,
|
||||
pub term: u64,
|
||||
pub leader_id: String,
|
||||
pub leader_grpc_addr: String,
|
||||
pub prev_log_index: u64,
|
||||
pub prev_log_term: u64,
|
||||
pub entries: Vec<SerializedRaftEntry>,
|
||||
pub leader_commit: u64,
|
||||
}
|
||||
|
||||
impl BytesConvertable for AppendEntriesRequest {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
// Version
|
||||
buf.extend(self.version.to_be_bytes());
|
||||
// Term, leader_id, leader_grpc_addr, prev_log_index, prev_log_term
|
||||
buf.extend(self.term.to_be_bytes());
|
||||
encode_string_bytes(&mut buf, &self.leader_id);
|
||||
encode_string_bytes(&mut buf, &self.leader_grpc_addr);
|
||||
buf.extend(self.prev_log_index.to_be_bytes());
|
||||
buf.extend(self.prev_log_term.to_be_bytes());
|
||||
buf.extend((self.entries.len() as u32).to_be_bytes());
|
||||
for entry in &self.entries {
|
||||
buf.extend(entry.term.to_be_bytes());
|
||||
buf.extend(entry.index.to_be_bytes());
|
||||
buf.extend(entry.checksum.to_be_bytes());
|
||||
buf.extend((entry.command_bytes.len() as u32).to_be_bytes());
|
||||
buf.extend(&entry.command_bytes);
|
||||
}
|
||||
buf.extend(self.leader_commit.to_be_bytes());
|
||||
buf
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let mut offset = 0;
|
||||
let version = read_u32(&bytes, &mut offset);
|
||||
let term = read_u64(&bytes, &mut offset);
|
||||
let leader_id = read_string(&bytes, &mut offset);
|
||||
let leader_grpc_addr = read_string(&bytes, &mut offset);
|
||||
let prev_log_index = read_u64(&bytes, &mut offset);
|
||||
let prev_log_term = read_u64(&bytes, &mut offset);
|
||||
let entry_count_raw = read_u32(&bytes, &mut offset) as usize;
|
||||
const MAX_ENTRIES_PER_BATCH: usize = 10_000;
|
||||
let entry_count = entry_count_raw.min(MAX_ENTRIES_PER_BATCH);
|
||||
if entry_count < entry_count_raw {
|
||||
tracing::warn!(
|
||||
claimed = entry_count_raw,
|
||||
capped = entry_count,
|
||||
"AppendEntries entry count capped to prevent DoS"
|
||||
);
|
||||
}
|
||||
let mut entries = Vec::with_capacity(entry_count);
|
||||
for _ in 0..entry_count {
|
||||
let eterm = read_u64(&bytes, &mut offset);
|
||||
let eindex = read_u64(&bytes, &mut offset);
|
||||
let echecksum = read_u32(&bytes, &mut offset);
|
||||
let cmd_len = read_u32(&bytes, &mut offset) as usize;
|
||||
if offset + cmd_len > bytes.len() {
|
||||
tracing::warn!(
|
||||
offset,
|
||||
cmd_len,
|
||||
total = bytes.len(),
|
||||
"AppendEntries entry truncated, stopping decode"
|
||||
);
|
||||
break;
|
||||
}
|
||||
if cmd_len > MAX_STRING_LEN {
|
||||
tracing::warn!(
|
||||
cmd_len,
|
||||
max = MAX_STRING_LEN,
|
||||
"AppendEntries entry too large, stopping decode"
|
||||
);
|
||||
break;
|
||||
}
|
||||
let command_bytes = bytes[offset..offset + cmd_len].to_vec();
|
||||
offset += cmd_len;
|
||||
entries.push(SerializedRaftEntry {
|
||||
term: eterm,
|
||||
index: eindex,
|
||||
command_bytes,
|
||||
checksum: echecksum,
|
||||
});
|
||||
}
|
||||
let leader_commit = read_u64(&bytes, &mut offset);
|
||||
Self {
|
||||
version,
|
||||
term,
|
||||
leader_id,
|
||||
leader_grpc_addr,
|
||||
prev_log_index,
|
||||
prev_log_term,
|
||||
entries,
|
||||
leader_commit,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// AppendEntries RPC response: Follower → Leader.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AppendEntriesResponse {
|
||||
/// Protocol version.
|
||||
pub version: u32,
|
||||
pub term: u64,
|
||||
pub success: bool,
|
||||
/// Follower's match_index after appending.
|
||||
pub match_index: u64,
|
||||
/// Hint for fast conflict resolution (optional).
|
||||
pub conflict_index: u64,
|
||||
pub conflict_term: u64,
|
||||
}
|
||||
|
||||
impl BytesConvertable for AppendEntriesResponse {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
buf.extend(self.version.to_be_bytes());
|
||||
buf.extend(self.term.to_be_bytes());
|
||||
buf.push(if self.success { 1 } else { 0 });
|
||||
buf.extend(self.match_index.to_be_bytes());
|
||||
buf.extend(self.conflict_index.to_be_bytes());
|
||||
buf.extend(self.conflict_term.to_be_bytes());
|
||||
buf
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let mut offset = 0;
|
||||
let version = read_u32(&bytes, &mut offset);
|
||||
let term = read_u64(&bytes, &mut offset);
|
||||
let success = bytes.get(offset).copied().unwrap_or(0) == 1;
|
||||
offset += 1;
|
||||
let match_index = read_u64(&bytes, &mut offset);
|
||||
let conflict_index = read_u64(&bytes, &mut offset);
|
||||
let conflict_term = read_u64(&bytes, &mut offset);
|
||||
Self {
|
||||
version,
|
||||
term,
|
||||
success,
|
||||
match_index,
|
||||
conflict_index,
|
||||
conflict_term,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// ReadIndex request: Client → Leader.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReadIndexRequest {
|
||||
pub relative_path: String,
|
||||
}
|
||||
|
||||
impl BytesConvertable for ReadIndexRequest {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
encode_strings(&[self.relative_path])
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let values = decode_strings(bytes);
|
||||
Self {
|
||||
relative_path: values.first().cloned().unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// ReadIndex response: Leader → Client.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ReadIndexResponse {
|
||||
pub commit_index: u64,
|
||||
pub leader_term: u64,
|
||||
pub is_leader: bool,
|
||||
}
|
||||
|
||||
impl BytesConvertable for ReadIndexResponse {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
buf.extend(self.commit_index.to_be_bytes());
|
||||
buf.extend(self.leader_term.to_be_bytes());
|
||||
buf.push(if self.is_leader { 1 } else { 0 });
|
||||
buf
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
let mut offset = 0;
|
||||
let commit_index = read_u64(&bytes, &mut offset);
|
||||
let leader_term = read_u64(&bytes, &mut offset);
|
||||
let is_leader = bytes.get(offset).copied().unwrap_or(0) == 1;
|
||||
Self {
|
||||
commit_index,
|
||||
leader_term,
|
||||
is_leader,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_strings(values: &[String]) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
for value in values {
|
||||
let bytes = value.as_bytes();
|
||||
buf.extend((bytes.len() as u64).to_be_bytes());
|
||||
buf.extend(bytes);
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
fn encode_string_bytes(buf: &mut Vec<u8>, s: &str) {
|
||||
let bytes = s.as_bytes();
|
||||
buf.extend((bytes.len() as u32).to_be_bytes());
|
||||
buf.extend(bytes);
|
||||
}
|
||||
|
||||
fn read_u32(data: &[u8], offset: &mut usize) -> u32 {
|
||||
if *offset + 4 > data.len() {
|
||||
return 0;
|
||||
}
|
||||
let val = u32::from_be_bytes(data[*offset..*offset + 4].try_into().unwrap_or([0; 4]));
|
||||
*offset += 4;
|
||||
val
|
||||
}
|
||||
|
||||
fn read_u64(data: &[u8], offset: &mut usize) -> u64 {
|
||||
if *offset + 8 > data.len() {
|
||||
return 0;
|
||||
}
|
||||
let val = u64::from_be_bytes(data[*offset..*offset + 8].try_into().unwrap_or([0; 8]));
|
||||
*offset += 8;
|
||||
val
|
||||
}
|
||||
|
||||
fn read_string(data: &[u8], offset: &mut usize) -> String {
|
||||
let len = read_u32(data, offset) as usize;
|
||||
if *offset + len > data.len() {
|
||||
return String::new();
|
||||
}
|
||||
let s = String::from_utf8_lossy(&data[*offset..*offset + len]).into_owned();
|
||||
*offset += len;
|
||||
s
|
||||
}
|
||||
|
||||
const MAX_STRING_LEN: usize = 10 * 1024 * 1024; // 10MB
|
||||
const MAX_TOTAL_SIZE: usize = 50 * 1024 * 1024; // 50MB
|
||||
|
||||
fn decode_strings(bytes: Vec<u8>) -> Vec<String> {
|
||||
let mut values = Vec::new();
|
||||
let mut offset = 0;
|
||||
|
||||
if bytes.len() > MAX_TOTAL_SIZE {
|
||||
tracing::warn!(
|
||||
total = bytes.len(),
|
||||
max = MAX_TOTAL_SIZE,
|
||||
"message exceeds maximum size, truncating"
|
||||
);
|
||||
return values;
|
||||
}
|
||||
|
||||
while offset + 8 <= bytes.len() {
|
||||
let len_bytes: [u8; 8] = bytes[offset..offset + 8].try_into().unwrap_or([0u8; 8]);
|
||||
let len_u64 = u64::from_be_bytes(len_bytes);
|
||||
|
||||
if len_u64 > MAX_STRING_LEN as u64 {
|
||||
tracing::warn!(
|
||||
offset,
|
||||
claimed_len = len_u64,
|
||||
max = MAX_STRING_LEN,
|
||||
"string length exceeds maximum, stopping decode"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
let len = len_u64 as usize;
|
||||
offset += 8;
|
||||
|
||||
let end_offset = match offset.checked_add(len) {
|
||||
Some(end) => end,
|
||||
None => {
|
||||
tracing::warn!(
|
||||
offset,
|
||||
len,
|
||||
"integer overflow in offset calculation, stopping decode"
|
||||
);
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
if end_offset > bytes.len() {
|
||||
tracing::warn!(
|
||||
offset,
|
||||
claimed_len = len,
|
||||
total = bytes.len(),
|
||||
"malformed bytes in decode_strings, stopping early"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
values.push(String::from_utf8_lossy(&bytes[offset..end_offset]).into_owned());
|
||||
offset = end_offset;
|
||||
}
|
||||
values
|
||||
}
|
||||
@@ -1,20 +0,0 @@
|
||||
pub mod handler;
|
||||
pub mod message;
|
||||
pub mod raft_log;
|
||||
pub mod server;
|
||||
pub mod snapshot;
|
||||
pub mod sync;
|
||||
|
||||
pub use handler::find_primary_in_cluster;
|
||||
pub use handler::{
|
||||
GitNodeActor, GitNodeArgs, RepoEntry, broadcast_append_entries, broadcast_ref_update,
|
||||
broadcast_role_changed, get_category_members, get_cluster_nodes, is_leader_lease_valid,
|
||||
list_all_groups, renew_leader_lease, route_group_for, start_node_actor,
|
||||
};
|
||||
pub use message::{
|
||||
AppendEntriesRequest, AppendEntriesResponse, ElectionRequest, ElectionResult, GitNodeMessage,
|
||||
NodeHealth, RAFT_MSG_VERSION, ROLE_PRIMARY, ROLE_REPLICA, ReadIndexRequest, ReadIndexResponse,
|
||||
RefUpdateEvent, RepoActorMessage, RoleChangedEvent, RouteDecision, SerializedRaftEntry,
|
||||
};
|
||||
pub use raft_log::{Command as RaftCommand, LogEntry as RaftLogEntry, RaftLog};
|
||||
pub use server::init_actor_cluster;
|
||||
@@ -1,764 +0,0 @@
|
||||
//! Raft log storage with disk persistence and CAS-based concurrent append.
|
||||
//!
|
||||
//! The log is stored in memory for fast access and persisted to disk for crash recovery.
|
||||
//! Each entry has a CRC32 checksum for integrity verification.
|
||||
//!
|
||||
//! Storage format:
|
||||
//! - `raft-log.dat`: Append-only log file containing serialized entries
|
||||
//! - `raft-index.dat`: Index file mapping entry index to file offset (for random access)
|
||||
|
||||
use std::io::{BufReader, BufWriter, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
|
||||
use ractor_cluster::BytesConvertable;
|
||||
|
||||
use crate::actor::handler::RepoEntry;
|
||||
use crate::actor::snapshot::{RaftSnapshot, SnapshotStorage};
|
||||
use crate::error::{GitError, GitResult};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Protocol version for forward/backward compatibility.
|
||||
pub const RAFT_PROTOCOL_VERSION: u32 = 1;
|
||||
|
||||
/// Maximum log entry size (1MB).
|
||||
const MAX_ENTRY_SIZE: usize = 1024 * 1024;
|
||||
|
||||
/// Log compression threshold (100MB).
|
||||
pub const LOG_COMPACT_THRESHOLD_BYTES: u64 = 100 * 1024 * 1024;
|
||||
|
||||
// ── Command ──────────────────────────────────────────────────
|
||||
|
||||
/// A Raft command that can be applied to the state machine.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Command {
|
||||
RefUpdate {
|
||||
relative_path: String,
|
||||
ref_name: String,
|
||||
old_oid: String,
|
||||
new_oid: String,
|
||||
},
|
||||
RegisterRepo {
|
||||
relative_path: String,
|
||||
storage_name: String,
|
||||
},
|
||||
RemoveRepo {
|
||||
relative_path: String,
|
||||
},
|
||||
SetPrimary {
|
||||
storage_name: String,
|
||||
relative_paths: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Serialize command to bytes.
|
||||
pub fn encode(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
match self {
|
||||
Command::RefUpdate {
|
||||
relative_path,
|
||||
ref_name,
|
||||
old_oid,
|
||||
new_oid,
|
||||
} => {
|
||||
buf.push(0); // tag
|
||||
encode_strings(&mut buf, &[relative_path, ref_name, old_oid, new_oid]);
|
||||
}
|
||||
Command::RegisterRepo {
|
||||
relative_path,
|
||||
storage_name,
|
||||
} => {
|
||||
buf.push(1);
|
||||
encode_strings(&mut buf, &[relative_path, storage_name]);
|
||||
}
|
||||
Command::RemoveRepo { relative_path } => {
|
||||
buf.push(2);
|
||||
encode_strings(&mut buf, &[relative_path]);
|
||||
}
|
||||
Command::SetPrimary {
|
||||
storage_name,
|
||||
relative_paths,
|
||||
} => {
|
||||
buf.push(3);
|
||||
encode_string(&mut buf, storage_name);
|
||||
buf.extend((relative_paths.len() as u32).to_be_bytes());
|
||||
for p in relative_paths {
|
||||
encode_string(&mut buf, p);
|
||||
}
|
||||
}
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
/// Deserialize command from bytes.
|
||||
pub fn decode(data: &[u8]) -> Option<Self> {
|
||||
if data.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let tag = data[0];
|
||||
let mut offset = 1;
|
||||
match tag {
|
||||
0 => {
|
||||
let (rp, o1) = decode_string(data, offset)?;
|
||||
offset = o1;
|
||||
let (rn, o2) = decode_string(data, offset)?;
|
||||
offset = o2;
|
||||
let (oo, o3) = decode_string(data, offset)?;
|
||||
offset = o3;
|
||||
let (no, _) = decode_string(data, offset)?;
|
||||
Some(Command::RefUpdate {
|
||||
relative_path: rp,
|
||||
ref_name: rn,
|
||||
old_oid: oo,
|
||||
new_oid: no,
|
||||
})
|
||||
}
|
||||
1 => {
|
||||
let (rp, o1) = decode_string(data, offset)?;
|
||||
offset = o1;
|
||||
let (sn, _) = decode_string(data, offset)?;
|
||||
Some(Command::RegisterRepo {
|
||||
relative_path: rp,
|
||||
storage_name: sn,
|
||||
})
|
||||
}
|
||||
2 => {
|
||||
let (rp, _) = decode_string(data, offset)?;
|
||||
Some(Command::RemoveRepo { relative_path: rp })
|
||||
}
|
||||
3 => {
|
||||
let (sn, o1) = decode_string(data, offset)?;
|
||||
offset = o1;
|
||||
if offset + 4 > data.len() {
|
||||
return None;
|
||||
}
|
||||
let count = u32::from_be_bytes(data[offset..offset + 4].try_into().ok()?) as usize;
|
||||
offset += 4;
|
||||
let mut paths = Vec::with_capacity(count);
|
||||
for _ in 0..count {
|
||||
let (p, o) = decode_string(data, offset)?;
|
||||
offset = o;
|
||||
paths.push(p);
|
||||
}
|
||||
Some(Command::SetPrimary {
|
||||
storage_name: sn,
|
||||
relative_paths: paths,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl BytesConvertable for Command {
|
||||
fn into_bytes(self) -> Vec<u8> {
|
||||
self.encode()
|
||||
}
|
||||
|
||||
fn from_bytes(bytes: Vec<u8>) -> Self {
|
||||
Self::decode(&bytes).unwrap_or(Command::RemoveRepo {
|
||||
relative_path: String::new(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_string(buf: &mut Vec<u8>, s: &str) {
|
||||
let bytes = s.as_bytes();
|
||||
buf.extend((bytes.len() as u32).to_be_bytes());
|
||||
buf.extend(bytes);
|
||||
}
|
||||
|
||||
fn encode_strings(buf: &mut Vec<u8>, strings: &[&str]) {
|
||||
buf.extend((strings.len() as u32).to_be_bytes());
|
||||
for s in strings {
|
||||
encode_string(buf, s);
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_string(data: &[u8], offset: usize) -> Option<(String, usize)> {
|
||||
if offset + 4 > data.len() {
|
||||
return None;
|
||||
}
|
||||
let len = u32::from_be_bytes(data[offset..offset + 4].try_into().ok()?) as usize;
|
||||
let start = offset + 4;
|
||||
if start + len > data.len() {
|
||||
return None;
|
||||
}
|
||||
let s = String::from_utf8_lossy(&data[start..start + len]).into_owned();
|
||||
Some((s, start + len))
|
||||
}
|
||||
|
||||
// ── LogEntry ─────────────────────────────────────────────────
|
||||
|
||||
/// A single Raft log entry.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct LogEntry {
|
||||
pub term: u64,
|
||||
pub index: u64,
|
||||
pub command: Command,
|
||||
pub checksum: u32,
|
||||
}
|
||||
|
||||
impl LogEntry {
|
||||
pub fn new(term: u64, index: u64, command: Command) -> Self {
|
||||
let checksum = Self::compute_checksum(term, index, &command);
|
||||
Self {
|
||||
term,
|
||||
index,
|
||||
command,
|
||||
checksum,
|
||||
}
|
||||
}
|
||||
|
||||
fn compute_checksum(term: u64, index: u64, command: &Command) -> u32 {
|
||||
let mut hasher = crc32fast::Hasher::new();
|
||||
hasher.update(&term.to_be_bytes());
|
||||
hasher.update(&index.to_be_bytes());
|
||||
hasher.update(&command.encode());
|
||||
hasher.finalize()
|
||||
}
|
||||
|
||||
/// Serialize entry to bytes (for disk storage).
|
||||
pub fn encode(&self) -> Vec<u8> {
|
||||
let cmd_bytes = self.command.encode();
|
||||
let total_len = 8 + 8 + 4 + 4 + cmd_bytes.len(); // term + index + checksum + cmd_len + cmd
|
||||
let mut buf = Vec::with_capacity(total_len);
|
||||
buf.extend(self.term.to_be_bytes());
|
||||
buf.extend(self.index.to_be_bytes());
|
||||
buf.extend(self.checksum.to_be_bytes());
|
||||
buf.extend((cmd_bytes.len() as u32).to_be_bytes());
|
||||
buf.extend(&cmd_bytes);
|
||||
buf
|
||||
}
|
||||
|
||||
/// Deserialize entry from bytes.
|
||||
pub fn decode(data: &[u8]) -> Option<Self> {
|
||||
if data.len() < 24 {
|
||||
return None;
|
||||
}
|
||||
let term = u64::from_be_bytes(data[0..8].try_into().ok()?);
|
||||
let index = u64::from_be_bytes(data[8..16].try_into().ok()?);
|
||||
let checksum = u32::from_be_bytes(data[16..20].try_into().ok()?);
|
||||
let cmd_len = u32::from_be_bytes(data[20..24].try_into().ok()?) as usize;
|
||||
if 24 + cmd_len > data.len() {
|
||||
return None;
|
||||
}
|
||||
let command = Command::decode(&data[24..24 + cmd_len])?;
|
||||
|
||||
// Verify checksum
|
||||
let expected = Self::compute_checksum(term, index, &command);
|
||||
if checksum != expected {
|
||||
tracing::warn!(
|
||||
term,
|
||||
index,
|
||||
expected,
|
||||
actual = checksum,
|
||||
"log entry checksum mismatch"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(LogEntry {
|
||||
term,
|
||||
index,
|
||||
command,
|
||||
checksum,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── IndexEntry ───────────────────────────────────────────────
|
||||
|
||||
/// Maps a log index to its file offset (for random access).
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct IndexEntry {
|
||||
index: u64,
|
||||
file_offset: u64,
|
||||
entry_size: u32,
|
||||
}
|
||||
|
||||
impl IndexEntry {
|
||||
const SIZE: usize = 20; // 8 + 8 + 4
|
||||
|
||||
fn encode(&self) -> [u8; Self::SIZE] {
|
||||
let mut buf = [0u8; Self::SIZE];
|
||||
buf[0..8].copy_from_slice(&self.index.to_be_bytes());
|
||||
buf[8..16].copy_from_slice(&self.file_offset.to_be_bytes());
|
||||
buf[16..20].copy_from_slice(&self.entry_size.to_be_bytes());
|
||||
buf
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn decode(data: &[u8; Self::SIZE]) -> Self {
|
||||
Self {
|
||||
index: u64::from_be_bytes(data[0..8].try_into().unwrap()),
|
||||
file_offset: u64::from_be_bytes(data[8..16].try_into().unwrap()),
|
||||
entry_size: u32::from_be_bytes(data[16..20].try_into().unwrap()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── RaftStorage ──────────────────────────────────────────────
|
||||
|
||||
/// Disk persistence layer for the Raft log.
|
||||
struct RaftStorage {
|
||||
log_path: PathBuf,
|
||||
index_path: PathBuf,
|
||||
}
|
||||
|
||||
impl RaftStorage {
|
||||
fn new(data_dir: &Path) -> Self {
|
||||
Self {
|
||||
log_path: data_dir.join("raft-log.dat"),
|
||||
index_path: data_dir.join("raft-index.dat"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Append an entry to the log file and update the index.
|
||||
fn append(&self, entry: &LogEntry) -> GitResult<u64> {
|
||||
let entry_bytes = entry.encode();
|
||||
let entry_size = entry_bytes.len() as u32;
|
||||
|
||||
// Append to log file
|
||||
let mut log_file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.log_path)
|
||||
.map_err(GitError::Io)?;
|
||||
let file_offset = log_file.metadata().map_err(GitError::Io)?.len();
|
||||
log_file.write_all(&entry_bytes).map_err(GitError::Io)?;
|
||||
log_file.flush().map_err(GitError::Io)?;
|
||||
|
||||
// Append to index file
|
||||
let index_entry = IndexEntry {
|
||||
index: entry.index,
|
||||
file_offset,
|
||||
entry_size,
|
||||
};
|
||||
let mut index_file = std::fs::OpenOptions::new()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&self.index_path)
|
||||
.map_err(GitError::Io)?;
|
||||
index_file
|
||||
.write_all(&index_entry.encode())
|
||||
.map_err(GitError::Io)?;
|
||||
index_file.flush().map_err(GitError::Io)?;
|
||||
|
||||
Ok(entry.index)
|
||||
}
|
||||
|
||||
/// Load all entries from disk.
|
||||
fn load_all(&self) -> GitResult<Vec<LogEntry>> {
|
||||
if !self.log_path.exists() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let log_file = std::fs::File::open(&self.log_path).map_err(GitError::Io)?;
|
||||
let mut reader = BufReader::new(log_file);
|
||||
let mut entries = Vec::new();
|
||||
|
||||
loop {
|
||||
// Read term (8 bytes)
|
||||
let mut term_buf = [0u8; 8];
|
||||
match reader.read_exact(&mut term_buf) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => break,
|
||||
Err(e) => return Err(GitError::Io(e)),
|
||||
}
|
||||
let term = u64::from_be_bytes(term_buf);
|
||||
|
||||
// Read index (8 bytes)
|
||||
let mut index_buf = [0u8; 8];
|
||||
reader.read_exact(&mut index_buf).map_err(GitError::Io)?;
|
||||
let index = u64::from_be_bytes(index_buf);
|
||||
|
||||
// Read checksum (4 bytes)
|
||||
let mut checksum_buf = [0u8; 4];
|
||||
reader.read_exact(&mut checksum_buf).map_err(GitError::Io)?;
|
||||
let checksum = u32::from_be_bytes(checksum_buf);
|
||||
|
||||
// Read command length (4 bytes)
|
||||
let mut cmd_len_buf = [0u8; 4];
|
||||
reader.read_exact(&mut cmd_len_buf).map_err(GitError::Io)?;
|
||||
let cmd_len = u32::from_be_bytes(cmd_len_buf) as usize;
|
||||
|
||||
if cmd_len > MAX_ENTRY_SIZE {
|
||||
tracing::warn!(index, cmd_len, "entry too large, stopping recovery");
|
||||
break;
|
||||
}
|
||||
|
||||
// Read command bytes
|
||||
let mut cmd_buf = vec![0u8; cmd_len];
|
||||
reader.read_exact(&mut cmd_buf).map_err(GitError::Io)?;
|
||||
|
||||
// Decode command
|
||||
match Command::decode(&cmd_buf) {
|
||||
Some(command) => {
|
||||
let expected_checksum = LogEntry::compute_checksum(term, index, &command);
|
||||
if checksum != expected_checksum {
|
||||
tracing::warn!(
|
||||
index,
|
||||
expected = expected_checksum,
|
||||
actual = checksum,
|
||||
"checksum mismatch during recovery, stopping"
|
||||
);
|
||||
break;
|
||||
}
|
||||
entries.push(LogEntry {
|
||||
term,
|
||||
index,
|
||||
command,
|
||||
checksum,
|
||||
});
|
||||
}
|
||||
None => {
|
||||
tracing::warn!(index, "failed to decode command during recovery, stopping");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
tracing::info!(count = entries.len(), "loaded raft log entries from disk");
|
||||
Ok(entries)
|
||||
}
|
||||
|
||||
/// Get the total size of the log file.
|
||||
fn log_file_size(&self) -> u64 {
|
||||
std::fs::metadata(&self.log_path)
|
||||
.map(|m| m.len())
|
||||
.unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Truncate the log file and rebuild the index.
|
||||
fn truncate_and_rebuild(&self, entries: &[LogEntry]) -> GitResult<()> {
|
||||
// Write new log file
|
||||
let log_file = std::fs::File::create(&self.log_path).map_err(GitError::Io)?;
|
||||
let mut writer = BufWriter::new(log_file);
|
||||
let mut index_entries = Vec::with_capacity(entries.len());
|
||||
let mut offset = 0u64;
|
||||
|
||||
for entry in entries {
|
||||
let entry_bytes = entry.encode();
|
||||
let entry_size = entry_bytes.len() as u32;
|
||||
index_entries.push(IndexEntry {
|
||||
index: entry.index,
|
||||
file_offset: offset,
|
||||
entry_size,
|
||||
});
|
||||
writer.write_all(&entry_bytes).map_err(GitError::Io)?;
|
||||
offset += entry_size as u64;
|
||||
}
|
||||
writer.flush().map_err(GitError::Io)?;
|
||||
|
||||
// Write new index file
|
||||
let index_file = std::fs::File::create(&self.index_path).map_err(GitError::Io)?;
|
||||
let mut writer = BufWriter::new(index_file);
|
||||
for ie in &index_entries {
|
||||
writer.write_all(&ie.encode()).map_err(GitError::Io)?;
|
||||
}
|
||||
writer.flush().map_err(GitError::Io)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ── RaftLog ──────────────────────────────────────────────────
|
||||
|
||||
/// Thread-safe Raft log with in-memory storage and disk persistence.
|
||||
pub struct RaftLog {
|
||||
entries: Vec<LogEntry>,
|
||||
commit_index: u64,
|
||||
last_applied: u64,
|
||||
/// Atomic next index for CAS-based concurrent append.
|
||||
next_index: AtomicU64,
|
||||
storage: RaftStorage,
|
||||
/// Snapshot storage for log compaction.
|
||||
snapshot_storage: SnapshotStorage,
|
||||
/// Last snapshot index (entries before this are discarded).
|
||||
snapshot_index: u64,
|
||||
/// Last snapshot term.
|
||||
snapshot_term: u64,
|
||||
/// Path for snapshot data.
|
||||
data_dir: PathBuf,
|
||||
}
|
||||
|
||||
impl RaftLog {
|
||||
/// Create a new RaftLog, loading existing entries from disk.
|
||||
pub fn new(data_dir: &Path) -> GitResult<Self> {
|
||||
std::fs::create_dir_all(data_dir).map_err(GitError::Io)?;
|
||||
let storage = RaftStorage::new(data_dir);
|
||||
let snapshot_storage = SnapshotStorage::new(data_dir);
|
||||
|
||||
// Load snapshot if exists
|
||||
let (snapshot_index, snapshot_term) = match snapshot_storage.load()? {
|
||||
Some(snapshot) => {
|
||||
tracing::info!(
|
||||
index = snapshot.last_included_index,
|
||||
term = snapshot.last_included_term,
|
||||
"loaded existing raft snapshot"
|
||||
);
|
||||
(snapshot.last_included_index, snapshot.last_included_term)
|
||||
}
|
||||
None => (0, 0),
|
||||
};
|
||||
|
||||
let entries = storage.load_all()?;
|
||||
|
||||
let next_index = entries
|
||||
.last()
|
||||
.map(|e| e.index + 1)
|
||||
.unwrap_or(snapshot_index + 1);
|
||||
let last_applied = entries.last().map(|e| e.index).unwrap_or(snapshot_index);
|
||||
|
||||
Ok(Self {
|
||||
entries,
|
||||
commit_index: snapshot_index,
|
||||
last_applied,
|
||||
next_index: AtomicU64::new(next_index),
|
||||
storage,
|
||||
snapshot_storage,
|
||||
snapshot_index,
|
||||
snapshot_term,
|
||||
data_dir: data_dir.to_path_buf(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Get the last log index (0 if empty).
|
||||
pub fn last_index(&self) -> u64 {
|
||||
self.entries.last().map(|e| e.index).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Get the last log term (0 if empty).
|
||||
pub fn last_term(&self) -> u64 {
|
||||
self.entries.last().map(|e| e.term).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Get the current commit index.
|
||||
pub fn commit_index(&self) -> u64 {
|
||||
self.commit_index
|
||||
}
|
||||
|
||||
/// Get the last applied index.
|
||||
pub fn last_applied(&self) -> u64 {
|
||||
self.last_applied
|
||||
}
|
||||
|
||||
/// Get the next index for a new entry.
|
||||
pub fn next_index(&self) -> u64 {
|
||||
self.next_index.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// CAS-based index reservation for concurrent append.
|
||||
/// Returns the reserved index. The caller must then call `append_reserved`.
|
||||
pub fn reserve_index(&self) -> u64 {
|
||||
self.next_index.fetch_add(1, Ordering::SeqCst)
|
||||
}
|
||||
|
||||
/// Append a pre-reserved entry to the log.
|
||||
/// The entry must have been assigned an index via `reserve_index`.
|
||||
pub fn append_reserved(&mut self, entry: LogEntry) -> GitResult<u64> {
|
||||
// Persist to disk
|
||||
self.storage.append(&entry)?;
|
||||
// Add to memory
|
||||
self.entries.push(entry.clone());
|
||||
Ok(entry.index)
|
||||
}
|
||||
|
||||
/// Convenience: create and append an entry in one call (non-concurrent).
|
||||
pub fn append(&mut self, term: u64, command: Command) -> GitResult<u64> {
|
||||
let index = self.reserve_index();
|
||||
let entry = LogEntry::new(term, index, command);
|
||||
self.append_reserved(entry)
|
||||
}
|
||||
|
||||
/// Get an entry by index.
|
||||
pub fn get(&self, index: u64) -> Option<&LogEntry> {
|
||||
if self.entries.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let first_index = self.entries[0].index;
|
||||
if index < first_index {
|
||||
return None;
|
||||
}
|
||||
let offset = (index - first_index) as usize;
|
||||
self.entries.get(offset)
|
||||
}
|
||||
|
||||
/// Get entries in range [from_index, to_index).
|
||||
pub fn get_range(&self, from_index: u64, to_index: u64) -> Vec<&LogEntry> {
|
||||
if self.entries.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
let first_index = self.entries[0].index;
|
||||
let start = if from_index < first_index {
|
||||
0
|
||||
} else {
|
||||
(from_index - first_index) as usize
|
||||
};
|
||||
let end = if to_index < first_index {
|
||||
0
|
||||
} else {
|
||||
((to_index - first_index) as usize).min(self.entries.len())
|
||||
};
|
||||
self.entries[start..end].iter().collect()
|
||||
}
|
||||
|
||||
/// Get the term of the entry at the given index.
|
||||
pub fn term_at(&self, index: u64) -> u64 {
|
||||
self.get(index).map(|e| e.term).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Advance commit_index to the given value.
|
||||
pub fn advance_commit_index(&mut self, new_commit_index: u64) {
|
||||
if new_commit_index > self.commit_index {
|
||||
self.commit_index = new_commit_index;
|
||||
tracing::debug!(commit_index = new_commit_index, "commit index advanced");
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark entries up to the given index as applied.
|
||||
pub fn advance_last_applied(&mut self, new_last_applied: u64) {
|
||||
if new_last_applied > self.last_applied {
|
||||
self.last_applied = new_last_applied;
|
||||
}
|
||||
}
|
||||
|
||||
/// Get committed entries that haven't been applied yet.
|
||||
pub fn unapplied_entries(&self) -> Vec<&LogEntry> {
|
||||
self.get_range(self.last_applied + 1, self.commit_index + 1)
|
||||
}
|
||||
|
||||
/// Check if the log needs compaction.
|
||||
pub fn needs_compaction(&self) -> bool {
|
||||
self.storage.log_file_size() >= LOG_COMPACT_THRESHOLD_BYTES
|
||||
}
|
||||
|
||||
/// Compact the log, keeping entries from `from_index` onwards.
|
||||
pub fn compact(&mut self, from_index: u64) -> GitResult<()> {
|
||||
if from_index <= self.entries[0].index {
|
||||
return Ok(()); // Nothing to compact
|
||||
}
|
||||
|
||||
let keep: Vec<LogEntry> = self
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| e.index >= from_index)
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
if keep.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.storage.truncate_and_rebuild(&keep)?;
|
||||
self.entries = keep;
|
||||
|
||||
tracing::info!(from_index, kept = self.entries.len(), "raft log compacted");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Truncate the log, removing entries from `from_index` onwards (inclusive).
|
||||
/// This is used during AppendEntries to resolve log conflicts per the Raft protocol:
|
||||
/// when a follower detects a term mismatch, it must delete the conflicting entry
|
||||
/// and all entries that follow.
|
||||
pub fn truncate_from(&mut self, from_index: u64) -> GitResult<()> {
|
||||
let keep: Vec<LogEntry> = self
|
||||
.entries
|
||||
.iter()
|
||||
.filter(|e| e.index < from_index)
|
||||
.cloned()
|
||||
.collect();
|
||||
|
||||
let removed = self.entries.len() - keep.len();
|
||||
if removed == 0 {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
self.storage.truncate_and_rebuild(&keep)?;
|
||||
self.entries = keep;
|
||||
|
||||
let new_next = self.entries.last().map(|e| e.index + 1).unwrap_or(1);
|
||||
self.next_index.store(new_next, Ordering::SeqCst);
|
||||
|
||||
tracing::info!(
|
||||
from_index,
|
||||
removed,
|
||||
remaining = self.entries.len(),
|
||||
"raft log truncated"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the data directory path (for snapshot storage).
|
||||
pub fn data_dir(&self) -> &Path {
|
||||
&self.data_dir
|
||||
}
|
||||
|
||||
/// Get the total number of entries in the log.
|
||||
pub fn len(&self) -> usize {
|
||||
self.entries.len()
|
||||
}
|
||||
|
||||
/// Check if the log is empty.
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.entries.is_empty()
|
||||
}
|
||||
|
||||
/// Create a snapshot of the current state and compact the log.
|
||||
pub fn create_snapshot(&mut self, repos: HashMap<String, RepoEntry>) -> GitResult<()> {
|
||||
let snapshot = RaftSnapshot::new(self.last_applied, self.term_at(self.last_applied), repos);
|
||||
|
||||
self.snapshot_storage.save(&snapshot)?;
|
||||
self.snapshot_index = snapshot.last_included_index;
|
||||
self.snapshot_term = snapshot.last_included_term;
|
||||
|
||||
// Compact the log: remove entries before the snapshot
|
||||
self.compact(self.snapshot_index + 1)?;
|
||||
|
||||
tracing::info!(
|
||||
snapshot_index = self.snapshot_index,
|
||||
snapshot_term = self.snapshot_term,
|
||||
remaining_entries = self.entries.len(),
|
||||
"raft snapshot created and log compacted"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Restore state from a snapshot.
|
||||
pub fn restore_snapshot(
|
||||
&mut self,
|
||||
snapshot: RaftSnapshot,
|
||||
) -> GitResult<HashMap<String, RepoEntry>> {
|
||||
self.snapshot_index = snapshot.last_included_index;
|
||||
self.snapshot_term = snapshot.last_included_term;
|
||||
self.commit_index = snapshot.last_included_index;
|
||||
self.last_applied = snapshot.last_included_index;
|
||||
self.next_index
|
||||
.store(snapshot.last_included_index + 1, Ordering::SeqCst);
|
||||
|
||||
// Clear all entries (they're covered by the snapshot)
|
||||
self.entries.clear();
|
||||
self.storage.truncate_and_rebuild(&[])?;
|
||||
|
||||
tracing::info!(
|
||||
snapshot_index = self.snapshot_index,
|
||||
snapshot_term = self.snapshot_term,
|
||||
"raft state restored from snapshot"
|
||||
);
|
||||
|
||||
Ok(snapshot.repos)
|
||||
}
|
||||
|
||||
/// Get the snapshot index.
|
||||
pub fn snapshot_index(&self) -> u64 {
|
||||
self.snapshot_index
|
||||
}
|
||||
|
||||
/// Get the snapshot term.
|
||||
pub fn snapshot_term(&self) -> u64 {
|
||||
self.snapshot_term
|
||||
}
|
||||
}
|
||||
@@ -1,17 +0,0 @@
|
||||
use crate::actor::handler::start_node_actor;
|
||||
use crate::actor::message::GitNodeMessage;
|
||||
use crate::server::GitksService;
|
||||
use ractor::ActorRef;
|
||||
use std::path::PathBuf;
|
||||
|
||||
pub async fn init_actor_cluster(
|
||||
service: GitksService,
|
||||
storage_name: String,
|
||||
grpc_addr: String,
|
||||
data_dir: PathBuf,
|
||||
) -> Result<(ActorRef<GitNodeMessage>, tokio::task::JoinHandle<()>), ractor::SpawnErr> {
|
||||
tracing::info!(storage_name = %storage_name, grpc_addr = %grpc_addr, "initializing actor cluster");
|
||||
let result = start_node_actor(service, storage_name.clone(), grpc_addr, data_dir).await?;
|
||||
tracing::info!(storage_name = %storage_name, "actor cluster ready");
|
||||
Ok(result)
|
||||
}
|
||||
@@ -1,205 +0,0 @@
|
||||
//! Raft log snapshot mechanism for log compaction.
|
||||
//!
|
||||
//! When the Raft log grows beyond the size threshold, a snapshot is created
|
||||
//! that captures the current state of all repositories. Old log entries before
|
||||
//! the snapshot are then discarded.
|
||||
//!
|
||||
//! Snapshot format:
|
||||
//! - `raft-snapshot.dat`: Contains the serialized state at a given log index
|
||||
//!
|
||||
//! The snapshot includes:
|
||||
//! - All repository entries (path, role, last_commit)
|
||||
//! - The log index at which the snapshot was taken
|
||||
//! - The term at which the snapshot was taken
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::{BufReader, BufWriter, Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::actor::handler::RepoEntry;
|
||||
use crate::error::{GitError, GitResult};
|
||||
|
||||
/// Snapshot metadata and data.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RaftSnapshot {
|
||||
/// The log index at which this snapshot was taken.
|
||||
pub last_included_index: u64,
|
||||
/// The term at which this snapshot was taken.
|
||||
pub last_included_term: u64,
|
||||
/// All repository entries at the time of the snapshot.
|
||||
pub repos: HashMap<String, RepoEntry>,
|
||||
}
|
||||
|
||||
impl RaftSnapshot {
|
||||
/// Create a new snapshot from the current state.
|
||||
pub fn new(
|
||||
last_included_index: u64,
|
||||
last_included_term: u64,
|
||||
repos: HashMap<String, RepoEntry>,
|
||||
) -> Self {
|
||||
Self {
|
||||
last_included_index,
|
||||
last_included_term,
|
||||
repos,
|
||||
}
|
||||
}
|
||||
|
||||
/// Serialize the snapshot to bytes.
|
||||
pub fn encode(&self) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
// Header
|
||||
buf.extend(self.last_included_index.to_be_bytes());
|
||||
buf.extend(self.last_included_term.to_be_bytes());
|
||||
// Repository count
|
||||
buf.extend((self.repos.len() as u32).to_be_bytes());
|
||||
// Each repository entry
|
||||
for (path, entry) in &self.repos {
|
||||
encode_string(&mut buf, path);
|
||||
encode_string(&mut buf, &entry.role);
|
||||
encode_string(&mut buf, &entry.last_commit);
|
||||
buf.push(if entry.read_only { 1 } else { 0 });
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
/// Deserialize a snapshot from bytes.
|
||||
pub fn decode(data: &[u8]) -> Option<Self> {
|
||||
if data.len() < 20 {
|
||||
return None;
|
||||
}
|
||||
let mut offset = 0;
|
||||
let last_included_index = read_u64(data, &mut offset)?;
|
||||
let last_included_term = read_u64(data, &mut offset)?;
|
||||
let repo_count = read_u32(data, &mut offset)? as usize;
|
||||
|
||||
let mut repos = HashMap::with_capacity(repo_count);
|
||||
for _ in 0..repo_count {
|
||||
let path = read_string(data, &mut offset)?;
|
||||
let role = read_string(data, &mut offset)?;
|
||||
let last_commit = read_string(data, &mut offset)?;
|
||||
let read_only = data.get(offset).copied().unwrap_or(0) == 1;
|
||||
offset += 1;
|
||||
repos.insert(
|
||||
path,
|
||||
RepoEntry {
|
||||
role,
|
||||
last_commit,
|
||||
read_only,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
Some(Self {
|
||||
last_included_index,
|
||||
last_included_term,
|
||||
repos,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Snapshot storage manager.
|
||||
pub struct SnapshotStorage {
|
||||
snapshot_path: PathBuf,
|
||||
}
|
||||
|
||||
impl SnapshotStorage {
|
||||
pub fn new(data_dir: &Path) -> Self {
|
||||
Self {
|
||||
snapshot_path: data_dir.join("raft-snapshot.dat"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Save a snapshot to disk.
|
||||
pub fn save(&self, snapshot: &RaftSnapshot) -> GitResult<()> {
|
||||
let data = snapshot.encode();
|
||||
let file = std::fs::File::create(&self.snapshot_path).map_err(GitError::Io)?;
|
||||
let mut writer = BufWriter::new(file);
|
||||
writer.write_all(&data).map_err(GitError::Io)?;
|
||||
writer.flush().map_err(GitError::Io)?;
|
||||
|
||||
tracing::info!(
|
||||
index = snapshot.last_included_index,
|
||||
term = snapshot.last_included_term,
|
||||
repos = snapshot.repos.len(),
|
||||
"raft snapshot saved"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load a snapshot from disk.
|
||||
pub fn load(&self) -> GitResult<Option<RaftSnapshot>> {
|
||||
if !self.snapshot_path.exists() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let file = std::fs::File::open(&self.snapshot_path).map_err(GitError::Io)?;
|
||||
let mut reader = BufReader::new(file);
|
||||
let mut data = Vec::new();
|
||||
reader.read_to_end(&mut data).map_err(GitError::Io)?;
|
||||
|
||||
match RaftSnapshot::decode(&data) {
|
||||
Some(snapshot) => {
|
||||
tracing::info!(
|
||||
index = snapshot.last_included_index,
|
||||
term = snapshot.last_included_term,
|
||||
repos = snapshot.repos.len(),
|
||||
"raft snapshot loaded"
|
||||
);
|
||||
Ok(Some(snapshot))
|
||||
}
|
||||
None => {
|
||||
tracing::warn!("failed to decode raft snapshot, ignoring");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a snapshot exists.
|
||||
pub fn exists(&self) -> bool {
|
||||
self.snapshot_path.exists()
|
||||
}
|
||||
|
||||
/// Delete the snapshot file.
|
||||
pub fn delete(&self) -> GitResult<()> {
|
||||
if self.snapshot_path.exists() {
|
||||
std::fs::remove_file(&self.snapshot_path).map_err(GitError::Io)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// ── Helper functions ─────────────────────────────────────────
|
||||
|
||||
fn encode_string(buf: &mut Vec<u8>, s: &str) {
|
||||
let bytes = s.as_bytes();
|
||||
buf.extend((bytes.len() as u32).to_be_bytes());
|
||||
buf.extend(bytes);
|
||||
}
|
||||
|
||||
fn read_u32(data: &[u8], offset: &mut usize) -> Option<u32> {
|
||||
if *offset + 4 > data.len() {
|
||||
return None;
|
||||
}
|
||||
let val = u32::from_be_bytes(data[*offset..*offset + 4].try_into().ok()?);
|
||||
*offset += 4;
|
||||
Some(val)
|
||||
}
|
||||
|
||||
fn read_u64(data: &[u8], offset: &mut usize) -> Option<u64> {
|
||||
if *offset + 8 > data.len() {
|
||||
return None;
|
||||
}
|
||||
let val = u64::from_be_bytes(data[*offset..*offset + 8].try_into().ok()?);
|
||||
*offset += 8;
|
||||
Some(val)
|
||||
}
|
||||
|
||||
fn read_string(data: &[u8], offset: &mut usize) -> Option<String> {
|
||||
let len = read_u32(data, offset)? as usize;
|
||||
if *offset + len > data.len() {
|
||||
return None;
|
||||
}
|
||||
let s = String::from_utf8_lossy(&data[*offset..*offset + len]).into_owned();
|
||||
*offset += len;
|
||||
Some(s)
|
||||
}
|
||||
-406
@@ -1,406 +0,0 @@
|
||||
use crate::actor::message::RefUpdateEvent;
|
||||
use crate::pb::Oid;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct BundleApplicator {
|
||||
pub repo_path: PathBuf,
|
||||
}
|
||||
|
||||
impl BundleApplicator {
|
||||
pub fn new(repo_path: PathBuf) -> Self {
|
||||
Self { repo_path }
|
||||
}
|
||||
|
||||
pub fn apply_bundle(&self, data: &[u8]) -> Result<(), String> {
|
||||
let mut child = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.repo_path.to_string_lossy(),
|
||||
"bundle",
|
||||
"unbundle",
|
||||
"-",
|
||||
])
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|e| format!("spawn git bundle unbundle: {e}"))?;
|
||||
use std::io::Write;
|
||||
if let Some(ref mut stdin) = child.stdin {
|
||||
stdin
|
||||
.write_all(data)
|
||||
.map_err(|e| format!("write bundle: {e}"))?;
|
||||
}
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("wait bundle: {e}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(String::from_utf8_lossy(&output.stderr).into_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply bundle from a file path (for streaming writes).
|
||||
pub fn apply_bundle_from_file(&self, path: &Path) -> Result<(), String> {
|
||||
let file = std::fs::File::open(path).map_err(|e| format!("open bundle file: {e}"))?;
|
||||
let mut child = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.repo_path.to_string_lossy(),
|
||||
"bundle",
|
||||
"unbundle",
|
||||
"-",
|
||||
])
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|e| format!("spawn git bundle unbundle: {e}"))?;
|
||||
|
||||
// Stream file contents to stdin in a background thread
|
||||
let mut stdin = child.stdin.take().ok_or("no stdin")?;
|
||||
let file_handle = file;
|
||||
let writer = std::thread::spawn(move || -> Result<(), String> {
|
||||
use std::io::{Read, Write};
|
||||
let mut reader = std::io::BufReader::new(file_handle);
|
||||
let mut buf = vec![0u8; 65536];
|
||||
loop {
|
||||
match reader.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
stdin
|
||||
.write_all(&buf[..n])
|
||||
.map_err(|e| format!("write to stdin: {e}"))?;
|
||||
}
|
||||
Err(e) => return Err(format!("read bundle file: {e}")),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("wait bundle: {e}"))?;
|
||||
|
||||
// Wait for writer thread
|
||||
let _ = writer.join().map_err(|_| "writer thread panicked")?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(String::from_utf8_lossy(&output.stderr).into_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn collect_local_haves(repo_path: &Path) -> Result<Vec<Oid>, String> {
|
||||
let result = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&repo_path.to_string_lossy(),
|
||||
"for-each-ref",
|
||||
"--format=%(objectname)",
|
||||
])
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.output()
|
||||
.map_err(|e| format!("git for-each-ref: {e}"))?;
|
||||
|
||||
if !result.status.success() {
|
||||
return Err(String::from_utf8_lossy(&result.stderr).into_owned());
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&result.stdout);
|
||||
let haves: Vec<Oid> = stdout
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty() && line.trim() != crate::oid::ZERO_OID)
|
||||
.map(|hex| {
|
||||
let hex = hex.trim().to_string();
|
||||
Oid {
|
||||
value: crate::oid::hex_to_bytes(&hex).unwrap_or_default(),
|
||||
hex,
|
||||
format: crate::pb::ObjectFormat::Sha1 as i32,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
tracing::debug!(
|
||||
repo = %repo_path.display(),
|
||||
haves_count = haves.len(),
|
||||
"collected local haves from refs"
|
||||
);
|
||||
Ok(haves)
|
||||
}
|
||||
|
||||
pub async fn sync_from_primary(event: RefUpdateEvent, local_repo_path: PathBuf) {
|
||||
tracing::info!(
|
||||
relative_path = %event.relative_path,
|
||||
ref_name = %event.ref_name,
|
||||
primary = %event.primary_grpc_addr,
|
||||
"replica sync starting"
|
||||
);
|
||||
|
||||
let grpc_addr = event.primary_grpc_addr.clone();
|
||||
let relative_path = event.relative_path.clone();
|
||||
let repo_for_haves = local_repo_path.clone();
|
||||
|
||||
// Collect haves in a blocking thread
|
||||
let haves = match tokio::task::spawn_blocking(move || collect_local_haves(&repo_for_haves))
|
||||
.await
|
||||
{
|
||||
Ok(Ok(h)) => h,
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!(relative_path = %event.relative_path, error = %e, "collect haves failed");
|
||||
return;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(relative_path = %event.relative_path, error = %e, "haves task failed");
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
// Stream pack data to a temporary file to avoid OOM
|
||||
let temp_dir = local_repo_path.join(".gitks_tmp");
|
||||
if let Err(e) = std::fs::create_dir_all(&temp_dir) {
|
||||
tracing::error!(relative_path = %event.relative_path, error = %e, "create temp dir failed");
|
||||
return;
|
||||
}
|
||||
|
||||
let pack_result =
|
||||
sync_via_pack_service_to_file(&grpc_addr, &relative_path, &haves, &temp_dir).await;
|
||||
|
||||
match pack_result {
|
||||
Ok(Some(pack_file)) => {
|
||||
let repo = local_repo_path.clone();
|
||||
let pack_path = pack_file.clone();
|
||||
match tokio::task::spawn_blocking(move || {
|
||||
let applicator = BundleApplicator::new(repo);
|
||||
applicator.apply_bundle_from_file(&pack_path)
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(Ok(())) => {
|
||||
update_local_ref(&local_repo_path, &event.ref_name, &event.new_oid);
|
||||
tracing::info!(
|
||||
relative_path = %event.relative_path,
|
||||
"replica sync done"
|
||||
);
|
||||
}
|
||||
Ok(Err(e)) => {
|
||||
tracing::error!(relative_path = %event.relative_path, error = %e, "pack apply failed")
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(relative_path = %event.relative_path, error = %e, "apply task failed")
|
||||
}
|
||||
}
|
||||
// Cleanup temp file
|
||||
let _ = std::fs::remove_file(&pack_file);
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::warn!(relative_path = %event.relative_path, "empty pack data from primary")
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(relative_path = %event.relative_path, error = %e, "pack fetch failed")
|
||||
}
|
||||
}
|
||||
|
||||
// Cleanup temp dir if empty
|
||||
let _ = std::fs::remove_dir(&temp_dir);
|
||||
}
|
||||
|
||||
/// Maximum pack size before we reject (10GB)
|
||||
const MAX_PACK_SIZE: u64 = 10 * 1024 * 1024 * 1024;
|
||||
|
||||
/// Stream pack data from primary to a temporary file.
|
||||
/// Returns Ok(Some(path)) on success, Ok(None) if empty, Err on failure.
|
||||
async fn sync_via_pack_service_to_file(
|
||||
grpc_addr: &str,
|
||||
relative_path: &str,
|
||||
haves: &[Oid],
|
||||
temp_dir: &Path,
|
||||
) -> Result<Option<PathBuf>, String> {
|
||||
use crate::pb::pack_service_client::PackServiceClient;
|
||||
use crate::pb::{
|
||||
AdvertiseRefsRequest, PackObjectsOptions, PackObjectsRequest, RepositoryHeader,
|
||||
};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio_stream::StreamExt;
|
||||
|
||||
let endpoint = crate::server::remote_endpoint(grpc_addr)
|
||||
.await
|
||||
.map_err(|e| e.to_string())?;
|
||||
|
||||
let mut client = PackServiceClient::connect(endpoint)
|
||||
.await
|
||||
.map_err(|e| format!("connect to primary: {e}"))?;
|
||||
|
||||
let header = RepositoryHeader {
|
||||
storage_name: String::new(),
|
||||
relative_path: relative_path.to_string(),
|
||||
storage_path: String::new(),
|
||||
};
|
||||
|
||||
let refs_resp = client
|
||||
.advertise_refs(AdvertiseRefsRequest {
|
||||
repository: Some(header.clone()),
|
||||
protocol: None,
|
||||
service: "upload-pack".to_string(),
|
||||
raw: false,
|
||||
})
|
||||
.await
|
||||
.map_err(|e| format!("AdvertiseRefs: {e}"))?;
|
||||
|
||||
let refs = refs_resp.into_inner().references;
|
||||
if refs.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let wants: Vec<Oid> = refs.iter().filter_map(|r| r.target_oid.clone()).collect();
|
||||
|
||||
let want_count = wants.len();
|
||||
let have_count = haves.len();
|
||||
|
||||
tracing::info!(
|
||||
relative_path = %relative_path,
|
||||
want_count,
|
||||
have_count,
|
||||
"requesting incremental pack from primary"
|
||||
);
|
||||
|
||||
let options = PackObjectsOptions {
|
||||
wants,
|
||||
haves: haves.to_vec(),
|
||||
shallow_revisions: Vec::new(),
|
||||
deepen: 0,
|
||||
thin_pack: false,
|
||||
include_tag: true,
|
||||
use_bitmaps: true,
|
||||
delta_base_offset: true,
|
||||
pathspec: Vec::new(),
|
||||
};
|
||||
|
||||
let req = PackObjectsRequest {
|
||||
repository: Some(header.clone()),
|
||||
options: Some(options),
|
||||
};
|
||||
|
||||
let resp = client
|
||||
.pack_objects(req)
|
||||
.await
|
||||
.map_err(|e| format!("PackObjects: {e}"))?;
|
||||
|
||||
let mut stream = resp.into_inner();
|
||||
|
||||
// Create a temporary file for streaming
|
||||
let temp_file = temp_dir.join(format!("pack_{}.bundle", std::process::id()));
|
||||
let mut file = tokio::fs::File::create(&temp_file)
|
||||
.await
|
||||
.map_err(|e| format!("create temp file: {e}"))?;
|
||||
|
||||
let mut total_bytes: u64 = 0;
|
||||
while let Some(chunk) = stream.next().await {
|
||||
match chunk {
|
||||
Ok(msg) => {
|
||||
total_bytes += msg.data.len() as u64;
|
||||
if total_bytes > MAX_PACK_SIZE {
|
||||
let _ = tokio::fs::remove_file(&temp_file).await;
|
||||
return Err(format!(
|
||||
"pack data exceeds maximum size ({}GB)",
|
||||
MAX_PACK_SIZE / (1024 * 1024 * 1024)
|
||||
));
|
||||
}
|
||||
file.write_all(&msg.data)
|
||||
.await
|
||||
.map_err(|e| format!("write pack data: {e}"))?;
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tokio::fs::remove_file(&temp_file).await;
|
||||
return Err(format!("pack stream: {e}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush and close the file
|
||||
file.flush()
|
||||
.await
|
||||
.map_err(|e| format!("flush pack file: {e}"))?;
|
||||
drop(file);
|
||||
|
||||
tracing::info!(
|
||||
relative_path = %relative_path,
|
||||
pack_bytes = total_bytes,
|
||||
"received pack data from primary"
|
||||
);
|
||||
|
||||
Ok(Some(temp_file))
|
||||
}
|
||||
|
||||
fn update_local_ref(repo_path: &Path, ref_name: &str, new_oid: &str) {
|
||||
if ref_name.is_empty() || new_oid.is_empty() {
|
||||
return;
|
||||
}
|
||||
match std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&repo_path.to_string_lossy(),
|
||||
"update-ref",
|
||||
ref_name,
|
||||
new_oid,
|
||||
])
|
||||
.output()
|
||||
{
|
||||
Ok(o) if o.status.success() => {
|
||||
tracing::info!(ref_name = %ref_name, new_oid = %new_oid, "ref updated")
|
||||
}
|
||||
Ok(o) => {
|
||||
tracing::error!(ref_name = %ref_name, error = %String::from_utf8_lossy(&o.stderr), "update-ref failed")
|
||||
}
|
||||
Err(e) => tracing::error!(ref_name = %ref_name, error = %e, "update-ref spawn failed"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Apply a committed Raft command to the local git repository.
|
||||
/// This is called on followers when they receive committed entries from the leader.
|
||||
pub fn apply_raft_command_to_repo(repo_prefix: &Path, command: &crate::actor::raft_log::Command) {
|
||||
match command {
|
||||
crate::actor::raft_log::Command::RefUpdate {
|
||||
relative_path,
|
||||
ref_name,
|
||||
old_oid: _,
|
||||
new_oid,
|
||||
} => {
|
||||
let repo_path = repo_prefix.join(relative_path);
|
||||
tracing::info!(
|
||||
relative_path = %relative_path,
|
||||
ref_name = %ref_name,
|
||||
new_oid = %new_oid,
|
||||
"applying RefUpdate from Raft log to local repo"
|
||||
);
|
||||
update_local_ref(&repo_path, ref_name, new_oid);
|
||||
}
|
||||
crate::actor::raft_log::Command::RegisterRepo {
|
||||
relative_path,
|
||||
storage_name: _,
|
||||
} => {
|
||||
tracing::info!(
|
||||
relative_path = %relative_path,
|
||||
"RegisterRepo from Raft log (no git action needed)"
|
||||
);
|
||||
}
|
||||
crate::actor::raft_log::Command::RemoveRepo { relative_path } => {
|
||||
tracing::info!(
|
||||
relative_path = %relative_path,
|
||||
"RemoveRepo from Raft log (no git action needed)"
|
||||
);
|
||||
}
|
||||
crate::actor::raft_log::Command::SetPrimary {
|
||||
storage_name,
|
||||
relative_paths,
|
||||
} => {
|
||||
tracing::info!(
|
||||
storage_name = %storage_name,
|
||||
paths = relative_paths.len(),
|
||||
"SetPrimary from Raft log (no git action needed)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
+15
-2
@@ -22,7 +22,11 @@ impl GitBare {
|
||||
|
||||
// Validate revision before spawning (cannot use ? inside spawn_blocking closure)
|
||||
let revision = match request.treeish.and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)
|
||||
.map_err(|e| tonic::Status::invalid_argument(e.to_string()))?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)
|
||||
.map_err(|e| tonic::Status::invalid_argument(e.to_string()))?;
|
||||
@@ -31,9 +35,18 @@ impl GitBare {
|
||||
None => "HEAD".into(),
|
||||
};
|
||||
|
||||
let options = request.options.unwrap_or_default();
|
||||
if !options.prefix.is_empty() {
|
||||
crate::sanitize::validate_file_path(&options.prefix)
|
||||
.map_err(|e| tonic::Status::invalid_argument(e.to_string()))?;
|
||||
}
|
||||
for path in &options.pathspec {
|
||||
crate::sanitize::validate_file_path(path)
|
||||
.map_err(|e| tonic::Status::invalid_argument(e.to_string()))?;
|
||||
}
|
||||
|
||||
// Spawn the blocking git subprocess in a dedicated thread
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let options = request.options.unwrap_or_default();
|
||||
let format = archive_options::Format::try_from(options.format)
|
||||
.unwrap_or(archive_options::Format::ArchiveFormatTar);
|
||||
let mut args = vec!["archive".to_string()];
|
||||
|
||||
@@ -2,8 +2,9 @@ use std::process::Command;
|
||||
|
||||
use crate::bare::GitBare;
|
||||
use crate::error::{GitError, GitResult};
|
||||
use crate::paginate;
|
||||
use crate::pb::{
|
||||
ArchiveEntry, ListArchiveEntriesRequest, ListArchiveEntriesResponse, ObjectType, PageInfo,
|
||||
ArchiveEntry, ListArchiveEntriesRequest, ListArchiveEntriesResponse, ObjectType,
|
||||
object_selector,
|
||||
};
|
||||
|
||||
@@ -13,7 +14,10 @@ impl GitBare {
|
||||
request: ListArchiveEntriesRequest,
|
||||
) -> GitResult<ListArchiveEntriesResponse> {
|
||||
let revision = match request.treeish.and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
@@ -22,8 +26,11 @@ impl GitBare {
|
||||
};
|
||||
let mut args = vec!["ls-tree".to_string(), "-r".into(), "-l".into(), revision];
|
||||
if !request.pathspec.is_empty() {
|
||||
for path in &request.pathspec {
|
||||
crate::sanitize::validate_file_path(path)?;
|
||||
}
|
||||
args.push("--".into());
|
||||
args.extend(request.pathspec);
|
||||
args.extend(request.pathspec.clone());
|
||||
}
|
||||
let output = Command::new("git")
|
||||
.arg("--git-dir")
|
||||
@@ -58,14 +65,10 @@ impl GitBare {
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let total_count = entries.len() as u64;
|
||||
let (entries, page_info) = paginate::paginate(&entries, request.pagination.as_ref());
|
||||
Ok(ListArchiveEntriesResponse {
|
||||
entries,
|
||||
page_info: Some(PageInfo {
|
||||
next_page_token: String::new(),
|
||||
has_next_page: false,
|
||||
total_count,
|
||||
}),
|
||||
page_info: Some(page_info),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
+15
-8
@@ -1,11 +1,16 @@
|
||||
use crate::bare::GitBare;
|
||||
use crate::error::{GitError, GitResult};
|
||||
use crate::pb::{BlameHunk, BlameLine, BlameRequest, BlameResponse, PageInfo};
|
||||
use crate::paginate;
|
||||
use crate::pb::{BlameHunk, BlameLine, BlameRequest, BlameResponse};
|
||||
|
||||
impl GitBare {
|
||||
pub fn blame(&self, request: BlameRequest) -> GitResult<BlameResponse> {
|
||||
crate::sanitize::validate_file_path(&request.path)?;
|
||||
let revision = match request.revision.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex.clone(),
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex.clone()
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision.clone()
|
||||
@@ -25,6 +30,12 @@ impl GitBare {
|
||||
"--porcelain".to_string(),
|
||||
];
|
||||
if let Some(range) = &request.range {
|
||||
if range.start == 0 || range.end < range.start {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"invalid blame range: {}..{}",
|
||||
range.start, range.end
|
||||
)));
|
||||
}
|
||||
args.push("-L".into());
|
||||
args.push(format!("{},{}", range.start, range.end));
|
||||
}
|
||||
@@ -44,14 +55,10 @@ impl GitBare {
|
||||
}
|
||||
let output = String::from_utf8_lossy(&result.stdout);
|
||||
let hunks = parse_porcelain_blame(&output, &request.path, self);
|
||||
let total_count = hunks.len() as u64;
|
||||
let (hunks, page_info) = paginate::paginate(&hunks, request.pagination.as_ref());
|
||||
Ok(BlameResponse {
|
||||
hunks,
|
||||
page_info: Some(PageInfo {
|
||||
next_page_token: String::new(),
|
||||
has_next_page: false,
|
||||
total_count,
|
||||
}),
|
||||
page_info: Some(page_info),
|
||||
truncated: false,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ impl GitBare {
|
||||
let repo = self.gix_repo()?;
|
||||
let revision = tree::resolve_revision(&request.revision)?;
|
||||
let (blob, mode, path) = if let Some(oid) = request.oid.as_ref() {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
let id = gix::hash::ObjectId::from_hex(oid.hex.as_bytes())
|
||||
.map_err(|e| GitError::InvalidOid(e.to_string()))?;
|
||||
(
|
||||
@@ -20,6 +21,7 @@ impl GitBare {
|
||||
request.path,
|
||||
)
|
||||
} else {
|
||||
crate::sanitize::validate_file_path(&request.path)?;
|
||||
let tree = repo
|
||||
.rev_parse_single(format!("{}^{{tree}}", revision).as_str())?
|
||||
.object()?
|
||||
|
||||
@@ -8,7 +8,10 @@ impl GitBare {
|
||||
pub fn create_branch(&self, request: CreateBranchRequest) -> GitResult<Branch> {
|
||||
crate::sanitize::validate_ref_name(&request.name)?;
|
||||
let revision = match request.start_point.and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
|
||||
@@ -4,6 +4,7 @@ use crate::pb::{Branch, GetBranchRequest};
|
||||
|
||||
impl GitBare {
|
||||
pub fn get_branch(&self, request: GetBranchRequest) -> GitResult<Branch> {
|
||||
crate::sanitize::validate_ref_name(&request.name)?;
|
||||
let repo = self.gix_repo()?;
|
||||
let refname = format!("refs/heads/{}", request.name);
|
||||
let mut r = repo.find_reference(refname.as_str())?;
|
||||
|
||||
@@ -6,6 +6,8 @@ impl GitBare {
|
||||
pub fn set_branch_upstream(&self, request: SetBranchUpstreamRequest) -> GitResult<Branch> {
|
||||
crate::sanitize::validate_ref_name(&request.name)?;
|
||||
if let Some(upstream) = request.upstream {
|
||||
crate::sanitize::validate_ref_name(&upstream.remote_name)?;
|
||||
crate::sanitize::validate_ref_name(&upstream.remote_branch_name)?;
|
||||
let tracking = format!("{}/{}", upstream.remote_name, upstream.remote_branch_name);
|
||||
let result = duct::cmd(
|
||||
"git",
|
||||
|
||||
@@ -14,11 +14,13 @@ impl GitBare {
|
||||
.ok_or_else(|| GitError::InvalidArgument("new_oid is required".into()))?
|
||||
.hex
|
||||
.clone();
|
||||
crate::sanitize::validate_oid_hex(&new_oid)?;
|
||||
let refname = format!("refs/heads/{}", request.name);
|
||||
let mut args = vec!["update-ref".to_string(), refname.clone(), new_oid];
|
||||
if !request.force
|
||||
&& let Some(old) = request.expected_old_oid.as_ref()
|
||||
{
|
||||
crate::sanitize::validate_oid_hex(&old.hex)?;
|
||||
args.push(old.hex.clone());
|
||||
}
|
||||
let output = Command::new("git")
|
||||
|
||||
@@ -1,232 +0,0 @@
|
||||
//! etcd-based peer discovery for ractor_cluster.
|
||||
//!
|
||||
//! Responsibilities:
|
||||
//! - Connect to etcd and create a Lease (TTL-based health check)
|
||||
//! - Register this node under `/gitks/nodes/{storage_name}`
|
||||
//! - Discover existing peers via prefix GET
|
||||
//! - Watch for peer join/leave events and invoke callbacks
|
||||
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use etcd_client::{Client, ConnectOptions, EventType, GetOptions, PutOptions, WatchOptions};
|
||||
|
||||
use super::types::PeerInfo;
|
||||
|
||||
/// Key prefix used for all gitks entries in etcd.
|
||||
const KEY_PREFIX: &str = "/gitks/nodes/";
|
||||
|
||||
/// Wraps an etcd client with lease-based registration and peer discovery.
|
||||
pub struct EtcdRegistry {
|
||||
client: Mutex<Client>,
|
||||
lease_id: i64,
|
||||
storage_name: String,
|
||||
}
|
||||
|
||||
impl EtcdRegistry {
|
||||
/// Connect to etcd, create a lease, and register this node.
|
||||
///
|
||||
/// Returns `None` if the connection fails (caller should fall back to standalone mode).
|
||||
pub async fn register(
|
||||
endpoints: Vec<String>,
|
||||
info: &PeerInfo,
|
||||
ttl_secs: i64,
|
||||
connect_timeout_ms: u64,
|
||||
) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let connect_opts = ConnectOptions::new()
|
||||
.with_connect_timeout(std::time::Duration::from_millis(connect_timeout_ms))
|
||||
.with_keep_alive(
|
||||
std::time::Duration::from_secs(5),
|
||||
std::time::Duration::from_secs(3),
|
||||
)
|
||||
.with_keep_alive_while_idle(true);
|
||||
|
||||
let mut client = Client::connect(endpoints.clone(), Some(connect_opts)).await?;
|
||||
|
||||
tracing::info!(endpoints = ?endpoints, "connected to etcd");
|
||||
|
||||
// Create lease
|
||||
let lease_resp = client.lease_grant(ttl_secs, None).await?;
|
||||
let lease_id = lease_resp.id();
|
||||
tracing::info!(lease_id, ttl = ttl_secs, "etcd lease granted");
|
||||
|
||||
// Register node info under the lease
|
||||
let key = format!("{KEY_PREFIX}{}", info.storage_name);
|
||||
let value = serde_json::to_string(info)?;
|
||||
client
|
||||
.put(key, value, Some(PutOptions::new().with_lease(lease_id)))
|
||||
.await?;
|
||||
tracing::info!(
|
||||
storage_name = %info.storage_name,
|
||||
cluster_addr = %info.cluster_addr,
|
||||
"registered in etcd"
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
client: Mutex::new(client),
|
||||
lease_id,
|
||||
storage_name: info.storage_name.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Start the lease keepalive loop in a background task.
|
||||
///
|
||||
/// The loop sends periodic heartbeats to etcd to prevent the lease from expiring.
|
||||
/// If keepalive fails, it logs a warning but does not panic — the node will
|
||||
/// eventually be removed from etcd when the lease expires.
|
||||
pub fn start_keepalive(self: &Arc<Self>) -> tokio::task::JoinHandle<()> {
|
||||
let this = Arc::clone(self);
|
||||
tokio::spawn(async move {
|
||||
let lease_id = this.lease_id;
|
||||
let (mut keeper, mut stream) = {
|
||||
let mut client = this.client.lock().await;
|
||||
match client.lease_keep_alive(lease_id).await {
|
||||
Ok(pair) => pair,
|
||||
Err(e) => {
|
||||
tracing::error!(lease_id, error = %e, "failed to start lease keepalive");
|
||||
return;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
if let Err(e) = keeper.keep_alive().await {
|
||||
tracing::warn!(lease_id, error = %e, "etcd lease keepalive failed");
|
||||
// Don't break — let the lease expire naturally if we can't recover
|
||||
}
|
||||
// Drain keepalive responses
|
||||
let _ = stream.message().await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Discover all currently registered peers (excluding this node).
|
||||
pub async fn discover_peers(
|
||||
&self,
|
||||
) -> Result<Vec<PeerInfo>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut client = self.client.lock().await;
|
||||
let resp = client
|
||||
.get(KEY_PREFIX, Some(GetOptions::new().with_prefix()))
|
||||
.await?;
|
||||
|
||||
let mut peers = Vec::new();
|
||||
for kv in resp.kvs() {
|
||||
match serde_json::from_slice::<PeerInfo>(kv.value()) {
|
||||
Ok(info) if info.storage_name != self.storage_name => {
|
||||
peers.push(info);
|
||||
}
|
||||
Ok(_) => {} // skip self
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
key = %String::from_utf8_lossy(kv.key()),
|
||||
error = %e,
|
||||
"failed to parse peer info from etcd"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(peers)
|
||||
}
|
||||
|
||||
/// Start a long-running watch loop that monitors peer join/leave events.
|
||||
///
|
||||
/// Callbacks are invoked synchronously within the watch task; keep them fast
|
||||
/// (prefer sending messages to actors rather than doing blocking work).
|
||||
pub fn start_watch(
|
||||
self: &Arc<Self>,
|
||||
on_peer_joined: impl Fn(PeerInfo) + Send + Sync + 'static,
|
||||
on_peer_left: impl Fn(String) + Send + Sync + 'static,
|
||||
) -> tokio::task::JoinHandle<()> {
|
||||
let this = Arc::clone(self);
|
||||
let my_name = self.storage_name.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let on_joined = Arc::new(on_peer_joined);
|
||||
let on_left = Arc::new(on_peer_left);
|
||||
|
||||
loop {
|
||||
// Create a fresh watch client each iteration (after reconnect)
|
||||
let mut watch_stream = {
|
||||
let mut client = this.client.lock().await;
|
||||
match client
|
||||
.watch(KEY_PREFIX, Some(WatchOptions::new().with_prefix()))
|
||||
.await
|
||||
{
|
||||
Ok(stream) => stream,
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "etcd watch failed, retrying in 3s");
|
||||
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
tracing::info!("etcd watch loop started");
|
||||
|
||||
loop {
|
||||
match watch_stream.message().await {
|
||||
Ok(Some(resp)) => {
|
||||
for event in resp.events() {
|
||||
match event.event_type() {
|
||||
EventType::Put => {
|
||||
if let Some(kv) = event.kv()
|
||||
&& let Ok(info) =
|
||||
serde_json::from_slice::<PeerInfo>(kv.value())
|
||||
&& info.storage_name != my_name
|
||||
{
|
||||
tracing::info!(
|
||||
peer = %info.storage_name,
|
||||
cluster_addr = %info.cluster_addr,
|
||||
"peer joined via etcd watch"
|
||||
);
|
||||
on_joined(info);
|
||||
}
|
||||
}
|
||||
EventType::Delete => {
|
||||
if let Some(kv) = event.kv() {
|
||||
let key = String::from_utf8_lossy(kv.key());
|
||||
let name = key
|
||||
.strip_prefix(KEY_PREFIX)
|
||||
.unwrap_or(&key)
|
||||
.to_string();
|
||||
if name != my_name {
|
||||
tracing::warn!(
|
||||
peer = %name,
|
||||
"peer left (etcd lease expired or key deleted)"
|
||||
);
|
||||
on_left(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::warn!("etcd watch stream ended");
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!(error = %e, "etcd watch stream error");
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Reconnect after a delay
|
||||
tracing::info!("etcd watch loop restarting in 3s");
|
||||
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// Check if the lease is still alive (for external health monitoring).
|
||||
pub async fn is_lease_alive(&self) -> bool {
|
||||
let mut client = self.client.lock().await;
|
||||
match client.lease_time_to_live(self.lease_id, None).await {
|
||||
Ok(resp) => resp.ttl() > 0,
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
-210
@@ -1,210 +0,0 @@
|
||||
//! Cluster discovery: etcd-driven ractor_cluster node discovery.
|
||||
//!
|
||||
//! Architecture:
|
||||
//! 1. Start a `ractor_cluster::NodeServer` (TCP listener for actor remoting)
|
||||
//! 2. Connect to etcd and register this node
|
||||
//! 3. Discover existing peers → `client_connect()` to each
|
||||
//! 4. Watch etcd for future peer join/leave → connect/disconnect dynamically
|
||||
//!
|
||||
//! Once ractor_cluster TCP connections are established, the existing
|
||||
//! `pg::get_members()` / `ractor::call_t!()` APIs automatically work
|
||||
//! cross-network — no changes needed in actor/handler.rs or server/mod.rs.
|
||||
|
||||
pub mod discovery;
|
||||
pub mod types;
|
||||
|
||||
pub use discovery::EtcdRegistry;
|
||||
pub use types::PeerInfo;
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use ractor::ActorRef;
|
||||
use ractor_cluster::node::NodeConnectionMode;
|
||||
use ractor_cluster::{NodeServer, NodeServerMessage, client_connect};
|
||||
|
||||
use crate::error::{GitError, GitResult};
|
||||
|
||||
/// Configuration for the cluster subsystem.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ClusterConfig {
|
||||
/// etcd endpoints (e.g. ["http://etcd1:2379", "http://etcd2:2379"])
|
||||
pub etcd_endpoints: Vec<String>,
|
||||
/// Logical name for this storage node
|
||||
pub storage_name: String,
|
||||
/// gRPC address advertised to clients
|
||||
pub grpc_addr: String,
|
||||
/// TCP port for ractor_cluster NodeServer
|
||||
pub cluster_port: u16,
|
||||
/// Shared authentication cookie for ractor_cluster
|
||||
pub cookie: String,
|
||||
/// etcd lease TTL in seconds
|
||||
pub lease_ttl_secs: i64,
|
||||
/// etcd connection timeout in milliseconds
|
||||
pub connect_timeout_ms: u64,
|
||||
/// Hostname used in the ractor_cluster node name (`name@hostname`).
|
||||
/// Also used by remote nodes to connect back via `{cluster_hostname}:{cluster_port}`.
|
||||
/// In K8s/Docker, this should be a resolvable address (Pod IP, service DNS, etc.)
|
||||
pub cluster_hostname: String,
|
||||
}
|
||||
|
||||
/// The running cluster manager. Holds references to the NodeServer and etcd registry.
|
||||
/// Dropping this will stop the background tasks.
|
||||
pub struct ClusterManager {
|
||||
/// The ractor_cluster NodeServer actor
|
||||
pub node_server: ActorRef<NodeServerMessage>,
|
||||
/// The etcd registry (for health checks, etc.)
|
||||
pub registry: Arc<EtcdRegistry>,
|
||||
/// Handles for background tasks (keepalive + watch)
|
||||
_keepalive_handle: tokio::task::JoinHandle<()>,
|
||||
_watch_handle: tokio::task::JoinHandle<()>,
|
||||
}
|
||||
|
||||
impl ClusterManager {
|
||||
/// Start the full cluster subsystem:
|
||||
/// 1. Spawn NodeServer (TCP listener)
|
||||
/// 2. Connect to etcd + register
|
||||
/// 3. Discover peers → client_connect
|
||||
/// 4. Start keepalive + watch loops
|
||||
///
|
||||
/// Returns `Err` if etcd is unreachable (caller should fall back to standalone).
|
||||
pub async fn start(config: ClusterConfig) -> GitResult<Self> {
|
||||
let node_server = spawn_node_server(&config).await?;
|
||||
tracing::info!(
|
||||
port = config.cluster_port,
|
||||
hostname = %config.cluster_hostname,
|
||||
"NodeServer started"
|
||||
);
|
||||
|
||||
let cluster_addr = format!("{}:{}", config.cluster_hostname, config.cluster_port);
|
||||
let peer_info = PeerInfo {
|
||||
storage_name: config.storage_name.clone(),
|
||||
cluster_addr: cluster_addr.clone(),
|
||||
grpc_addr: config.grpc_addr.clone(),
|
||||
version: env!("CARGO_PKG_VERSION").to_string(),
|
||||
};
|
||||
|
||||
let registry = Arc::new(
|
||||
EtcdRegistry::register(
|
||||
config.etcd_endpoints.clone(),
|
||||
&peer_info,
|
||||
config.lease_ttl_secs,
|
||||
config.connect_timeout_ms,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| GitError::Internal(format!("etcd registration failed: {e}")))?,
|
||||
);
|
||||
|
||||
let peers = registry
|
||||
.discover_peers()
|
||||
.await
|
||||
.map_err(|e| GitError::Internal(format!("peer discovery failed: {e}")))?;
|
||||
|
||||
for peer in &peers {
|
||||
connect_to_peer(&node_server, peer, &config.storage_name).await;
|
||||
}
|
||||
|
||||
let keepalive_handle = registry.start_keepalive();
|
||||
|
||||
let ns_for_watch = node_server.clone();
|
||||
let my_name_for_watch = config.storage_name.clone();
|
||||
let watch_handle = registry.start_watch(
|
||||
move |peer| {
|
||||
let ns = ns_for_watch.clone();
|
||||
let my_name = my_name_for_watch.clone();
|
||||
tokio::spawn(async move {
|
||||
connect_to_peer(&ns, &peer, &my_name).await;
|
||||
});
|
||||
},
|
||||
move |name| {
|
||||
tracing::info!(
|
||||
peer = %name,
|
||||
"peer left etcd registry (ractor_cluster will cleanup TCP session)"
|
||||
);
|
||||
// ractor_cluster automatically:
|
||||
// 1. Detects TCP disconnection
|
||||
// 2. Stops the NodeSession actor
|
||||
// 3. Stops all RemoteActors for that session
|
||||
// 4. Removes them from Process Groups
|
||||
// No manual cleanup needed.
|
||||
},
|
||||
);
|
||||
|
||||
tracing::info!(
|
||||
storage_name = %config.storage_name,
|
||||
peers_found = peers.len(),
|
||||
"cluster manager started"
|
||||
);
|
||||
|
||||
Ok(Self {
|
||||
node_server,
|
||||
registry,
|
||||
_keepalive_handle: keepalive_handle,
|
||||
_watch_handle: watch_handle,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawn the ractor_cluster NodeServer actor (TCP listener for inter-node communication).
|
||||
async fn spawn_node_server(config: &ClusterConfig) -> GitResult<ActorRef<NodeServerMessage>> {
|
||||
let server = NodeServer::new(
|
||||
config.cluster_port,
|
||||
config.cookie.clone(),
|
||||
config.storage_name.clone(),
|
||||
config.cluster_hostname.clone(),
|
||||
None, // no encryption (internal network)
|
||||
Some(NodeConnectionMode::Transitive),
|
||||
);
|
||||
|
||||
let (actor_ref, _handle) = ractor::Actor::spawn(
|
||||
Some(format!("node_server_{}", config.storage_name)),
|
||||
server,
|
||||
(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| GitError::Internal(format!("failed to spawn NodeServer: {e}")))?;
|
||||
|
||||
Ok(actor_ref)
|
||||
}
|
||||
|
||||
/// Establish a ractor_cluster TCP connection to a remote peer.
|
||||
///
|
||||
/// Uses ordering optimization: only the node with the lexicographically
|
||||
/// smaller `storage_name` initiates the connection. The other side will
|
||||
/// accept the incoming connection. This prevents duplicate connections.
|
||||
async fn connect_to_peer(
|
||||
node_server: &ActorRef<NodeServerMessage>,
|
||||
peer: &PeerInfo,
|
||||
my_name: &str,
|
||||
) {
|
||||
// Ordering optimization: only smaller-named node connects
|
||||
if my_name >= peer.storage_name.as_str() {
|
||||
tracing::debug!(
|
||||
peer = %peer.storage_name,
|
||||
"skipping connect (peer has lower/equal name, they connect to us)"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
peer = %peer.storage_name,
|
||||
cluster_addr = %peer.cluster_addr,
|
||||
"connecting to peer via ractor_cluster"
|
||||
);
|
||||
|
||||
match client_connect(node_server, peer.cluster_addr.as_str()).await {
|
||||
Ok(()) => {
|
||||
tracing::info!(
|
||||
peer = %peer.storage_name,
|
||||
"ractor_cluster connection initiated"
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
peer = %peer.storage_name,
|
||||
cluster_addr = %peer.cluster_addr,
|
||||
error = %e,
|
||||
"failed to connect to peer (will retry on next watch event)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// Information about a peer node, registered in etcd.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PeerInfo {
|
||||
/// Logical storage name (e.g. "node-a", "default")
|
||||
pub storage_name: String,
|
||||
/// ractor_cluster TCP address (e.g. "10.0.1.4:4697")
|
||||
pub cluster_addr: String,
|
||||
/// gRPC service address (e.g. "http://10.0.1.4:50051")
|
||||
pub grpc_addr: String,
|
||||
/// Software version
|
||||
pub version: String,
|
||||
}
|
||||
@@ -11,7 +11,10 @@ impl GitBare {
|
||||
let target_branch = request.branch.clone();
|
||||
crate::sanitize::validate_ref_name(&target_branch)?;
|
||||
let cp_revision = match request.commit.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
|
||||
@@ -110,7 +110,7 @@ impl GitBare {
|
||||
commits.push(read_commit_from_repo(self, &repo, id)?);
|
||||
}
|
||||
|
||||
let end = start_offset + page_ids.len();
|
||||
let end = start_offset.saturating_add(page_ids.len());
|
||||
let has_next = end < total;
|
||||
let page_info = crate::pb::PageInfo {
|
||||
next_page_token: if has_next {
|
||||
|
||||
+15
-2
@@ -29,6 +29,7 @@ impl GitBare {
|
||||
args.push(revision.to_string());
|
||||
|
||||
if !request.path.is_empty() {
|
||||
crate::sanitize::validate_file_path(&request.path)?;
|
||||
args.push("--".to_string());
|
||||
args.push(request.path.clone());
|
||||
}
|
||||
@@ -36,12 +37,18 @@ impl GitBare {
|
||||
let output = std::process::Command::new("git")
|
||||
.args(&args)
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.output()
|
||||
.map_err(|e| crate::error::GitError::CommandFailed {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let count = String::from_utf8_lossy(&output.stdout)
|
||||
.trim()
|
||||
@@ -69,12 +76,18 @@ impl GitBare {
|
||||
&format!("{}...{}", request.left, request.right),
|
||||
])
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.output()
|
||||
.map_err(|e| crate::error::GitError::CommandFailed {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
// Format: "<left_count>\t<right_count>"
|
||||
|
||||
@@ -16,7 +16,9 @@ impl GitBare {
|
||||
Some(object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
}
|
||||
Some(object_selector::Selector::Oid(_)) => {} // OID is always safe
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
}
|
||||
None => {} // will use branch name, already validated
|
||||
}
|
||||
}
|
||||
@@ -30,7 +32,10 @@ impl GitBare {
|
||||
"creating commit"
|
||||
);
|
||||
let start_rev = match request.start_revision.clone().and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => name.revision,
|
||||
None => request.branch.clone(),
|
||||
};
|
||||
|
||||
@@ -6,7 +6,10 @@ impl GitBare {
|
||||
/// Find a single commit by revision.
|
||||
pub fn find_commit(&self, request: FindCommitRequest) -> GitResult<Commit> {
|
||||
let revision = match request.revision.and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => name.revision,
|
||||
None => "HEAD".to_string(),
|
||||
};
|
||||
|
||||
@@ -7,7 +7,7 @@ impl GitBare {
|
||||
pub fn list_commits(&self, request: ListCommitsRequest) -> GitResult<ListCommitsResponse> {
|
||||
let revision = resolve_revision!(request.revision.clone());
|
||||
|
||||
let base_args = build_rev_list_args(self, &request, &revision);
|
||||
let base_args = build_rev_list_args(self, &request, &revision)?;
|
||||
|
||||
// 1. Get total count via rev-list --count (lightweight, no object parsing)
|
||||
let total = {
|
||||
@@ -87,7 +87,7 @@ impl GitBare {
|
||||
commits
|
||||
};
|
||||
|
||||
let end = start_offset + page_ids.len();
|
||||
let end = start_offset.saturating_add(page_ids.len());
|
||||
let has_next = end < total;
|
||||
let page_info = crate::pb::PageInfo {
|
||||
next_page_token: if has_next {
|
||||
@@ -106,7 +106,11 @@ impl GitBare {
|
||||
}
|
||||
}
|
||||
|
||||
fn build_rev_list_args(gb: &GitBare, request: &ListCommitsRequest, revision: &str) -> Vec<String> {
|
||||
fn build_rev_list_args(
|
||||
gb: &GitBare,
|
||||
request: &ListCommitsRequest,
|
||||
revision: &str,
|
||||
) -> GitResult<Vec<String>> {
|
||||
let mut args = vec![
|
||||
"--git-dir".to_string(),
|
||||
gb.bare_dir.to_string_lossy().into_owned(),
|
||||
@@ -136,10 +140,11 @@ fn build_rev_list_args(gb: &GitBare, request: &ListCommitsRequest, revision: &st
|
||||
args.push(revision.to_string());
|
||||
}
|
||||
if !request.path.is_empty() {
|
||||
crate::sanitize::validate_file_path(&request.path)?;
|
||||
args.push("--".into());
|
||||
args.push(request.path.clone());
|
||||
}
|
||||
args
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
/// Read a single commit from an already-opened gix repo (no subprocess).
|
||||
|
||||
+29
-7
@@ -20,12 +20,13 @@ impl GitBare {
|
||||
} else {
|
||||
request.limit.min(200)
|
||||
};
|
||||
let max_count = request.offset.saturating_add(limit).min(10_000);
|
||||
|
||||
let mut args = vec![
|
||||
"--git-dir".to_string(),
|
||||
self.bare_dir.to_string_lossy().into_owned(),
|
||||
"log".to_string(),
|
||||
format!("--max-count={}", limit),
|
||||
format!("--max-count={max_count}"),
|
||||
"--format=%H".to_string(),
|
||||
];
|
||||
|
||||
@@ -51,6 +52,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let repo = self.gix_repo()?;
|
||||
@@ -94,7 +101,10 @@ impl GitBare {
|
||||
/// Get stats for a single commit.
|
||||
pub fn get_commit_stats(&self, request: GetCommitStatsRequest) -> GitResult<CommitStats> {
|
||||
let revision = match request.revision.and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => name.revision,
|
||||
None => "HEAD".to_string(),
|
||||
};
|
||||
@@ -109,12 +119,18 @@ impl GitBare {
|
||||
&format!("{revision}^!"),
|
||||
])
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.output()
|
||||
.map_err(|e| crate::error::GitError::CommandFailed {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut additions = 0u32;
|
||||
@@ -125,12 +141,12 @@ impl GitBare {
|
||||
let parts: Vec<&str> = line.split('\t').collect();
|
||||
if parts.len() >= 2 {
|
||||
if let Ok(add) = parts[0].parse::<u32>() {
|
||||
additions += add;
|
||||
additions = additions.saturating_add(add);
|
||||
}
|
||||
if let Ok(del) = parts[1].parse::<u32>() {
|
||||
deletions += del;
|
||||
deletions = deletions.saturating_add(del);
|
||||
}
|
||||
changed_files += 1;
|
||||
changed_files = changed_files.saturating_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -170,12 +186,18 @@ impl GitBare {
|
||||
let output = std::process::Command::new("git")
|
||||
.args(&args)
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::null())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.output()
|
||||
.map_err(|e| crate::error::GitError::CommandFailed {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let hex = stdout.lines().next().unwrap_or("").trim().to_string();
|
||||
|
||||
@@ -8,7 +8,10 @@ impl GitBare {
|
||||
let target_branch = request.branch.clone();
|
||||
crate::sanitize::validate_ref_name(&target_branch)?;
|
||||
let revert_revision = match request.commit.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
|
||||
+14
-7
@@ -19,16 +19,17 @@ impl GitBare {
|
||||
"-r".to_string(),
|
||||
];
|
||||
|
||||
if !request.paths.is_empty() {
|
||||
args.push("--".to_string());
|
||||
for p in &request.paths {
|
||||
args.push(p.clone());
|
||||
}
|
||||
}
|
||||
|
||||
args.push(request.base.clone());
|
||||
args.push(request.head.clone());
|
||||
|
||||
if !request.paths.is_empty() {
|
||||
args.push("--".to_string());
|
||||
for path in &request.paths {
|
||||
crate::sanitize::validate_file_path(path)?;
|
||||
args.push(path.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let output = std::process::Command::new("git")
|
||||
.args(&args)
|
||||
.stdout(std::process::Stdio::piped())
|
||||
@@ -38,6 +39,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut paths = Vec::new();
|
||||
|
||||
+25
-2
@@ -22,10 +22,15 @@ struct RawDiffEntry {
|
||||
/// Type alias for diff raw output: (entries, numstat_map)
|
||||
type DiffRawOutput = (Vec<RawDiffEntry>, HashMap<String, (u32, u32, bool)>);
|
||||
|
||||
const MAX_DIFF_COMMAND_OUTPUT_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
impl GitBare {
|
||||
pub fn get_diff(&self, request: GetDiffRequest) -> GitResult<GetDiffResponse> {
|
||||
let base = match request.base.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex.clone(),
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex.clone()
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision.clone()
|
||||
@@ -33,7 +38,10 @@ impl GitBare {
|
||||
None => "HEAD".into(),
|
||||
};
|
||||
let head = match request.head.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex.clone(),
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex.clone()
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision.clone()
|
||||
@@ -48,6 +56,11 @@ impl GitBare {
|
||||
);
|
||||
|
||||
let options = request.options.as_ref();
|
||||
if let Some(options) = options {
|
||||
for path in &options.pathspec {
|
||||
crate::sanitize::validate_file_path(path)?;
|
||||
}
|
||||
}
|
||||
let want_patch = options.is_some_and(|o| o.include_patch);
|
||||
|
||||
let (raw_entries, numstat_map) = self.diff_raw_and_numstat(&base, &head, options)?;
|
||||
@@ -177,6 +190,11 @@ impl GitBare {
|
||||
stderr: String::from_utf8_lossy(&result.stderr).into_owned(),
|
||||
});
|
||||
}
|
||||
if result.stdout.len() > MAX_DIFF_COMMAND_OUTPUT_BYTES {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"diff metadata output too large (max {MAX_DIFF_COMMAND_OUTPUT_BYTES} bytes)"
|
||||
)));
|
||||
}
|
||||
|
||||
// Split by NUL — each record is NUL-terminated
|
||||
let records: Vec<&[u8]> = result.stdout.split(|b| *b == 0).collect();
|
||||
@@ -343,6 +361,11 @@ impl GitBare {
|
||||
stderr: String::from_utf8_lossy(&result.stderr).into_owned(),
|
||||
});
|
||||
}
|
||||
if result.stdout.len() > MAX_DIFF_COMMAND_OUTPUT_BYTES {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"diff patch output too large (max {MAX_DIFF_COMMAND_OUTPUT_BYTES} bytes)"
|
||||
)));
|
||||
}
|
||||
|
||||
// Split combined patch output by "diff --git" headers
|
||||
let mut map = HashMap::new();
|
||||
|
||||
@@ -17,6 +17,12 @@ pub(crate) fn diff_stats_for_range(
|
||||
head: &str,
|
||||
options: Option<&crate::pb::DiffOptions>,
|
||||
) -> GitResult<crate::pb::DiffStats> {
|
||||
if let Some(options) = options {
|
||||
for path in &options.pathspec {
|
||||
crate::sanitize::validate_file_path(path)?;
|
||||
}
|
||||
}
|
||||
|
||||
let mut args = vec![
|
||||
"--git-dir".to_string(),
|
||||
repo.bare_dir.to_string_lossy().into_owned(),
|
||||
|
||||
@@ -3,6 +3,8 @@ use crate::error::{GitError, GitResult};
|
||||
use crate::pb::{GetPatchRequest, GetPatchResponse};
|
||||
use crate::resolve_revision;
|
||||
|
||||
const MAX_PATCH_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
impl GitBare {
|
||||
pub fn get_patch(&self, request: GetPatchRequest) -> GitResult<Vec<GetPatchResponse>> {
|
||||
let base = resolve_revision!(request.base);
|
||||
@@ -28,6 +30,11 @@ impl GitBare {
|
||||
stderr: String::from_utf8_lossy(&result.stderr).into_owned(),
|
||||
});
|
||||
}
|
||||
if result.stdout.len() > MAX_PATCH_BYTES {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"patch output too large (max {MAX_PATCH_BYTES} bytes)"
|
||||
)));
|
||||
}
|
||||
Ok(vec![GetPatchResponse {
|
||||
data: result.stdout,
|
||||
}])
|
||||
|
||||
+34
-3
@@ -2,6 +2,8 @@ use crate::bare::GitBare;
|
||||
use crate::error::GitResult;
|
||||
use crate::pb::*;
|
||||
|
||||
const MAX_RAW_DIFF_OUTPUT_BYTES: usize = 64 * 1024 * 1024;
|
||||
|
||||
impl GitBare {
|
||||
/// Stream raw diff output.
|
||||
pub fn raw_diff(&self, request: RawDiffRequest) -> GitResult<Vec<RawDiffResponse>> {
|
||||
@@ -16,6 +18,7 @@ impl GitBare {
|
||||
"diff".to_string(),
|
||||
];
|
||||
|
||||
let mut pathspecs = Vec::new();
|
||||
// Apply options if present
|
||||
if let Some(ref opts) = request.options {
|
||||
if opts.recursive {
|
||||
@@ -26,14 +29,18 @@ impl GitBare {
|
||||
} else {
|
||||
args.push("--no-binary".to_string());
|
||||
}
|
||||
for ps in &opts.pathspec {
|
||||
args.push("--".to_string());
|
||||
args.push(ps.clone());
|
||||
for pathspec in &opts.pathspec {
|
||||
crate::sanitize::validate_file_path(pathspec)?;
|
||||
pathspecs.push(pathspec.clone());
|
||||
}
|
||||
}
|
||||
|
||||
args.push(base.clone());
|
||||
args.push(head.clone());
|
||||
if !pathspecs.is_empty() {
|
||||
args.push("--".to_string());
|
||||
args.extend(pathspecs);
|
||||
}
|
||||
|
||||
let output = std::process::Command::new("git")
|
||||
.args(&args)
|
||||
@@ -44,6 +51,18 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if output.stdout.len() > MAX_RAW_DIFF_OUTPUT_BYTES {
|
||||
return Err(crate::error::GitError::InvalidArgument(format!(
|
||||
"raw diff output too large (max {MAX_RAW_DIFF_OUTPUT_BYTES} bytes)"
|
||||
)));
|
||||
}
|
||||
|
||||
// Chunk the output for streaming
|
||||
const CHUNK_SIZE: usize = 32768;
|
||||
@@ -78,6 +97,18 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
if output.stdout.len() > MAX_RAW_DIFF_OUTPUT_BYTES {
|
||||
return Err(crate::error::GitError::InvalidArgument(format!(
|
||||
"raw patch output too large (max {MAX_RAW_DIFF_OUTPUT_BYTES} bytes)"
|
||||
)));
|
||||
}
|
||||
|
||||
const CHUNK_SIZE: usize = 32768;
|
||||
let data = output.stdout;
|
||||
|
||||
+59
-29
@@ -29,18 +29,22 @@ const INFO_REFS_DIR_RELATIVE: &str = "+gitks-cache/info_refs";
|
||||
fn random_value() -> String {
|
||||
use std::fmt::Write;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
let mut buf = [0u8; 16];
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(SystemTime::UNIX_EPOCH)
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_nanos() as u64;
|
||||
buf[..8].copy_from_slice(&nanos.to_le_bytes());
|
||||
static COUNTER: AtomicU64 = AtomicU64::new(0);
|
||||
let c = COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
buf[8..].copy_from_slice(&c.to_le_bytes());
|
||||
let counter = COUNTER.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
let mut buf = [0u8; 16];
|
||||
buf[..8].copy_from_slice(&nanos.to_le_bytes());
|
||||
buf[8..].copy_from_slice(&counter.to_le_bytes());
|
||||
|
||||
let mut s = String::with_capacity(32);
|
||||
for byte in &buf {
|
||||
write!(s, "{byte:02x}").unwrap();
|
||||
let _ = write!(s, "{byte:02x}");
|
||||
}
|
||||
s
|
||||
}
|
||||
@@ -56,16 +60,17 @@ fn sha256_digest(parts: &[&str]) -> String {
|
||||
let mut s = String::with_capacity(64);
|
||||
for byte in result {
|
||||
use std::fmt::Write;
|
||||
write!(s, "{byte:02x}").unwrap();
|
||||
let _ = write!(s, "{byte:02x}");
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
/// Convert a digest into a two-level file path: `${digest:0:2}/${digest:2}`.
|
||||
pub fn digest_to_path(digest: &str) -> PathBuf {
|
||||
let prefix = &digest[..2];
|
||||
let rest = &digest[2..];
|
||||
PathBuf::from(prefix).join(rest)
|
||||
match (digest.get(..2), digest.get(2..)) {
|
||||
(Some(prefix), Some(rest)) => PathBuf::from(prefix).join(rest),
|
||||
_ => PathBuf::from(digest),
|
||||
}
|
||||
}
|
||||
|
||||
/// DiskCache manages per-repository state and cached response files on local disk.
|
||||
@@ -116,7 +121,7 @@ impl DiskCache {
|
||||
}
|
||||
|
||||
/// Ensure the state directory for a repository exists and has a `latest` file.
|
||||
/// If `latest` does not exist, create it with a random value.
|
||||
/// If `latest` does not exist, create it atomically with a random value.
|
||||
pub fn ensure_state(&self, relative_path: &str) -> GitResult<String> {
|
||||
if !self.enabled {
|
||||
return Ok(random_value());
|
||||
@@ -129,12 +134,15 @@ impl DiskCache {
|
||||
let latest_path = self.latest_path_for(relative_path);
|
||||
if latest_path.exists() {
|
||||
let val = std::fs::read_to_string(&latest_path).map_err(GitError::Io)?;
|
||||
Ok(val.trim().to_string())
|
||||
} else {
|
||||
let val = random_value();
|
||||
std::fs::write(&latest_path, &val).map_err(GitError::Io)?;
|
||||
Ok(val)
|
||||
return Ok(val.trim().to_string());
|
||||
}
|
||||
|
||||
// Atomic write: create temp file, then rename into place
|
||||
let val = random_value();
|
||||
let tmp_path = latest_path.with_extension("tmp");
|
||||
std::fs::write(&tmp_path, &val).map_err(GitError::Io)?;
|
||||
std::fs::rename(&tmp_path, &latest_path).map_err(GitError::Io)?;
|
||||
Ok(val)
|
||||
}
|
||||
|
||||
/// Create a lease file for a mutating RPC.
|
||||
@@ -448,30 +456,52 @@ impl DiskCache {
|
||||
if !dir.exists() {
|
||||
continue;
|
||||
}
|
||||
for prefix_entry in std::fs::read_dir(&dir).map_err(GitError::Io)? {
|
||||
let prefix_entry = prefix_entry.map_err(GitError::Io)?;
|
||||
let prefix_dir = prefix_entry.path();
|
||||
let prefix_iter = match std::fs::read_dir(&dir) {
|
||||
Ok(iter) => iter,
|
||||
Err(_) => continue,
|
||||
};
|
||||
for prefix_entry in prefix_iter {
|
||||
let prefix_dir = match prefix_entry {
|
||||
Ok(e) => e.path(),
|
||||
Err(_) => continue,
|
||||
};
|
||||
if !prefix_dir.is_dir() {
|
||||
continue;
|
||||
}
|
||||
for entry in std::fs::read_dir(&prefix_dir).map_err(GitError::Io)? {
|
||||
let entry = entry.map_err(GitError::Io)?;
|
||||
let path = entry.path();
|
||||
if let Ok(metadata) = entry.metadata()
|
||||
&& let Ok(modified) = metadata.modified()
|
||||
&& let Ok(age) = now.duration_since(modified)
|
||||
&& age > self.max_age
|
||||
{
|
||||
// Process all entries in this prefix directory
|
||||
let entries = match std::fs::read_dir(&prefix_dir) {
|
||||
Ok(iter) => iter,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let mut prefix_empty = true;
|
||||
for entry in entries {
|
||||
let path = match entry {
|
||||
Ok(e) => e.path(),
|
||||
Err(_) => continue,
|
||||
};
|
||||
let expired = match std::fs::metadata(&path) {
|
||||
Ok(meta) => meta
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|mtime| now.duration_since(mtime).ok())
|
||||
.is_some_and(|age| age > self.max_age),
|
||||
Err(_) => false,
|
||||
};
|
||||
if expired {
|
||||
tracing::debug!(
|
||||
path = %path.display(),
|
||||
age_secs = age.as_secs(),
|
||||
"removing expired cache entry"
|
||||
);
|
||||
std::fs::remove_file(&path).ok();
|
||||
removed += 1;
|
||||
} else {
|
||||
prefix_empty = false;
|
||||
}
|
||||
}
|
||||
std::fs::remove_dir(&prefix_dir).ok();
|
||||
// Remove empty prefix directory
|
||||
if prefix_empty {
|
||||
std::fs::remove_dir(&prefix_dir).ok();
|
||||
}
|
||||
}
|
||||
}
|
||||
if removed > 0 {
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
pub mod actor;
|
||||
pub mod archive;
|
||||
pub mod bare;
|
||||
pub mod blame;
|
||||
pub mod blob;
|
||||
pub mod branch;
|
||||
pub mod cluster;
|
||||
pub mod commit;
|
||||
pub mod diff;
|
||||
pub mod disk_cache;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
//! and validating revision strings before use.
|
||||
|
||||
/// Extract a revision string from an optional ObjectSelector, applying
|
||||
/// validation to revision names. OID hex strings are always safe.
|
||||
/// validation to revision names and OID hex strings.
|
||||
///
|
||||
/// Returns "HEAD" when selector is None.
|
||||
///
|
||||
@@ -15,7 +15,10 @@
|
||||
macro_rules! resolve_revision {
|
||||
($selector:expr) => {{
|
||||
match $selector.and_then(|s| s.selector) {
|
||||
Some($crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some($crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
$crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some($crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
$crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
@@ -25,7 +28,10 @@ macro_rules! resolve_revision {
|
||||
}};
|
||||
($selector:expr, $default:expr) => {{
|
||||
match $selector.and_then(|s| s.selector) {
|
||||
Some($crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some($crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
$crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some($crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
$crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use gitks::actor::init_actor_cluster;
|
||||
use gitks::cluster::{ClusterConfig, ClusterManager};
|
||||
use gitks::disk_cache::DiskCache;
|
||||
use gitks::hooks::HookManager;
|
||||
use gitks::metrics;
|
||||
use gitks::server::{GitksService, serve};
|
||||
|
||||
use etcd_client::{Client, PutOptions};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use tracing_subscriber::EnvFilter;
|
||||
use tracing_subscriber::fmt;
|
||||
use tracing_subscriber::prelude::*;
|
||||
@@ -16,6 +18,88 @@ const DEFAULT_HOST: &str = "0.0.0.0";
|
||||
const DEFAULT_PORT: &str = "50051";
|
||||
const DEFAULT_STORAGE_NAME: &str = "default";
|
||||
|
||||
/// etcd-backed config reader. Priority: etcd > env > default.
|
||||
struct EtcdConfig {
|
||||
client: Arc<Mutex<Client>>,
|
||||
prefix: String,
|
||||
}
|
||||
|
||||
impl EtcdConfig {
|
||||
async fn connect(endpoints: Vec<String>, prefix: &str) -> Result<Self, String> {
|
||||
let client = Client::connect(endpoints, None)
|
||||
.await
|
||||
.map_err(|e| format!("etcd connect: {e}"))?;
|
||||
Ok(Self {
|
||||
client: Arc::new(Mutex::new(client)),
|
||||
prefix: prefix.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Get config: etcd first, env second, default last.
|
||||
async fn get(&self, key: &str, default: &str) -> String {
|
||||
let etcd_key = format!("{}config/{}", self.prefix, key);
|
||||
if let Ok(mut c) = self.client.try_lock()
|
||||
&& let Ok(resp) = c.get(etcd_key.as_str(), None).await
|
||||
&& let Some(kv) = resp.kvs().first()
|
||||
&& let Ok(v) = kv.value_str()
|
||||
&& !v.is_empty()
|
||||
{
|
||||
return v.to_string();
|
||||
}
|
||||
std::env::var(key).unwrap_or_else(|_| default.to_string())
|
||||
}
|
||||
|
||||
/// Register this service under the common prefix for discovery by other services.
|
||||
async fn register(&self, service_name: &str, addr: &str) -> Result<(), String> {
|
||||
let instance_id = uuid::Uuid::now_v7().to_string();
|
||||
let addr = addr.to_string();
|
||||
let key = format!("{}services/{}/{}", self.prefix, service_name, instance_id);
|
||||
let instance =
|
||||
serde_json::json!({"addr": &addr, "port": 0, "version": env!("CARGO_PKG_VERSION")});
|
||||
let value = serde_json::to_string(&instance).map_err(|e| format!("json: {e}"))?;
|
||||
|
||||
let lease = {
|
||||
let mut c = self.client.lock().await;
|
||||
c.lease_grant(15, None)
|
||||
.await
|
||||
.map_err(|e| format!("lease: {e}"))?
|
||||
};
|
||||
{
|
||||
let mut c = self.client.lock().await;
|
||||
let opts = PutOptions::new().with_lease(lease.id());
|
||||
c.put(key.clone(), value, Some(opts))
|
||||
.await
|
||||
.map_err(|e| format!("put: {e}"))?;
|
||||
}
|
||||
tracing::info!(service = service_name, addr = %addr, "registered in etcd");
|
||||
|
||||
let c = self.client.clone();
|
||||
tokio::spawn(async move {
|
||||
loop {
|
||||
let r = {
|
||||
let mut cl = c.lock().await;
|
||||
cl.lease_keep_alive(lease.id()).await
|
||||
};
|
||||
drop(r);
|
||||
tokio::time::sleep(std::time::Duration::from_secs(5)).await;
|
||||
if let Ok(lr) = {
|
||||
let mut cl = c.lock().await;
|
||||
cl.lease_grant(15, None).await
|
||||
} {
|
||||
let inst = serde_json::json!({"addr": &addr, "port": 0, "version": env!("CARGO_PKG_VERSION")});
|
||||
if let Ok(v) = serde_json::to_string(&inst) {
|
||||
let mut cl = c.lock().await;
|
||||
let _ = cl
|
||||
.put(key.clone(), v, Some(PutOptions::new().with_lease(lr.id())))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn env_or(key: &str, default: &str) -> String {
|
||||
std::env::var(key).unwrap_or_else(|_| default.into())
|
||||
}
|
||||
@@ -75,9 +159,17 @@ fn init_tracing() -> Option<tracing_appender::non_blocking::WorkerGuard> {
|
||||
builder = builder.max_log_files(retention);
|
||||
}
|
||||
|
||||
let file_appender = builder
|
||||
.build(&log_dir)
|
||||
.expect("failed to create log directory");
|
||||
let file_appender = match builder.build(&log_dir) {
|
||||
Ok(file_appender) => file_appender,
|
||||
Err(err) => {
|
||||
eprintln!("failed to create log directory '{log_dir}': {err}");
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(fmt_layer)
|
||||
.init();
|
||||
return None;
|
||||
}
|
||||
};
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
let file_layer = fmt::layer()
|
||||
@@ -119,9 +211,40 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let host = env_or("GITKS_HOST", DEFAULT_HOST);
|
||||
let port = env_or("GITKS_PORT", DEFAULT_PORT);
|
||||
let storage_name = env_or("STORAGE_NAME", DEFAULT_STORAGE_NAME);
|
||||
|
||||
// --- etcd config overlay: connect etcd, override key settings ---
|
||||
let etcd_endpoints: Vec<String> = std::env::var("GITKS_ETCD_ENDPOINTS")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.split(',').map(str::trim).map(String::from).collect())
|
||||
.unwrap_or_else(|| vec!["http://localhost:2379".to_string()]);
|
||||
let etcd_prefix = std::env::var("ETCD_KEY_PREFIX").unwrap_or_else(|_| "/appks/".to_string());
|
||||
|
||||
let etcd = EtcdConfig::connect(etcd_endpoints, &etcd_prefix).await.ok();
|
||||
let host = if let Some(ref e) = etcd {
|
||||
e.get("GITKS_HOST", &host).await
|
||||
} else {
|
||||
host
|
||||
};
|
||||
let port = if let Some(ref e) = etcd {
|
||||
e.get("GITKS_PORT", &port).await
|
||||
} else {
|
||||
port
|
||||
};
|
||||
let storage_name = if let Some(ref e) = etcd {
|
||||
e.get("GITKS_STORAGE_NAME", &storage_name).await
|
||||
} else {
|
||||
storage_name
|
||||
};
|
||||
let grpc_addr =
|
||||
std::env::var("GITKS_ADVERTISE_ADDR").unwrap_or_else(|_| format!("http://{host}:{port}"));
|
||||
|
||||
// Register this service so other services (appks) can discover us
|
||||
if let Some(ref e) = etcd {
|
||||
let addr_str = format!("{host}:{port}");
|
||||
e.register("gitks", &addr_str).await.ok();
|
||||
}
|
||||
|
||||
let repo_prefix = std::env::var("REPO_PREFIX_PATH")
|
||||
.map_err(|_| "REPO_PREFIX_PATH environment variable is required (e.g. /data/repos)")?;
|
||||
let repo_prefix = PathBuf::from(&repo_prefix);
|
||||
@@ -197,16 +320,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
None
|
||||
};
|
||||
|
||||
// Health check / election configuration
|
||||
let health_check_interval = env_u64("GITKS_HEALTH_CHECK_INTERVAL", 1);
|
||||
let max_health_failures = env_u64("GITKS_MAX_HEALTH_FAILURES", 10);
|
||||
|
||||
tracing::info!(
|
||||
interval_secs = health_check_interval,
|
||||
max_failures = max_health_failures,
|
||||
"health check configured"
|
||||
);
|
||||
|
||||
let metrics_port = env_u64("GITKS_METRICS_PORT", 9100) as u16;
|
||||
let http_cancel = tokio_util::sync::CancellationToken::new();
|
||||
metrics::set_http_cancel_token(http_cancel.clone());
|
||||
@@ -221,60 +334,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
"slow request detection configured"
|
||||
);
|
||||
|
||||
let etcd_endpoints = std::env::var("GITKS_ETCD_ENDPOINTS")
|
||||
.ok()
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| {
|
||||
s.split(',')
|
||||
.map(str::trim)
|
||||
.map(String::from)
|
||||
.collect::<Vec<_>>()
|
||||
});
|
||||
|
||||
let cluster_port = env_or("GITKS_CLUSTER_PORT", "4697")
|
||||
.parse::<u16>()
|
||||
.unwrap_or(4697);
|
||||
let cluster_cookie = env_or("GITKS_CLUSTER_COOKIE", "gitks-default-cookie");
|
||||
let lease_ttl = env_u64("GITKS_LEASE_TTL", 15) as i64;
|
||||
let connect_timeout_ms = env_u64("GITKS_ETCD_CONNECT_TIMEOUT", 5000);
|
||||
|
||||
let cluster_hostname = std::env::var("GITKS_CLUSTER_HOSTNAME")
|
||||
.or_else(|_| std::env::var("POD_IP"))
|
||||
.or_else(|_| std::env::var("HOSTNAME"))
|
||||
.unwrap_or_else(|_| "localhost".to_string());
|
||||
|
||||
let _cluster: Option<ClusterManager> = if let Some(endpoints) = etcd_endpoints {
|
||||
tracing::info!(
|
||||
?endpoints,
|
||||
cluster_port,
|
||||
cluster_hostname = %cluster_hostname,
|
||||
"starting cluster discovery via etcd"
|
||||
);
|
||||
let config = ClusterConfig {
|
||||
etcd_endpoints: endpoints,
|
||||
storage_name: storage_name.clone(),
|
||||
grpc_addr: grpc_addr.clone(),
|
||||
cluster_port,
|
||||
cookie: cluster_cookie,
|
||||
lease_ttl_secs: lease_ttl,
|
||||
connect_timeout_ms,
|
||||
cluster_hostname,
|
||||
};
|
||||
match ClusterManager::start(config).await {
|
||||
Ok(cm) => {
|
||||
tracing::info!("cluster discovery active");
|
||||
Some(cm)
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(error = %e, "etcd unavailable, running in standalone mode");
|
||||
None
|
||||
}
|
||||
}
|
||||
} else {
|
||||
tracing::info!("GITKS_ETCD_ENDPOINTS not set, running in standalone mode");
|
||||
None
|
||||
};
|
||||
|
||||
let addr: std::net::SocketAddr = format!("{host}:{port}").parse()?;
|
||||
let mut svc = GitksService::new(repo_prefix.clone());
|
||||
|
||||
@@ -288,17 +347,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
svc = svc.with_hook_manager(hm);
|
||||
}
|
||||
|
||||
let raft_data_dir = repo_prefix.join(".gitks_raft");
|
||||
let (node_actor, node_handle) = init_actor_cluster(
|
||||
svc.clone(),
|
||||
storage_name.clone(),
|
||||
grpc_addr.clone(),
|
||||
raft_data_dir,
|
||||
)
|
||||
.await?;
|
||||
let svc = svc
|
||||
.with_actor(node_actor.clone())
|
||||
.with_grpc_addr(grpc_addr.clone());
|
||||
let svc = svc.with_grpc_addr(grpc_addr.clone());
|
||||
|
||||
tracing::info!(
|
||||
addr = %addr,
|
||||
@@ -308,16 +357,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
"starting gitks gRPC server"
|
||||
);
|
||||
|
||||
let _route_cache_cleanup = gitks::server::GitksService::start_route_cache_cleanup(svc.clone());
|
||||
|
||||
serve(addr, svc).await?;
|
||||
|
||||
// Gracefully shut down the HTTP metrics server
|
||||
http_cancel.cancel();
|
||||
|
||||
node_actor.stop(None);
|
||||
node_handle.await?;
|
||||
|
||||
tracing::info!("gitks shut down");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
+4
-1
@@ -7,7 +7,10 @@ impl GitBare {
|
||||
let target_branch = request.target_branch.clone();
|
||||
crate::sanitize::validate_ref_name(&target_branch)?;
|
||||
let source_revision = match request.source.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex.clone(),
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex.clone()
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision.clone()
|
||||
|
||||
@@ -9,7 +9,10 @@ impl GitBare {
|
||||
request: ListMergeConflictsRequest,
|
||||
) -> GitResult<ListMergeConflictsResponse> {
|
||||
let target = match request.target.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
@@ -17,7 +20,10 @@ impl GitBare {
|
||||
None => return Err(GitError::InvalidArgument("target is required".into())),
|
||||
};
|
||||
let source = match request.source.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
|
||||
+4
-1
@@ -8,7 +8,10 @@ impl GitBare {
|
||||
let branch = request.branch.clone();
|
||||
crate::sanitize::validate_ref_name(&branch)?;
|
||||
let upstream_revision = match request.upstream.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
|
||||
@@ -11,7 +11,10 @@ impl GitBare {
|
||||
let target_branch = request.target_branch.clone();
|
||||
crate::sanitize::validate_ref_name(&target_branch)?;
|
||||
let source_revision = match request.source.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
@@ -48,6 +51,7 @@ impl GitBare {
|
||||
command_ok(read_tree)?;
|
||||
|
||||
for resolution in &request.resolutions {
|
||||
crate::sanitize::validate_file_path(&resolution.path)?;
|
||||
let hash = duct::cmd(
|
||||
"git",
|
||||
["--git-dir", bare.as_str(), "hash-object", "-w", "--stdin"],
|
||||
|
||||
+88
-124
@@ -59,24 +59,12 @@ struct MetricsInner {
|
||||
hook_count: DashMap<String, AtomicU64>,
|
||||
/// Counter: slow requests by method
|
||||
slow_request_count: DashMap<String, AtomicU64>,
|
||||
|
||||
raft_term: AtomicU64,
|
||||
/// Gauge: current Raft commit index
|
||||
raft_commit_index: AtomicU64,
|
||||
/// Gauge: current Raft last applied index
|
||||
raft_last_applied: AtomicU64,
|
||||
/// Gauge: whether this node is the Raft leader (1 = yes, 0 = no)
|
||||
raft_is_leader: AtomicU64,
|
||||
/// Gauge: number of entries in the Raft log
|
||||
raft_log_entries: AtomicU64,
|
||||
/// Counter: total AppendEntries RPCs sent
|
||||
raft_append_entries_total: AtomicU64,
|
||||
/// Counter: successful AppendEntries RPCs
|
||||
raft_append_entries_success: AtomicU64,
|
||||
/// Counter: total elections triggered
|
||||
raft_elections_total: AtomicU64,
|
||||
/// Counter: elections won
|
||||
raft_elections_won: AtomicU64,
|
||||
/// Counter: cache evictions by (cause, namespace)
|
||||
cache_eviction_count: DashMap<String, AtomicU64>,
|
||||
/// Counter: cache hits by namespace
|
||||
cache_hit_by_namespace: DashMap<String, AtomicU64>,
|
||||
/// Counter: cache misses by namespace
|
||||
cache_miss_by_namespace: DashMap<String, AtomicU64>,
|
||||
}
|
||||
|
||||
static METRICS: OnceLock<Arc<MetricsInner>> = OnceLock::new();
|
||||
@@ -108,16 +96,9 @@ fn metrics() -> &'static Arc<MetricsInner> {
|
||||
hook_duration_buckets: DashMap::new(),
|
||||
hook_count: DashMap::new(),
|
||||
slow_request_count: DashMap::new(),
|
||||
// Raft metrics
|
||||
raft_term: AtomicU64::new(0),
|
||||
raft_commit_index: AtomicU64::new(0),
|
||||
raft_last_applied: AtomicU64::new(0),
|
||||
raft_is_leader: AtomicU64::new(0),
|
||||
raft_log_entries: AtomicU64::new(0),
|
||||
raft_append_entries_total: AtomicU64::new(0),
|
||||
raft_append_entries_success: AtomicU64::new(0),
|
||||
raft_elections_total: AtomicU64::new(0),
|
||||
raft_elections_won: AtomicU64::new(0),
|
||||
cache_eviction_count: DashMap::new(),
|
||||
cache_hit_by_namespace: DashMap::new(),
|
||||
cache_miss_by_namespace: DashMap::new(),
|
||||
})
|
||||
})
|
||||
}
|
||||
@@ -243,6 +224,37 @@ pub fn record_cache_op(cache: &str, result: &str, duration: Duration) {
|
||||
record_duration_bucket(&m.cache_op_duration_buckets, cache, duration_ms);
|
||||
}
|
||||
|
||||
/// Record a cache entry eviction.
|
||||
pub fn record_cache_eviction(namespace: &str, cause: &str) {
|
||||
let m = metrics();
|
||||
let key = format!("{cause}:{namespace}");
|
||||
m.cache_eviction_count
|
||||
.entry(key)
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.value()
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record a per-namespace cache hit.
|
||||
pub fn record_cache_hit_ns(namespace: &str) {
|
||||
metrics()
|
||||
.cache_hit_by_namespace
|
||||
.entry(namespace.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.value()
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record a per-namespace cache miss.
|
||||
pub fn record_cache_miss_ns(namespace: &str) {
|
||||
metrics()
|
||||
.cache_miss_by_namespace
|
||||
.entry(namespace.to_string())
|
||||
.or_insert_with(|| AtomicU64::new(0))
|
||||
.value()
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record a hook execution.
|
||||
pub fn record_hook_execution(hook_type: &str, result: &str, duration: Duration) {
|
||||
let m = metrics();
|
||||
@@ -258,41 +270,6 @@ pub fn record_hook_execution(hook_type: &str, result: &str, duration: Duration)
|
||||
record_duration_bucket(&m.hook_duration_buckets, hook_type, duration_ms);
|
||||
}
|
||||
|
||||
pub fn set_raft_state(
|
||||
term: u64,
|
||||
commit_index: u64,
|
||||
last_applied: u64,
|
||||
is_leader: bool,
|
||||
log_entries: u64,
|
||||
) {
|
||||
let m = metrics();
|
||||
m.raft_term.store(term, Ordering::Relaxed);
|
||||
m.raft_commit_index.store(commit_index, Ordering::Relaxed);
|
||||
m.raft_last_applied.store(last_applied, Ordering::Relaxed);
|
||||
m.raft_is_leader
|
||||
.store(if is_leader { 1 } else { 0 }, Ordering::Relaxed);
|
||||
m.raft_log_entries.store(log_entries, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Record an AppendEntries RPC attempt.
|
||||
pub fn inc_raft_append_entries(success: bool) {
|
||||
let m = metrics();
|
||||
m.raft_append_entries_total.fetch_add(1, Ordering::Relaxed);
|
||||
if success {
|
||||
m.raft_append_entries_success
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Record an election trigger.
|
||||
pub fn inc_raft_election(won: bool) {
|
||||
let m = metrics();
|
||||
m.raft_elections_total.fetch_add(1, Ordering::Relaxed);
|
||||
if won {
|
||||
m.raft_elections_won.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Escape a string for use as a Prometheus label value.
|
||||
/// Replaces `\` → `\\`, `"` → `\"`, `\n` → `\n` per the Prometheus spec.
|
||||
fn prom_escape(value: &str) -> String {
|
||||
@@ -447,6 +424,33 @@ pub fn render_metrics() -> String {
|
||||
&m.cache_op_duration_buckets,
|
||||
);
|
||||
|
||||
// Cache evictions by cause and namespace
|
||||
render_counter_map(
|
||||
&mut out,
|
||||
"gitks_cache_evictions_total",
|
||||
"Cache evictions by cause and namespace",
|
||||
&m.cache_eviction_count,
|
||||
&["cause", "namespace"],
|
||||
);
|
||||
|
||||
// Per-namespace cache hits
|
||||
render_counter_map(
|
||||
&mut out,
|
||||
"gitks_cache_hits_by_namespace_total",
|
||||
"Cache hits by namespace",
|
||||
&m.cache_hit_by_namespace,
|
||||
&["namespace"],
|
||||
);
|
||||
|
||||
// Per-namespace cache misses
|
||||
render_counter_map(
|
||||
&mut out,
|
||||
"gitks_cache_misses_by_namespace_total",
|
||||
"Cache misses by namespace",
|
||||
&m.cache_miss_by_namespace,
|
||||
&["namespace"],
|
||||
);
|
||||
|
||||
// Hook execution
|
||||
render_counter_map(
|
||||
&mut out,
|
||||
@@ -462,59 +466,6 @@ pub fn render_metrics() -> String {
|
||||
&m.hook_duration_buckets,
|
||||
);
|
||||
|
||||
// Raft consensus metrics
|
||||
let raft_term = m.raft_term.load(Ordering::Relaxed);
|
||||
let raft_commit = m.raft_commit_index.load(Ordering::Relaxed);
|
||||
let raft_applied = m.raft_last_applied.load(Ordering::Relaxed);
|
||||
let raft_leader = m.raft_is_leader.load(Ordering::Relaxed);
|
||||
let raft_entries = m.raft_log_entries.load(Ordering::Relaxed);
|
||||
let raft_ae_total = m.raft_append_entries_total.load(Ordering::Relaxed);
|
||||
let raft_ae_success = m.raft_append_entries_success.load(Ordering::Relaxed);
|
||||
let raft_elections = m.raft_elections_total.load(Ordering::Relaxed);
|
||||
let raft_elections_won = m.raft_elections_won.load(Ordering::Relaxed);
|
||||
|
||||
out.push_str("# HELP gitks_raft_term Current Raft term\n");
|
||||
out.push_str("# TYPE gitks_raft_term gauge\n");
|
||||
out.push_str(&format!("gitks_raft_term {raft_term}\n\n"));
|
||||
|
||||
out.push_str("# HELP gitks_raft_commit_index Current Raft commit index\n");
|
||||
out.push_str("# TYPE gitks_raft_commit_index gauge\n");
|
||||
out.push_str(&format!("gitks_raft_commit_index {raft_commit}\n\n"));
|
||||
|
||||
out.push_str("# HELP gitks_raft_last_applied Current Raft last applied index\n");
|
||||
out.push_str("# TYPE gitks_raft_last_applied gauge\n");
|
||||
out.push_str(&format!("gitks_raft_last_applied {raft_applied}\n\n"));
|
||||
|
||||
out.push_str("# HELP gitks_raft_is_leader Whether this node is the Raft leader\n");
|
||||
out.push_str("# TYPE gitks_raft_is_leader gauge\n");
|
||||
out.push_str(&format!("gitks_raft_is_leader {raft_leader}\n\n"));
|
||||
|
||||
out.push_str("# HELP gitks_raft_log_entries Number of entries in the Raft log\n");
|
||||
out.push_str("# TYPE gitks_raft_log_entries gauge\n");
|
||||
out.push_str(&format!("gitks_raft_log_entries {raft_entries}\n\n"));
|
||||
|
||||
out.push_str("# HELP gitks_raft_append_entries_total Total AppendEntries RPCs sent\n");
|
||||
out.push_str("# TYPE gitks_raft_append_entries_total counter\n");
|
||||
out.push_str(&format!(
|
||||
"gitks_raft_append_entries_total {raft_ae_total}\n\n"
|
||||
));
|
||||
|
||||
out.push_str("# HELP gitks_raft_append_entries_success Successful AppendEntries RPCs\n");
|
||||
out.push_str("# TYPE gitks_raft_append_entries_success counter\n");
|
||||
out.push_str(&format!(
|
||||
"gitks_raft_append_entries_success {raft_ae_success}\n\n"
|
||||
));
|
||||
|
||||
out.push_str("# HELP gitks_raft_elections_total Total elections triggered\n");
|
||||
out.push_str("# TYPE gitks_raft_elections_total counter\n");
|
||||
out.push_str(&format!("gitks_raft_elections_total {raft_elections}\n\n"));
|
||||
|
||||
out.push_str("# HELP gitks_raft_elections_won Elections won by this node\n");
|
||||
out.push_str("# TYPE gitks_raft_elections_won counter\n");
|
||||
out.push_str(&format!(
|
||||
"gitks_raft_elections_won {raft_elections_won}\n\n"
|
||||
));
|
||||
|
||||
out
|
||||
}
|
||||
|
||||
@@ -548,21 +499,35 @@ impl Service<Request<Incoming>> for Router {
|
||||
}
|
||||
|
||||
fn json_response(status: u16, body: &str) -> Response<Full<Bytes>> {
|
||||
Response::builder()
|
||||
match Response::builder()
|
||||
.status(status)
|
||||
.header("Content-Type", "application/json")
|
||||
.header("Connection", "close")
|
||||
.body(Full::new(Bytes::from(body.to_string())))
|
||||
.unwrap()
|
||||
{
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
tracing::error!(error = %err, "failed to build JSON response");
|
||||
Response::new(Full::new(Bytes::from_static(
|
||||
br#"{"error":"response build failed"}"#,
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn text_response(status: u16, content_type: &str, body: String) -> Response<Full<Bytes>> {
|
||||
Response::builder()
|
||||
match Response::builder()
|
||||
.status(status)
|
||||
.header("Content-Type", content_type)
|
||||
.header("Connection", "close")
|
||||
.body(Full::new(Bytes::from(body)))
|
||||
.unwrap()
|
||||
{
|
||||
Ok(response) => response,
|
||||
Err(err) => {
|
||||
tracing::error!(error = %err, "failed to build text response");
|
||||
Response::new(Full::new(Bytes::from_static(b"response build failed")))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_request(req: Request<Incoming>) -> Result<Response<Full<Bytes>>, Infallible> {
|
||||
@@ -592,10 +557,9 @@ async fn handle_request(req: Request<Incoming>) -> Result<Response<Full<Bytes>>,
|
||||
};
|
||||
json_response(200, &format!(r#"{{"log_level":"{msg}"}}"#))
|
||||
}
|
||||
(Method::PUT, "/debug/log-level") => match handle_log_level_update(req).await {
|
||||
Ok(resp) => resp,
|
||||
Err(e) => json_response(400, &format!(r#"{{"error":"{e}"}}"#)),
|
||||
},
|
||||
(Method::PUT, "/debug/log-level") => handle_log_level_update(req)
|
||||
.await
|
||||
.unwrap_or_else(|e| json_response(400, &format!(r#"{{"error":"{e}"}}"#))),
|
||||
(Method::GET, "/debug/config") => {
|
||||
let threshold = metrics().slow_request_threshold_ms.load(Ordering::Relaxed);
|
||||
let ready = metrics().ready.load(Ordering::Relaxed);
|
||||
|
||||
@@ -34,10 +34,12 @@ pub fn hex_to_bytes(hex: &str) -> Result<Vec<u8>, crate::error::GitError> {
|
||||
));
|
||||
}
|
||||
|
||||
(0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|idx| {
|
||||
u8::from_str_radix(&hex[idx..idx + 2], 16)
|
||||
hex.as_bytes()
|
||||
.chunks_exact(2)
|
||||
.map(|pair| {
|
||||
let part = std::str::from_utf8(pair)
|
||||
.map_err(|e| crate::error::GitError::InvalidOid(e.to_string()))?;
|
||||
u8::from_str_radix(part, 16)
|
||||
.map_err(|e| crate::error::GitError::InvalidOid(e.to_string()))
|
||||
})
|
||||
.collect()
|
||||
|
||||
+19
-4
@@ -7,6 +7,8 @@ use tokio_stream::wrappers::ReceiverStream;
|
||||
use crate::bare::GitBare;
|
||||
use crate::pb::PackfileChunk;
|
||||
|
||||
const MAX_PACK_OBJECTS_STDERR_BYTES: u64 = 64 * 1024;
|
||||
|
||||
impl GitBare {
|
||||
/// Pack objects using git-pack-objects --stdout.
|
||||
///
|
||||
@@ -52,7 +54,15 @@ impl GitBare {
|
||||
if opts.is_some_and(|o| o.delta_base_offset) {
|
||||
args.push("--delta-base-offset".into());
|
||||
}
|
||||
let stdin_data = generate_pack_input(&request);
|
||||
let stdin_data = match generate_pack_input(&request) {
|
||||
Ok(data) => data,
|
||||
Err(err) => {
|
||||
let _ = tx
|
||||
.send(Err(tonic::Status::invalid_argument(err.to_string())))
|
||||
.await;
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
let mut child = match Command::new("git")
|
||||
.args(&args)
|
||||
@@ -118,7 +128,8 @@ impl GitBare {
|
||||
let stderr_task = {
|
||||
let tx = tx.clone();
|
||||
async move {
|
||||
if let Some(mut stderr) = stderr.take() {
|
||||
if let Some(stderr) = stderr.take() {
|
||||
let mut stderr = stderr.take(MAX_PACK_OBJECTS_STDERR_BYTES);
|
||||
let mut s = String::new();
|
||||
if stderr.read_to_string(&mut s).await.is_ok() && !s.is_empty() {
|
||||
let _ = tx.send(Err(tonic::Status::internal(s))).await;
|
||||
@@ -150,15 +161,19 @@ impl GitBare {
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_pack_input(req: &crate::pb::PackObjectsRequest) -> Vec<u8> {
|
||||
fn generate_pack_input(
|
||||
req: &crate::pb::PackObjectsRequest,
|
||||
) -> Result<Vec<u8>, crate::error::GitError> {
|
||||
let mut input = String::new();
|
||||
if let Some(opts) = req.options.as_ref() {
|
||||
for want in &opts.wants {
|
||||
crate::sanitize::validate_oid_hex(&want.hex)?;
|
||||
input.push_str(&format!("{}\n", want.hex));
|
||||
}
|
||||
for have in &opts.haves {
|
||||
crate::sanitize::validate_oid_hex(&have.hex)?;
|
||||
input.push_str(&format!("^{}\n", have.hex));
|
||||
}
|
||||
}
|
||||
input.into_bytes()
|
||||
Ok(input.into_bytes())
|
||||
}
|
||||
|
||||
@@ -12,6 +12,8 @@ use crate::pb::ReceivePackResponse;
|
||||
|
||||
/// Maximum time allowed for a git receive-pack process before it is killed.
|
||||
const RECEIVE_PACK_TIMEOUT: Duration = Duration::from_secs(1800); // 30 minutes
|
||||
const MAX_RECEIVE_PACKET_BYTES: usize = 16 * 1024 * 1024;
|
||||
const MAX_RECEIVE_STDERR_BYTES: u64 = 64 * 1024;
|
||||
|
||||
impl GitBare {
|
||||
/// Receive pack data using git-receive-pack with true concurrent streaming.
|
||||
@@ -85,6 +87,9 @@ impl GitBare {
|
||||
}
|
||||
match result {
|
||||
Ok(req) => {
|
||||
if req.packet.len() > MAX_RECEIVE_PACKET_BYTES {
|
||||
break;
|
||||
}
|
||||
if stdin.write_all(&req.packet).await.is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -134,7 +139,8 @@ impl GitBare {
|
||||
let stderr_task = {
|
||||
let tx = tx.clone();
|
||||
async move {
|
||||
if let Some(mut stderr) = stderr.take() {
|
||||
if let Some(stderr) = stderr.take() {
|
||||
let mut stderr = stderr.take(MAX_RECEIVE_STDERR_BYTES);
|
||||
let mut s = String::new();
|
||||
if stderr.read_to_string(&mut s).await.is_ok() && !s.is_empty() {
|
||||
let _ = tx
|
||||
|
||||
+7
-1
@@ -12,6 +12,8 @@ use crate::pb::UploadPackResponse;
|
||||
|
||||
/// Maximum time allowed for a git upload-pack process before it is killed.
|
||||
const UPLOAD_PACK_TIMEOUT: Duration = Duration::from_secs(600); // 10 minutes
|
||||
const MAX_UPLOAD_PACKET_BYTES: usize = 16 * 1024 * 1024;
|
||||
const MAX_UPLOAD_STDERR_BYTES: u64 = 64 * 1024;
|
||||
|
||||
impl GitBare {
|
||||
/// Upload pack data using git-upload-pack with true concurrent streaming.
|
||||
@@ -87,6 +89,9 @@ impl GitBare {
|
||||
}
|
||||
match result {
|
||||
Ok(req) => {
|
||||
if req.packet.len() > MAX_UPLOAD_PACKET_BYTES {
|
||||
break;
|
||||
}
|
||||
if stdin.write_all(&req.packet).await.is_err() {
|
||||
break;
|
||||
}
|
||||
@@ -137,7 +142,8 @@ impl GitBare {
|
||||
let stderr_task = {
|
||||
let tx = tx.clone();
|
||||
async move {
|
||||
if let Some(mut stderr) = stderr.take() {
|
||||
if let Some(stderr) = stderr.take() {
|
||||
let mut stderr = stderr.take(MAX_UPLOAD_STDERR_BYTES);
|
||||
let mut s = String::new();
|
||||
if stderr.read_to_string(&mut s).await.is_ok() && !s.is_empty() {
|
||||
let _ = tx
|
||||
|
||||
+1
-1
@@ -19,7 +19,7 @@ pub fn paginate<T: Clone>(items: &[T], pagination: Option<&Pagination>) -> (Vec<
|
||||
.unwrap_or(0)
|
||||
.min(items.len());
|
||||
|
||||
let end = std::cmp::min(start_offset + page_size, items.len());
|
||||
let end = start_offset.saturating_add(page_size).min(items.len());
|
||||
let has_next = end < items.len();
|
||||
let next_page_token = if has_next {
|
||||
end.to_string()
|
||||
|
||||
+10
-5
@@ -106,11 +106,16 @@ pub async fn acquire(repo_relative_path: Option<&str>) -> Option<RateLimitGuard>
|
||||
"rate limit semaphore closed, recreating"
|
||||
);
|
||||
let new_sem = Arc::new(Semaphore::new(get_max_concurrent()));
|
||||
let permit = new_sem
|
||||
.clone()
|
||||
.acquire_owned()
|
||||
.await
|
||||
.expect("newly created semaphore should have permits");
|
||||
let permit = match new_sem.clone().acquire_owned().await {
|
||||
Ok(permit) => permit,
|
||||
Err(_closed) => {
|
||||
tracing::warn!(
|
||||
repo = %repo_relative_path.unwrap_or(""),
|
||||
"new rate limit semaphore closed unexpectedly"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
limiter()
|
||||
.semaphores
|
||||
.insert(repo_relative_path.unwrap_or("").to_string(), new_sem);
|
||||
|
||||
+27
-1
@@ -35,6 +35,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut refs = Vec::new();
|
||||
@@ -92,6 +98,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut all_refs: Vec<FoundRef> = Vec::new();
|
||||
@@ -127,7 +139,6 @@ impl GitBare {
|
||||
}
|
||||
}
|
||||
|
||||
/// Simple glob match. Supports `*` (any chars) and `?` (single char).
|
||||
fn simple_glob_match(pattern: &str, name: &str) -> bool {
|
||||
let pat_bytes = pattern.as_bytes();
|
||||
let name_bytes = name.as_bytes();
|
||||
@@ -148,6 +159,9 @@ fn simple_glob_match(pattern: &str, name: &str) -> bool {
|
||||
pi += 1;
|
||||
ni += 1;
|
||||
} else if let Some(sp) = star_pi {
|
||||
if star_ni >= name_bytes.len() {
|
||||
return false;
|
||||
}
|
||||
pi = sp + 1;
|
||||
star_ni += 1;
|
||||
ni = star_ni;
|
||||
@@ -157,3 +171,15 @@ fn simple_glob_match(pattern: &str, name: &str) -> bool {
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::simple_glob_match;
|
||||
|
||||
#[test]
|
||||
fn glob_with_trailing_literal_does_not_loop_on_short_name() {
|
||||
assert!(!simple_glob_match("*a", ""));
|
||||
assert!(!simple_glob_match("refs/*/main", "refs/heads"));
|
||||
assert!(simple_glob_match("refs/*/main", "refs/heads/main"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,6 +175,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !result.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: result.status.code(),
|
||||
stderr: String::from_utf8_lossy(&result.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
let name = String::from_utf8_lossy(&result.stdout)
|
||||
.trim()
|
||||
.strip_prefix("refs/heads/")
|
||||
|
||||
@@ -11,6 +11,7 @@ pub fn find_remote_repository(
|
||||
exists: false,
|
||||
});
|
||||
}
|
||||
crate::sanitize::validate_remote_url(&request.remote_url)?;
|
||||
|
||||
let output = std::process::Command::new("git")
|
||||
.args(["ls-remote", "--symref", &request.remote_url])
|
||||
@@ -78,6 +79,11 @@ pub fn find_remote_repository(
|
||||
pub fn find_remote_root_ref(
|
||||
request: FindRemoteRootRefRequest,
|
||||
) -> GitResult<FindRemoteRootRefResponse> {
|
||||
if request.remote_url.is_empty() {
|
||||
return Ok(FindRemoteRootRefResponse::default());
|
||||
}
|
||||
crate::sanitize::validate_remote_url(&request.remote_url)?;
|
||||
|
||||
let output = std::process::Command::new("git")
|
||||
.args(["ls-remote", "--symref", &request.remote_url, "HEAD"])
|
||||
.stdout(std::process::Stdio::piped())
|
||||
@@ -87,6 +93,9 @@ pub fn find_remote_root_ref(
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Ok(FindRemoteRootRefResponse::default());
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
for line in stdout.lines() {
|
||||
|
||||
+37
-33
@@ -20,7 +20,7 @@ impl GitBare {
|
||||
crate::sanitize::validate_refspec(rs)?;
|
||||
}
|
||||
|
||||
let remote_check = std::process::Command::new("git")
|
||||
let remote_exists = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.bare_dir.to_string_lossy(),
|
||||
@@ -28,38 +28,33 @@ impl GitBare {
|
||||
"get-url",
|
||||
remote_name,
|
||||
])
|
||||
.output();
|
||||
.output()
|
||||
.map(|output| output.status.success())
|
||||
.unwrap_or(false);
|
||||
|
||||
if remote_check.is_err() || !remote_check.unwrap().status.success() {
|
||||
std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.bare_dir.to_string_lossy(),
|
||||
"remote",
|
||||
"add",
|
||||
remote_name,
|
||||
&request.remote_url,
|
||||
])
|
||||
.output()
|
||||
.map_err(|e| crate::error::GitError::CommandFailed {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
} else {
|
||||
std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.bare_dir.to_string_lossy(),
|
||||
"remote",
|
||||
"set-url",
|
||||
remote_name,
|
||||
&request.remote_url,
|
||||
])
|
||||
.output()
|
||||
.map_err(|e| crate::error::GitError::CommandFailed {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
let remote_command = if remote_exists { "set-url" } else { "add" };
|
||||
let remote_output = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.bare_dir.to_string_lossy(),
|
||||
"remote",
|
||||
remote_command,
|
||||
remote_name,
|
||||
&request.remote_url,
|
||||
])
|
||||
.output()
|
||||
.map_err(|e| crate::error::GitError::CommandFailed {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
|
||||
if !remote_output.status.success() {
|
||||
return Ok(UpdateRemoteMirrorResponse {
|
||||
ok: false,
|
||||
error: String::from_utf8_lossy(&remote_output.stderr)
|
||||
.trim()
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut fetch_args = vec![
|
||||
@@ -156,7 +151,7 @@ impl GitBare {
|
||||
.unwrap_or(false);
|
||||
|
||||
if !exists {
|
||||
std::process::Command::new("git")
|
||||
let remote_output = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.bare_dir.to_string_lossy(),
|
||||
@@ -170,6 +165,15 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
|
||||
if !remote_output.status.success() {
|
||||
return Ok(FetchRemoteResponse {
|
||||
ok: false,
|
||||
error: String::from_utf8_lossy(&remote_output.stderr)
|
||||
.trim()
|
||||
.to_string(),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let mut args = vec![
|
||||
|
||||
@@ -12,11 +12,21 @@ impl GitBare {
|
||||
return Ok(FindMergeBaseResponse::default());
|
||||
}
|
||||
|
||||
const MAX_MERGE_BASE_REVISIONS: usize = 64;
|
||||
if request.revisions.len() > MAX_MERGE_BASE_REVISIONS {
|
||||
return Err(crate::error::GitError::InvalidArgument(format!(
|
||||
"too many revisions (max {MAX_MERGE_BASE_REVISIONS})"
|
||||
)));
|
||||
}
|
||||
|
||||
let revisions: Vec<String> = request
|
||||
.revisions
|
||||
.iter()
|
||||
.map(|b| String::from_utf8_lossy(b).to_string())
|
||||
.collect();
|
||||
for revision in &revisions {
|
||||
crate::sanitize::validate_revision(revision)?;
|
||||
}
|
||||
|
||||
if revisions.len() < 2 {
|
||||
return Ok(FindMergeBaseResponse {
|
||||
|
||||
+25
-13
@@ -12,6 +12,7 @@ include!(concat!(env!("OUT_DIR"), "/linguist_generated.rs"));
|
||||
|
||||
/// Default max file size for line counting (512 KB).
|
||||
const DEFAULT_MAX_FILE_SIZE: u32 = 512 * 1024;
|
||||
const MAX_TREE_WALK_DEPTH: usize = 256;
|
||||
|
||||
/// Look up a language by file extension (case-insensitive, includes leading dot).
|
||||
fn lookup_by_extension(ext: &str) -> Option<(&'static str, &'static str)> {
|
||||
@@ -122,7 +123,10 @@ impl GitBare {
|
||||
) -> GitResult<GetLanguageStatsResponse> {
|
||||
let repo = self.gix_repo()?;
|
||||
let revision = match request.revision.clone().and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
@@ -144,6 +148,7 @@ impl GitBare {
|
||||
|
||||
// If path is specified, descend into subdirectory
|
||||
if !request.path.is_empty() {
|
||||
crate::sanitize::validate_file_path(&request.path)?;
|
||||
let entry = tree
|
||||
.lookup_entry_by_path(&request.path)?
|
||||
.ok_or_else(|| GitError::NotFound(request.path.clone()))?;
|
||||
@@ -166,7 +171,7 @@ impl GitBare {
|
||||
total_bytes: &mut total_bytes,
|
||||
total_lines: &mut total_lines,
|
||||
};
|
||||
self.walk_tree(&repo, &tree, &prefix, &mut ctx)?;
|
||||
self.walk_tree(&repo, &tree, &prefix, 0, &mut ctx)?;
|
||||
|
||||
// Resolve groups: merge child language stats into parent group
|
||||
tracing::info!(
|
||||
@@ -185,9 +190,9 @@ impl GitBare {
|
||||
lang_type: s.lang_type.clone(),
|
||||
..Default::default()
|
||||
});
|
||||
entry.file_count += s.file_count;
|
||||
entry.bytes += s.bytes;
|
||||
entry.lines += s.lines;
|
||||
entry.file_count = entry.file_count.saturating_add(s.file_count);
|
||||
entry.bytes = entry.bytes.saturating_add(s.bytes);
|
||||
entry.lines = entry.lines.saturating_add(s.lines);
|
||||
// Keep the lang_type from the parent (or first encountered)
|
||||
if entry.lang_type.is_empty() {
|
||||
entry.lang_type = s.lang_type;
|
||||
@@ -233,8 +238,15 @@ impl GitBare {
|
||||
_repo: &gix::Repository,
|
||||
tree: &gix::Tree<'_>,
|
||||
prefix: &str,
|
||||
depth: usize,
|
||||
ctx: &mut WalkContext<'_>,
|
||||
) -> GitResult<()> {
|
||||
if depth > MAX_TREE_WALK_DEPTH {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"tree depth exceeds maximum of {MAX_TREE_WALK_DEPTH}"
|
||||
)));
|
||||
}
|
||||
|
||||
for entry in tree.iter() {
|
||||
let entry = entry?;
|
||||
let name = String::from_utf8_lossy(entry.filename()).into_owned();
|
||||
@@ -250,7 +262,7 @@ impl GitBare {
|
||||
.object()?
|
||||
.try_into_tree()
|
||||
.map_err(|e| GitError::Gix(e.to_string()))?;
|
||||
self.walk_tree(_repo, &child_tree, &path, ctx)?;
|
||||
self.walk_tree(_repo, &child_tree, &path, depth + 1, ctx)?;
|
||||
}
|
||||
EntryKind::Blob | EntryKind::BlobExecutable => {
|
||||
let blob = entry
|
||||
@@ -277,15 +289,15 @@ impl GitBare {
|
||||
let lang_key = lang_name.to_string();
|
||||
|
||||
// Count code lines only for non-binary files within size limit
|
||||
let lines = if !is_binary && (size as u32) <= ctx.max_file_size {
|
||||
let lines = if !is_binary && size <= u64::from(ctx.max_file_size) {
|
||||
count_code_lines(data)
|
||||
} else {
|
||||
0
|
||||
};
|
||||
|
||||
*ctx.total_files += 1;
|
||||
*ctx.total_bytes += size;
|
||||
*ctx.total_lines += lines;
|
||||
*ctx.total_files = ctx.total_files.saturating_add(1);
|
||||
*ctx.total_bytes = ctx.total_bytes.saturating_add(size);
|
||||
*ctx.total_lines = ctx.total_lines.saturating_add(lines);
|
||||
|
||||
let s = ctx
|
||||
.stats
|
||||
@@ -294,9 +306,9 @@ impl GitBare {
|
||||
lang_type: lang_type.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
s.file_count += 1;
|
||||
s.bytes += size;
|
||||
s.lines += lines;
|
||||
s.file_count = s.file_count.saturating_add(1);
|
||||
s.bytes = s.bytes.saturating_add(size);
|
||||
s.lines = s.lines.saturating_add(lines);
|
||||
}
|
||||
_ => {} // Skip symlinks, submodules
|
||||
}
|
||||
|
||||
@@ -9,9 +9,16 @@ impl GitBare {
|
||||
return Ok(ObjectsSizeResponse::default());
|
||||
}
|
||||
|
||||
const MAX_OBJECTS_SIZE_OIDS: usize = 10_000;
|
||||
if request.oids.len() > MAX_OBJECTS_SIZE_OIDS {
|
||||
return Err(crate::error::GitError::InvalidArgument(format!(
|
||||
"too many oids (max {MAX_OBJECTS_SIZE_OIDS})"
|
||||
)));
|
||||
}
|
||||
|
||||
let mut input = String::new();
|
||||
for oid in &request.oids {
|
||||
crate::sanitize::validate_revision(oid)?;
|
||||
crate::sanitize::validate_oid_hex(oid)?;
|
||||
input.push_str(oid);
|
||||
input.push('\n');
|
||||
}
|
||||
@@ -49,6 +56,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut sizes = Vec::new();
|
||||
@@ -81,6 +94,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let size = stdout
|
||||
|
||||
@@ -29,6 +29,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut changes = Vec::new();
|
||||
|
||||
@@ -2,23 +2,29 @@ use crate::bare::GitBare;
|
||||
use crate::error::GitResult;
|
||||
use crate::pb::*;
|
||||
|
||||
const MAX_SEARCH_RESULTS: u32 = 1000;
|
||||
|
||||
impl GitBare {
|
||||
/// Search file contents with a regex pattern.
|
||||
pub fn search_files_by_content(
|
||||
&self,
|
||||
request: SearchFilesByContentRequest,
|
||||
) -> GitResult<SearchFilesByContentResponse> {
|
||||
crate::sanitize::validate_revision(&request.revision)?;
|
||||
|
||||
let revision = if request.revision.is_empty() {
|
||||
"HEAD"
|
||||
} else {
|
||||
&request.revision
|
||||
};
|
||||
crate::sanitize::validate_revision(revision)?;
|
||||
if request.query.is_empty() {
|
||||
return Err(crate::error::GitError::InvalidArgument(
|
||||
"search query cannot be empty".into(),
|
||||
));
|
||||
}
|
||||
let max_results = if request.max_results == 0 {
|
||||
100
|
||||
} else {
|
||||
request.max_results
|
||||
request.max_results.min(MAX_SEARCH_RESULTS)
|
||||
};
|
||||
|
||||
let mut args = vec![
|
||||
@@ -51,6 +57,12 @@ impl GitBare {
|
||||
})?;
|
||||
|
||||
// git grep returns exit code 1 when no matches found — that's not an error
|
||||
if !output.status.success() && output.status.code() != Some(1) {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut results = Vec::new();
|
||||
|
||||
@@ -59,13 +71,16 @@ impl GitBare {
|
||||
if let Some((path_and_rest, matched)) = line.rsplit_once(':') {
|
||||
let prefix_parts: Vec<&str> = path_and_rest.rsplitn(3, ':').collect();
|
||||
if prefix_parts.len() >= 3
|
||||
&& let Ok(line_num) = prefix_parts[0].parse::<u32>()
|
||||
&& let Ok(line_num) = prefix_parts[1].parse::<u32>()
|
||||
{
|
||||
results.push(SearchResult {
|
||||
path: prefix_parts[2].to_string(),
|
||||
line: line_num,
|
||||
matched_text: matched.to_string(),
|
||||
});
|
||||
if results.len() >= max_results as usize {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -88,7 +103,7 @@ impl GitBare {
|
||||
let max_results = if request.max_results == 0 {
|
||||
100
|
||||
} else {
|
||||
request.max_results
|
||||
request.max_results.min(MAX_SEARCH_RESULTS)
|
||||
};
|
||||
|
||||
let mut args = vec![
|
||||
@@ -113,6 +128,12 @@ impl GitBare {
|
||||
status_code: None,
|
||||
stderr: e.to_string(),
|
||||
})?;
|
||||
if !output.status.success() {
|
||||
return Err(crate::error::GitError::CommandFailed {
|
||||
status_code: output.status.code(),
|
||||
stderr: String::from_utf8_lossy(&output.stderr).trim().to_string(),
|
||||
});
|
||||
}
|
||||
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let mut results = Vec::new();
|
||||
|
||||
+33
-5
@@ -11,9 +11,7 @@ use crate::error::GitResult;
|
||||
/// Git disallows: space, `~`, `^`, `:`, `?`, `*`, `[`, `\`, and all ASCII
|
||||
/// control characters (bytes 0–31 and 127). The control characters are
|
||||
/// checked separately via `is_ascii_control()`.
|
||||
const FORBIDDEN_REF_CHARS: &[char] = &[
|
||||
'~', '^', ':', '?', '*', '[', '\\', ' ',
|
||||
];
|
||||
const FORBIDDEN_REF_CHARS: &[char] = &['~', '^', ':', '?', '*', '[', '\\', ' '];
|
||||
|
||||
/// Returns true if `c` is an ASCII control character (bytes 0–31, 127).
|
||||
fn is_ascii_control(c: char) -> bool {
|
||||
@@ -30,6 +28,24 @@ fn is_ascii_control(c: char) -> bool {
|
||||
/// - Cannot contain '..'
|
||||
/// - Cannot contain '@{'
|
||||
/// - Cannot be empty
|
||||
pub fn validate_oid_hex(hex: &str) -> GitResult<()> {
|
||||
if hex.is_empty() {
|
||||
return Err(GitError::InvalidArgument("oid hex cannot be empty".into()));
|
||||
}
|
||||
if !(4..=64).contains(&hex.len()) {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"oid hex length must be 4..=64 chars: {}",
|
||||
hex.len()
|
||||
)));
|
||||
}
|
||||
if !hex.chars().all(|c| c.is_ascii_hexdigit()) {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"oid hex contains non-hex character: {hex}"
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn validate_ref_name(name: &str) -> GitResult<()> {
|
||||
if name.is_empty() {
|
||||
return Err(GitError::InvalidArgument("ref name cannot be empty".into()));
|
||||
@@ -253,8 +269,10 @@ pub fn validate_config_key(key: &str) -> GitResult<()> {
|
||||
for pattern in DANGEROUS_CONFIG_KEYS {
|
||||
if pattern.contains('*') {
|
||||
// e.g. "remote.*.url" — match any "remote.<something>.url"
|
||||
let (prefix, suffix) = pattern.split_once('*').unwrap();
|
||||
if key.starts_with(prefix) && key.ends_with(suffix) {
|
||||
if let Some((prefix, suffix)) = pattern.split_once('*')
|
||||
&& key.starts_with(prefix)
|
||||
&& key.ends_with(suffix)
|
||||
{
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"config key '{key}' matches dangerous pattern '{pattern}'"
|
||||
)));
|
||||
@@ -347,6 +365,16 @@ pub fn validate_relative_path(path: &str) -> GitResult<()> {
|
||||
"relative_path must be relative, not absolute".into(),
|
||||
));
|
||||
}
|
||||
if path.contains('\0') {
|
||||
return Err(GitError::InvalidArgument(
|
||||
"relative_path cannot contain null byte".into(),
|
||||
));
|
||||
}
|
||||
if path.len() > 4096 {
|
||||
return Err(GitError::InvalidArgument(
|
||||
"relative_path too long (max 4096 chars)".into(),
|
||||
));
|
||||
}
|
||||
if path.contains("..") {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"path traversal detected: relative_path contains '..': {path}"
|
||||
|
||||
+1
-1
@@ -77,7 +77,7 @@ impl archive_service_server::ArchiveService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.treeish) {
|
||||
cache::cached_response("archive.list_archive_entries", &inner, || {
|
||||
cache::cached_response("archive.list_archive_entries", &repo, &inner, || {
|
||||
gb.list_archive_entries(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
|
||||
+2
-2
@@ -43,7 +43,7 @@ impl blame_service_server::BlameService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("blame.blame", &inner, || {
|
||||
cache::cached_response("blame.blame", &repo, &inner, || {
|
||||
gb.blame(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -85,7 +85,7 @@ impl blame_service_server::BlameService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("blame.blame", &inner, || {
|
||||
cache::cached_response("blame.blame", &repo, &inner, || {
|
||||
gb.blame(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
|
||||
+267
-86
@@ -1,40 +1,165 @@
|
||||
//! In-memory response cache layer for GitKS.
|
||||
//!
|
||||
//! Two-tier architecture:
|
||||
//! 1. **Moka in-memory cache** (this module) — sub-microsecond lookups for hot data
|
||||
//! 2. **Disk cache** (disk_cache.rs) — persistent cache for pack-objects / info-refs
|
||||
//!
|
||||
//! # Cache Key Format
|
||||
//!
|
||||
//! Keys are structured to enable efficient repo-scoped invalidation:
|
||||
//!
|
||||
//! ```text
|
||||
//! [namespace_len: u8][namespace: &[u8]][repo_path_len: u16 LE][repo_path: &[u8]][request_proto: &[u8]]
|
||||
//! ```
|
||||
//!
|
||||
//! This allows `invalidate_repo` to extract and match the repo_path without
|
||||
//! protobuf decoding or substring scanning.
|
||||
//!
|
||||
//! # Eviction Policy
|
||||
//!
|
||||
//! - **Weight-based**: total memory capped at 256 MB (weighed by key+value capacity)
|
||||
//! - **TTI** (time-to-idle): 2 minutes — frequently accessed entries stay hot
|
||||
//! - **TTL** (time-to-live): 10 minutes — hard upper bound for safety
|
||||
//! - Evictions are tracked via metrics for observability
|
||||
|
||||
use std::sync::OnceLock;
|
||||
use std::time::Duration;
|
||||
|
||||
use moka::sync::Cache;
|
||||
use prost::Message;
|
||||
|
||||
use crate::pb::{ObjectSelector, object_selector};
|
||||
/// Maximum total cache weight (key + value allocated bytes): 256 MB.
|
||||
const CACHE_MAX_WEIGHT: u64 = 256 * 1024 * 1024;
|
||||
|
||||
const GLOBAL_CACHE_MAX: u64 = 65_536;
|
||||
const CACHE_TTL: Duration = Duration::from_secs(300);
|
||||
/// Hard time-to-live: entries older than this are unconditionally evicted.
|
||||
const CACHE_MAX_TTL: Duration = Duration::from_secs(600); // 10 min
|
||||
|
||||
static GLOBAL_CACHE: OnceLock<Cache<Vec<u8>, Vec<u8>>> = OnceLock::new();
|
||||
/// Time-to-idle: entries not accessed within this window are evicted.
|
||||
/// Frequently accessed entries survive up to TTL, cold entries expire quickly.
|
||||
const CACHE_TTI: Duration = Duration::from_secs(120); // 2 min
|
||||
|
||||
fn cache() -> &'static Cache<Vec<u8>, Vec<u8>> {
|
||||
GLOBAL_CACHE.get_or_init(|| {
|
||||
Cache::builder()
|
||||
.max_capacity(GLOBAL_CACHE_MAX)
|
||||
.time_to_live(CACHE_TTL)
|
||||
.build()
|
||||
/// Estimated per-entry overhead (Moka internal Arc + metadata).
|
||||
/// Added to the weigher result to prevent underestimation.
|
||||
const ENTRY_OVERHEAD: u32 = 128;
|
||||
|
||||
struct CacheState {
|
||||
store: Cache<Vec<u8>, Vec<u8>>,
|
||||
}
|
||||
|
||||
static CACHE: OnceLock<CacheState> = OnceLock::new();
|
||||
|
||||
fn state() -> &'static CacheState {
|
||||
CACHE.get_or_init(|| {
|
||||
let store = Cache::builder()
|
||||
.weigher(|key: &Vec<u8>, value: &Vec<u8>| -> u32 {
|
||||
// capacity() reflects actual allocation including spare capacity
|
||||
key.capacity() as u32 + value.capacity() as u32 + ENTRY_OVERHEAD
|
||||
})
|
||||
.max_capacity(CACHE_MAX_WEIGHT)
|
||||
.time_to_live(CACHE_MAX_TTL)
|
||||
.time_to_idle(CACHE_TTI)
|
||||
.eviction_listener(|key: std::sync::Arc<Vec<u8>>, _value: Vec<u8>, cause| {
|
||||
let cause_str = match cause {
|
||||
moka::notification::RemovalCause::Expired => "expired",
|
||||
moka::notification::RemovalCause::Explicit => "explicit",
|
||||
moka::notification::RemovalCause::Replaced => "replaced",
|
||||
moka::notification::RemovalCause::Size => "size",
|
||||
};
|
||||
// Extract namespace for per-namespace metrics
|
||||
let namespace = decode_namespace(&key);
|
||||
crate::metrics::record_cache_eviction(namespace, cause_str);
|
||||
})
|
||||
.build();
|
||||
|
||||
tracing::info!(
|
||||
max_weight_mb = CACHE_MAX_WEIGHT / (1024 * 1024),
|
||||
ttl_secs = CACHE_MAX_TTL.as_secs(),
|
||||
tti_secs = CACHE_TTI.as_secs(),
|
||||
"Moka in-memory cache initialized"
|
||||
);
|
||||
|
||||
CacheState { store }
|
||||
})
|
||||
}
|
||||
|
||||
fn cache_key<Req>(namespace: &str, request: &Req) -> Vec<u8>
|
||||
where
|
||||
Req: Message,
|
||||
{
|
||||
let mut key = Vec::with_capacity(namespace.len() + 1 + request.encoded_len());
|
||||
key.extend_from_slice(namespace.as_bytes());
|
||||
key.push(0);
|
||||
request
|
||||
.encode(&mut key)
|
||||
.expect("encoding a prost message into Vec cannot fail");
|
||||
key
|
||||
#[inline]
|
||||
fn cache() -> &'static Cache<Vec<u8>, Vec<u8>> {
|
||||
&state().store
|
||||
}
|
||||
|
||||
// Key encoding
|
||||
|
||||
/// Encode a structured cache key.
|
||||
///
|
||||
/// Format: `namespace_len(u8) + namespace + repo_path_len(u16 LE) + repo_path + request_proto`
|
||||
///
|
||||
fn encode_key(namespace: &str, repo_path: &str, request_bytes: &[u8]) -> Option<Vec<u8>> {
|
||||
let ns = namespace.as_bytes();
|
||||
let rp = repo_path.as_bytes();
|
||||
if ns.len() > u8::MAX as usize || rp.len() > u16::MAX as usize {
|
||||
tracing::warn!(
|
||||
namespace_len = ns.len(),
|
||||
repo_path_len = rp.len(),
|
||||
"cache key too long, bypassing cache"
|
||||
);
|
||||
return None;
|
||||
}
|
||||
|
||||
let total = 1 + ns.len() + 2 + rp.len() + request_bytes.len();
|
||||
let mut key = Vec::with_capacity(total);
|
||||
key.push(ns.len() as u8);
|
||||
key.extend_from_slice(ns);
|
||||
key.extend_from_slice(&(rp.len() as u16).to_le_bytes());
|
||||
key.extend_from_slice(rp);
|
||||
key.extend_from_slice(request_bytes);
|
||||
Some(key)
|
||||
}
|
||||
|
||||
/// Extract the namespace string from a cache key.
|
||||
fn decode_namespace(key: &[u8]) -> &str {
|
||||
if key.is_empty() {
|
||||
return "unknown";
|
||||
}
|
||||
let ns_len = key[0] as usize;
|
||||
let end = (1 + ns_len).min(key.len());
|
||||
std::str::from_utf8(&key[1..end]).unwrap_or("unknown")
|
||||
}
|
||||
|
||||
/// Extract the repo_path from a cache key (returns slice into the key).
|
||||
fn extract_repo_path_bytes(key: &[u8]) -> Option<&[u8]> {
|
||||
if key.len() < 3 {
|
||||
return None;
|
||||
}
|
||||
let ns_len = key[0] as usize;
|
||||
let rp_len_offset = 1 + ns_len;
|
||||
if key.len() < rp_len_offset + 2 {
|
||||
return None;
|
||||
}
|
||||
let rp_len = u16::from_le_bytes([key[rp_len_offset], key[rp_len_offset + 1]]) as usize;
|
||||
let rp_start = rp_len_offset + 2;
|
||||
let rp_end = rp_start.checked_add(rp_len)?;
|
||||
if rp_end > key.len() {
|
||||
return None;
|
||||
}
|
||||
Some(&key[rp_start..rp_end])
|
||||
}
|
||||
|
||||
/// Check if a cache key belongs to the given repository.
|
||||
fn key_matches_repo(key: &[u8], target_repo: &[u8]) -> bool {
|
||||
extract_repo_path_bytes(key).is_some_and(|rp| rp == target_repo)
|
||||
}
|
||||
|
||||
// Single-message cache
|
||||
|
||||
/// Cache a single protobuf response.
|
||||
///
|
||||
/// On cache hit, decodes and returns the cached response.
|
||||
/// On cache miss, calls `build`, caches the result, and returns it.
|
||||
///
|
||||
/// `repo_path` should be the repository's relative path (used for scoped invalidation).
|
||||
pub(crate) fn cached_response<Req, Res, E, F>(
|
||||
namespace: &'static str,
|
||||
repo_path: &str,
|
||||
request: &Req,
|
||||
build: F,
|
||||
) -> Result<Res, E>
|
||||
@@ -43,14 +168,21 @@ where
|
||||
Res: Message + Default,
|
||||
F: FnOnce() -> Result<Res, E>,
|
||||
{
|
||||
let key = cache_key(namespace, request);
|
||||
let req_bytes = encode_request(request);
|
||||
let Some(key) = encode_key(namespace, repo_path, &req_bytes) else {
|
||||
return build();
|
||||
};
|
||||
|
||||
if let Some(bytes) = cache().get(&key)
|
||||
&& let Ok(response) = Res::decode(bytes.as_slice())
|
||||
{
|
||||
let elapsed = std::time::Duration::ZERO; // Moka get is memory-only, effectively instant
|
||||
crate::metrics::record_cache_op("moka", "hit", elapsed);
|
||||
tracing::debug!(
|
||||
namespace = %namespace,
|
||||
repo = %repo_path,
|
||||
key_len = key.len(),
|
||||
value_len = bytes.len(),
|
||||
"cache hit"
|
||||
);
|
||||
return Ok(response);
|
||||
@@ -58,20 +190,41 @@ where
|
||||
|
||||
tracing::debug!(
|
||||
namespace = %namespace,
|
||||
repo = %repo_path,
|
||||
key_len = key.len(),
|
||||
"cache miss, building response"
|
||||
);
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let response = build()?;
|
||||
let build_elapsed = start.elapsed();
|
||||
|
||||
let mut bytes = Vec::with_capacity(response.encoded_len());
|
||||
response
|
||||
.encode(&mut bytes)
|
||||
.expect("encoding a prost message into Vec cannot fail");
|
||||
cache().insert(key, bytes);
|
||||
if let Err(err) = response.encode(&mut bytes) {
|
||||
tracing::warn!(
|
||||
namespace = %namespace,
|
||||
repo = %repo_path,
|
||||
error = %err,
|
||||
"failed to encode cache response"
|
||||
);
|
||||
} else {
|
||||
cache().insert(key, bytes);
|
||||
}
|
||||
|
||||
crate::metrics::record_cache_op("moka", "miss", build_elapsed);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
// Vec-message cache
|
||||
|
||||
/// Cache a `Vec<Item>` protobuf response using length-delimited encoding.
|
||||
///
|
||||
/// Each item is stored sequentially with length-delimited framing, allowing
|
||||
/// partial decode resilience: if any single item fails to decode, the entire
|
||||
/// entry is discarded and rebuilt.
|
||||
pub(crate) fn cached_vec_response<Req, Item, E, F>(
|
||||
namespace: &'static str,
|
||||
repo_path: &str,
|
||||
request: &Req,
|
||||
build: F,
|
||||
) -> Result<Vec<Item>, E>
|
||||
@@ -80,90 +233,125 @@ where
|
||||
Item: Message + Default,
|
||||
F: FnOnce() -> Result<Vec<Item>, E>,
|
||||
{
|
||||
let key = cache_key(namespace, request);
|
||||
let req_bytes = encode_request(request);
|
||||
let Some(key) = encode_key(namespace, repo_path, &req_bytes) else {
|
||||
return build();
|
||||
};
|
||||
|
||||
// Try cache hit
|
||||
if let Some(bytes) = cache().get(&key) {
|
||||
let mut remaining = bytes.as_slice();
|
||||
let mut items = Vec::new();
|
||||
let mut remaining = bytes.as_slice();
|
||||
let mut valid = true;
|
||||
while !remaining.is_empty() {
|
||||
match Item::decode_length_delimited(&mut remaining) {
|
||||
Ok(item) => items.push(item),
|
||||
Err(_) => {
|
||||
valid = false;
|
||||
break;
|
||||
|
||||
// Pre-allocate based on first size hint
|
||||
if let Ok(first) = Item::decode_length_delimited(&mut remaining) {
|
||||
items.push(first);
|
||||
while !remaining.is_empty() {
|
||||
match Item::decode_length_delimited(&mut remaining) {
|
||||
Ok(item) => items.push(item),
|
||||
Err(_) => {
|
||||
valid = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else if !remaining.is_empty() {
|
||||
valid = false;
|
||||
}
|
||||
|
||||
if valid {
|
||||
crate::metrics::record_cache_op("moka", "hit", std::time::Duration::ZERO);
|
||||
tracing::debug!(
|
||||
namespace = %namespace,
|
||||
key_len = key.len(),
|
||||
repo = %repo_path,
|
||||
item_count = items.len(),
|
||||
"vec cache hit"
|
||||
);
|
||||
return Ok(items);
|
||||
}
|
||||
|
||||
tracing::warn!(
|
||||
namespace = %namespace,
|
||||
repo = %repo_path,
|
||||
"vec cache decode failed, rebuilding"
|
||||
);
|
||||
// Invalidate the corrupt entry
|
||||
cache().invalidate(&key);
|
||||
}
|
||||
|
||||
tracing::debug!(
|
||||
namespace = %namespace,
|
||||
key_len = key.len(),
|
||||
repo = %repo_path,
|
||||
"vec cache miss, building response"
|
||||
);
|
||||
|
||||
let start = std::time::Instant::now();
|
||||
let response = build()?;
|
||||
let mut bytes = Vec::new();
|
||||
let build_elapsed = start.elapsed();
|
||||
|
||||
// Encode all items into a single buffer with length-delimited framing
|
||||
let total_est: usize = response
|
||||
.iter()
|
||||
.map(|item| item.encoded_len() + 10) // 10 = prost length-delimited overhead
|
||||
.sum();
|
||||
let mut bytes = Vec::with_capacity(total_est);
|
||||
let mut encode_ok = true;
|
||||
for item in &response {
|
||||
item.encode_length_delimited(&mut bytes)
|
||||
.expect("encoding a prost message into Vec cannot fail");
|
||||
if let Err(err) = item.encode_length_delimited(&mut bytes) {
|
||||
tracing::warn!(
|
||||
namespace = %namespace,
|
||||
repo = %repo_path,
|
||||
error = %err,
|
||||
"failed to encode vec cache item"
|
||||
);
|
||||
encode_ok = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
cache().insert(key, bytes);
|
||||
|
||||
if encode_ok {
|
||||
cache().insert(key, bytes);
|
||||
}
|
||||
crate::metrics::record_cache_op("moka", "miss", build_elapsed);
|
||||
Ok(response)
|
||||
}
|
||||
|
||||
/// Invalidate all cache entries related to a specific repository.
|
||||
/// Called when refs are updated (create branch, create commit, etc.)
|
||||
/// so that stale data is not served.
|
||||
// Request encoding helpers
|
||||
|
||||
/// Encode a protobuf request into a byte vector.
|
||||
#[inline]
|
||||
fn encode_request<Req: Message>(request: &Req) -> Vec<u8> {
|
||||
let mut buf = Vec::with_capacity(request.encoded_len());
|
||||
if let Err(err) = request.encode(&mut buf) {
|
||||
tracing::warn!(error = %err, "failed to encode cache request");
|
||||
}
|
||||
buf
|
||||
}
|
||||
|
||||
// Repository-scoped invalidation
|
||||
|
||||
/// Invalidate all cache entries for a specific repository.
|
||||
///
|
||||
/// Uses the structured key format to extract and match repository paths
|
||||
/// without protobuf decoding or substring scanning. O(n) where n is the
|
||||
/// number of cached entries, with O(1) per-key comparison.
|
||||
///
|
||||
/// Called by `notify_ref_update` after any mutator RPC (create commit,
|
||||
/// create branch, etc.) to prevent serving stale data.
|
||||
pub(crate) fn invalidate_repo(relative_path: &str) {
|
||||
let c = cache();
|
||||
let target = relative_path.as_bytes();
|
||||
let mut keys_to_remove: Vec<std::sync::Arc<Vec<u8>>> = Vec::with_capacity(64);
|
||||
|
||||
// Encode the relative_path to match how it appears in cache keys
|
||||
let target_path_bytes = relative_path.as_bytes();
|
||||
|
||||
// Remove all keys that reference this repository
|
||||
// Cache keys are: namespace\0 + prost-encoded request
|
||||
let keys_to_remove: Vec<std::sync::Arc<Vec<u8>>> = c
|
||||
.iter()
|
||||
.filter_map(|(key, _)| {
|
||||
// Find the null byte separator
|
||||
if let Some(null_pos) = key.iter().position(|&b| b == 0) {
|
||||
let encoded_request = &key[null_pos + 1..];
|
||||
|
||||
// Check if this encoded request contains the repository path
|
||||
// We use a sliding window to find the path bytes in the encoded protobuf
|
||||
// This is conservative but correct: we may invalidate slightly more than
|
||||
// necessary, but we won't miss any entries for this repository.
|
||||
//
|
||||
// The encoded protobuf format embeds string fields as length-prefixed data,
|
||||
// so the relative_path bytes should appear verbatim somewhere in the message.
|
||||
if contains_subslice(encoded_request, target_path_bytes) {
|
||||
return Some(key);
|
||||
}
|
||||
} else {
|
||||
// Malformed key without separator, remove it to be safe
|
||||
tracing::warn!("found cache key without null separator, removing");
|
||||
return Some(key);
|
||||
}
|
||||
None
|
||||
})
|
||||
.collect();
|
||||
for (key, _value) in c.iter() {
|
||||
if key_matches_repo(&key, target) {
|
||||
keys_to_remove.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
let removed = keys_to_remove.len();
|
||||
for key in keys_to_remove {
|
||||
for key in &keys_to_remove {
|
||||
c.invalidate(key.as_ref());
|
||||
}
|
||||
|
||||
@@ -176,20 +364,12 @@ pub(crate) fn invalidate_repo(relative_path: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if a byte slice contains a subslice
|
||||
fn contains_subslice(haystack: &[u8], needle: &[u8]) -> bool {
|
||||
if needle.is_empty() {
|
||||
return true;
|
||||
}
|
||||
if needle.len() > haystack.len() {
|
||||
return false;
|
||||
}
|
||||
// Selector helpers
|
||||
|
||||
haystack
|
||||
.windows(needle.len())
|
||||
.any(|window| window == needle)
|
||||
}
|
||||
use crate::pb::{ObjectSelector, object_selector};
|
||||
|
||||
/// Returns true if the selector is an OID-based reference.
|
||||
/// OID-based selectors are cacheable because they are immutable.
|
||||
pub(crate) fn selector_is_oid(selector: &Option<ObjectSelector>) -> bool {
|
||||
matches!(
|
||||
selector.as_ref().and_then(|s| s.selector.as_ref()),
|
||||
@@ -197,6 +377,7 @@ pub(crate) fn selector_is_oid(selector: &Option<ObjectSelector>) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
/// Returns true if both selectors are OID-based.
|
||||
pub(crate) fn selectors_are_oid(
|
||||
left: &Option<ObjectSelector>,
|
||||
right: &Option<ObjectSelector>,
|
||||
|
||||
+4
-4
@@ -39,7 +39,7 @@ impl commit_service_server::CommitService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if !inner.all && cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("commit.list_commits", &inner, || {
|
||||
cache::cached_response("commit.list_commits", &repo, &inner, || {
|
||||
gb.list_commits(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -78,7 +78,7 @@ impl commit_service_server::CommitService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("commit.get_commit", &inner, || {
|
||||
cache::cached_response("commit.get_commit", &repo, &inner, || {
|
||||
gb.get_commit(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -116,7 +116,7 @@ impl commit_service_server::CommitService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("commit.get_commit_ancestors", &inner, || {
|
||||
cache::cached_response("commit.get_commit_ancestors", &repo, &inner, || {
|
||||
gb.get_commit_ancestors(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -265,7 +265,7 @@ impl commit_service_server::CommitService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
|
||||
cache::cached_response("commit.compare_commits", &inner, || {
|
||||
cache::cached_response("commit.compare_commits", &repo, &inner, || {
|
||||
gb.compare_commits(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
|
||||
+4
-4
@@ -42,7 +42,7 @@ impl diff_service_server::DiffService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
|
||||
cache::cached_response("diff.get_diff", &inner, || {
|
||||
cache::cached_response("diff.get_diff", &repo, &inner, || {
|
||||
gb.get_diff(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -81,7 +81,7 @@ impl diff_service_server::DiffService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.commit) {
|
||||
cache::cached_response("diff.get_commit_diff", &inner, || {
|
||||
cache::cached_response("diff.get_commit_diff", &repo, &inner, || {
|
||||
gb.get_commit_diff(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -122,7 +122,7 @@ impl diff_service_server::DiffService for GitksService {
|
||||
}
|
||||
};
|
||||
let items = if cache::selectors_are_oid(&inner.base, &inner.head) {
|
||||
cache::cached_vec_response("diff.get_patch", &inner, || {
|
||||
cache::cached_vec_response("diff.get_patch", &repo, &inner, || {
|
||||
gb.get_patch(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -160,7 +160,7 @@ impl diff_service_server::DiffService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
|
||||
cache::cached_response("diff.get_diff_stats", &inner, || {
|
||||
cache::cached_response("diff.get_diff_stats", &repo, &inner, || {
|
||||
gb.get_diff_stats(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
|
||||
+8
-269
@@ -1,31 +1,12 @@
|
||||
/// Generate a `remote_<service>_client` helper function that resolves a repository
|
||||
/// route and returns a connected gRPC client for the given service.
|
||||
/// Single-machine mode: no cluster forwarding.
|
||||
macro_rules! remote_client {
|
||||
($fn_name:ident, $client:ty, $svc_label:literal) => {
|
||||
async fn $fn_name(
|
||||
svc: &super::GitksService,
|
||||
header: Option<&crate::pb::RepositoryHeader>,
|
||||
is_write: bool,
|
||||
_svc: &super::GitksService,
|
||||
_header: Option<&crate::pb::RepositoryHeader>,
|
||||
_is_write: bool,
|
||||
) -> Result<Option<$client>, tonic::Status> {
|
||||
let header = match header {
|
||||
Some(h) => h,
|
||||
None => return Ok(None),
|
||||
};
|
||||
let Some(route) = svc.route_repository(header, is_write).await? else {
|
||||
return Ok(None);
|
||||
};
|
||||
tracing::info!(
|
||||
storage_name = %route.storage_name,
|
||||
relative_path = %route.relative_path,
|
||||
actor_name = %route.actor_name,
|
||||
grpc_addr = %route.grpc_addr,
|
||||
concat!("forwarding ", $svc_label, " rpc")
|
||||
);
|
||||
let endpoint = super::remote_endpoint(&route.grpc_addr).await?;
|
||||
let client = <$client>::connect(endpoint)
|
||||
.await
|
||||
.map_err(|e| tonic::Status::unavailable(e.to_string()))?;
|
||||
Ok(Some(client))
|
||||
Ok(None)
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -45,14 +26,10 @@ mod repository_maint;
|
||||
mod tag;
|
||||
mod tree;
|
||||
|
||||
use dashmap::DashMap;
|
||||
use gix::discover::is_git;
|
||||
use ractor::{ActorCell, ActorRef};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
|
||||
use crate::actor::message::{GitNodeMessage, RouteDecision};
|
||||
use crate::bare::GitBare;
|
||||
use crate::error::{GitError, GitResult};
|
||||
use crate::pb::{
|
||||
@@ -61,45 +38,26 @@ use crate::pb::{
|
||||
remote_service_server, repository_service_server, tag_service_server, tree_service_server,
|
||||
};
|
||||
|
||||
/// TTL for route cache entries.
|
||||
const ROUTE_CACHE_TTL: Duration = Duration::from_secs(60); // 1 minute
|
||||
|
||||
/// A cached route entry with creation time.
|
||||
#[derive(Clone)]
|
||||
pub struct CachedRoute {
|
||||
pub decision: RouteDecision,
|
||||
pub created_at: Instant,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct GitksService {
|
||||
pub repo_prefix: PathBuf,
|
||||
pub node_actor: Option<ActorRef<GitNodeMessage>>,
|
||||
pub grpc_addr: String,
|
||||
pub disk_cache: Option<crate::disk_cache::DiskCache>,
|
||||
pub pack_cache: Option<crate::pack_cache::PackCache>,
|
||||
pub hook_manager: Option<crate::hooks::HookManager>,
|
||||
pub route_cache: DashMap<String, CachedRoute>,
|
||||
}
|
||||
|
||||
impl GitksService {
|
||||
pub fn new(repo_prefix: PathBuf) -> Self {
|
||||
Self {
|
||||
repo_prefix,
|
||||
node_actor: None,
|
||||
grpc_addr: String::new(),
|
||||
disk_cache: None,
|
||||
pack_cache: None,
|
||||
hook_manager: None,
|
||||
route_cache: DashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_actor(mut self, node_actor: ActorRef<GitNodeMessage>) -> Self {
|
||||
self.node_actor = Some(node_actor);
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_disk_cache(mut self, dc: crate::disk_cache::DiskCache) -> Self {
|
||||
self.disk_cache = Some(dc);
|
||||
self
|
||||
@@ -120,30 +78,6 @@ impl GitksService {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn cleanup_route_cache(&self) {
|
||||
let before = self.route_cache.len();
|
||||
self.route_cache
|
||||
.retain(|_key, cached| cached.created_at.elapsed() < ROUTE_CACHE_TTL);
|
||||
let removed = before - self.route_cache.len();
|
||||
if removed > 0 {
|
||||
tracing::debug!(
|
||||
removed,
|
||||
remaining = self.route_cache.len(),
|
||||
"route cache cleaned"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_route_cache_cleanup(svc: Self) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(120));
|
||||
loop {
|
||||
interval.tick().await;
|
||||
svc.cleanup_route_cache();
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scan_all_repo(&self) -> GitResult<Vec<String>> {
|
||||
let root = self.repo_prefix.as_ref();
|
||||
let mut repos = Vec::new();
|
||||
@@ -157,82 +91,6 @@ impl GitksService {
|
||||
.filter_map(|path| path.to_str().map(str::to_owned))
|
||||
.collect())
|
||||
}
|
||||
pub async fn route_repository(
|
||||
&self,
|
||||
header: &crate::pb::RepositoryHeader,
|
||||
is_write: bool,
|
||||
) -> Result<Option<RouteDecision>, tonic::Status> {
|
||||
use crate::actor::message::{ROLE_PRIMARY, ROLE_REPLICA};
|
||||
|
||||
// Check route cache for read requests
|
||||
if !is_write
|
||||
&& let Some(cached) = self.route_cache.get(&header.relative_path)
|
||||
&& !cached.decision.grpc_addr.is_empty()
|
||||
&& cached.decision.found
|
||||
&& cached.created_at.elapsed() < ROUTE_CACHE_TTL
|
||||
{
|
||||
tracing::debug!(
|
||||
relative_path = %header.relative_path,
|
||||
grpc_addr = %cached.decision.grpc_addr,
|
||||
"route cache hit"
|
||||
);
|
||||
return Ok(Some(cached.decision.clone()));
|
||||
}
|
||||
|
||||
let members = ractor::pg::get_members(&"gitks_nodes".to_string());
|
||||
let local = self.node_actor.as_ref().map(|actor| actor.get_cell());
|
||||
let mut primary: Option<RouteDecision> = None;
|
||||
let mut replica: Option<RouteDecision> = None;
|
||||
for member in members {
|
||||
if local.as_ref().is_some_and(|actor| actor == &member) {
|
||||
continue;
|
||||
}
|
||||
if let Some(decision) = query_find_primary(member.clone(), header.clone()).await?
|
||||
&& decision.found
|
||||
&& !decision.grpc_addr.is_empty()
|
||||
{
|
||||
primary = Some(decision);
|
||||
if is_write {
|
||||
return Ok(primary);
|
||||
}
|
||||
}
|
||||
if !is_write
|
||||
&& replica.is_none()
|
||||
&& let Some(decision) = query_find_replica(member.clone(), header.clone()).await?
|
||||
&& decision.found
|
||||
&& !decision.grpc_addr.is_empty()
|
||||
&& decision.role == ROLE_REPLICA
|
||||
{
|
||||
replica = Some(decision);
|
||||
}
|
||||
}
|
||||
let result = if let Some(p) = primary {
|
||||
Some(p)
|
||||
} else if let Some(r) = replica {
|
||||
tracing::info!(
|
||||
storage_name = %r.storage_name,
|
||||
relative_path = %r.relative_path,
|
||||
"read request routed to replica"
|
||||
);
|
||||
Some(r)
|
||||
} else {
|
||||
let _ = ROLE_PRIMARY;
|
||||
None
|
||||
};
|
||||
|
||||
// Cache result for read requests
|
||||
if let Some(ref decision) = result {
|
||||
self.route_cache.insert(
|
||||
header.relative_path.clone(),
|
||||
CachedRoute {
|
||||
decision: decision.clone(),
|
||||
created_at: Instant::now(),
|
||||
},
|
||||
);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn repo_label(&self, header: Option<&crate::pb::RepositoryHeader>) -> String {
|
||||
header
|
||||
.and_then(|h| {
|
||||
@@ -349,101 +207,17 @@ impl GitksService {
|
||||
pub fn notify_ref_update(
|
||||
&self,
|
||||
relative_path: &str,
|
||||
ref_name: &str,
|
||||
old_oid: &str,
|
||||
new_oid: &str,
|
||||
_ref_name: &str,
|
||||
_old_oid: &str,
|
||||
_new_oid: &str,
|
||||
) {
|
||||
// Invalidate moka caches
|
||||
crate::server::cache::invalidate_repo(relative_path);
|
||||
|
||||
// Invalidate route cache
|
||||
self.route_cache.remove(relative_path);
|
||||
|
||||
// Invalidate disk cache
|
||||
if let Some(ref pc) = self.pack_cache {
|
||||
pc.invalidate_repo(relative_path);
|
||||
}
|
||||
|
||||
if let Some(ref actor) = self.node_actor {
|
||||
let event = crate::actor::message::RefUpdateEvent {
|
||||
relative_path: relative_path.to_string(),
|
||||
ref_name: ref_name.to_string(),
|
||||
old_oid: old_oid.to_string(),
|
||||
new_oid: new_oid.to_string(),
|
||||
primary_grpc_addr: self.grpc_addr.clone(),
|
||||
primary_storage_name: String::new(),
|
||||
};
|
||||
crate::actor::handler::broadcast_ref_update(actor, event);
|
||||
}
|
||||
}
|
||||
|
||||
/// Submit a write command through Raft consensus.
|
||||
/// This method:
|
||||
/// 1. Checks if this node is the Leader (via leader lease)
|
||||
/// 2. Creates a LogEntry with the command
|
||||
/// 3. Appends to local raft_log
|
||||
/// 4. Broadcasts AppendEntries to all followers
|
||||
/// 5. Waits for majority ACK (10 second timeout)
|
||||
/// 6. Advances commit_index and applies the command
|
||||
///
|
||||
/// Returns Ok(()) on success, or an error if consensus fails.
|
||||
pub async fn raft_consensus_write(
|
||||
&self,
|
||||
command: crate::actor::raft_log::Command,
|
||||
) -> Result<(), tonic::Status> {
|
||||
let actor = self
|
||||
.node_actor
|
||||
.as_ref()
|
||||
.ok_or_else(|| tonic::Status::failed_precondition("node actor not initialized"))?;
|
||||
|
||||
// Send the command to the actor for Raft processing
|
||||
let result = ractor::call_t!(
|
||||
actor,
|
||||
GitNodeMessage::RaftWrite,
|
||||
10000, // 10 second timeout
|
||||
command
|
||||
);
|
||||
|
||||
match result {
|
||||
Ok(success) => {
|
||||
if success {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(tonic::Status::aborted(
|
||||
"Raft consensus failed: not leader or timeout",
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(e) => Err(tonic::Status::internal(format!("Raft write error: {e}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Perform a ReadIndex check to ensure this node can serve consistent reads.
|
||||
/// This confirms the Leader is still valid before reading from local state.
|
||||
pub async fn raft_read_index(&self) -> Result<(), tonic::Status> {
|
||||
let actor = self
|
||||
.node_actor
|
||||
.as_ref()
|
||||
.ok_or_else(|| tonic::Status::failed_precondition("node actor not initialized"))?;
|
||||
|
||||
let request = crate::actor::message::ReadIndexRequest {
|
||||
relative_path: String::new(),
|
||||
};
|
||||
|
||||
let result = ractor::call_t!(actor, GitNodeMessage::ReadIndex, 5000, request);
|
||||
|
||||
match result {
|
||||
Ok(response) => {
|
||||
if response.is_leader {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(tonic::Status::failed_precondition(
|
||||
"not leader, cannot serve consistent read",
|
||||
))
|
||||
}
|
||||
}
|
||||
Err(e) => Err(tonic::Status::internal(format!("ReadIndex error: {e}"))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Inject repo_prefix as storage_path into the client-provided header
|
||||
@@ -456,13 +230,6 @@ impl GitksService {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn remote_endpoint(addr: &str) -> Result<tonic::transport::Endpoint, tonic::Status> {
|
||||
let uri: tonic::codegen::http::Uri = addr
|
||||
.parse()
|
||||
.map_err(|e| tonic::Status::invalid_argument(format!("invalid URI: {e}")))?;
|
||||
tonic::transport::Endpoint::new(uri).map_err(|e| tonic::Status::internal(e.to_string()))
|
||||
}
|
||||
|
||||
pub(super) fn bridge_server_stream<T: Send + 'static>(
|
||||
mut remote: tonic::Streaming<T>,
|
||||
) -> tokio_stream::wrappers::ReceiverStream<Result<T, tonic::Status>> {
|
||||
@@ -478,34 +245,6 @@ pub(super) fn bridge_server_stream<T: Send + 'static>(
|
||||
tokio_stream::wrappers::ReceiverStream::new(rx)
|
||||
}
|
||||
|
||||
async fn query_find_primary(
|
||||
member: ActorCell,
|
||||
header: crate::pb::RepositoryHeader,
|
||||
) -> Result<Option<RouteDecision>, tonic::Status> {
|
||||
let actor_ref: ActorRef<GitNodeMessage> = member.into();
|
||||
match ractor::call_t!(actor_ref, GitNodeMessage::FindPrimary, 500, header) {
|
||||
Ok(decision) => Ok(Some(decision)),
|
||||
Err(err) => {
|
||||
tracing::warn!(error = %err, "find primary query failed");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn query_find_replica(
|
||||
member: ActorCell,
|
||||
header: crate::pb::RepositoryHeader,
|
||||
) -> Result<Option<RouteDecision>, tonic::Status> {
|
||||
let actor_ref: ActorRef<GitNodeMessage> = member.into();
|
||||
match ractor::call_t!(actor_ref, GitNodeMessage::FindReplica, 500, header) {
|
||||
Ok(decision) => Ok(Some(decision)),
|
||||
Err(err) => {
|
||||
tracing::warn!(error = %err, "find replica query failed");
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn scan_bare_repos_recursively(dir: &Path, repos: &mut Vec<PathBuf>) -> GitResult<()> {
|
||||
for entry in std::fs::read_dir(dir)? {
|
||||
let entry = entry?;
|
||||
|
||||
+36
-35
@@ -13,6 +13,8 @@ remote_client!(
|
||||
"pack"
|
||||
);
|
||||
|
||||
const MAX_INDEX_PACK_BUFFER_BYTES: usize = 512 * 1024 * 1024;
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl pack_service_server::PackService for GitksService {
|
||||
type UploadPackStream = CancellableReceiverStream<Result<UploadPackResponse, tonic::Status>>;
|
||||
@@ -276,43 +278,33 @@ impl pack_service_server::PackService for GitksService {
|
||||
m.record("ok");
|
||||
let (tx, rx) = tokio::sync::mpsc::channel(16);
|
||||
tokio::spawn(async move {
|
||||
let result = tokio::task::spawn_blocking(move || {
|
||||
use std::io::Read;
|
||||
let mut file = file;
|
||||
let mut buf = vec![0u8; 65536];
|
||||
let mut chunks = Vec::new();
|
||||
loop {
|
||||
match file.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => chunks.push(Ok(PackfileChunk {
|
||||
data: buf[..n].to_vec(),
|
||||
})),
|
||||
Err(e) => {
|
||||
chunks.push(Err(tonic::Status::internal(format!(
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
let mut file = tokio::fs::File::from_std(file);
|
||||
let mut buf = vec![0u8; 65536];
|
||||
loop {
|
||||
match file.read(&mut buf).await {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
if tx
|
||||
.send(Ok(PackfileChunk {
|
||||
data: buf[..n].to_vec(),
|
||||
}))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Err(tonic::Status::internal(format!(
|
||||
"cache read error: {e}"
|
||||
))));
|
||||
break;
|
||||
}
|
||||
))))
|
||||
.await;
|
||||
break;
|
||||
}
|
||||
}
|
||||
chunks
|
||||
})
|
||||
.await;
|
||||
match result {
|
||||
Ok(chunks) => {
|
||||
for chunk in chunks {
|
||||
if tx.send(chunk).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = tx
|
||||
.send(Err(tonic::Status::internal(format!(
|
||||
"cache read task failed: {e}"
|
||||
))))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
return Ok(tonic::Response::new(ReceiverStream::new(rx)));
|
||||
@@ -340,8 +332,17 @@ impl pack_service_server::PackService for GitksService {
|
||||
let m = crate::metrics::RequestMetrics::new("gitks.PackService/IndexPack");
|
||||
let mut stream = request.into_inner();
|
||||
let mut inputs = Vec::new();
|
||||
let mut total_bytes = 0usize;
|
||||
while let Some(msg) = stream.next().await {
|
||||
inputs.push(msg?);
|
||||
let msg = msg?;
|
||||
total_bytes = total_bytes.saturating_add(msg.data.len());
|
||||
if total_bytes > MAX_INDEX_PACK_BUFFER_BYTES {
|
||||
return Err(tonic::Status::resource_exhausted(format!(
|
||||
"index-pack input too large (max {} bytes)",
|
||||
MAX_INDEX_PACK_BUFFER_BYTES
|
||||
)));
|
||||
}
|
||||
inputs.push(msg);
|
||||
}
|
||||
let _rate = self
|
||||
.acquire_rate_limit(
|
||||
|
||||
@@ -567,9 +567,11 @@ impl repository_service_server::RepositoryService for GitksService {
|
||||
.list_snapshots(&repo)
|
||||
.map_err(tonic::Status::internal)?;
|
||||
|
||||
let limit = (inner.limit > 0).then_some(inner.limit as usize);
|
||||
let resp = ListSnapshotsResponse {
|
||||
snapshots: snapshots
|
||||
.into_iter()
|
||||
.take(limit.unwrap_or(usize::MAX))
|
||||
.map(|s| crate::pb::SnapshotInfo {
|
||||
snapshot_id: s.snapshot_id,
|
||||
relative_path: s.relative_path,
|
||||
@@ -678,7 +680,7 @@ impl repository_service_server::RepositoryService for GitksService {
|
||||
return;
|
||||
}
|
||||
for offset in (0..total).step_by(CHUNK_SIZE) {
|
||||
let end = (offset + CHUNK_SIZE).min(total);
|
||||
let end = offset.saturating_add(CHUNK_SIZE).min(total);
|
||||
let chunk_data = bundle_data[offset..end].to_vec();
|
||||
let is_done = end >= total;
|
||||
if tx
|
||||
|
||||
+7
-7
@@ -42,7 +42,7 @@ impl tree_service_server::TreeService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("tree.list_tree", &inner, || {
|
||||
cache::cached_response("tree.list_tree", &repo, &inner, || {
|
||||
gb.list_tree(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -81,7 +81,7 @@ impl tree_service_server::TreeService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("tree.get_tree", &inner, || {
|
||||
cache::cached_response("tree.get_tree", &repo, &inner, || {
|
||||
gb.get_tree(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -120,7 +120,7 @@ impl tree_service_server::TreeService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("tree.get_blob", &inner, || {
|
||||
cache::cached_response("tree.get_blob", &repo, &inner, || {
|
||||
gb.get_blob(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -160,11 +160,11 @@ impl tree_service_server::TreeService for GitksService {
|
||||
}
|
||||
};
|
||||
let items = if inner.oid.is_some() {
|
||||
cache::cached_vec_response("tree.get_raw_blob", &inner, || {
|
||||
cache::cached_vec_response("tree.get_raw_blob", &repo, &inner, || {
|
||||
gb.get_raw_blob(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_vec_response("tree.get_raw_blob", &inner, || {
|
||||
cache::cached_vec_response("tree.get_raw_blob", &repo, &inner, || {
|
||||
gb.get_raw_blob(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -202,7 +202,7 @@ impl tree_service_server::TreeService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("tree.get_file_metadata", &inner, || {
|
||||
cache::cached_response("tree.get_file_metadata", &repo, &inner, || {
|
||||
gb.get_file_metadata(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
@@ -240,7 +240,7 @@ impl tree_service_server::TreeService for GitksService {
|
||||
}
|
||||
};
|
||||
let resp = if cache::selector_is_oid(&inner.revision) {
|
||||
cache::cached_response("tree.find_files", &inner, || {
|
||||
cache::cached_response("tree.find_files", &repo, &inner, || {
|
||||
gb.find_files(inner.clone()).map_err(into_status)
|
||||
})?
|
||||
} else {
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
pub mod ops;
|
||||
pub mod storage;
|
||||
pub mod sync;
|
||||
|
||||
pub use ops::{create_snapshot, restore_snapshot, verify_snapshot};
|
||||
pub use storage::{LocalSnapshotStorage, SnapshotInfo, SnapshotStorageBackend};
|
||||
|
||||
+2
-2
@@ -81,7 +81,7 @@ pub fn create_and_store_snapshot(
|
||||
pub fn restore_snapshot(repo_path: &Path, data: &[u8]) -> GitResult<()> {
|
||||
tracing::info!(path = %repo_path.display(), size_bytes = data.len(), "restoring snapshot");
|
||||
|
||||
let applicator = crate::actor::sync::BundleApplicator::new(repo_path.to_path_buf());
|
||||
let applicator = crate::snapshot::sync::BundleApplicator::new(repo_path.to_path_buf());
|
||||
applicator.apply_bundle(data).map_err(GitError::Internal)?;
|
||||
|
||||
tracing::info!(path = %repo_path.display(), "snapshot restored");
|
||||
@@ -163,7 +163,7 @@ fn generate_snapshot_id(relative_path: &str, head_oid: &str) -> String {
|
||||
let mut s = String::with_capacity(16);
|
||||
for byte in &hash[..8] {
|
||||
use std::fmt::Write;
|
||||
write!(s, "{byte:02x}").unwrap();
|
||||
let _ = write!(s, "{byte:02x}");
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
+18
-1
@@ -40,8 +40,22 @@ impl LocalSnapshotStorage {
|
||||
Self { base_dir }
|
||||
}
|
||||
|
||||
fn validate_snapshot_id(snapshot_id: &str) -> Result<(), String> {
|
||||
if snapshot_id.is_empty() {
|
||||
return Err("snapshot_id cannot be empty".into());
|
||||
}
|
||||
if snapshot_id.len() > 64
|
||||
|| !snapshot_id
|
||||
.chars()
|
||||
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
|
||||
{
|
||||
return Err(format!("invalid snapshot_id: {snapshot_id}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn snapshot_dir(&self, snapshot_id: &str) -> PathBuf {
|
||||
let prefix = &snapshot_id[..2.min(snapshot_id.len())];
|
||||
let prefix = snapshot_id.get(..2).unwrap_or(snapshot_id);
|
||||
self.base_dir.join(prefix).join(snapshot_id)
|
||||
}
|
||||
|
||||
@@ -62,6 +76,7 @@ impl SnapshotStorageBackend for LocalSnapshotStorage {
|
||||
head_oid: &str,
|
||||
data: &[u8],
|
||||
) -> Result<(), String> {
|
||||
Self::validate_snapshot_id(snapshot_id)?;
|
||||
let dir = self.snapshot_dir(snapshot_id);
|
||||
std::fs::create_dir_all(&dir).map_err(|e| format!("create dir: {e}"))?;
|
||||
|
||||
@@ -95,6 +110,7 @@ impl SnapshotStorageBackend for LocalSnapshotStorage {
|
||||
}
|
||||
|
||||
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String> {
|
||||
Self::validate_snapshot_id(snapshot_id)?;
|
||||
let path = self.data_path(snapshot_id);
|
||||
if !path.exists() {
|
||||
return Err(format!("snapshot not found: {snapshot_id}"));
|
||||
@@ -155,6 +171,7 @@ impl SnapshotStorageBackend for LocalSnapshotStorage {
|
||||
}
|
||||
|
||||
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String> {
|
||||
Self::validate_snapshot_id(snapshot_id)?;
|
||||
let dir = self.snapshot_dir(snapshot_id);
|
||||
if !dir.exists() {
|
||||
return Err(format!("snapshot not found: {snapshot_id}"));
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
//! Bundle applicator for restoring snapshots to git repositories.
|
||||
//!
|
||||
//! Uses `git bundle unbundle` to apply pack data to a bare repository.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
pub struct BundleApplicator {
|
||||
pub repo_path: PathBuf,
|
||||
}
|
||||
|
||||
impl BundleApplicator {
|
||||
pub fn new(repo_path: PathBuf) -> Self {
|
||||
Self { repo_path }
|
||||
}
|
||||
|
||||
pub fn apply_bundle(&self, data: &[u8]) -> Result<(), String> {
|
||||
let mut child = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.repo_path.to_string_lossy(),
|
||||
"bundle",
|
||||
"unbundle",
|
||||
"-",
|
||||
])
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|e| format!("spawn git bundle unbundle: {e}"))?;
|
||||
use std::io::Write;
|
||||
if let Some(ref mut stdin) = child.stdin {
|
||||
stdin
|
||||
.write_all(data)
|
||||
.map_err(|e| format!("write bundle: {e}"))?;
|
||||
}
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("wait bundle: {e}"))?;
|
||||
if !output.status.success() {
|
||||
return Err(String::from_utf8_lossy(&output.stderr).into_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Apply bundle from a file path (for streaming writes).
|
||||
pub fn apply_bundle_from_file(&self, path: &Path) -> Result<(), String> {
|
||||
let file = std::fs::File::open(path).map_err(|e| format!("open bundle file: {e}"))?;
|
||||
let mut child = std::process::Command::new("git")
|
||||
.args([
|
||||
"--git-dir",
|
||||
&self.repo_path.to_string_lossy(),
|
||||
"bundle",
|
||||
"unbundle",
|
||||
"-",
|
||||
])
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.map_err(|e| format!("spawn git bundle unbundle: {e}"))?;
|
||||
|
||||
// Stream file contents to stdin in a background thread
|
||||
let mut stdin = child.stdin.take().ok_or("no stdin")?;
|
||||
let file_handle = file;
|
||||
let writer = std::thread::spawn(move || -> Result<(), String> {
|
||||
use std::io::{Read, Write};
|
||||
let mut reader = std::io::BufReader::new(file_handle);
|
||||
let mut buf = vec![0u8; 65536];
|
||||
loop {
|
||||
match reader.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
stdin
|
||||
.write_all(&buf[..n])
|
||||
.map_err(|e| format!("write to stdin: {e}"))?;
|
||||
}
|
||||
Err(e) => return Err(format!("read bundle file: {e}")),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
});
|
||||
|
||||
let output = child
|
||||
.wait_with_output()
|
||||
.map_err(|e| format!("wait bundle: {e}"))?;
|
||||
|
||||
// Wait for writer thread
|
||||
let _ = writer.join().map_err(|_| "writer thread panicked")?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(String::from_utf8_lossy(&output.stderr).into_owned());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
+4
-1
@@ -6,7 +6,10 @@ impl GitBare {
|
||||
pub fn create_tag(&self, request: CreateTagRequest) -> GitResult<Tag> {
|
||||
crate::sanitize::validate_ref_name(&request.name)?;
|
||||
let target = match request.target.and_then(|s| s.selector) {
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(crate::pb::object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(crate::pb::object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
|
||||
@@ -4,6 +4,7 @@ use crate::pb::DeleteTagRequest;
|
||||
|
||||
impl GitBare {
|
||||
pub fn delete_tag(&self, request: DeleteTagRequest) -> GitResult<()> {
|
||||
crate::sanitize::validate_ref_name(&request.name)?;
|
||||
let result = duct::cmd(
|
||||
"git",
|
||||
[
|
||||
|
||||
@@ -6,6 +6,7 @@ use crate::pb::{GetTagRequest, ObjectType, Tag};
|
||||
|
||||
impl GitBare {
|
||||
pub fn get_tag(&self, request: GetTagRequest) -> GitResult<Tag> {
|
||||
crate::sanitize::validate_ref_name(&request.name)?;
|
||||
let repo = self.gix_repo()?;
|
||||
let refname = format!("refs/tags/{}", request.name);
|
||||
let mut r = repo.find_reference(refname.as_str())?;
|
||||
|
||||
@@ -4,6 +4,7 @@ use crate::pb::{VerifiedSignature, VerifyTagRequest};
|
||||
|
||||
impl GitBare {
|
||||
pub fn verify_tag(&self, request: VerifyTagRequest) -> GitResult<VerifiedSignature> {
|
||||
crate::sanitize::validate_ref_name(&request.name)?;
|
||||
let result = duct::cmd(
|
||||
"git",
|
||||
[
|
||||
|
||||
@@ -1,89 +0,0 @@
|
||||
#[cfg(test)]
|
||||
mod cluster_test {
|
||||
use gitks::pb::{
|
||||
CreateBranchRequest, GetRepositoryRequest, InitRepositoryRequest, ObjectName,
|
||||
ObjectSelector, RepositoryHeader, branch_service_client::BranchServiceClient,
|
||||
object_selector, repository_service_client::RepositoryServiceClient,
|
||||
};
|
||||
|
||||
const N1: &str = "http://localhost:50051";
|
||||
const N2: &str = "http://localhost:50052";
|
||||
const N3: &str = "http://localhost:50053";
|
||||
|
||||
fn hdr(path: &str) -> RepositoryHeader {
|
||||
RepositoryHeader {
|
||||
storage_name: String::new(),
|
||||
relative_path: path.into(),
|
||||
storage_path: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cluster_routing() {
|
||||
let ts = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_secs();
|
||||
let repo = format!("cluster-test-{ts}");
|
||||
|
||||
// ── Init via node1 ──
|
||||
let mut n1 = RepositoryServiceClient::connect(N1).await.unwrap();
|
||||
let r = n1
|
||||
.init_repository(tonic::Request::new(InitRepositoryRequest {
|
||||
repository: Some(hdr(&repo)),
|
||||
bare: true,
|
||||
object_format: 0,
|
||||
initial_branch: "main".into(),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
println!("✅ n1 init: bare={}", r.bare);
|
||||
|
||||
// ── Read via node2 (should forward to PRIMARY n1) ──
|
||||
let mut n2 = RepositoryServiceClient::connect(N2).await.unwrap();
|
||||
let r2 = n2
|
||||
.get_repository(tonic::Request::new(GetRepositoryRequest {
|
||||
repository: Some(hdr(&repo)),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
println!("✅ n2 get routed→primary: bare={}", r2.bare);
|
||||
|
||||
// ── Read via node3 ──
|
||||
let mut n3 = RepositoryServiceClient::connect(N3).await.unwrap();
|
||||
let r3 = n3
|
||||
.get_repository(tonic::Request::new(GetRepositoryRequest {
|
||||
repository: Some(hdr(&repo)),
|
||||
}))
|
||||
.await
|
||||
.unwrap()
|
||||
.into_inner();
|
||||
println!("✅ n3 get routed→primary: bare={}", r3.bare);
|
||||
|
||||
// ── Write (create branch) via node2 → primary ──
|
||||
let mut n2b = BranchServiceClient::connect(N2).await.unwrap();
|
||||
let b = n2b
|
||||
.create_branch(tonic::Request::new(CreateBranchRequest {
|
||||
repository: Some(hdr(&repo)),
|
||||
name: "feature/x".into(),
|
||||
start_point: Some(ObjectSelector {
|
||||
selector: Some(object_selector::Selector::Revision(ObjectName {
|
||||
revision: "main".into(),
|
||||
})),
|
||||
}),
|
||||
force: false,
|
||||
}))
|
||||
.await;
|
||||
match b {
|
||||
Ok(branch) => println!(
|
||||
"✅ n2 create-branch routed→primary: name={}",
|
||||
branch.into_inner().name
|
||||
),
|
||||
Err(e) => println!("⚠️ create-branch: {e} (expected — empty repo has no commits)"),
|
||||
}
|
||||
|
||||
println!("\n🎉 Cluster routing verified: init/read/write all proxied to PRIMARY");
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,12 @@ fn test_hex_to_bytes_invalid_hex() {
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hex_to_bytes_non_ascii_does_not_panic() {
|
||||
let result = hex_to_bytes("aéx");
|
||||
assert!(result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hex_to_bytes_with_whitespace() {
|
||||
let bytes = hex_to_bytes(" abcd ").unwrap();
|
||||
|
||||
@@ -140,3 +140,17 @@ fn test_local_snapshot_storage_delete() {
|
||||
// Delete non-existent
|
||||
assert!(storage.delete_snapshot("nonexistent").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_local_snapshot_storage_rejects_traversal_id() {
|
||||
let dir = tempfile::tempdir().unwrap().path().to_path_buf();
|
||||
let storage = LocalSnapshotStorage::new(dir);
|
||||
|
||||
assert!(storage.read_snapshot("../escape").is_err());
|
||||
assert!(storage.delete_snapshot("../escape").is_err());
|
||||
assert!(
|
||||
storage
|
||||
.write_snapshot("../escape", "repo.git", "abc123", b"data")
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ use crate::tree;
|
||||
|
||||
impl GitBare {
|
||||
pub fn get_file_metadata(&self, request: GetFileMetadataRequest) -> GitResult<FileMetadata> {
|
||||
crate::sanitize::validate_file_path(&request.path)?;
|
||||
let repo = self.gix_repo()?;
|
||||
let revision = resolve_revision!(request.revision);
|
||||
let tree = repo
|
||||
|
||||
+18
-2
@@ -5,11 +5,16 @@ use crate::error::{GitError, GitResult};
|
||||
use crate::paginate;
|
||||
use crate::pb::{ListTreeRequest, ListTreeResponse, TreeEntry, object_selector, tree_entry};
|
||||
|
||||
const MAX_RECURSIVE_TREE_DEPTH: usize = 256;
|
||||
|
||||
impl GitBare {
|
||||
pub fn list_tree(&self, request: ListTreeRequest) -> GitResult<ListTreeResponse> {
|
||||
let repo = self.gix_repo()?;
|
||||
let revision = match request.revision.clone().and_then(|s| s.selector) {
|
||||
Some(object_selector::Selector::Oid(oid)) => oid.hex,
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
oid.hex
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
name.revision
|
||||
@@ -22,6 +27,17 @@ impl GitBare {
|
||||
.try_into_tree()
|
||||
.map_err(|e| GitError::Gix(e.to_string()))?;
|
||||
if !request.path.is_empty() {
|
||||
crate::sanitize::validate_file_path(&request.path)?;
|
||||
let depth = request
|
||||
.path
|
||||
.split('/')
|
||||
.filter(|part| !part.is_empty())
|
||||
.count();
|
||||
if depth > MAX_RECURSIVE_TREE_DEPTH {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"tree depth exceeds maximum of {MAX_RECURSIVE_TREE_DEPTH}"
|
||||
)));
|
||||
}
|
||||
let entry = tree
|
||||
.lookup_entry_by_path(&request.path)?
|
||||
.ok_or_else(|| GitError::NotFound(request.path.clone()))?;
|
||||
@@ -43,6 +59,7 @@ impl GitBare {
|
||||
};
|
||||
let kind = entry.kind();
|
||||
let hex = entry.id().to_string();
|
||||
let child_path = path.clone();
|
||||
entries.push(TreeEntry {
|
||||
name,
|
||||
path,
|
||||
@@ -55,7 +72,6 @@ impl GitBare {
|
||||
});
|
||||
|
||||
if request.recursive && matches!(kind, EntryKind::Tree) {
|
||||
let child_path = entries.last().unwrap().path.clone();
|
||||
let child = self.list_tree(ListTreeRequest {
|
||||
repository: request.repository.clone(),
|
||||
revision: request.revision.clone(),
|
||||
|
||||
+4
-1
@@ -10,7 +10,10 @@ pub(crate) fn resolve_revision(
|
||||
sel: &Option<pb::ObjectSelector>,
|
||||
) -> Result<String, crate::error::GitError> {
|
||||
match sel.as_ref().and_then(|s| s.selector.as_ref()) {
|
||||
Some(object_selector::Selector::Oid(oid)) => Ok(oid.hex.clone()),
|
||||
Some(object_selector::Selector::Oid(oid)) => {
|
||||
crate::sanitize::validate_oid_hex(&oid.hex)?;
|
||||
Ok(oid.hex.clone())
|
||||
}
|
||||
Some(object_selector::Selector::Revision(name)) => {
|
||||
crate::sanitize::validate_revision(&name.revision)?;
|
||||
Ok(name.revision.clone())
|
||||
|
||||
Reference in New Issue
Block a user