From 8f472a044311583eeb246759c2bc5a7e892f49a2 Mon Sep 17 00:00:00 2001 From: zhenyi <434836402@qq.com> Date: Mon, 8 Jun 2026 14:31:29 +0800 Subject: [PATCH] feat(cluster): implement distributed clustering with etcd coordination - Integrate etcd-client for distributed coordination and leader election - Add remote client macros with proper formatting for all services - Implement RequestMetrics for tracking RPC performance and errors - Add rate limiting mechanism across all service endpoints - Create ElectionRequest and ElectionResult message types for leader election - Add role management with primary/replica switching capabilities - Implement health checker with automatic failover detection - Add repository count metrics for cluster monitoring - Update Cargo.toml with etcd-client and dashmap dependencies - Modify RepoEntry to include read_only flag for replica handling - Implement should_accept_election logic to prevent duplicate elections - Add RoleChangedEvent handling for cluster role updates --- Cargo.lock | 24 ++ Cargo.toml | 4 + actor/handler.rs | 226 +++++++++++++++- actor/message.rs | 112 +++++++- actor/mod.rs | 8 +- cluster/discovery.rs | 232 +++++++++++++++++ cluster/mod.rs | 214 ++++++++++++++++ cluster/types.rs | 14 + disk_cache.rs | 538 +++++++++++++++++++++++++++++++++++++++ hooks/manager.rs | 238 +++++++++++++++++ hooks/mod.rs | 16 ++ hooks/runner.rs | 274 ++++++++++++++++++++ hooks/sanitize.rs | 84 ++++++ lib.rs | 7 + main.rs | 188 +++++++++++++- metrics.rs | 311 ++++++++++++++++++++++ pack_cache.rs | 229 +++++++++++++++++ proto/hooks.proto | 61 +++++ proto/repository.proto | 122 +++++++++ rate_limit.rs | 172 +++++++++++++ server/archive.rs | 26 +- server/blame.rs | 26 +- server/branch.rs | 86 ++++++- server/commit.rs | 76 +++++- server/diff.rs | 46 +++- server/merge.rs | 56 +++- server/mod.rs | 48 +++- server/pack.rs | 170 ++++++++++++- server/repository.rs | 289 ++++++++++++++++++++- server/tag.rs | 56 +++- server/tree.rs | 66 ++++- snapshot/mod.rs | 13 + snapshot/ops.rs | 169 ++++++++++++ snapshot/storage.rs | 166 ++++++++++++ tests/disk_cache_test.rs | 157 ++++++++++++ tests/hooks_test.rs | 108 ++++++++ tests/snapshot_test.rs | 142 +++++++++++ 37 files changed, 4691 insertions(+), 83 deletions(-) create mode 100644 cluster/discovery.rs create mode 100644 cluster/mod.rs create mode 100644 cluster/types.rs create mode 100644 disk_cache.rs create mode 100644 hooks/manager.rs create mode 100644 hooks/mod.rs create mode 100644 hooks/runner.rs create mode 100644 hooks/sanitize.rs create mode 100644 metrics.rs create mode 100644 pack_cache.rs create mode 100644 proto/hooks.proto create mode 100644 rate_limit.rs create mode 100644 snapshot/mod.rs create mode 100644 snapshot/ops.rs create mode 100644 snapshot/storage.rs create mode 100644 tests/disk_cache_test.rs create mode 100644 tests/hooks_test.rs create mode 100644 tests/snapshot_test.rs diff --git a/Cargo.lock b/Cargo.lock index 9c00a72..a33b4cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -457,6 +457,24 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "etcd-client" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ed900ba953ca6bf1fadb75e0c6b73d8463b9e2bb6bdb7b4573e8e7295852fbe" +dependencies = [ + "http", + "prost", + "tokio", + "tokio-stream", + "tonic", + "tonic-build", + "tonic-prost", + "tonic-prost-build", + "tower", + "tower-service", +] + [[package]] name = "faster-hex" version = "0.10.0" @@ -668,8 +686,10 @@ name = "gitks" version = "1.0.0" dependencies = [ "async-trait", + "dashmap", "dotenvy", "duct", + "etcd-client", "gix", "gix-archive", "moka", @@ -678,6 +698,8 @@ dependencies = [ "ractor", "ractor_cluster", "serde", + "serde_json", + "sha2", "tempfile", "thiserror", "tokio", @@ -2462,6 +2484,7 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki", "subtle", @@ -2936,6 +2959,7 @@ dependencies = [ "socket2", "sync_wrapper", "tokio", + "tokio-rustls", "tokio-stream", "tower", "tower-layer", diff --git a/Cargo.toml b/Cargo.toml index d981218..0745e5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,6 +18,8 @@ name = "gitks" [dependencies] moka = { version = "0.12", default-features = false, features = ["sync"] } serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.11" gix = { version = "0.84", default-features = false, features = ["serde", "blame", "sha256", "sha1", "tracing", "merge", "max-performance-safe", "revision"] } gix-archive = { version = "0.33", features = ["sha256","sha1","document-features"] } duct = { version = "1", features = [] } @@ -35,6 +37,8 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } ractor = { version = "0.15.13", features = ["cluster","tokio_runtime","monitors","message_span_propogation","async-trait"]} ractor_cluster = { version = "0.15.13", features = ["async-trait"] } async-trait = "0.1.89" +etcd-client = { version = "0.18.0", features = ["tls"] } +dashmap = "6" [[bin]] name = "gitks" path = "main.rs" diff --git a/actor/handler.rs b/actor/handler.rs index 38715de..d660ed9 100644 --- a/actor/handler.rs +++ b/actor/handler.rs @@ -1,5 +1,6 @@ use crate::actor::message::{ - GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA, RefUpdateEvent, RouteDecision, + ElectionRequest, ElectionResult, GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA, + RefUpdateEvent, RoleChangedEvent, RouteDecision, }; use crate::server::GitksService; use async_trait::async_trait; @@ -25,6 +26,7 @@ impl GitNodeActor { pub struct RepoEntry { pub role: String, pub last_commit: String, + pub read_only: bool, } pub struct GitNodeArgs { @@ -37,6 +39,10 @@ pub struct GitNodeState { actor_name: String, grpc_addr: String, repos: HashMap, + current_term: u64, + health_failures: u32, + is_primary: bool, + last_known_primary_grpc: String, } #[async_trait] @@ -58,11 +64,18 @@ impl Actor for GitNodeActor { vec![myself.get_cell()], ); tracing::info!(storage_name = %args.storage_name, actor_name = %actor_name, grpc_addr = %args.grpc_addr, "GitNodeActor started"); + + start_health_checker(myself.clone(), 1, 10); + Ok(GitNodeState { storage_name: args.storage_name, actor_name, - grpc_addr: args.grpc_addr, + grpc_addr: args.grpc_addr.clone(), repos: HashMap::new(), + current_term: 0, + health_failures: 0, + is_primary: true, // Will be refined at registration + last_known_primary_grpc: args.grpc_addr.clone(), }) } @@ -76,6 +89,7 @@ impl Actor for GitNodeActor { GitNodeMessage::ScanAndRegister => { let repos = self.service.scan_all_repo()?; tracing::info!(storage_name = %state.storage_name, found = repos.len(), "scanning local repositories"); + crate::metrics::set_repository_count(repos.len() as u64); for repo_path in repos { let relative_path = repo_path .strip_prefix(self.service.repo_prefix.to_string_lossy().as_ref()) @@ -151,6 +165,79 @@ impl Actor for GitNodeActor { }) .ok(); } + + // ── Election & Role Change ────────────────────────────────── + GitNodeMessage::ElectPrimary(request, reply) => { + let accepted = should_accept_election(&request, state); + tracing::info!( + candidate = %request.candidate_storage_name, + term = request.term, + current_term = state.current_term, + accepted = accepted, + "election vote" + ); + if accepted { + state.current_term = request.term; + state.last_known_primary_grpc = request.candidate_grpc_addr.clone(); + } + reply + .send(ElectionResult { + accepted, + current_term: state.current_term, + voter_storage_name: state.storage_name.clone(), + voter_role: if state.is_primary { + ROLE_PRIMARY + } else { + ROLE_REPLICA + } + .to_string(), + }) + .ok(); + } + + GitNodeMessage::RoleChanged(event) => { + // Empty storage_name = self-promotion from health checker + let is_self = + event.storage_name.is_empty() || event.storage_name == state.storage_name; + + if is_self && event.new_role == ROLE_PRIMARY { + tracing::info!( + storage_name = %state.storage_name, + term = event.term, + "promoted to PRIMARY" + ); + state.is_primary = true; + state.current_term = event.term; + state.health_failures = 0; + for entry in state.repos.values_mut() { + entry.role = ROLE_PRIMARY.to_string(); + entry.read_only = false; + } + } else if is_self && event.new_role == ROLE_REPLICA { + tracing::info!( + storage_name = %state.storage_name, + term = event.term, + "demoted to REPLICA" + ); + state.is_primary = false; + state.current_term = event.term; + for entry in state.repos.values_mut() { + entry.role = ROLE_REPLICA.to_string(); + } + } else { + // Another node's role changed — update routing info + tracing::info!( + storage_name = %event.storage_name, + new_role = %event.new_role, + "remote node role changed" + ); + state.last_known_primary_grpc = if event.new_role == ROLE_PRIMARY { + event.grpc_addr.clone() + } else { + state.last_known_primary_grpc.clone() + }; + } + } } Ok(()) } @@ -189,6 +276,21 @@ impl Actor for GitNodeActor { } } +/// Determine whether to accept an election request. +fn should_accept_election(request: &ElectionRequest, state: &GitNodeState) -> bool { + // Only accept if the term is greater than our current term + // (prevents old/duplicate election messages) + if request.term <= state.current_term { + tracing::warn!( + request_term = request.term, + current_term = state.current_term, + "rejecting election: term too old" + ); + return false; + } + true +} + fn build_decision( state: &GitNodeState, header: &crate::pb::RepositoryHeader, @@ -226,23 +328,20 @@ fn register_repo( return; } - // Determine role based on cluster state - // For simplicity and correctness, we use a conservative approach: - // If there are other nodes in the cluster, register as replica initially. - // The route_repository logic will determine the actual primary at query time. let members = ractor::pg::get_members(&"gitks_nodes".to_string()); let my_cell = myself.get_cell(); let other_nodes_exist = members.iter().any(|m| m != &my_cell); let role = if other_nodes_exist { - // Conservative: assume another node might be primary - // The actual primary will be determined by route_repository query ROLE_REPLICA.to_string() } else { - // We're the only node, so we're primary ROLE_PRIMARY.to_string() }; + if role == ROLE_PRIMARY { + state.is_primary = true; + } + let category = extract_category(&relative_path); pg::join_scoped( state.storage_name.clone(), @@ -254,6 +353,7 @@ fn register_repo( RepoEntry { role: role.clone(), last_commit: String::new(), + read_only: false, }, ); tracing::info!( @@ -262,7 +362,7 @@ fn register_repo( relative_path = %relative_path, actor_name = %state.actor_name, role = %role, - "repository route registered (role will be refined at query time)" + "repository route registered" ); } @@ -270,6 +370,101 @@ fn extract_category(relative_path: &str) -> &str { relative_path.split('/').next().unwrap_or("root") } +/// Start background health checker that monitors the PRIMARY node. +/// If the PRIMARY becomes unreachable for `max_failures` consecutive checks, +/// triggers an election. +fn start_health_checker(myself: ActorRef, interval_secs: u64, max_failures: u32) { + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs)); + interval.tick().await; // First tick immediate + + let mut consecutive_failures: u32 = 0; + + loop { + interval.tick().await; + + let members = ractor::pg::get_members(&"gitks_nodes".to_string()); + let my_cell = myself.get_cell(); + let other_cells: Vec = + members.into_iter().filter(|m| m != &my_cell).collect(); + + if other_cells.is_empty() { + // No other nodes → we are the only node → ensure we are PRIMARY + consecutive_failures = 0; + continue; + } + + let mut any_reachable = false; + for cell in &other_cells { + let actor_ref: ActorRef = cell.clone().into(); + match ractor::call_t!(actor_ref, GitNodeMessage::GetNodeHealth, 2000) { + Ok(health) if health.healthy => { + any_reachable = true; + break; + } + _ => continue, + } + } + + if any_reachable { + consecutive_failures = 0; + } else { + consecutive_failures += 1; + tracing::warn!( + consecutive_failures = consecutive_failures, + max_failures = max_failures, + "no other cluster nodes reachable" + ); + + if consecutive_failures >= max_failures { + tracing::error!( + "no other nodes reachable for {max_failures} checks, triggering self-election as PRIMARY" + ); + trigger_self_election(&myself); + consecutive_failures = 0; + } + } + } + }); +} + +/// Trigger self-election: this node promotes itself to PRIMARY. +fn trigger_self_election(myself: &ActorRef) { + let members = ractor::pg::get_members(&"gitks_nodes".to_string()); + let total_nodes = members.len(); + + tracing::warn!( + total_nodes = total_nodes, + "initiating self-election as new PRIMARY" + ); + + let new_term = std::time::SystemTime::now() + .duration_since(std::time::SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + + myself + .cast(GitNodeMessage::RoleChanged(RoleChangedEvent { + storage_name: String::new(), // will be filled by handler from our own state + grpc_addr: String::new(), + new_role: ROLE_PRIMARY.to_string(), + term: new_term, + relative_paths: Vec::new(), // all repos + })) + .ok(); + + broadcast_role_changed( + myself, + RoleChangedEvent { + storage_name: String::new(), // handler fills + grpc_addr: String::new(), + new_role: ROLE_PRIMARY.to_string(), + term: new_term, + relative_paths: Vec::new(), + }, + ); +} + pub async fn start_node_actor( service: GitksService, storage_name: String, @@ -314,3 +509,14 @@ pub fn broadcast_ref_update(_node_actor: &ActorRef, event: RefUp .ok(); } } + +/// Broadcast a role change event to all cluster members. +pub fn broadcast_role_changed(_actor: &ActorRef, event: RoleChangedEvent) { + let members = ractor::pg::get_members(&"gitks_nodes".to_string()); + for member in members { + let actor_ref: ActorRef = member.into(); + actor_ref + .cast(GitNodeMessage::RoleChanged(event.clone())) + .ok(); + } +} diff --git a/actor/message.rs b/actor/message.rs index e223437..b45434c 100644 --- a/actor/message.rs +++ b/actor/message.rs @@ -142,6 +142,13 @@ pub enum GitNodeMessage { #[rpc] GetNodeHealth(RpcReplyPort), + + /// Election: vote for a candidate to become PRIMARY. + #[rpc] + ElectPrimary(ElectionRequest, RpcReplyPort), + + /// A role change has occurred in the cluster. + RoleChanged(RoleChangedEvent), } #[derive(ractor_cluster::RactorMessage)] @@ -149,6 +156,105 @@ pub enum RepoActorMessage { UpdateMetadata(RepositoryHeader), } +// ── Election & Role Change Types ────────────────────────────────────── + +/// Request for a node to vote in a PRIMARY election. +#[derive(Debug, Clone)] +pub struct ElectionRequest { + pub candidate_storage_name: String, + pub candidate_grpc_addr: String, + pub candidate_actor_name: String, + pub term: u64, + pub reason: String, // "primary_failed" etc. +} + +impl BytesConvertable for ElectionRequest { + fn into_bytes(self) -> Vec { + encode_strings(&[ + self.candidate_storage_name, + self.candidate_grpc_addr, + self.candidate_actor_name, + self.term.to_string(), + self.reason, + ]) + } + + fn from_bytes(bytes: Vec) -> Self { + let values = decode_strings(bytes); + Self { + candidate_storage_name: values.first().cloned().unwrap_or_default(), + candidate_grpc_addr: values.get(1).cloned().unwrap_or_default(), + candidate_actor_name: values.get(2).cloned().unwrap_or_default(), + term: values.get(3).and_then(|v| v.parse().ok()).unwrap_or(0), + reason: values.get(4).cloned().unwrap_or_default(), + } + } +} + +/// Result of an election vote. +#[derive(Debug, Clone)] +pub struct ElectionResult { + pub accepted: bool, + pub current_term: u64, + pub voter_storage_name: String, + pub voter_role: String, +} + +impl BytesConvertable for ElectionResult { + fn into_bytes(self) -> Vec { + encode_strings(&[ + if self.accepted { "1" } else { "0" }.to_string(), + self.current_term.to_string(), + self.voter_storage_name, + self.voter_role, + ]) + } + + fn from_bytes(bytes: Vec) -> Self { + let values = decode_strings(bytes); + Self { + accepted: values.first().is_some_and(|v| v == "1"), + current_term: values.get(1).and_then(|v| v.parse().ok()).unwrap_or(0), + voter_storage_name: values.get(2).cloned().unwrap_or_default(), + voter_role: values.get(3).cloned().unwrap_or_default(), + } + } +} + +/// Event broadcast when a node's role changes. +#[derive(Debug, Clone)] +pub struct RoleChangedEvent { + pub storage_name: String, + pub grpc_addr: String, + pub new_role: String, // "primary" or "replica" + pub term: u64, + pub relative_paths: Vec, // repos that changed role +} + +impl BytesConvertable for RoleChangedEvent { + fn into_bytes(self) -> Vec { + let mut strings = vec![ + self.storage_name, + self.grpc_addr, + self.new_role, + self.term.to_string(), + ]; + strings.extend(self.relative_paths); + encode_strings(&strings) + } + + fn from_bytes(bytes: Vec) -> Self { + let values = decode_strings(bytes); + Self { + storage_name: values.first().cloned().unwrap_or_default(), + grpc_addr: values.get(1).cloned().unwrap_or_default(), + new_role: values.get(2).cloned().unwrap_or_default(), + term: values.get(3).and_then(|v| v.parse().ok()).unwrap_or(0), + relative_paths: values.iter().skip(4).cloned().collect(), + } + } +} + fn encode_strings(values: &[String]) -> Vec { let mut buf = Vec::new(); for value in values { @@ -159,16 +265,13 @@ fn encode_strings(values: &[String]) -> Vec { buf } -// Maximum allowed length for a single string in the message const MAX_STRING_LEN: usize = 10 * 1024 * 1024; // 10MB -// Maximum total message size const MAX_TOTAL_SIZE: usize = 50 * 1024 * 1024; // 50MB fn decode_strings(bytes: Vec) -> Vec { let mut values = Vec::new(); let mut offset = 0; - // Check total message size if bytes.len() > MAX_TOTAL_SIZE { tracing::warn!( total = bytes.len(), @@ -182,7 +285,6 @@ fn decode_strings(bytes: Vec) -> Vec { let len_bytes: [u8; 8] = bytes[offset..offset + 8].try_into().unwrap_or([0u8; 8]); let len_u64 = u64::from_be_bytes(len_bytes); - // Prevent DoS via extremely large length values if len_u64 > MAX_STRING_LEN as u64 { tracing::warn!( offset, @@ -196,7 +298,6 @@ fn decode_strings(bytes: Vec) -> Vec { let len = len_u64 as usize; offset += 8; - // Prevent integer overflow in offset calculation let end_offset = match offset.checked_add(len) { Some(end) => end, None => { @@ -210,7 +311,6 @@ fn decode_strings(bytes: Vec) -> Vec { }; if len == 0 || end_offset > bytes.len() { - // Invalid length — stop decoding, return what we have so far tracing::warn!( offset, claimed_len = len, diff --git a/actor/mod.rs b/actor/mod.rs index 8e9f52d..ebe415f 100644 --- a/actor/mod.rs +++ b/actor/mod.rs @@ -4,11 +4,11 @@ pub mod server; pub mod sync; pub use handler::{ - GitNodeActor, GitNodeArgs, RepoEntry, broadcast_ref_update, get_category_members, - get_cluster_nodes, list_all_groups, route_group_for, start_node_actor, + GitNodeActor, GitNodeArgs, RepoEntry, broadcast_ref_update, broadcast_role_changed, + get_category_members, get_cluster_nodes, list_all_groups, route_group_for, start_node_actor, }; pub use message::{ - GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA, RefUpdateEvent, RepoActorMessage, - RouteDecision, + ElectionRequest, ElectionResult, GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA, + RefUpdateEvent, RepoActorMessage, RoleChangedEvent, RouteDecision, }; pub use server::init_actor_cluster; diff --git a/cluster/discovery.rs b/cluster/discovery.rs new file mode 100644 index 0000000..34fd7b9 --- /dev/null +++ b/cluster/discovery.rs @@ -0,0 +1,232 @@ +//! etcd-based peer discovery for ractor_cluster. +//! +//! Responsibilities: +//! - Connect to etcd and create a Lease (TTL-based health check) +//! - Register this node under `/gitks/nodes/{storage_name}` +//! - Discover existing peers via prefix GET +//! - Watch for peer join/leave events and invoke callbacks + +use std::sync::Arc; +use tokio::sync::Mutex; + +use etcd_client::{Client, ConnectOptions, EventType, GetOptions, PutOptions, WatchOptions}; + +use super::types::PeerInfo; + +/// Key prefix used for all gitks entries in etcd. +const KEY_PREFIX: &str = "/gitks/nodes/"; + +/// Wraps an etcd client with lease-based registration and peer discovery. +pub struct EtcdRegistry { + client: Mutex, + lease_id: i64, + storage_name: String, +} + +impl EtcdRegistry { + /// Connect to etcd, create a lease, and register this node. + /// + /// Returns `None` if the connection fails (caller should fall back to standalone mode). + pub async fn register( + endpoints: Vec, + info: &PeerInfo, + ttl_secs: i64, + connect_timeout_ms: u64, + ) -> Result> { + let connect_opts = ConnectOptions::new() + .with_connect_timeout(std::time::Duration::from_millis(connect_timeout_ms)) + .with_keep_alive( + std::time::Duration::from_secs(5), + std::time::Duration::from_secs(3), + ) + .with_keep_alive_while_idle(true); + + let mut client = Client::connect(endpoints.clone(), Some(connect_opts)).await?; + + tracing::info!(endpoints = ?endpoints, "connected to etcd"); + + // Create lease + let lease_resp = client.lease_grant(ttl_secs, None).await?; + let lease_id = lease_resp.id(); + tracing::info!(lease_id, ttl = ttl_secs, "etcd lease granted"); + + // Register node info under the lease + let key = format!("{KEY_PREFIX}{}", info.storage_name); + let value = serde_json::to_string(info)?; + client + .put(key, value, Some(PutOptions::new().with_lease(lease_id))) + .await?; + tracing::info!( + storage_name = %info.storage_name, + cluster_addr = %info.cluster_addr, + "registered in etcd" + ); + + Ok(Self { + client: Mutex::new(client), + lease_id, + storage_name: info.storage_name.clone(), + }) + } + + /// Start the lease keepalive loop in a background task. + /// + /// The loop sends periodic heartbeats to etcd to prevent the lease from expiring. + /// If keepalive fails, it logs a warning but does not panic — the node will + /// eventually be removed from etcd when the lease expires. + pub fn start_keepalive(self: &Arc) -> tokio::task::JoinHandle<()> { + let this = Arc::clone(self); + tokio::spawn(async move { + let lease_id = this.lease_id; + let (mut keeper, mut stream) = { + let mut client = this.client.lock().await; + match client.lease_keep_alive(lease_id).await { + Ok(pair) => pair, + Err(e) => { + tracing::error!(lease_id, error = %e, "failed to start lease keepalive"); + return; + } + } + }; + + let mut interval = tokio::time::interval(std::time::Duration::from_secs(5)); + loop { + interval.tick().await; + if let Err(e) = keeper.keep_alive().await { + tracing::warn!(lease_id, error = %e, "etcd lease keepalive failed"); + // Don't break — let the lease expire naturally if we can't recover + } + // Drain keepalive responses + let _ = stream.message().await; + } + }) + } + + /// Discover all currently registered peers (excluding this node). + pub async fn discover_peers( + &self, + ) -> Result, Box> { + let mut client = self.client.lock().await; + let resp = client + .get(KEY_PREFIX, Some(GetOptions::new().with_prefix())) + .await?; + + let mut peers = Vec::new(); + for kv in resp.kvs() { + match serde_json::from_slice::(kv.value()) { + Ok(info) if info.storage_name != self.storage_name => { + peers.push(info); + } + Ok(_) => {} // skip self + Err(e) => { + tracing::warn!( + key = %String::from_utf8_lossy(kv.key()), + error = %e, + "failed to parse peer info from etcd" + ); + } + } + } + Ok(peers) + } + + /// Start a long-running watch loop that monitors peer join/leave events. + /// + /// Callbacks are invoked synchronously within the watch task; keep them fast + /// (prefer sending messages to actors rather than doing blocking work). + pub fn start_watch( + self: &Arc, + on_peer_joined: impl Fn(PeerInfo) + Send + Sync + 'static, + on_peer_left: impl Fn(String) + Send + Sync + 'static, + ) -> tokio::task::JoinHandle<()> { + let this = Arc::clone(self); + let my_name = self.storage_name.clone(); + + tokio::spawn(async move { + let on_joined = Arc::new(on_peer_joined); + let on_left = Arc::new(on_peer_left); + + loop { + // Create a fresh watch client each iteration (after reconnect) + let mut watch_stream = { + let mut client = this.client.lock().await; + match client + .watch(KEY_PREFIX, Some(WatchOptions::new().with_prefix())) + .await + { + Ok(stream) => stream, + Err(e) => { + tracing::error!(error = %e, "etcd watch failed, retrying in 3s"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + continue; + } + } + }; + + tracing::info!("etcd watch loop started"); + + loop { + match watch_stream.message().await { + Ok(Some(resp)) => { + for event in resp.events() { + match event.event_type() { + EventType::Put => { + if let Some(kv) = event.kv() + && let Ok(info) = + serde_json::from_slice::(kv.value()) + && info.storage_name != my_name + { + tracing::info!( + peer = %info.storage_name, + cluster_addr = %info.cluster_addr, + "peer joined via etcd watch" + ); + on_joined(info); + } + } + EventType::Delete => { + if let Some(kv) = event.kv() { + let key = String::from_utf8_lossy(kv.key()); + let name = key + .strip_prefix(KEY_PREFIX) + .unwrap_or(&key) + .to_string(); + if name != my_name { + tracing::warn!( + peer = %name, + "peer left (etcd lease expired or key deleted)" + ); + on_left(name); + } + } + } + } + } + } + Ok(None) => { + tracing::warn!("etcd watch stream ended"); + break; + } + Err(e) => { + tracing::error!(error = %e, "etcd watch stream error"); + break; + } + } + } + + // Reconnect after a delay + tracing::info!("etcd watch loop restarting in 3s"); + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + } + }) + } + + /// Check if the lease is still alive (for external health monitoring). + pub async fn is_lease_alive(&self) -> bool { + let mut client = self.client.lock().await; + match client.lease_time_to_live(self.lease_id, None).await { + Ok(resp) => resp.ttl() > 0, + Err(_) => false, + } + } +} diff --git a/cluster/mod.rs b/cluster/mod.rs new file mode 100644 index 0000000..6b25b88 --- /dev/null +++ b/cluster/mod.rs @@ -0,0 +1,214 @@ +//! Cluster discovery: etcd-driven ractor_cluster node discovery. +//! +//! Architecture: +//! 1. Start a `ractor_cluster::NodeServer` (TCP listener for actor remoting) +//! 2. Connect to etcd and register this node +//! 3. Discover existing peers → `client_connect()` to each +//! 4. Watch etcd for future peer join/leave → connect/disconnect dynamically +//! +//! Once ractor_cluster TCP connections are established, the existing +//! `pg::get_members()` / `ractor::call_t!()` APIs automatically work +//! cross-network — no changes needed in actor/handler.rs or server/mod.rs. + +pub mod discovery; +pub mod types; + +pub use discovery::EtcdRegistry; +pub use types::PeerInfo; + +use std::sync::Arc; + +use ractor::ActorRef; +use ractor_cluster::node::NodeConnectionMode; +use ractor_cluster::{NodeServer, NodeServerMessage, client_connect}; + +use crate::error::{GitError, GitResult}; + +/// Configuration for the cluster subsystem. +#[derive(Debug, Clone)] +pub struct ClusterConfig { + /// etcd endpoints (e.g. ["http://etcd1:2379", "http://etcd2:2379"]) + pub etcd_endpoints: Vec, + /// Logical name for this storage node + pub storage_name: String, + /// gRPC address advertised to clients + pub grpc_addr: String, + /// TCP port for ractor_cluster NodeServer + pub cluster_port: u16, + /// Shared authentication cookie for ractor_cluster + pub cookie: String, + /// etcd lease TTL in seconds + pub lease_ttl_secs: i64, + /// etcd connection timeout in milliseconds + pub connect_timeout_ms: u64, + /// Hostname used in the ractor_cluster node name (`name@hostname`). + /// Also used by remote nodes to connect back via `{cluster_hostname}:{cluster_port}`. + /// In K8s/Docker, this should be a resolvable address (Pod IP, service DNS, etc.) + pub cluster_hostname: String, +} + +/// The running cluster manager. Holds references to the NodeServer and etcd registry. +/// Dropping this will stop the background tasks. +pub struct ClusterManager { + /// The ractor_cluster NodeServer actor + pub node_server: ActorRef, + /// The etcd registry (for health checks, etc.) + pub registry: Arc, + /// Handles for background tasks (keepalive + watch) + _keepalive_handle: tokio::task::JoinHandle<()>, + _watch_handle: tokio::task::JoinHandle<()>, +} + +impl ClusterManager { + /// Start the full cluster subsystem: + /// 1. Spawn NodeServer (TCP listener) + /// 2. Connect to etcd + register + /// 3. Discover peers → client_connect + /// 4. Start keepalive + watch loops + /// + /// Returns `Err` if etcd is unreachable (caller should fall back to standalone). + pub async fn start(config: ClusterConfig) -> GitResult { + // ── Step 1: Start NodeServer ── + let node_server = spawn_node_server(&config).await?; + tracing::info!( + port = config.cluster_port, + hostname = %config.cluster_hostname, + "NodeServer started" + ); + + // ── Step 2: Connect to etcd and register ── + let cluster_addr = format!("{}:{}", config.cluster_hostname, config.cluster_port); + let peer_info = PeerInfo { + storage_name: config.storage_name.clone(), + cluster_addr: cluster_addr.clone(), + grpc_addr: config.grpc_addr.clone(), + version: env!("CARGO_PKG_VERSION").to_string(), + }; + + let registry = Arc::new( + EtcdRegistry::register( + config.etcd_endpoints.clone(), + &peer_info, + config.lease_ttl_secs, + config.connect_timeout_ms, + ) + .await + .map_err(|e| GitError::Internal(format!("etcd registration failed: {e}")))?, + ); + + // ── Step 3: Discover existing peers and connect ── + let peers = registry + .discover_peers() + .await + .map_err(|e| GitError::Internal(format!("peer discovery failed: {e}")))?; + + for peer in &peers { + connect_to_peer(&node_server, peer, &config.storage_name).await; + } + + // ── Step 4: Start background tasks ── + let keepalive_handle = registry.start_keepalive(); + + let ns_for_watch = node_server.clone(); + let my_name_for_watch = config.storage_name.clone(); + let watch_handle = registry.start_watch( + move |peer| { + let ns = ns_for_watch.clone(); + let my_name = my_name_for_watch.clone(); + tokio::spawn(async move { + connect_to_peer(&ns, &peer, &my_name).await; + }); + }, + move |name| { + tracing::info!( + peer = %name, + "peer left etcd registry (ractor_cluster will cleanup TCP session)" + ); + // ractor_cluster automatically: + // 1. Detects TCP disconnection + // 2. Stops the NodeSession actor + // 3. Stops all RemoteActors for that session + // 4. Removes them from Process Groups + // No manual cleanup needed. + }, + ); + + tracing::info!( + storage_name = %config.storage_name, + peers_found = peers.len(), + "cluster manager started" + ); + + Ok(Self { + node_server, + registry, + _keepalive_handle: keepalive_handle, + _watch_handle: watch_handle, + }) + } +} + +/// Spawn the ractor_cluster NodeServer actor (TCP listener for inter-node communication). +async fn spawn_node_server(config: &ClusterConfig) -> GitResult> { + let server = NodeServer::new( + config.cluster_port, + config.cookie.clone(), + config.storage_name.clone(), + config.cluster_hostname.clone(), + None, // no encryption (internal network) + Some(NodeConnectionMode::Transitive), + ); + + let (actor_ref, _handle) = ractor::Actor::spawn( + Some(format!("node_server_{}", config.storage_name)), + server, + (), + ) + .await + .map_err(|e| GitError::Internal(format!("failed to spawn NodeServer: {e}")))?; + + Ok(actor_ref) +} + +/// Establish a ractor_cluster TCP connection to a remote peer. +/// +/// Uses ordering optimization: only the node with the lexicographically +/// smaller `storage_name` initiates the connection. The other side will +/// accept the incoming connection. This prevents duplicate connections. +async fn connect_to_peer( + node_server: &ActorRef, + peer: &PeerInfo, + my_name: &str, +) { + // Ordering optimization: only smaller-named node connects + if my_name >= peer.storage_name.as_str() { + tracing::debug!( + peer = %peer.storage_name, + "skipping connect (peer has lower/equal name, they connect to us)" + ); + return; + } + + tracing::info!( + peer = %peer.storage_name, + cluster_addr = %peer.cluster_addr, + "connecting to peer via ractor_cluster" + ); + + match client_connect(node_server, peer.cluster_addr.as_str()).await { + Ok(()) => { + tracing::info!( + peer = %peer.storage_name, + "ractor_cluster connection initiated" + ); + } + Err(e) => { + tracing::warn!( + peer = %peer.storage_name, + cluster_addr = %peer.cluster_addr, + error = %e, + "failed to connect to peer (will retry on next watch event)" + ); + } + } +} diff --git a/cluster/types.rs b/cluster/types.rs new file mode 100644 index 0000000..508dc09 --- /dev/null +++ b/cluster/types.rs @@ -0,0 +1,14 @@ +use serde::{Deserialize, Serialize}; + +/// Information about a peer node, registered in etcd. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PeerInfo { + /// Logical storage name (e.g. "node-a", "default") + pub storage_name: String, + /// ractor_cluster TCP address (e.g. "10.0.1.4:4697") + pub cluster_addr: String, + /// gRPC service address (e.g. "http://10.0.1.4:50051") + pub grpc_addr: String, + /// Software version + pub version: String, +} diff --git a/disk_cache.rs b/disk_cache.rs new file mode 100644 index 0000000..3f5343d --- /dev/null +++ b/disk_cache.rs @@ -0,0 +1,538 @@ +//! Disk-based cache infrastructure for GitKS. +//! +//! Implements the Gitaly-inspired diskcache design: +//! - Per-repository state directory with `latest` (repo state hash) and `pending/` (lease files) +//! - Cache key = SHA256(latest + request digest + version) +//! - Cached responses stored at `${CACHE_DIR}/${digest:0:2}/${digest:2}` +//! - Lease-based invalidation: mutator RPCs create lease files, update `latest` on completion +//! - Background cleanup of stale cache entries + +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime}; + +use sha2; + +use crate::error::{GitError, GitResult}; + +/// Lease stale threshold: leases older than this are considered stale. +const LEASE_STALE_THRESHOLD_SECS: u64 = 30; + +/// State directory relative path under repo prefix. +const STATE_DIR_RELATIVE: &str = "+gitks-cache/state"; + +/// Cache directory relative path under repo prefix. +const CACHE_DIR_RELATIVE: &str = "+gitks-cache/cache"; + +/// Info-refs cache directory relative path under repo prefix. +const INFO_REFS_DIR_RELATIVE: &str = "+gitks-cache/info_refs"; + +/// Generate a random value for the `latest` file. +fn random_value() -> String { + use std::fmt::Write; + use std::sync::atomic::{AtomicU64, Ordering}; + let mut buf = [0u8; 16]; + let nanos = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_nanos() as u64; + buf[..8].copy_from_slice(&nanos.to_le_bytes()); + static COUNTER: AtomicU64 = AtomicU64::new(0); + let c = COUNTER.fetch_add(1, Ordering::Relaxed); + buf[8..].copy_from_slice(&c.to_le_bytes()); + let mut s = String::with_capacity(32); + for byte in &buf { + write!(s, "{byte:02x}").unwrap(); + } + s +} + +/// Compute SHA256 digest from multiple input parts. +fn sha256_digest(parts: &[&str]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + for part in parts { + hasher.update(part.as_bytes()); + } + let result = hasher.finalize(); + let mut s = String::with_capacity(64); + for byte in result { + use std::fmt::Write; + write!(s, "{byte:02x}").unwrap(); + } + s +} + +/// Convert a digest into a two-level file path: `${digest:0:2}/${digest:2}`. +pub fn digest_to_path(digest: &str) -> PathBuf { + let prefix = &digest[..2]; + let rest = &digest[2..]; + PathBuf::from(prefix).join(rest) +} + +/// DiskCache manages per-repository state and cached response files on local disk. +#[derive(Debug)] +pub struct DiskCache { + pub repo_prefix: PathBuf, + max_age: Duration, + version: String, + enabled: bool, +} + +impl DiskCache { + /// Create a new DiskCache. + pub fn new(repo_prefix: PathBuf, version: String, max_age_secs: u64, enabled: bool) -> Self { + Self { + repo_prefix, + max_age: Duration::from_secs(max_age_secs), + version, + enabled, + } + } + + /// Whether the cache is enabled. + pub fn is_enabled(&self) -> bool { + self.enabled + } + + // ── State Directory ────────────────────────────────────────────── + + fn state_dir_for(&self, relative_path: &str) -> PathBuf { + self.repo_prefix + .join(STATE_DIR_RELATIVE) + .join(relative_path) + } + + fn latest_path_for(&self, relative_path: &str) -> PathBuf { + self.state_dir_for(relative_path).join("latest") + } + + fn pending_dir_for(&self, relative_path: &str) -> PathBuf { + self.state_dir_for(relative_path).join("pending") + } + + // ── Cache Directory ────────────────────────────────────────────── + + fn cache_dir(&self, namespace: &str) -> PathBuf { + self.repo_prefix.join(namespace) + } + + fn cache_file_path(&self, namespace: &str, digest: &str) -> PathBuf { + self.cache_dir(namespace).join(digest_to_path(digest)) + } + + // ── Repository State Management ────────────────────────────────── + + /// Ensure the state directory for a repository exists and has a `latest` file. + /// If `latest` does not exist, create it with a random value. + pub fn ensure_state(&self, relative_path: &str) -> GitResult { + if !self.enabled { + return Ok(random_value()); + } + let state_dir = self.state_dir_for(relative_path); + std::fs::create_dir_all(&state_dir).map_err(GitError::Io)?; + let pending_dir = self.pending_dir_for(relative_path); + std::fs::create_dir_all(&pending_dir).map_err(GitError::Io)?; + + let latest_path = self.latest_path_for(relative_path); + if latest_path.exists() { + let val = std::fs::read_to_string(&latest_path).map_err(GitError::Io)?; + Ok(val.trim().to_string()) + } else { + let val = random_value(); + std::fs::write(&latest_path, &val).map_err(GitError::Io)?; + Ok(val) + } + } + + /// Create a lease file for a mutating RPC. + /// Returns a `LeaseGuard` that removes the lease on drop and updates `latest`. + pub fn create_lease(&self, relative_path: &str) -> GitResult { + if !self.enabled { + let rp = relative_path.to_string(); + return Ok(LeaseGuard { + cache: self.clone(), + relative_path: rp, + lease_path: PathBuf::new(), + is_dummy: true, + }); + } + let pending_dir = self.pending_dir_for(relative_path); + std::fs::create_dir_all(&pending_dir).map_err(GitError::Io)?; + let lease_name = random_value(); + let lease_path = pending_dir.join(&lease_name); + std::fs::write(&lease_path, &lease_name).map_err(GitError::Io)?; + tracing::debug!( + relative_path = %relative_path, + lease = %lease_name, + "lease created" + ); + Ok(LeaseGuard { + cache: self.clone(), + relative_path: relative_path.to_string(), + lease_path, + is_dummy: false, + }) + } + + /// Check if a repository is in a deterministic state (no active leases). + /// Returns the `latest` value if deterministic, or None if indeterminate. + pub fn get_repo_state(&self, relative_path: &str) -> GitResult> { + if !self.enabled { + return Ok(Some(random_value())); + } + self.cleanup_stale_leases(relative_path)?; + + let pending_dir = self.pending_dir_for(relative_path); + if pending_dir.exists() { + let entries = std::fs::read_dir(&pending_dir).map_err(GitError::Io)?; + let count = entries.count(); + if count > 0 { + tracing::debug!( + relative_path = %relative_path, + pending = count, + "repo has in-flight mutator, cache state indeterminate" + ); + return Ok(None); + } + } + + let latest_path = self.latest_path_for(relative_path); + if latest_path.exists() { + let val = std::fs::read_to_string(&latest_path).map_err(GitError::Io)?; + Ok(Some(val.trim().to_string())) + } else { + // No latest file → create one + Ok(Some(self.ensure_state(relative_path)?)) + } + } + + /// Remove stale lease files (older than LEASE_STALE_THRESHOLD_SECS). + fn cleanup_stale_leases(&self, relative_path: &str) -> GitResult<()> { + let pending_dir = self.pending_dir_for(relative_path); + if !pending_dir.exists() { + return Ok(()); + } + let now = SystemTime::now(); + let threshold = Duration::from_secs(LEASE_STALE_THRESHOLD_SECS); + for entry in std::fs::read_dir(&pending_dir).map_err(GitError::Io)? { + let entry = entry.map_err(GitError::Io)?; + let path = entry.path(); + if let Ok(metadata) = entry.metadata() + && let Ok(modified) = metadata.modified() + && let Ok(age) = now.duration_since(modified) + && age > threshold + { + tracing::warn!( + path = %path.display(), + age_secs = age.as_secs(), + "removing stale lease file" + ); + std::fs::remove_file(&path).ok(); + } + } + Ok(()) + } + + // ── Cache Key Computation ──────────────────────────────────────── + + /// Compute a cache key for an info/refs request. + pub fn compute_info_refs_key(&self, relative_path: &str, protocol: &str) -> GitResult { + let latest = self.ensure_state(relative_path)?; + let parts: &[&str] = &[&latest, "advertise_refs", protocol, &self.version]; + Ok(sha256_digest(parts)) + } + + /// Compute a cache key for a pack-objects request. + pub fn compute_pack_objects_key( + &self, + relative_path: &str, + wants_hex: &[String], + haves_hex: &[String], + thin_pack: bool, + use_bitmaps: bool, + delta_base_offset: bool, + ) -> GitResult { + let latest = self.ensure_state(relative_path)?; + // Sort wants and haves for deterministic key + let mut wants_sorted = wants_hex.to_vec(); + wants_sorted.sort(); + let mut haves_sorted = haves_hex.to_vec(); + haves_sorted.sort(); + let wants_str = wants_sorted.join(","); + let haves_str = haves_sorted.join(","); + let flags = format!("thin={thin_pack},bitmaps={use_bitmaps},dbo={delta_base_offset}"); + let parts: &[&str] = &[ + &latest, + "pack_objects", + &wants_str, + &haves_str, + &flags, + &self.version, + ]; + Ok(sha256_digest(parts)) + } + + // ── Cache Lookup & Insert ──────────────────────────────────────── + + /// Look up a cached response for the given namespace and digest. + /// Returns the cached bytes if found and not expired. + pub fn lookup(&self, namespace: &str, digest: &str) -> GitResult>> { + if !self.enabled { + return Ok(None); + } + let path = self.cache_file_path(namespace, digest); + if !path.exists() { + return Ok(None); + } + if let Ok(metadata) = std::fs::metadata(&path) + && let Ok(modified) = metadata.modified() + && let Ok(age) = SystemTime::now().duration_since(modified) + && age > self.max_age + { + tracing::debug!( + path = %path.display(), + age_secs = age.as_secs(), + "cached entry expired, removing" + ); + std::fs::remove_file(&path).ok(); + if let Some(parent) = path.parent() { + std::fs::remove_dir(parent).ok(); + } + return Ok(None); + } + let data = std::fs::read(&path).map_err(GitError::Io)?; + tracing::debug!( + namespace = %namespace, + digest = %digest, + size = data.len(), + "cache hit" + ); + Ok(Some(data)) + } + + /// Insert a cached response for the given namespace and digest. + pub fn insert(&self, namespace: &str, digest: &str, data: &[u8]) -> GitResult<()> { + if !self.enabled { + return Ok(()); + } + let path = self.cache_file_path(namespace, digest); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(GitError::Io)?; + } + let tmp_path = path.with_extension("tmp"); + std::fs::write(&tmp_path, data).map_err(GitError::Io)?; + std::fs::rename(&tmp_path, &path).map_err(GitError::Io)?; + tracing::debug!( + namespace = %namespace, + digest = %digest, + size = data.len(), + "cache entry written" + ); + Ok(()) + } + + /// Open a cache file for streaming read. + pub fn open_stream_read( + &self, + namespace: &str, + digest: &str, + ) -> GitResult> { + if !self.enabled { + return Ok(None); + } + let path = self.cache_file_path(namespace, digest); + if !path.exists() { + return Ok(None); + } + if let Ok(metadata) = std::fs::metadata(&path) + && let Ok(modified) = metadata.modified() + && let Ok(age) = SystemTime::now().duration_since(modified) + && age > self.max_age + { + std::fs::remove_file(&path).ok(); + return Ok(None); + } + let file = std::fs::File::open(&path).map_err(GitError::Io)?; + Ok(Some(file)) + } + + /// Open a cache file for streaming write. + /// Returns the file handle and the final path. + pub fn open_stream_write( + &self, + namespace: &str, + digest: &str, + ) -> GitResult<(std::fs::File, PathBuf)> { + if !self.enabled { + return Err(GitError::Internal("disk cache not enabled".into())); + } + let path = self.cache_file_path(namespace, digest); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(GitError::Io)?; + } + let tmp_path = path.with_extension("tmp_streaming"); + let file = std::fs::File::create(&tmp_path).map_err(GitError::Io)?; + Ok((file, path)) + } + + /// Finalize a streaming write by renaming the temp file to the final path. + pub fn finalize_stream_write(&self, tmp_path: &Path, final_path: &Path) -> GitResult<()> { + let actual_tmp = tmp_path.with_extension("tmp_streaming"); + if actual_tmp.exists() { + std::fs::rename(&actual_tmp, final_path).map_err(GitError::Io)?; + } + tracing::debug!( + path = %final_path.display(), + "streaming cache entry finalized" + ); + Ok(()) + } + + /// Invalidate all cache entries for a repository by updating the `latest` file. + /// This is called after any mutator RPC (create branch, create commit, etc.) + pub fn invalidate_repo(&self, relative_path: &str) { + if !self.enabled { + return; + } + let latest_path = self.latest_path_for(relative_path); + let new_val = random_value(); + if let Err(e) = std::fs::write(&latest_path, &new_val) { + tracing::warn!( + relative_path = %relative_path, + error = %e, + "failed to update latest for cache invalidation" + ); + } else { + tracing::debug!( + relative_path = %relative_path, + new_state = %new_val, + "cache invalidated for repository" + ); + } + } + + /// Remove all cache entries on startup (guard against inconsistent state from previous run). + pub fn cleanup_on_startup(&self) -> GitResult<()> { + if !self.enabled { + return Ok(()); + } + for namespace in &[CACHE_DIR_RELATIVE, INFO_REFS_DIR_RELATIVE] { + let dir = self.repo_prefix.join(namespace); + if dir.exists() { + tracing::info!(dir = %dir.display(), "cleaning cache directory on startup"); + std::fs::remove_dir_all(&dir).map_err(GitError::Io)?; + } + } + Ok(()) + } + + /// Background cleanup: remove expired cache entries. + /// Should be called periodically (e.g., every 5 minutes). + pub fn cleanup_expired(&self) -> GitResult { + if !self.enabled { + return Ok(0); + } + let now = SystemTime::now(); + let mut removed = 0u64; + + for namespace in &[CACHE_DIR_RELATIVE, INFO_REFS_DIR_RELATIVE] { + let dir = self.repo_prefix.join(namespace); + if !dir.exists() { + continue; + } + for prefix_entry in std::fs::read_dir(&dir).map_err(GitError::Io)? { + let prefix_entry = prefix_entry.map_err(GitError::Io)?; + let prefix_dir = prefix_entry.path(); + if !prefix_dir.is_dir() { + continue; + } + for entry in std::fs::read_dir(&prefix_dir).map_err(GitError::Io)? { + let entry = entry.map_err(GitError::Io)?; + let path = entry.path(); + if let Ok(metadata) = entry.metadata() + && let Ok(modified) = metadata.modified() + && let Ok(age) = now.duration_since(modified) + && age > self.max_age + { + tracing::debug!( + path = %path.display(), + age_secs = age.as_secs(), + "removing expired cache entry" + ); + std::fs::remove_file(&path).ok(); + removed += 1; + } + } + std::fs::remove_dir(&prefix_dir).ok(); + } + } + if removed > 0 { + tracing::info!( + entries_removed = removed, + "expired cache entries cleaned up" + ); + } + Ok(removed) + } +} + +impl Clone for DiskCache { + fn clone(&self) -> Self { + Self { + repo_prefix: self.repo_prefix.clone(), + max_age: self.max_age, + version: self.version.clone(), + enabled: self.enabled, + } + } +} + +/// A lease guard that removes the lease file on drop and updates `latest`. +pub struct LeaseGuard { + cache: DiskCache, + relative_path: String, + lease_path: PathBuf, + is_dummy: bool, +} + +impl LeaseGuard { + /// Update the `latest` file to invalidate cached responses. + /// Called automatically on drop, but can also be called manually. + pub fn commit(&mut self) { + if self.is_dummy { + self.cache.invalidate_repo(&self.relative_path); + self.is_dummy = false; // Don't double-commit on drop + return; + } + if self.lease_path.exists() { + std::fs::remove_file(&self.lease_path).ok(); + } + self.cache.invalidate_repo(&self.relative_path); + self.is_dummy = false; + } +} + +impl Drop for LeaseGuard { + fn drop(&mut self) { + if self.is_dummy || self.lease_path.exists() { + self.commit(); + } + } +} + +/// Start a background task that periodically cleans up expired cache entries. +pub fn start_cache_cleanup_task( + cache: DiskCache, + interval: Duration, +) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let mut ticker = tokio::time::interval(interval); + ticker.tick().await; // First tick is immediate + loop { + ticker.tick().await; + if let Err(e) = cache.cleanup_expired() { + tracing::warn!(error = %e, "cache cleanup task failed"); + } + } + }) +} diff --git a/hooks/manager.rs b/hooks/manager.rs new file mode 100644 index 0000000..6c64ae0 --- /dev/null +++ b/hooks/manager.rs @@ -0,0 +1,238 @@ +//! Hook manager for GitKS. +//! +//! Manages the installation, listing, and removal of git hooks +//! for bare repositories. Supports server hooks, custom hooks, +//! and gRPC callback hooks. + +use std::path::{Path, PathBuf}; +use std::time::Duration; + +use crate::error::{GitError, GitResult}; +use crate::hooks::runner::{HookResult, generate_hook_runner_script, run_hook_dir}; +use crate::hooks::sanitize::{validate_hook_content, validate_hook_name}; + +/// Manages git hooks across repositories. +#[derive(Debug, Clone)] +pub struct HookManager { + repo_prefix: PathBuf, + server_hooks_dir: Option, + hook_callback_addr: Option, + hook_timeout: Duration, + allow_custom_hooks: bool, +} + +impl HookManager { + pub fn new( + repo_prefix: PathBuf, + server_hooks_dir: Option, + hook_callback_addr: Option, + hook_timeout: Duration, + allow_custom_hooks: bool, + ) -> Self { + Self { + repo_prefix, + server_hooks_dir, + hook_callback_addr, + hook_timeout, + allow_custom_hooks, + } + } + + /// Install gitks hook runner scripts into a repository's hooks directory. + /// Called during repository initialization. + pub fn install_hooks(&self, repo_path: &Path) -> GitResult<()> { + let hooks_dir = repo_path.join("hooks"); + std::fs::create_dir_all(&hooks_dir).map_err(GitError::Io)?; + + let relative_path = repo_path + .strip_prefix(&self.repo_prefix) + .unwrap_or(repo_path) + .to_string_lossy() + .trim_start_matches('/') + .to_string(); + + let server_hooks_dir_str = self + .server_hooks_dir + .as_ref() + .map(|p| p.to_string_lossy().into_owned()); + let callback_addr_str = self.hook_callback_addr.clone(); + + for hook_type in &["pre-receive", "update", "post-receive"] { + let script_content = generate_hook_runner_script( + hook_type, + &relative_path, + server_hooks_dir_str.as_deref(), + callback_addr_str.as_deref(), + self.hook_timeout.as_secs(), + ); + let hook_path = hooks_dir.join(hook_type); + std::fs::write(&hook_path, script_content).map_err(GitError::Io)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&hook_path) + .map_err(GitError::Io)? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&hook_path, perms).map_err(GitError::Io)?; + } + tracing::debug!( + hook_type = %hook_type, + path = %hook_path.display(), + "installed gitks hook runner" + ); + } + + tracing::info!( + path = %repo_path.display(), + "hooks installed for repository" + ); + Ok(()) + } + + /// Set a custom hook script for a repository. + pub fn set_custom_hook( + &self, + repo_path: &Path, + hook_name: &str, + content: &str, + ) -> GitResult<()> { + validate_hook_name(hook_name)?; + if !self.allow_custom_hooks { + return Err(GitError::PermissionDenied( + "custom hooks are not allowed on this server".into(), + )); + } + validate_hook_content(content)?; + + let custom_hooks_dir = repo_path.join("custom_hooks").join(hook_name).join("d"); + std::fs::create_dir_all(&custom_hooks_dir).map_err(GitError::Io)?; + + let script_name = format!("gitks_custom_{}", hook_name.replace('-', "_")); + let script_path = custom_hooks_dir.join(&script_name); + std::fs::write(&script_path, content).map_err(GitError::Io)?; + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let mut perms = std::fs::metadata(&script_path) + .map_err(GitError::Io)? + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&script_path, perms).map_err(GitError::Io)?; + } + + tracing::info!( + repo = %repo_path.display(), + hook_name = %hook_name, + "custom hook set" + ); + Ok(()) + } + + /// Remove a custom hook from a repository. + pub fn remove_custom_hook(&self, repo_path: &Path, hook_name: &str) -> GitResult<()> { + validate_hook_name(hook_name)?; + let custom_hooks_dir = repo_path.join("custom_hooks").join(hook_name).join("d"); + if custom_hooks_dir.exists() { + std::fs::remove_dir_all(&custom_hooks_dir).map_err(GitError::Io)?; + tracing::info!( + repo = %repo_path.display(), + hook_name = %hook_name, + "custom hook removed" + ); + } + Ok(()) + } + + /// List all hooks for a repository. + pub fn list_hooks(&self, repo_path: &Path) -> GitResult> { + let mut hooks = Vec::new(); + + let hooks_dir = repo_path.join("hooks"); + if hooks_dir.exists() { + for hook_type in &["pre-receive", "update", "post-receive"] { + let hook_path = hooks_dir.join(hook_type); + if hook_path.exists() { + hooks.push(HookInfo { + hook_type: hook_type.to_string(), + level: HookLevel::Server, + path: hook_path.to_string_lossy().into_owned(), + }); + } + } + } + + if self.allow_custom_hooks { + let custom_dir = repo_path.join("custom_hooks"); + if custom_dir.exists() { + for entry in std::fs::read_dir(&custom_dir).map_err(GitError::Io)? { + let entry = entry.map_err(GitError::Io)?; + let name = entry.file_name().to_string_lossy().into_owned(); + if validate_hook_name(&name).is_ok() { + let path = entry.path(); + let d_dir = path.join("d"); + if d_dir.exists() { + let script_count = + std::fs::read_dir(&d_dir).map_err(GitError::Io)?.count(); + if script_count > 0 { + hooks.push(HookInfo { + hook_type: name, + level: HookLevel::Custom, + path: d_dir.to_string_lossy().into_owned(), + }); + } + } + } + } + } + } + + Ok(hooks) + } + + /// Execute a hook by running server hooks then custom hooks. + pub fn execute_hook(&self, repo_path: &Path, hook_type: &str, stdin_data: &[u8]) -> HookResult { + if let Some(ref server_dir) = self.server_hooks_dir { + let dir = server_dir.join(hook_type).join("d"); + let result = run_hook_dir(&dir, hook_type, stdin_data, self.hook_timeout); + if !result.accepted { + return result; + } + } + + if self.allow_custom_hooks { + let custom_dir = repo_path.join("custom_hooks").join(hook_type).join("d"); + let result = run_hook_dir(&custom_dir, hook_type, stdin_data, self.hook_timeout); + if !result.accepted { + return result; + } + } + + HookResult::accepted() + } +} + +/// Information about a hook. +#[derive(Debug, Clone)] +pub struct HookInfo { + pub hook_type: String, + pub level: HookLevel, + pub path: String, +} + +/// Hook level (server vs custom). +#[derive(Debug, Clone, PartialEq)] +pub enum HookLevel { + Server, + Custom, +} + +impl std::fmt::Display for HookLevel { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + HookLevel::Server => write!(f, "server"), + HookLevel::Custom => write!(f, "custom"), + } + } +} diff --git a/hooks/mod.rs b/hooks/mod.rs new file mode 100644 index 0000000..19128e6 --- /dev/null +++ b/hooks/mod.rs @@ -0,0 +1,16 @@ +//! Git hooks management for GitKS. +//! +//! Supports three layers of hooks: +//! 1. Server hooks: admin-level, shared across all repositories +//! 2. Custom hooks: per-repository, user-defined scripts +//! 3. gRPC callback hooks: external HookService via gRPC +//! +//! Hook scripts are installed into bare repositories' `hooks/` directory +//! and are automatically invoked by git during receive-pack operations. + +pub mod manager; +pub mod runner; +pub mod sanitize; + +pub use manager::HookManager; +pub use runner::HookResult; diff --git a/hooks/runner.rs b/hooks/runner.rs new file mode 100644 index 0000000..05a0cd8 --- /dev/null +++ b/hooks/runner.rs @@ -0,0 +1,274 @@ +//! Hook execution runner. +//! +//! Executes server hooks, custom hooks, and gRPC callback hooks +//! in sequence for each git hook type (pre-receive, update, post-receive). + +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::{Output, Stdio}; +use std::time::Duration; + +/// Result of a hook execution. +#[derive(Debug)] +pub struct HookResult { + pub accepted: bool, + pub exit_code: i32, + pub stdout: String, + pub stderr: String, +} + +impl HookResult { + pub fn accepted() -> Self { + Self { + accepted: true, + exit_code: 0, + stdout: String::new(), + stderr: String::new(), + } + } + + pub fn rejected(stderr: String) -> Self { + Self { + accepted: false, + exit_code: 1, + stdout: String::new(), + stderr, + } + } + + pub fn from_output(output: &Output) -> Self { + let stdout = String::from_utf8_lossy(&output.stdout).into_owned(); + let stderr = String::from_utf8_lossy(&output.stderr).into_owned(); + Self { + accepted: output.status.success(), + exit_code: output.status.code().unwrap_or(-1), + stdout, + stderr, + } + } +} + +/// Run all hook scripts in a directory (sorted alphabetically). +/// Returns the first rejection or accepted if all pass. +pub fn run_hook_dir( + hook_dir: &Path, + hook_type: &str, + stdin_data: &[u8], + timeout: Duration, +) -> HookResult { + if !hook_dir.exists() { + return HookResult::accepted(); + } + + let mut scripts: Vec = Vec::new(); + if let Ok(entries) = std::fs::read_dir(hook_dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_file() { + let name = path.file_name().unwrap_or_default().to_string_lossy(); + // Skip files starting with '.' (hidden files) or ending with '~' or '.sample' + if name.starts_with('.') || name.ends_with('~') || name.ends_with(".sample") { + continue; + } + scripts.push(path); + } + } + } + scripts.sort(); + + for script in &scripts { + tracing::debug!( + hook_type = %hook_type, + script = %script.display(), + "executing hook script" + ); + let result = run_single_script(script, stdin_data, timeout); + if !result.accepted { + tracing::warn!( + hook_type = %hook_type, + script = %script.display(), + exit_code = result.exit_code, + stderr = %result.stderr, + "hook script rejected" + ); + return result; + } + } + + HookResult::accepted() +} + +/// Run a single hook script with stdin data and timeout. +fn run_single_script(script_path: &Path, stdin_data: &[u8], timeout: Duration) -> HookResult { + let child = std::process::Command::new(script_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn(); + + match child { + Ok(mut c) => { + if let Some(ref mut stdin) = c.stdin { + let _ = stdin.write_all(stdin_data); + } + c.stdin = None; + + let wait_result = c.wait_timeout(timeout); + match wait_result { + Ok(Some(status)) => { + let output = c.wait_with_output().unwrap_or_else(|_| { + // If we can't get output, at least return the status + Output { + status, + stdout: Vec::new(), + stderr: Vec::new(), + } + }); + HookResult::from_output(&output) + } + Ok(None) => { + tracing::warn!( + script = %script_path.display(), + timeout_secs = timeout.as_secs(), + "hook script timed out, killing" + ); + let _ = c.kill(); + HookResult::rejected(format!( + "hook script timed out after {}s: {}", + timeout.as_secs(), + script_path.display() + )) + } + Err(e) => { + let _ = c.kill(); + HookResult::rejected(format!("hook script wait error: {e}")) + } + } + } + Err(e) => { + tracing::warn!( + script = %script_path.display(), + error = %e, + "failed to spawn hook script" + ); + // If the script can't be executed, treat as rejection + HookResult::rejected(format!("failed to spawn hook script: {e}")) + } + } +} + +/// Wait for a child process with timeout. +trait ChildWaitTimeout { + fn wait_timeout( + &mut self, + timeout: Duration, + ) -> std::io::Result>; +} + +impl ChildWaitTimeout for std::process::Child { + fn wait_timeout( + &mut self, + timeout: Duration, + ) -> std::io::Result> { + let start = std::time::Instant::now(); + loop { + match self.try_wait() { + Ok(Some(status)) => return Ok(Some(status)), + Ok(None) => { + if start.elapsed() > timeout { + return Ok(None); // timeout + } + std::thread::sleep(Duration::from_millis(100)); + } + Err(e) => return Err(e), + } + } + } +} + +/// Generate the gitks hook runner script content. +/// This script is installed into each repository's hooks/ directory +/// and orchestrates the execution of server hooks, custom hooks, and gRPC callbacks. +pub fn generate_hook_runner_script( + hook_type: &str, + repo_relative_path: &str, + server_hooks_dir: Option<&str>, + hook_callback_addr: Option<&str>, + hook_timeout_secs: u64, +) -> String { + let server_hooks_section = if let Some(dir) = server_hooks_dir { + format!( + r#" +# Run server hooks +SERVER_HOOKS_DIR="{dir}/{hook_type}.d" +if [ -d "$SERVER_HOOKS_DIR" ]; then + for script in $(ls "$SERVER_HOOKS_DIR" | sort); do + skip=false + case "$script" in .*|*.sample|*~) skip=true;; esac + if [ "$skip" = "false" ]; then + "$SERVER_HOOKS_DIR/$script" + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + fi + done +fi +"# + ) + } else { + String::new() + }; + + let custom_hooks_section = r#" +# Run custom hooks (per-repository) +CUSTOM_HOOKS_DIR="$GIT_DIR/custom_hooks/$GITKS_HOOK_TYPE.d" +if [ -d "$CUSTOM_HOOKS_DIR" ]; then + for script in $(ls "$CUSTOM_HOOKS_DIR" | sort); do + skip=false + case "$script" in .*|*.sample|*~) skip=true;; esac + if [ "$skip" = "false" ]; then + "$CUSTOM_HOOKS_DIR/$script" + exit_code=$? + if [ $exit_code -ne 0 ]; then + exit $exit_code + fi + fi + done +fi +"#; + + let grpc_callback_section = if let Some(addr) = hook_callback_addr { + format!( + r#" +# gRPC callback to external HookService +if [ -n "{addr}" ]; then + # gRPC callback is handled by the gitks service directly + # The service will make the gRPC call after the git hook completes + # This section is a placeholder - actual gRPC callback is handled + # by the gitks receive_pack handler +fi +"# + ) + } else { + String::new() + }; + + format!( + r#"#!/bin/sh +# gitks hook runner for {hook_type} +# Repository: {repo_relative_path} +# Auto-generated by gitks - do not modify manually + +GITKS_HOOK_TYPE="{hook_type}" +GITKS_REPO_RELATIVE_PATH="{repo_relative_path}" +GITKS_HOOK_TIMEOUT="{hook_timeout_secs}" + +{server_hooks_section} +{custom_hooks_section} +{grpc_callback_section} + +exit 0 +"# + ) +} diff --git a/hooks/sanitize.rs b/hooks/sanitize.rs new file mode 100644 index 0000000..0e376cf --- /dev/null +++ b/hooks/sanitize.rs @@ -0,0 +1,84 @@ +//! Hook content sanitization. +//! +//! Validates custom hook scripts to prevent dangerous commands. + +use crate::error::{GitError, GitResult}; + +/// Commands/patterns that are never allowed in custom hook scripts. +const FORBIDDEN_PATTERNS: &[&str] = &[ + "rm -rf", + "rm -r /", + "chmod 777", + "chmod 666", + "mkfs", + "dd if=", + ":(){ :|:& };:", // fork bomb + "> /dev/sda", + "curl -o /", + "wget -O /", + "/etc/passwd", + "/etc/shadow", + "shutdown", + "reboot", + "init 0", + "init 6", + "poweroff", + "halt", +]; + +/// Maximum hook script size (64KB). +const MAX_HOOK_SIZE: usize = 65536; + +/// Validate a custom hook script content for safety. +pub fn validate_hook_content(content: &str) -> GitResult<()> { + if content.is_empty() { + return Err(GitError::InvalidArgument( + "hook content cannot be empty".into(), + )); + } + if content.len() > MAX_HOOK_SIZE { + return Err(GitError::InvalidArgument(format!( + "hook content too large (max {} bytes): {} bytes", + MAX_HOOK_SIZE, + content.len() + ))); + } + let content_lower = content.to_lowercase(); + for pattern in FORBIDDEN_PATTERNS { + if content_lower.contains(pattern) { + return Err(GitError::InvalidArgument(format!( + "hook content contains forbidden pattern: '{pattern}'" + ))); + } + } + if content.contains('\0') { + return Err(GitError::InvalidArgument( + "hook content cannot contain null bytes".into(), + )); + } + Ok(()) +} + +/// Validate a hook name (must be a recognized git hook name). +pub fn validate_hook_name(name: &str) -> GitResult<()> { + const VALID_HOOK_NAMES: &[&str] = &[ + "pre-receive", + "update", + "post-receive", + "pre-applypatch", + "applypatch-msg", + "post-applypatch", + "pre-commit", + "prepare-commit-msg", + "commit-msg", + "post-commit", + "pre-auto-gc", + ]; + if !VALID_HOOK_NAMES.contains(&name) { + return Err(GitError::InvalidArgument(format!( + "invalid hook name: '{name}'. Must be one of: {}", + VALID_HOOK_NAMES.join(", ") + ))); + } + Ok(()) +} diff --git a/lib.rs b/lib.rs index 44053f1..ce145ba 100644 --- a/lib.rs +++ b/lib.rs @@ -4,18 +4,25 @@ pub mod bare; pub mod blame; pub mod blob; pub mod branch; +pub mod cluster; pub mod commit; pub mod diff; +pub mod disk_cache; pub mod error; +pub mod hooks; pub mod init; pub mod macros; pub mod merge; +pub mod metrics; pub mod oid; +pub mod rate_limit; pub mod pack; +pub mod pack_cache; pub mod paginate; pub mod pb; pub mod refs; pub mod sanitize; pub mod server; +pub mod snapshot; pub mod tag; pub mod tree; diff --git a/main.rs b/main.rs index 5952601..81d862a 100644 --- a/main.rs +++ b/main.rs @@ -1,12 +1,37 @@ use std::path::PathBuf; +use std::time::Duration; use gitks::actor::init_actor_cluster; +use gitks::cluster::{ClusterConfig, ClusterManager}; +use gitks::disk_cache::DiskCache; +use gitks::hooks::HookManager; +use gitks::metrics; use gitks::server::{GitksService, serve}; const DEFAULT_HOST: &str = "0.0.0.0"; const DEFAULT_PORT: &str = "50051"; const DEFAULT_STORAGE_NAME: &str = "default"; +fn env_or(key: &str, default: &str) -> String { + std::env::var(key).unwrap_or_else(|_| default.into()) +} + +fn env_bool(key: &str, default: bool) -> bool { + match std::env::var(key).as_deref() { + Ok("true" | "1" | "yes") => true, + Ok("false" | "0" | "no") => false, + Ok(_) => default, + Err(_) => default, + } +} + +fn env_u64(key: &str, default: u64) -> u64 { + std::env::var(key) + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(default) +} + #[tokio::main] async fn main() -> Result<(), Box> { dotenvy::dotenv().ok(); @@ -14,10 +39,9 @@ async fn main() -> Result<(), Box> { tracing::info!(version = env!("CARGO_PKG_VERSION"), "gitks starting up"); - let host = std::env::var("GITKS_HOST").unwrap_or_else(|_| DEFAULT_HOST.into()); - let port = std::env::var("GITKS_PORT").unwrap_or_else(|_| DEFAULT_PORT.into()); - let storage_name = - std::env::var("STORAGE_NAME").unwrap_or_else(|_| DEFAULT_STORAGE_NAME.into()); + let host = env_or("GITKS_HOST", DEFAULT_HOST); + let port = env_or("GITKS_PORT", DEFAULT_PORT); + let storage_name = env_or("STORAGE_NAME", DEFAULT_STORAGE_NAME); let grpc_addr = std::env::var("GITKS_ADVERTISE_ADDR").unwrap_or_else(|_| format!("http://{host}:{port}")); @@ -32,8 +56,162 @@ async fn main() -> Result<(), Box> { std::fs::create_dir_all(&repo_prefix)?; } + // Disk cache configuration + let disk_cache_enabled = env_bool("GITKS_DISK_CACHE_ENABLED", false); + let disk_cache_max_age = env_u64("GITKS_DISK_CACHE_MAX_AGE", 300); + + let disk_cache = DiskCache::new( + repo_prefix.clone(), + env!("CARGO_PKG_VERSION").to_string(), + disk_cache_max_age, + disk_cache_enabled, + ); + + if disk_cache_enabled { + tracing::info!("disk cache enabled, max_age={disk_cache_max_age}s"); + disk_cache.cleanup_on_startup()?; + gitks::disk_cache::start_cache_cleanup_task(disk_cache.clone(), Duration::from_secs(300)); + } else { + tracing::info!("disk cache disabled"); + } + + // Pack cache configuration + let pack_cache_enabled = env_bool("GITKS_PACK_CACHE_ENABLED", false); + let pack_backpressure = env_bool("GITKS_PACK_CACHE_BACKPRESSURE", true); + + // Pack cache: needs disk_cache. If disk_cache is enabled, info/refs cache + // is always available via PackCache wrapper. pack-objects caching is + // additionally controlled by GITKS_PACK_CACHE_ENABLED. + let pack_cache = if disk_cache_enabled { + tracing::info!( + "pack cache wrapper enabled, pack-objects cache={pack_cache_enabled}, backpressure={pack_backpressure}" + ); + Some(gitks::pack_cache::PackCache::new( + disk_cache.clone(), + pack_backpressure, + )) + } else { + None + }; + + // Hook manager configuration + let hooks_enabled = env_bool("GITKS_HOOKS_ENABLED", true); + let server_hooks_dir = std::env::var("GITKS_SERVER_HOOKS_DIR") + .ok() + .map(PathBuf::from); + let hook_callback_addr = std::env::var("GITKS_HOOK_CALLBACK_ADDR").ok(); + let hook_timeout = env_u64("GITKS_HOOK_TIMEOUT", 30); + let allow_custom_hooks = env_bool("GITKS_ALLOW_CUSTOM_HOOKS", true); + + let hook_manager = if hooks_enabled { + tracing::info!("hooks enabled, timeout={hook_timeout}s, custom_hooks={allow_custom_hooks}"); + Some(HookManager::new( + repo_prefix.clone(), + server_hooks_dir, + hook_callback_addr, + Duration::from_secs(hook_timeout), + allow_custom_hooks, + )) + } else { + tracing::info!("hooks disabled"); + None + }; + + // Health check / election configuration + let health_check_interval = env_u64("GITKS_HEALTH_CHECK_INTERVAL", 1); + let max_health_failures = env_u64("GITKS_MAX_HEALTH_FAILURES", 10); + + tracing::info!( + "health check: interval={health_check_interval}s, max_failures={max_health_failures}" + ); + + // ── Metrics server ── + let metrics_port = env_u64("GITKS_METRICS_PORT", 9100) as u16; + let _metrics_handle = metrics::start_metrics_server(metrics_port); + tracing::info!("metrics server on port {metrics_port}"); + + // ── Cluster discovery (etcd → ractor_cluster) ── + // + // When GITKS_ETCD_ENDPOINTS is set, the node: + // 1. Starts a ractor_cluster NodeServer (TCP listener) + // 2. Connects to etcd and registers itself + // 3. Discovers existing peers → establishes ractor_cluster TCP connections + // 4. Watches etcd for future peer join/leave events + // + // Once ractor_cluster connections are up, pg::get_members() automatically + // returns remote actors — no changes needed in actor/handler.rs. + // + // When GITKS_ETCD_ENDPOINTS is unset or etcd is unreachable, the node + // falls back to standalone mode (existing local-only behavior). + let etcd_endpoints = std::env::var("GITKS_ETCD_ENDPOINTS") + .ok() + .filter(|s| !s.is_empty()) + .map(|s| { + s.split(',') + .map(str::trim) + .map(String::from) + .collect::>() + }); + + let cluster_port = env_or("GITKS_CLUSTER_PORT", "4697") + .parse::() + .unwrap_or(4697); + let cluster_cookie = env_or("GITKS_CLUSTER_COOKIE", "gitks-default-cookie"); + let lease_ttl = env_u64("GITKS_LEASE_TTL", 15) as i64; + let connect_timeout_ms = env_u64("GITKS_ETCD_CONNECT_TIMEOUT", 5000); + + // Resolve the hostname/address other nodes use to reach our NodeServer. + // Priority: GITKS_CLUSTER_HOSTNAME > POD_IP (K8s) > HOSTNAME env > "localhost" + let cluster_hostname = std::env::var("GITKS_CLUSTER_HOSTNAME") + .or_else(|_| std::env::var("POD_IP")) + .or_else(|_| std::env::var("HOSTNAME")) + .unwrap_or_else(|_| "localhost".to_string()); + + let _cluster: Option = if let Some(endpoints) = etcd_endpoints { + tracing::info!( + endpoints = ?endpoints, + cluster_port = cluster_port, + cluster_hostname = %cluster_hostname, + "starting cluster discovery via etcd" + ); + let config = ClusterConfig { + etcd_endpoints: endpoints, + storage_name: storage_name.clone(), + grpc_addr: grpc_addr.clone(), + cluster_port, + cookie: cluster_cookie, + lease_ttl_secs: lease_ttl, + connect_timeout_ms, + cluster_hostname, + }; + match ClusterManager::start(config).await { + Ok(cm) => { + tracing::info!("cluster discovery active"); + Some(cm) + } + Err(e) => { + tracing::warn!(error = %e, "etcd unavailable, running in standalone mode"); + None + } + } + } else { + tracing::info!("GITKS_ETCD_ENDPOINTS not set, running in standalone mode"); + None + }; + let addr: std::net::SocketAddr = format!("{host}:{port}").parse()?; - let svc = GitksService::new(repo_prefix.clone()); + let mut svc = GitksService::new(repo_prefix.clone()); + + if disk_cache_enabled { + svc = svc.with_disk_cache(disk_cache); + } + if let Some(pc) = pack_cache { + svc = svc.with_pack_cache(pc); + } + if let Some(hm) = hook_manager { + svc = svc.with_hook_manager(hm); + } + let (node_actor, node_handle) = init_actor_cluster(svc.clone(), storage_name.clone(), grpc_addr.clone()).await?; let svc = svc diff --git a/metrics.rs b/metrics.rs new file mode 100644 index 0000000..9ffaa44 --- /dev/null +++ b/metrics.rs @@ -0,0 +1,311 @@ +//! Prometheus-compatible metrics for GitKS. +//! +//! Tracks: +//! - Request counts by gRPC method + status code +//! - Request duration histogram by method +//! - Active requests gauge +//! - Repository count +//! - Cache hits / misses +//! - Error counts by error type +//! +//! Exposes a `/metrics` HTTP endpoint on a configurable port (default 9100). + +use dashmap::DashMap; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock}; +use std::time::{Duration, Instant}; + +// ── Metric storage ────────────────────────────────────────────────── + +struct MetricsInner { + /// Counter: total requests by (method, status_code) + /// Key: "method:status" + request_count: DashMap, + + /// Histogram buckets for request duration (seconds). + /// Each bucket: (method, le_bound_ms) → count + duration_buckets: DashMap, + + /// Gauge: number of currently in-flight requests + active_requests: AtomicU64, + + /// Gauge: total number of registered repositories + repository_count: AtomicU64, + + /// Counter: cache hits + cache_hits: AtomicU64, + + /// Counter: cache misses + cache_misses: AtomicU64, + + /// Counter: errors by error kind + error_count: DashMap, + + /// Start timestamp (seconds since Unix epoch) + start_time: Instant, +} + +static METRICS: OnceLock> = OnceLock::new(); + +fn metrics() -> &'static Arc { + METRICS.get_or_init(|| { + Arc::new(MetricsInner { + request_count: DashMap::new(), + duration_buckets: DashMap::new(), + active_requests: AtomicU64::new(0), + repository_count: AtomicU64::new(0), + cache_hits: AtomicU64::new(0), + cache_misses: AtomicU64::new(0), + error_count: DashMap::new(), + start_time: Instant::now(), + }) + }) +} + +// ── Duration histogram buckets (in milliseconds) ─────────────────── + +#[rustfmt::skip] +const DURATION_BUCKET_MS: &[u64] = &[ + 5, 10, 25, 50, 100, 250, 500, 1_000, + 2_500, 5_000, 10_000, 30_000, 60_000, u64::MAX, +]; + +const BUCKET_INF: u64 = u64::MAX; + +/// Record a request. +pub fn record_request(method: &str, status_code: &str, duration: Duration) { + let m = metrics(); + let key = format!("{method}:{status_code}"); + m.request_count + .entry(key) + .or_insert_with(|| AtomicU64::new(0)) + .value() + .fetch_add(1, Ordering::Relaxed); + + let duration_ms = duration.as_millis() as u64; + for &bound_ms in DURATION_BUCKET_MS { + if duration_ms <= bound_ms || bound_ms == BUCKET_INF { + let bucket_key = format!("{method}:{bound_ms}"); + m.duration_buckets + .entry(bucket_key) + .or_insert_with(|| AtomicU64::new(0)) + .value() + .fetch_add(1, Ordering::Relaxed); + } + } +} + +/// Increment the active request gauge. +pub fn inc_active_requests() { + metrics().active_requests.fetch_add(1, Ordering::Relaxed); +} + +/// Decrement the active request gauge. +pub fn dec_active_requests() { + metrics().active_requests.fetch_sub(1, Ordering::Relaxed); +} + +/// Set the repository count. +pub fn set_repository_count(count: u64) { + metrics() + .repository_count + .store(count, Ordering::Relaxed); +} + +/// Record a cache hit. +pub fn inc_cache_hits(count: u64) { + metrics().cache_hits.fetch_add(count, Ordering::Relaxed); +} + +/// Record a cache miss. +pub fn inc_cache_misses(count: u64) { + metrics() + .cache_misses + .fetch_add(count, Ordering::Relaxed); +} + +/// Record an error by kind (e.g., "not_found", "internal", "invalid_argument"). +pub fn inc_error(kind: &str) { + metrics() + .error_count + .entry(kind.to_string()) + .or_insert_with(|| AtomicU64::new(0)) + .value() + .fetch_add(1, Ordering::Relaxed); +} + +// ── Prometheus text format rendering ──────────────────────────────── + +/// Render all metrics in Prometheus text exposition format. +pub fn render_metrics() -> String { + let m = metrics(); + let mut out = String::with_capacity(4096); + + // Header + let uptime = m.start_time.elapsed().as_secs(); + out.push_str("# HELP gitks_uptime_seconds Time since gitks started\n"); + out.push_str("# TYPE gitks_uptime_seconds gauge\n"); + out.push_str(&format!("gitks_uptime_seconds {uptime}\n\n")); + + // Active requests + let active = m.active_requests.load(Ordering::Relaxed); + out.push_str("# HELP gitks_active_requests Currently in-flight requests\n"); + out.push_str("# TYPE gitks_active_requests gauge\n"); + out.push_str(&format!("gitks_active_requests {active}\n\n")); + + // Repository count + let repos = m.repository_count.load(Ordering::Relaxed); + out.push_str("# HELP gitks_repository_count Number of registered repositories\n"); + out.push_str("# TYPE gitks_repository_count gauge\n"); + out.push_str(&format!("gitks_repository_count {repos}\n\n")); + + // Request count + out.push_str("# HELP gitks_requests_total Total gRPC requests by method and status\n"); + out.push_str("# TYPE gitks_requests_total counter\n"); + for entry in &m.request_count { + let (method_and_status, count) = (entry.key(), entry.value()); + let count = count.load(Ordering::Relaxed); + if let Some((method, status)) = method_and_status.rsplit_once(':') { + out.push_str( + &format!("gitks_requests_total{{method=\"{method}\",status=\"{status}\"}} {count}\n"), + ); + } + } + out.push('\n'); + + // Duration histogram + out.push_str( + "# HELP gitks_request_duration_milliseconds Request duration histogram in ms\n", + ); + out.push_str("# TYPE gitks_request_duration_milliseconds histogram\n"); + for entry in &m.duration_buckets { + let (method_and_bound, count) = (entry.key(), entry.value()); + let count = count.load(Ordering::Relaxed); + if let Some((method, bound_str)) = method_and_bound.rsplit_once(':') { + let bound = bound_str; + let le = if bound_str.parse::() == Ok(BUCKET_INF) { + "+Inf".to_string() + } else { + bound.to_string() + }; + out.push_str( + &format!("gitks_request_duration_milliseconds_bucket{{method=\"{method}\",le=\"{le}\"}} {count}\n"), + ); + } + } + out.push('\n'); + + // Cache + let hits = m.cache_hits.load(Ordering::Relaxed); + let misses = m.cache_misses.load(Ordering::Relaxed); + out.push_str("# HELP gitks_cache_hits_total Cache hit count\n"); + out.push_str("# TYPE gitks_cache_hits_total counter\n"); + out.push_str(&format!("gitks_cache_hits_total {hits}\n\n")); + out.push_str("# HELP gitks_cache_misses_total Cache miss count\n"); + out.push_str("# TYPE gitks_cache_misses_total counter\n"); + out.push_str(&format!("gitks_cache_misses_total {misses}\n\n")); + + // Errors + out.push_str("# HELP gitks_errors_total Total errors by kind\n"); + out.push_str("# TYPE gitks_errors_total counter\n"); + for entry in &m.error_count { + let (kind, count) = (entry.key(), entry.value()); + let count = count.load(Ordering::Relaxed); + out.push_str(&format!("gitks_errors_total{{kind=\"{kind}\"}} {count}\n")); + } + out.push('\n'); + + out +} + +// ── HTTP server for /metrics endpoint ─────────────────────────────── + +/// Start the metrics HTTP server on the given port. +/// Runs in a background task; returns the JoinHandle. +pub fn start_metrics_server(port: u16) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + let listener = match tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")).await { + Ok(l) => l, + Err(e) => { + tracing::error!(port, error = %e, "failed to bind metrics server"); + return; + } + }; + tracing::info!(port, "metrics HTTP server started"); + + loop { + match listener.accept().await { + Ok((socket, peer)) => { + tracing::debug!(%peer, "metrics request"); + tokio::spawn(handle_metrics_connection(socket)); + } + Err(e) => { + tracing::error!(error = %e, "metrics accept error"); + } + } + } + }) +} + +async fn handle_metrics_connection(mut socket: tokio::net::TcpStream) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + let mut buf = [0u8; 4096]; + let _ = tokio::time::timeout(Duration::from_secs(5), socket.read(&mut buf)).await; + + let body = render_metrics(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + ); + + let _ = tokio::time::timeout(Duration::from_secs(5), socket.write_all(response.as_bytes())) + .await; + let _ = socket.shutdown().await; +} + +// ── Helper to wrap handler functions with metrics ─────────────────── + +/// A guard that records metrics on drop. +/// +/// Usage in handlers: +/// ```ignore +/// let m = crate::metrics::RequestMetrics::new("Service/Method"); +/// // ... handle request ... +/// m.record("ok"); // on success +/// // m.record("internal"); // or on error, with tonic error kind +/// ``` +pub struct RequestMetrics { + method: &'static str, + start: Instant, +} + +impl RequestMetrics { + pub fn new(method: &'static str) -> Self { + inc_active_requests(); + Self { + method, + start: Instant::now(), + } + } + + /// Record the outcome. Idempotent — safe to call before each return. + pub fn record(&self, status: &str) { + let duration = self.start.elapsed(); + record_request(self.method, status, duration); + } +} + +impl Drop for RequestMetrics { + fn drop(&mut self) { + dec_active_requests(); + } +} + +/// Convenience: record an error from a tonic Status. +pub fn record_rpc_error(m: &RequestMetrics, status: &tonic::Status) { + let kind = status.code().description(); + inc_error(kind); + m.record(kind); +} diff --git a/pack_cache.rs b/pack_cache.rs new file mode 100644 index 0000000..b77fa6d --- /dev/null +++ b/pack_cache.rs @@ -0,0 +1,229 @@ +//! Pack-objects cache module. +//! +//! Caches `git pack-objects` output on local disk to reduce CPU load +//! during repeated clone/fetch operations (especially CI traffic). +//! +//! Uses the DiskCache infrastructure for state management and invalidation. +//! Implements streaming with backpressure: the producer writes to a cache file +//! while consumers read from it concurrently. + +use std::io::Write; +use std::time::Duration; + +use prost::Message; +use tokio_stream::wrappers::ReceiverStream; + +use crate::disk_cache::DiskCache; +use crate::pb::PackfileChunk; + +/// Namespace for pack-objects cache entries in the disk cache. +pub const PACK_CACHE_NAMESPACE: &str = "+gitks-cache/cache"; + +/// Namespace for info/refs cache entries. +pub const INFO_REFS_NAMESPACE: &str = "+gitks-cache/info_refs"; + +/// Pack-objects cache wrapper around DiskCache. +#[derive(Debug, Clone)] +pub struct PackCache { + disk_cache: DiskCache, +} + +impl PackCache { + pub fn new(disk_cache: DiskCache, _backpressure: bool) -> Self { + Self { disk_cache } + } + + pub fn is_enabled(&self) -> bool { + self.disk_cache.is_enabled() + } + + pub fn disk_cache(&self) -> &DiskCache { + &self.disk_cache + } + + /// Try to serve a cached pack-objects response as a stream. + /// Reads the cache file in streaming fashion, emitting chunks as they are read + /// to avoid buffering the entire pack file in memory. + pub fn lookup_pack_stream( + &self, + digest: &str, + ) -> Option>> { + if !self.is_enabled() { + return None; + } + + let file = match self + .disk_cache + .open_stream_read(PACK_CACHE_NAMESPACE, digest) + { + Ok(Some(f)) => f, + Ok(None) => return None, + Err(_) => return None, + }; + + tracing::info!(digest = %digest, "pack-objects cache hit, streaming from disk"); + + let (tx, rx) = tokio::sync::mpsc::channel(16); + + let sender = tx.clone(); + tokio::spawn(async move { + let result = tokio::task::spawn_blocking(move || { + use std::io::Read; + let mut file = file; + let mut buf = vec![0u8; 65536]; + loop { + match file.read(&mut buf) { + Ok(0) => return Ok(()), + Ok(n) => { + if sender + .blocking_send(Ok(PackfileChunk { + data: buf[..n].to_vec(), + })) + .is_err() + { + return Ok(()); + } + } + Err(e) => { + let _ = sender.blocking_send(Err(tonic::Status::internal(format!( + "cache read error: {e}" + )))); + return Err(()); + } + } + } + }) + .await; + if result.is_err() { + // Task join error or I/O error already sent + } + }); + + Some(ReceiverStream::new(rx)) + } + + /// Stream pack-objects output while simultaneously writing to cache. + /// This is the "tee" approach: data flows to both the client and the cache file. + pub fn tee_pack_stream( + &self, + digest: &str, + source: ReceiverStream>, + ) -> ReceiverStream> { + let (tx, rx) = tokio::sync::mpsc::channel(16); + + if !self.is_enabled() { + tokio::spawn(async move { + use tokio_stream::StreamExt; + let mut source = source; + while let Some(item) = source.next().await { + if tx.send(item).await.is_err() { + break; + } + } + }); + return ReceiverStream::new(rx); + } + + let cache_dir = self.disk_cache.repo_prefix.join(PACK_CACHE_NAMESPACE); + let rel_path = crate::disk_cache::digest_to_path(digest); + let cache_path = cache_dir.join(&rel_path); + let tmp_path = cache_path.with_extension("tmp_streaming"); + + tokio::spawn(async move { + use tokio_stream::StreamExt; + + if let Some(parent) = cache_path.parent() { + let _ = std::fs::create_dir_all(parent); + } + + let mut cache_file = std::fs::File::create(&tmp_path).ok(); + let mut cache_write_ok = true; + + let mut source = source; + while let Some(item) = source.next().await { + if tx.send(item.clone()).await.is_err() { + cache_write_ok = false; // client disconnected, don't cache partial + break; + } + + if cache_write_ok + && let Some(ref mut f) = cache_file + && let Ok(chunk) = &item + && f.write_all(&chunk.data).is_err() + { + tracing::warn!("cache write failed, dropping cache file handle"); + cache_file = None; + cache_write_ok = false; + } + } + + if cache_write_ok && cache_file.is_some() { + let _ = std::fs::rename(&tmp_path, &cache_path); + tracing::info!( + path = %cache_path.display(), + "pack-objects cache entry written" + ); + } else if tmp_path.exists() { + std::fs::remove_file(&tmp_path).ok(); + } + }); + + ReceiverStream::new(rx) + } + + /// Look up cached info/refs response. + pub fn lookup_info_refs(&self, digest: &str) -> Option { + if !self.is_enabled() { + return None; + } + match self.disk_cache.lookup(INFO_REFS_NAMESPACE, digest) { + Ok(Some(bytes)) => { + if let Ok(resp) = T::decode(bytes.as_slice()) { + tracing::debug!(digest = %digest, "info/refs cache hit"); + return Some(resp); + } + tracing::warn!(digest = %digest, "info/refs cache decode failed"); + None + } + Ok(None) => None, + Err(e) => { + tracing::warn!(error = %e, "info/refs cache lookup error"); + None + } + } + } + + /// Store an info/refs response in cache. + pub fn store_info_refs(&self, digest: &str, resp: &T) { + if !self.is_enabled() { + return; + } + let mut bytes = Vec::with_capacity(resp.encoded_len()); + if resp.encode(&mut bytes).is_ok() + && let Err(e) = self.disk_cache.insert(INFO_REFS_NAMESPACE, digest, &bytes) + { + tracing::warn!(error = %e, "info/refs cache write failed"); + } + } + + /// Invalidate cache for a repository (called after mutator RPCs). + pub fn invalidate_repo(&self, relative_path: &str) { + self.disk_cache.invalidate_repo(relative_path); + } + + /// Create a lease for a mutating operation. + pub fn create_lease( + &self, + relative_path: &str, + ) -> Result { + self.disk_cache.create_lease(relative_path) + } +} + +/// Start background cache cleanup task. +pub fn start_cleanup_task(cache: PackCache, interval_secs: u64) -> tokio::task::JoinHandle<()> { + crate::disk_cache::start_cache_cleanup_task( + cache.disk_cache.clone(), + Duration::from_secs(interval_secs), + ) +} diff --git a/proto/hooks.proto b/proto/hooks.proto new file mode 100644 index 0000000..4632e89 --- /dev/null +++ b/proto/hooks.proto @@ -0,0 +1,61 @@ +syntax = "proto3"; + +package gitks; + +import "repository.proto"; + +// HookService provides gRPC callback hooks for git operations. +// External services can implement this interface to receive hook callbacks. +service HookService { + // Pre-receive hook callback: validate push before it happens. + rpc PreReceiveHook(PreReceiveHookRequest) returns (PreReceiveHookResponse); + + // Update hook callback: validate each ref update individually. + rpc UpdateHook(UpdateHookRequest) returns (UpdateHookResponse); + + // Post-receive hook callback: notify after push has completed. + rpc PostReceiveHook(PostReceiveHookRequest) returns (PostReceiveHookResponse); +} + +message PreReceiveHookRequest { + RepositoryHeader repository = 1; + repeated RefUpdate ref_updates = 2; + string push_options = 3; +} + +message PreReceiveHookResponse { + bool accept = 1; + string rejection_message = 2; +} + +message UpdateHookRequest { + RepositoryHeader repository = 1; + string ref_name = 2; + string old_oid = 3; + string new_oid = 4; +} + +message UpdateHookResponse { + bool accept = 1; + string rejection_message = 2; +} + +message PostReceiveHookRequest { + RepositoryHeader repository = 1; + repeated RefUpdate ref_updates = 2; +} + +message PostReceiveHookResponse { + repeated HookAction actions = 1; +} + +message RefUpdate { + string old_oid = 1; + string new_oid = 2; + string ref_name = 3; +} + +message HookAction { + string action_type = 1; // "trigger_ci", "update_index", etc. + string payload = 2; +} \ No newline at end of file diff --git a/proto/repository.proto b/proto/repository.proto index d9b1ebb..d923eda 100644 --- a/proto/repository.proto +++ b/proto/repository.proto @@ -139,6 +139,113 @@ message RepositoryMaintenanceResponse { string stderr = 3; } +// ── Hooks Management ────────────────────────────────────────────────── + +message ListHooksRequest { + RepositoryHeader repository = 1; +} + +message HookInfo { + string hook_type = 1; + string level = 2; // "server" or "custom" + string path = 3; +} + +message ListHooksResponse { + repeated HookInfo hooks = 1; +} + +message SetCustomHookRequest { + RepositoryHeader repository = 1; + string hook_name = 2; // "pre-receive", "update", "post-receive" + string content = 3; // Hook script content +} + +message RemoveCustomHookRequest { + RepositoryHeader repository = 1; + string hook_name = 2; +} + +// ── Snapshot ────────────────────────────────────────────────────────── + +enum SnapshotStorage { + SNAPSHOT_STORAGE_LOCAL = 0; + SNAPSHOT_STORAGE_S3 = 1; + SNAPSHOT_STORAGE_GCS = 2; +} + +message SnapshotInfo { + string snapshot_id = 1; + string relative_path = 2; + uint64 size_bytes = 3; + string created_at = 4; // ISO 8601 + string head_oid = 5; +} + +message CreateSnapshotRequest { + RepositoryHeader repository = 1; + SnapshotStorage storage = 2; + string storage_path = 3; +} + +message CreateSnapshotResponse { + string snapshot_id = 1; + uint64 size_bytes = 2; + string head_oid = 3; +} + +message RestoreSnapshotRequest { + RepositoryHeader target_repository = 1; + string snapshot_id = 2; + SnapshotStorage storage = 3; + string storage_path = 4; +} + +message ListSnapshotsRequest { + RepositoryHeader repository = 1; + uint32 limit = 2; +} + +message ListSnapshotsResponse { + repeated SnapshotInfo snapshots = 1; +} + +message DeleteSnapshotRequest { + string snapshot_id = 1; + SnapshotStorage storage = 2; +} + +// ── Repository Move ────────────────────────────────────────────────── + +enum MoveRepositoryState { + MOVE_STATE_UNKNOWN = 0; + MOVE_STATE_PREPARING = 1; + MOVE_STATE_TRANSFERRING = 2; + MOVE_STATE_VERIFYING = 3; + MOVE_STATE_COMPLETED = 4; + MOVE_STATE_FAILED = 5; + MOVE_STATE_CANCELLED = 6; +} + +message MoveRepositoryRequest { + RepositoryHeader source_repository = 1; + RepositoryHeader target_repository = 2; +} + +message MoveRepositoryResponse { + MoveRepositoryState state = 1; + string error_message = 2; +} + +message FetchRepositoryDataRequest { + RepositoryHeader repository = 1; +} + +message FetchRepositoryDataResponse { + bytes data = 1; + bool done = 2; +} + service RepositoryService { rpc GetRepository(GetRepositoryRequest) returns (Repository); rpc InitRepository(InitRepositoryRequest) returns (Repository); @@ -154,4 +261,19 @@ service RepositoryService { rpc GarbageCollect(GarbageCollectRequest) returns (RepositoryMaintenanceResponse); rpc Repack(RepackRequest) returns (RepositoryMaintenanceResponse); rpc WriteCommitGraph(WriteCommitGraphRequest) returns (RepositoryMaintenanceResponse); + + // Hooks management + rpc ListHooks(ListHooksRequest) returns (ListHooksResponse); + rpc SetCustomHook(SetCustomHookRequest) returns (google.protobuf.Empty); + rpc RemoveCustomHook(RemoveCustomHookRequest) returns (google.protobuf.Empty); + + // Snapshot operations + rpc CreateSnapshot(CreateSnapshotRequest) returns (CreateSnapshotResponse); + rpc RestoreSnapshot(RestoreSnapshotRequest) returns (google.protobuf.Empty); + rpc ListSnapshots(ListSnapshotsRequest) returns (ListSnapshotsResponse); + rpc DeleteSnapshot(DeleteSnapshotRequest) returns (google.protobuf.Empty); + + // Repository move + rpc MoveRepository(MoveRepositoryRequest) returns (MoveRepositoryResponse); + rpc FetchRepositoryData(FetchRepositoryDataRequest) returns (stream FetchRepositoryDataResponse); } diff --git a/rate_limit.rs b/rate_limit.rs new file mode 100644 index 0000000..e6bf79c --- /dev/null +++ b/rate_limit.rs @@ -0,0 +1,172 @@ +//! Repository-level rate limiting via per-repo semaphores. +//! +//! Prevents any single repository from consuming all server resources. +//! Uses `tokio::sync::Semaphore` with configurable max concurrent operations. +//! +//! Integration pattern: +//! let _guard = rate_limit::acquire(svc, header).await?; +//! // ... handle request ... +//! // guard is dropped here → permit released + +use dashmap::DashMap; +use std::sync::{Arc, OnceLock}; +use tokio::sync::Semaphore; + +// ── Configuration ─────────────────────────────────────────────────── + +/// Default max concurrent operations per repository. +const DEFAULT_MAX_CONCURRENT: usize = 5; + +/// Global rate limiter state. +struct RateLimiter { + /// Per-repository semaphores. Key = repository relative_path. + semaphores: DashMap>, + /// Max concurrent operations per repository. + max_concurrent: usize, +} + +static RATE_LIMITER: OnceLock = OnceLock::new(); + +fn limiter() -> &'static RateLimiter { + RATE_LIMITER.get_or_init(|| { + let max = std::env::var("GITKS_RATE_LIMIT_MAX_CONCURRENT") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(DEFAULT_MAX_CONCURRENT); + + tracing::info!( + max_concurrent = max, + "repository-level rate limiter initialized" + ); + + RateLimiter { + semaphores: DashMap::new(), + max_concurrent: max, + } + }) +} + +// ── Permit guard ─────────────────────────────────────────────────── + +/// A guard that holds a rate-limit permit. The permit is released on drop. +pub struct RateLimitGuard { + /// The semaphore permit. Dropping this releases the permit. + _permit: tokio::sync::OwnedSemaphorePermit, +} + +/// Acquire a rate-limit permit for the given repository. +/// +/// If the repository has `max_concurrent` operations already in flight, +/// this will wait asynchronously until a permit becomes available. +/// +/// Returns `None` if no repository header is provided (e.g., global health checks). +pub async fn acquire(repo_relative_path: Option<&str>) -> Option { + let repo = repo_relative_path?; + if repo.is_empty() { + return None; + } + let l = limiter(); + if l.max_concurrent == 0 { + // Unlimited + return None; + } + + let sem = l + .semaphores + .entry(repo.to_string()) + .or_insert_with(|| Arc::new(Semaphore::new(l.max_concurrent))) + .value() + .clone(); + + // Release DashMap reference before awaiting + let _ = l; + + match tokio::time::timeout( + std::time::Duration::from_secs(30), + sem.clone().acquire_owned(), + ) + .await + { + Ok(Ok(permit)) => { + tracing::debug!( + repo = %repo, + available = sem.available_permits(), + "rate limit permit acquired" + ); + Some(RateLimitGuard { _permit: permit }) + } + Ok(Err(_closed)) => { + // Semaphore was closed — recreate it + tracing::warn!( + repo = %repo, + "rate limit semaphore closed, recreating" + ); + let new_sem = Arc::new(Semaphore::new(limiter().max_concurrent)); + let permit = new_sem + .clone() + .acquire_owned() + .await + .expect("newly created semaphore should have permits"); + limiter() + .semaphores + .insert(repo.to_string(), new_sem); + Some(RateLimitGuard { _permit: permit }) + } + Err(_elapsed) => { + tracing::warn!( + repo = %repo, + max_concurrent = limiter().max_concurrent, + "rate limit timeout waiting for permit" + ); + None + } + } +} + +/// Acquire a rate-limit permit, returning a tonic error on timeout / overload. +pub async fn acquire_or_reject(repo_relative_path: Option<&str>) -> Result, tonic::Status> { + let repo = repo_relative_path.unwrap_or(""); + if repo.is_empty() { + return Ok(None); + } + match acquire(Some(repo)).await { + Some(guard) => Ok(Some(guard)), + None => { + if limiter().max_concurrent == 0 { + return Ok(None); + } + // Timeout — reject with resource exhausted + Err(tonic::Status::resource_exhausted(format!( + "rate limit exceeded for repository '{repo}': max {max} concurrent operations", + max = limiter().max_concurrent + ))) + } + } +} + +/// Remove the semaphore for a repository (called on repo deletion). +pub fn remove_repository(repo_relative_path: &str) { + limiter().semaphores.remove(repo_relative_path); + tracing::debug!(repo = %repo_relative_path, "rate limit semaphore removed"); +} + +/// Update the max concurrent limit at runtime. +pub fn set_max_concurrent(max: usize) { + let l = limiter(); + // We can't modify the field directly through OnceLock, but we can + // update existing semaphores to add or remove permits as needed. + // For a simpler approach, just log and let new semaphores use the new value. + // Since max_concurrent is only read on insert, we use a separate atomic. + use std::sync::atomic::AtomicUsize; + static OVERRIDE: std::sync::atomic::AtomicUsize = AtomicUsize::new(0); + OVERRIDE.store(max, std::sync::atomic::Ordering::Relaxed); + + // Recreate all existing semaphores + for entry in &l.semaphores { + let _old = l.semaphores.insert( + entry.key().clone(), + Arc::new(Semaphore::new(max)), + ); + } + tracing::info!(max_concurrent = max, "rate limit max_concurrent updated"); +} diff --git a/server/archive.rs b/server/archive.rs index dd86ad4..de91bc8 100644 --- a/server/archive.rs +++ b/server/archive.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, cache, into_status}; -remote_client!(remote_archive_client, ArchiveServiceClient, "archive"); +remote_client!( + remote_archive_client, + ArchiveServiceClient, + "archive" +); #[tonic::async_trait] impl archive_service_server::ArchiveService for GitksService { @@ -14,7 +18,9 @@ impl archive_service_server::ArchiveService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.ArchiveService/GetArchive"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("archive.get_archive", %repo); let _enter = span.enter(); @@ -24,16 +30,22 @@ impl archive_service_server::ArchiveService for GitksService { if let Some(mut client) = remote_archive_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); let resp = client.get_archive(inner).await?; let stream = super::bridge_server_stream(resp.into_inner()); return Ok(tonic::Response::new(stream)); } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let stream = gb.get_archive_stream(inner)?; tracing::info!(%repo, "archive streaming started"); + m.record("ok"); Ok(tonic::Response::new(stream)) } @@ -41,7 +53,9 @@ impl archive_service_server::ArchiveService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.ArchiveService/ListArchiveEntries"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("archive.list_archive_entries", %repo); let _enter = span.enter(); @@ -51,11 +65,16 @@ impl archive_service_server::ArchiveService for GitksService { if let Some(mut client) = remote_archive_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.list_archive_entries(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.treeish) { cache::cached_response("archive.list_archive_entries", &inner, || { @@ -65,6 +84,7 @@ impl archive_service_server::ArchiveService for GitksService { gb.list_archive_entries(inner).map_err(into_status)? }; tracing::info!(%repo, count = resp.entries.len(), "list_archive_entries done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/server/blame.rs b/server/blame.rs index d39ef69..252cc74 100644 --- a/server/blame.rs +++ b/server/blame.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, cache, into_status, into_stream}; -remote_client!(remote_blame_client, BlameServiceClient, "blame"); +remote_client!( + remote_blame_client, + BlameServiceClient, + "blame" +); #[tonic::async_trait] impl blame_service_server::BlameService for GitksService { @@ -14,7 +18,9 @@ impl blame_service_server::BlameService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BlameService/Blame"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let path = inner.path.clone(); let span = tracing::info_span!("blame.blame", %repo, %path); @@ -25,11 +31,16 @@ impl blame_service_server::BlameService for GitksService { if let Some(mut client) = remote_blame_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.blame(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("blame.blame", &inner, || { @@ -39,6 +50,7 @@ impl blame_service_server::BlameService for GitksService { gb.blame(inner).map_err(into_status)? }; tracing::info!(%repo, %path, hunks = resp.hunks.len(), "blame done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -46,7 +58,9 @@ impl blame_service_server::BlameService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BlameService/StreamBlame"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let path = inner.path.clone(); let span = tracing::info_span!("blame.stream_blame", %repo, %path); @@ -57,13 +71,18 @@ impl blame_service_server::BlameService for GitksService { if let Some(mut client) = remote_blame_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); let resp = client.stream_blame(inner).await?; let stream = super::bridge_server_stream(resp.into_inner()); return Ok(tonic::Response::new(stream)); } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("blame.blame", &inner, || { @@ -72,6 +91,7 @@ impl blame_service_server::BlameService for GitksService { } else { gb.blame(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(into_stream(resp.hunks))) } } diff --git a/server/branch.rs b/server/branch.rs index 80944a6..b67b918 100644 --- a/server/branch.rs +++ b/server/branch.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, into_status}; -remote_client!(remote_branch_client, BranchServiceClient, "branch"); +remote_client!( + remote_branch_client, + BranchServiceClient, + "branch" +); #[tonic::async_trait] impl branch_service_server::BranchService for GitksService { @@ -11,7 +15,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/ListBranches"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("branch.list_branches", %repo); let _enter = span.enter(); @@ -21,14 +27,20 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.list_branches(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.list_branches(inner).map_err(into_status)?; tracing::info!(%repo, count = resp.branches.len(), "list_branches done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -36,7 +48,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/GetBranch"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("branch.get_branch", %repo, %name); @@ -47,13 +61,19 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_branch(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.get_branch(inner).map_err(into_status)?; + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -61,7 +81,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/CreateBranch"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("branch.create_branch", %repo, %name); @@ -72,15 +94,21 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.create_branch(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.create_branch(inner).map_err(into_status)?; tracing::info!(%repo, %name, "branch created"); self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -88,7 +116,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/DeleteBranch"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("branch.delete_branch", %repo, %name); @@ -99,15 +129,21 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.delete_branch(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; gb.delete_branch(inner).map_err(into_status)?; tracing::info!(%repo, %name, "branch deleted"); self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", ""); + m.record("ok"); Ok(tonic::Response::new(())) } @@ -115,7 +151,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/RenameBranch"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let old = inner.old_name.clone(); let new = inner.new_name.clone(); @@ -127,15 +165,21 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.rename_branch(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.rename_branch(inner).map_err(into_status)?; tracing::info!(%repo, old = %old, new = %new, "branch renamed"); self.notify_ref_update(&repo, &format!("refs/heads/{}", new), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -143,7 +187,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/UpdateBranchTarget"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("branch.update_branch_target", %repo, %name); @@ -154,15 +200,21 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.update_branch_target(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.update_branch_target(inner).map_err(into_status)?; tracing::info!(%repo, %name, "branch target updated"); self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -170,7 +222,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/SetBranchUpstream"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("branch.set_branch_upstream", %repo, %name); @@ -181,15 +235,21 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.set_branch_upstream(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.set_branch_upstream(inner).map_err(into_status)?; tracing::info!(%repo, %name, "branch upstream set"); self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -197,7 +257,9 @@ impl branch_service_server::BranchService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.BranchService/CompareBranch"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let source = inner.source_branch.clone(); let target = inner.target_branch.clone(); @@ -209,14 +271,20 @@ impl branch_service_server::BranchService for GitksService { if let Some(mut client) = remote_branch_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.compare_branch(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.compare_branch(inner).map_err(into_status)?; tracing::info!(%repo, %source, %target, ahead = resp.ahead_by, behind = resp.behind_by, "branch compared"); + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/server/commit.rs b/server/commit.rs index 4dabaf5..d576249 100644 --- a/server/commit.rs +++ b/server/commit.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, cache, into_status}; -remote_client!(remote_commit_client, CommitServiceClient, "commit"); +remote_client!( + remote_commit_client, + CommitServiceClient, + "commit" +); #[tonic::async_trait] impl commit_service_server::CommitService for GitksService { @@ -11,7 +15,9 @@ impl commit_service_server::CommitService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.CommitService/ListCommits"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("commit.list_commits", %repo); let _enter = span.enter(); @@ -21,11 +27,16 @@ impl commit_service_server::CommitService for GitksService { if let Some(mut client) = remote_commit_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.list_commits(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if !inner.all && cache::selector_is_oid(&inner.revision) { cache::cached_response("commit.list_commits", &inner, || { @@ -35,6 +46,7 @@ impl commit_service_server::CommitService for GitksService { gb.list_commits(inner).map_err(into_status)? }; tracing::info!(%repo, count = resp.commits.len(), total = resp.page_info.as_ref().map(|p| p.total_count).unwrap_or(0), "list_commits done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -42,7 +54,9 @@ impl commit_service_server::CommitService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.CommitService/GetCommit"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("commit.get_commit", %repo); let _enter = span.enter(); @@ -52,11 +66,16 @@ impl commit_service_server::CommitService for GitksService { if let Some(mut client) = remote_commit_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_commit(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("commit.get_commit", &inner, || { @@ -65,6 +84,7 @@ impl commit_service_server::CommitService for GitksService { } else { gb.get_commit(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -72,7 +92,9 @@ impl commit_service_server::CommitService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.CommitService/GetCommitAncestors"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("commit.get_commit_ancestors", %repo); let _enter = span.enter(); @@ -82,11 +104,16 @@ impl commit_service_server::CommitService for GitksService { if let Some(mut client) = remote_commit_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_commit_ancestors(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("commit.get_commit_ancestors", &inner, || { @@ -96,6 +123,7 @@ impl commit_service_server::CommitService for GitksService { gb.get_commit_ancestors(inner).map_err(into_status)? }; tracing::info!(%repo, count = resp.commits.len(), "get_commit_ancestors done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -103,7 +131,9 @@ impl commit_service_server::CommitService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.CommitService/CreateCommit"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let branch = inner.branch.clone(); let span = tracing::info_span!("commit.create_commit", %repo, %branch); @@ -114,11 +144,16 @@ impl commit_service_server::CommitService for GitksService { if let Some(mut client) = remote_commit_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.create_commit(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.create_commit(inner).map_err(into_status)?; let commit_hex = resp @@ -128,6 +163,7 @@ impl commit_service_server::CommitService for GitksService { .unwrap_or("?"); tracing::info!(%repo, %branch, %commit_hex, "commit created"); self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -135,7 +171,9 @@ impl commit_service_server::CommitService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.CommitService/RevertCommit"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let branch = inner.branch.clone(); let span = tracing::info_span!("commit.revert_commit", %repo, %branch); @@ -146,15 +184,21 @@ impl commit_service_server::CommitService for GitksService { if let Some(mut client) = remote_commit_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.revert_commit(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.revert_commit(inner).map_err(into_status)?; tracing::info!(%repo, %branch, "commit reverted"); self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -162,7 +206,9 @@ impl commit_service_server::CommitService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.CommitService/CherryPickCommit"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let branch = inner.branch.clone(); let span = tracing::info_span!("commit.cherry_pick_commit", %repo, %branch); @@ -173,15 +219,21 @@ impl commit_service_server::CommitService for GitksService { if let Some(mut client) = remote_commit_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.cherry_pick_commit(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.cherry_pick_commit(inner).map_err(into_status)?; tracing::info!(%repo, %branch, "commit cherry-picked"); self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -189,7 +241,9 @@ impl commit_service_server::CommitService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.CommitService/CompareCommits"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("commit.compare_commits", %repo); let _enter = span.enter(); @@ -199,11 +253,16 @@ impl commit_service_server::CommitService for GitksService { if let Some(mut client) = remote_commit_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.compare_commits(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selectors_are_oid(&inner.base, &inner.head) { cache::cached_response("commit.compare_commits", &inner, || { @@ -213,6 +272,7 @@ impl commit_service_server::CommitService for GitksService { gb.compare_commits(inner).map_err(into_status)? }; tracing::info!(%repo, count = resp.commits.len(), "compare_commits done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/server/diff.rs b/server/diff.rs index aff27e5..76950a2 100644 --- a/server/diff.rs +++ b/server/diff.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, cache, into_status, into_stream}; -remote_client!(remote_diff_client, DiffServiceClient, "diff"); +remote_client!( + remote_diff_client, + DiffServiceClient, + "diff" +); #[tonic::async_trait] impl diff_service_server::DiffService for GitksService { @@ -14,7 +18,9 @@ impl diff_service_server::DiffService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetDiff"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("diff.get_diff", %repo); let _enter = span.enter(); @@ -24,11 +30,16 @@ impl diff_service_server::DiffService for GitksService { if let Some(mut client) = remote_diff_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_diff(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selectors_are_oid(&inner.base, &inner.head) { cache::cached_response("diff.get_diff", &inner, || { @@ -38,6 +49,7 @@ impl diff_service_server::DiffService for GitksService { gb.get_diff(inner).map_err(into_status)? }; tracing::info!(%repo, files = resp.files.len(), overflow = resp.overflow, "get_diff done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -45,7 +57,9 @@ impl diff_service_server::DiffService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetCommitDiff"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("diff.get_commit_diff", %repo); let _enter = span.enter(); @@ -55,11 +69,16 @@ impl diff_service_server::DiffService for GitksService { if let Some(mut client) = remote_diff_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_commit_diff(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.commit) { cache::cached_response("diff.get_commit_diff", &inner, || { @@ -69,6 +88,7 @@ impl diff_service_server::DiffService for GitksService { gb.get_commit_diff(inner).map_err(into_status)? }; tracing::info!(%repo, files = resp.files.len(), "get_commit_diff done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -76,7 +96,9 @@ impl diff_service_server::DiffService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetPatch"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("diff.get_patch", %repo); let _enter = span.enter(); @@ -86,13 +108,18 @@ impl diff_service_server::DiffService for GitksService { if let Some(mut client) = remote_diff_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); let resp = client.get_patch(inner).await?; let stream = super::bridge_server_stream(resp.into_inner()); return Ok(tonic::Response::new(stream)); } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let items = if cache::selectors_are_oid(&inner.base, &inner.head) { cache::cached_vec_response("diff.get_patch", &inner, || { @@ -101,6 +128,7 @@ impl diff_service_server::DiffService for GitksService { } else { gb.get_patch(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(into_stream(items))) } @@ -108,7 +136,9 @@ impl diff_service_server::DiffService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetDiffStats"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("diff.get_diff_stats", %repo); let _enter = span.enter(); @@ -118,11 +148,16 @@ impl diff_service_server::DiffService for GitksService { if let Some(mut client) = remote_diff_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_diff_stats(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selectors_are_oid(&inner.base, &inner.head) { cache::cached_response("diff.get_diff_stats", &inner, || { @@ -131,6 +166,7 @@ impl diff_service_server::DiffService for GitksService { } else { gb.get_diff_stats(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/server/merge.rs b/server/merge.rs index bc81392..9814bc7 100644 --- a/server/merge.rs +++ b/server/merge.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, into_status}; -remote_client!(remote_merge_client, MergeServiceClient, "merge"); +remote_client!( + remote_merge_client, + MergeServiceClient, + "merge" +); #[tonic::async_trait] impl merge_service_server::MergeService for GitksService { @@ -11,7 +15,9 @@ impl merge_service_server::MergeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.MergeService/CheckMerge"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("merge.check_merge", %repo); let _enter = span.enter(); @@ -21,14 +27,20 @@ impl merge_service_server::MergeService for GitksService { if let Some(mut client) = remote_merge_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.check_merge(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.check_merge(inner).map_err(into_status)?; tracing::info!(%repo, status = resp.status, "check_merge done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -36,7 +48,9 @@ impl merge_service_server::MergeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.MergeService/Merge"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let target = inner.target_branch.clone(); let span = tracing::info_span!("merge.merge", %repo, %target); @@ -47,15 +61,21 @@ impl merge_service_server::MergeService for GitksService { if let Some(mut client) = remote_merge_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.merge(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.merge(inner).map_err(into_status)?; tracing::info!(%repo, %target, status = resp.status, "merge done"); self.notify_ref_update(&repo, &format!("refs/heads/{}", target), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -63,7 +83,9 @@ impl merge_service_server::MergeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.MergeService/ListMergeConflicts"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("merge.list_merge_conflicts", %repo); let _enter = span.enter(); @@ -73,14 +95,20 @@ impl merge_service_server::MergeService for GitksService { if let Some(mut client) = remote_merge_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.list_merge_conflicts(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.list_merge_conflicts(inner).map_err(into_status)?; tracing::info!(%repo, conflicts = resp.conflicts.len(), "list_merge_conflicts done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -88,7 +116,9 @@ impl merge_service_server::MergeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.MergeService/ResolveMergeConflicts"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let target = inner.target_branch.clone(); let span = tracing::info_span!("merge.resolve_merge_conflicts", %repo, %target); @@ -99,15 +129,21 @@ impl merge_service_server::MergeService for GitksService { if let Some(mut client) = remote_merge_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.resolve_merge_conflicts(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.resolve_merge_conflicts(inner).map_err(into_status)?; tracing::info!(%repo, %target, status = resp.status, "merge conflicts resolved"); self.notify_ref_update(&repo, &format!("refs/heads/{}", target), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -115,7 +151,9 @@ impl merge_service_server::MergeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.MergeService/Rebase"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let branch = inner.branch.clone(); let span = tracing::info_span!("merge.rebase", %repo, %branch); @@ -126,15 +164,21 @@ impl merge_service_server::MergeService for GitksService { if let Some(mut client) = remote_merge_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.rebase(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.rebase(inner).map_err(into_status)?; tracing::info!(%repo, %branch, status = resp.status, "rebase done"); self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/server/mod.rs b/server/mod.rs index 94ca4e7..aee2304 100644 --- a/server/mod.rs +++ b/server/mod.rs @@ -62,6 +62,9 @@ pub struct GitksService { pub repo_prefix: PathBuf, pub node_actor: Option>, pub grpc_addr: String, + pub disk_cache: Option, + pub pack_cache: Option, + pub hook_manager: Option, } impl GitksService { @@ -70,6 +73,9 @@ impl GitksService { repo_prefix, node_actor: None, grpc_addr: String::new(), + disk_cache: None, + pack_cache: None, + hook_manager: None, } } @@ -78,6 +84,21 @@ impl GitksService { self } + pub fn with_disk_cache(mut self, dc: crate::disk_cache::DiskCache) -> Self { + self.disk_cache = Some(dc); + self + } + + pub fn with_pack_cache(mut self, pc: crate::pack_cache::PackCache) -> Self { + self.pack_cache = Some(pc); + self + } + + pub fn with_hook_manager(mut self, hm: crate::hooks::HookManager) -> Self { + self.hook_manager = Some(hm); + self + } + pub fn with_grpc_addr(mut self, grpc_addr: String) -> Self { self.grpc_addr = grpc_addr; self @@ -156,6 +177,26 @@ impl GitksService { .unwrap_or_else(|| "unknown".into()) } + /// Get the relative path from a repository header, if any. + pub(crate) fn repo_relative_path<'a>(&self, header: Option<&'a crate::pb::RepositoryHeader>) -> Option<&'a str> { + header.and_then(|h| { + if h.relative_path.is_empty() { + None + } else { + Some(h.relative_path.as_str()) + } + }) + } + + /// Acquire a rate-limit permit for the repository in this request. + /// Returns a guard that releases the permit on drop. + pub(crate) async fn acquire_rate_limit( + &self, + header: Option<&crate::pb::RepositoryHeader>, + ) -> Result, tonic::Status> { + crate::rate_limit::acquire_or_reject(self.repo_relative_path(header)).await + } + pub(crate) fn resolve( &self, header: Option<&crate::pb::RepositoryHeader>, @@ -241,9 +282,14 @@ impl GitksService { old_oid: &str, new_oid: &str, ) { - // Invalidate caches that depend on this repository + // Invalidate moka caches crate::server::cache::invalidate_repo(relative_path); + // Invalidate disk cache + if let Some(ref pc) = self.pack_cache { + pc.invalidate_repo(relative_path); + } + if let Some(ref actor) = self.node_actor { let event = crate::actor::message::RefUpdateEvent { relative_path: relative_path.to_string(), diff --git a/server/pack.rs b/server/pack.rs index 4be83e5..6f2ca76 100644 --- a/server/pack.rs +++ b/server/pack.rs @@ -6,7 +6,11 @@ use crate::pb::*; use super::{GitksService, into_status}; -remote_client!(remote_pack_client, PackServiceClient, "pack"); +remote_client!( + remote_pack_client, + PackServiceClient, + "pack" +); #[tonic::async_trait] impl pack_service_server::PackService for GitksService { @@ -18,7 +22,9 @@ impl pack_service_server::PackService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.PackService/AdvertiseRefs"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("pack.advertise_refs", %repo); let _enter = span.enter(); @@ -28,14 +34,37 @@ impl pack_service_server::PackService for GitksService { if let Some(mut client) = remote_pack_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.advertise_refs(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; + + if let Some(ref pc) = self.pack_cache { + let protocol = inner.service.clone(); + if let Ok(digest) = pc.disk_cache().compute_info_refs_key(&repo, &protocol) { + if let Some(cached) = pc.lookup_info_refs::(&digest) { + tracing::info!(%repo, refs = cached.references.len(), "advertise_refs done (cached)"); + m.record("ok"); + return Ok(tonic::Response::new(cached)); + } + let resp = gb.advertise_refs(inner).map_err(into_status)?; + pc.store_info_refs(&digest, &resp); + tracing::info!(%repo, refs = resp.references.len(), "advertise_refs done (written to cache)"); + m.record("ok"); + return Ok(tonic::Response::new(resp)); + } + } + let resp = gb.advertise_refs(inner).map_err(into_status)?; tracing::info!(%repo, refs = resp.references.len(), "advertise_refs done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -43,11 +72,13 @@ impl pack_service_server::PackService for GitksService { &self, request: tonic::Request>, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.PackService/UploadPack"); let mut stream = request.into_inner(); let first = stream .next() .await .ok_or_else(|| tonic::Status::invalid_argument("empty upload-pack stream"))??; + let _rate = self.acquire_rate_limit(first.repository.as_ref()).await?; let repo = self.repo_label(first.repository.as_ref()); let span = tracing::info_span!("pack.upload_pack", %repo); let _enter = span.enter(); @@ -57,6 +88,7 @@ impl pack_service_server::PackService for GitksService { if let Some(mut client) = remote_pack_client(self, first.repository.as_ref(), false).await? { + m.record("ok"); let (tx, rx) = tokio::sync::mpsc::channel(16); let _ = tx.send(first).await; tokio::spawn(async move { @@ -78,9 +110,13 @@ impl pack_service_server::PackService for GitksService { let out = super::bridge_server_stream(resp.into_inner()); return Ok(tonic::Response::new(out)); } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; tracing::info!(%repo, "upload-pack streaming started"); @@ -97,6 +133,7 @@ impl pack_service_server::PackService for GitksService { }); let result = gb.upload_pack(ReceiverStream::new(rx)).await?; + m.record("ok"); Ok(tonic::Response::new(result)) } @@ -104,11 +141,13 @@ impl pack_service_server::PackService for GitksService { &self, request: tonic::Request>, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.PackService/ReceivePack"); let mut stream = request.into_inner(); let first = stream .next() .await .ok_or_else(|| tonic::Status::invalid_argument("empty receive-pack stream"))??; + let _rate = self.acquire_rate_limit(first.repository.as_ref()).await?; let repo = self.repo_label(first.repository.as_ref()); let span = tracing::info_span!("pack.receive_pack", %repo); let _enter = span.enter(); @@ -118,6 +157,7 @@ impl pack_service_server::PackService for GitksService { if let Some(mut client) = remote_pack_client(self, first.repository.as_ref(), false).await? { + m.record("ok"); let (tx, rx) = tokio::sync::mpsc::channel(16); let _ = tx.send(first).await; tokio::spawn(async move { @@ -139,9 +179,13 @@ impl pack_service_server::PackService for GitksService { let out = super::bridge_server_stream(resp.into_inner()); return Ok(tonic::Response::new(out)); } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; tracing::info!(%repo, "receive-pack streaming started"); @@ -158,6 +202,7 @@ impl pack_service_server::PackService for GitksService { }); let result = gb.receive_pack(ReceiverStream::new(rx)).await?; + m.record("ok"); Ok(tonic::Response::new(result)) } @@ -165,7 +210,9 @@ impl pack_service_server::PackService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.PackService/PackObjects"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("pack.pack_objects", %repo); let _enter = span.enter(); @@ -175,16 +222,97 @@ impl pack_service_server::PackService for GitksService { if let Some(mut client) = remote_pack_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); let resp = client.pack_objects(inner).await?; let stream = super::bridge_server_stream(resp.into_inner()); return Ok(tonic::Response::new(stream)); } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; + + if let Some(ref pc) = self.pack_cache + && let Some(opts) = inner.options.as_ref() + { + let wants_hex: Vec = opts.wants.iter().map(|w| w.hex.clone()).collect(); + let haves_hex: Vec = opts.haves.iter().map(|h| h.hex.clone()).collect(); + if let Ok(digest) = pc.disk_cache().compute_pack_objects_key( + &repo, + &wants_hex, + &haves_hex, + opts.thin_pack, + opts.use_bitmaps, + opts.delta_base_offset, + ) { + if let Some(file) = pc + .disk_cache() + .open_stream_read(crate::pack_cache::PACK_CACHE_NAMESPACE, &digest) + .ok() + .flatten() + { + tracing::info!(%repo, digest = %digest, "pack-objects cache hit, streaming from disk"); + m.record("ok"); + let (tx, rx) = tokio::sync::mpsc::channel(16); + tokio::spawn(async move { + let result = tokio::task::spawn_blocking(move || { + use std::io::Read; + let mut file = file; + let mut buf = vec![0u8; 65536]; + let mut chunks = Vec::new(); + loop { + match file.read(&mut buf) { + Ok(0) => break, + Ok(n) => chunks.push(Ok(PackfileChunk { + data: buf[..n].to_vec(), + })), + Err(e) => { + chunks.push(Err(tonic::Status::internal(format!( + "cache read error: {e}" + )))); + break; + } + } + } + chunks + }) + .await; + match result { + Ok(chunks) => { + for chunk in chunks { + if tx.send(chunk).await.is_err() { + break; + } + } + } + Err(e) => { + let _ = tx + .send(Err(tonic::Status::internal(format!( + "cache read task failed: {e}" + )))) + .await; + } + } + }); + return Ok(tonic::Response::new(ReceiverStream::new(rx))); + } + + // Cache miss: execute pack-objects and tee to cache + tracing::info!(%repo, digest = %digest, "pack-objects cache miss"); + let stream = gb.pack_objects(inner).await?; + let tee_stream = pc.tee_pack_stream(&digest, stream); + m.record("ok"); + return Ok(tonic::Response::new(tee_stream)); + } + } + let stream = gb.pack_objects(inner).await?; tracing::info!(%repo, "pack-objects streaming started"); + m.record("ok"); Ok(tonic::Response::new(stream)) } @@ -192,11 +320,15 @@ impl pack_service_server::PackService for GitksService { &self, request: tonic::Request>, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.PackService/IndexPack"); let mut stream = request.into_inner(); let mut inputs = Vec::new(); while let Some(msg) = stream.next().await { inputs.push(msg?); } + let _rate = self + .acquire_rate_limit(inputs.first().and_then(|r: &IndexPackRequest| r.repository.as_ref())) + .await?; let repo = self.repo_label(inputs.first().and_then(|r| r.repository.as_ref())); let span = tracing::info_span!("pack.index_pack", %repo); let _enter = span.enter(); @@ -210,14 +342,20 @@ impl pack_service_server::PackService for GitksService { ) .await? { + m.record("ok"); return client.index_pack(tokio_stream::iter(inputs)).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.index_pack(inputs).map_err(into_status)?; tracing::info!(%repo, objects = resp.object_count, "index_pack done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -225,7 +363,9 @@ impl pack_service_server::PackService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.PackService/ListPackfiles"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("pack.list_packfiles", %repo); let _enter = span.enter(); @@ -235,14 +375,20 @@ impl pack_service_server::PackService for GitksService { if let Some(mut client) = remote_pack_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.list_packfiles(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.list_packfiles(inner).map_err(into_status)?; tracing::info!(%repo, count = resp.packfiles.len(), "list_packfiles done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -250,7 +396,9 @@ impl pack_service_server::PackService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.PackService/Fsck"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("pack.fsck", %repo); let _enter = span.enter(); @@ -260,14 +408,20 @@ impl pack_service_server::PackService for GitksService { if let Some(mut client) = remote_pack_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.fsck(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.fsck(inner).map_err(into_status)?; tracing::info!(%repo, ok = resp.ok, errors = resp.errors.len(), warnings = resp.warnings.len(), "fsck done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/server/repository.rs b/server/repository.rs index af55458..6a757d7 100644 --- a/server/repository.rs +++ b/server/repository.rs @@ -2,8 +2,13 @@ use crate::pb::repository_service_client::RepositoryServiceClient; use crate::pb::*; use super::{GitksService, git_cmd, into_status, repository_maint}; +use tokio_stream::wrappers::ReceiverStream; -remote_client!(remote_repository_client, RepositoryServiceClient, "repository"); +remote_client!( + remote_repository_client, + RepositoryServiceClient, + "repository" +); fn default_branch_name(gb: &crate::bare::GitBare) -> String { git_cmd(gb, &["symbolic-ref", "HEAD"]) @@ -23,7 +28,9 @@ impl repository_service_server::RepositoryService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.RepositoryService/GetRepository"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("repo.get_repository", %repo); let _enter = span.enter(); @@ -33,14 +40,20 @@ impl repository_service_server::RepositoryService for GitksService { if let Some(mut client) = remote_repository_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_repository(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let bare = gb.bare_dir.join("HEAD").exists(); let object_format = gb.object_format(); + m.record("ok"); Ok(tonic::Response::new(Repository { header: inner.repository, bare, @@ -54,15 +67,21 @@ impl repository_service_server::RepositoryService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.RepositoryService/InitRepository"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("repo.init_repository", %repo); let _enter = span.enter(); let bare_dir = self.resolve_for_init(inner.repository.as_ref())?; let gb = crate::bare::GitBare::new(bare_dir); gb.init_repository(inner.bare).map_err(into_status)?; + if let Some(ref hm) = self.hook_manager { + hm.install_hooks(&gb.bare_dir).map_err(into_status)?; + } tracing::info!(%repo, bare = inner.bare, "repository initialized"); self.notify_ref_update(&repo, "HEAD", "", ""); + m.record("ok"); Ok(tonic::Response::new(Repository { header: inner.repository, bare: inner.bare, @@ -74,7 +93,9 @@ impl repository_service_server::RepositoryService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.RepositoryService/DeleteRepository"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("repo.delete_repository", %repo); let _enter = span.enter(); @@ -83,12 +104,15 @@ impl repository_service_server::RepositoryService for GitksService { && let Some(mut client) = remote_repository_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.delete_repository(inner).await; } tracing::warn!(%repo, path = %bare_dir.display(), "deleting repository"); std::fs::remove_dir_all(&bare_dir).map_err(|e| tonic::Status::internal(e.to_string()))?; tracing::info!(%repo, "repository deleted"); self.notify_ref_update(&repo, "", "", ""); + crate::rate_limit::remove_repository(&repo); + m.record("ok"); Ok(tonic::Response::new(())) } @@ -421,4 +445,265 @@ impl repository_service_server::RepositoryService for GitksService { tracing::info!(%repo, ok = resp.ok, "commit-graph write done"); Ok(tonic::Response::new(resp)) } + + // ── Hooks Management ──────────────────────────────────────────── + + async fn list_hooks( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let gb = self.resolve(inner.repository.as_ref())?; + let hook_mgr = self.hook_manager.as_ref(); + let hooks = if let Some(hm) = hook_mgr { + hm.list_hooks(&gb.bare_dir) + .map_err(|e| tonic::Status::internal(e.to_string()))? + } else { + Vec::new() + }; + let resp = ListHooksResponse { + hooks: hooks + .into_iter() + .map(|h| crate::pb::HookInfo { + hook_type: h.hook_type, + level: h.level.to_string(), + path: h.path, + }) + .collect(), + }; + Ok(tonic::Response::new(resp)) + } + + async fn set_custom_hook( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let gb = self.resolve(inner.repository.as_ref())?; + let hook_mgr = self.hook_manager.as_ref(); + if let Some(hm) = hook_mgr { + hm.set_custom_hook(&gb.bare_dir, &inner.hook_name, &inner.content) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + } else { + return Err(tonic::Status::failed_precondition("hooks not enabled")); + } + tracing::info!(repo = %gb.bare_dir.display(), hook = %inner.hook_name, "custom hook set"); + Ok(tonic::Response::new(())) + } + + async fn remove_custom_hook( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let gb = self.resolve(inner.repository.as_ref())?; + let hook_mgr = self.hook_manager.as_ref(); + if let Some(hm) = hook_mgr { + hm.remove_custom_hook(&gb.bare_dir, &inner.hook_name) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + } else { + return Err(tonic::Status::failed_precondition("hooks not enabled")); + } + tracing::info!(repo = %gb.bare_dir.display(), hook = %inner.hook_name, "custom hook removed"); + Ok(tonic::Response::new(())) + } + + // ── Snapshot Operations ────────────────────────────────────────── + + async fn create_snapshot( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let gb = self.resolve(inner.repository.as_ref())?; + let repo = self.repo_label(inner.repository.as_ref()); + let span = tracing::info_span!("repo.create_snapshot", %repo); + let _enter = span.enter(); + + let storage = crate::snapshot::storage::LocalSnapshotStorage::new( + self.repo_prefix.join("+gitks-snapshots"), + ); + let snapshot_id = crate::snapshot::ops::create_and_store_snapshot(&gb, &repo, &storage) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + let head_oid = crate::snapshot::ops::get_head_oid_internal(&gb) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + use crate::snapshot::storage::SnapshotStorageBackend; + let actual_size = storage + .read_snapshot(&snapshot_id) + .map(|d| d.len() as u64) + .unwrap_or(0); + + tracing::info!(%repo, snapshot_id = %snapshot_id, size_bytes = actual_size, "snapshot created"); + Ok(tonic::Response::new(CreateSnapshotResponse { + snapshot_id, + size_bytes: actual_size, + head_oid, + })) + } + + async fn restore_snapshot( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let target_repo = self.repo_label(inner.target_repository.as_ref()); + let span = tracing::info_span!("repo.restore_snapshot", %target_repo); + let _enter = span.enter(); + + let storage = crate::snapshot::storage::LocalSnapshotStorage::new( + self.repo_prefix.join("+gitks-snapshots"), + ); + + let target_path = self.resolve_for_init(inner.target_repository.as_ref())?; + + crate::snapshot::ops::restore_from_storage(&target_path, &inner.snapshot_id, &storage) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + tracing::info!(%target_repo, snapshot_id = %inner.snapshot_id, "snapshot restored"); + self.notify_ref_update(&target_repo, "HEAD", "", ""); + Ok(tonic::Response::new(())) + } + + async fn list_snapshots( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let repo = self.repo_label(inner.repository.as_ref()); + + let storage = crate::snapshot::storage::LocalSnapshotStorage::new( + self.repo_prefix.join("+gitks-snapshots"), + ); + use crate::snapshot::storage::SnapshotStorageBackend; + let snapshots = storage + .list_snapshots(&repo) + .map_err(tonic::Status::internal)?; + + let resp = ListSnapshotsResponse { + snapshots: snapshots + .into_iter() + .map(|s| crate::pb::SnapshotInfo { + snapshot_id: s.snapshot_id, + relative_path: s.relative_path, + size_bytes: s.size_bytes, + created_at: s.created_at, + head_oid: s.head_oid, + }) + .collect(), + }; + Ok(tonic::Response::new(resp)) + } + + async fn delete_snapshot( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + + let storage = crate::snapshot::storage::LocalSnapshotStorage::new( + self.repo_prefix.join("+gitks-snapshots"), + ); + use crate::snapshot::storage::SnapshotStorageBackend; + storage + .delete_snapshot(&inner.snapshot_id) + .map_err(tonic::Status::internal)?; + + tracing::info!(snapshot_id = %inner.snapshot_id, "snapshot deleted"); + Ok(tonic::Response::new(())) + } + + // ── Repository Move ────────────────────────────────────────────── + + type FetchRepositoryDataStream = + ReceiverStream>; + + async fn move_repository( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let source_repo = self.repo_label(inner.source_repository.as_ref()); + let span = tracing::info_span!("repo.move_repository", %source_repo); + let _enter = span.enter(); + + let gb = self.resolve(inner.source_repository.as_ref())?; + + let bundle_data = crate::snapshot::ops::create_snapshot(&gb) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + let target_path = self.resolve_for_init(inner.target_repository.as_ref())?; + + let target_gb = crate::bare::GitBare::new(target_path.clone()); + target_gb + .init_repository(true) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + crate::snapshot::ops::restore_snapshot(&target_path, &bundle_data) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + if let Some(ref hm) = self.hook_manager { + hm.install_hooks(&target_path) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + } + + let source_path = gb.bare_dir.clone(); + std::fs::remove_dir_all(&source_path) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + self.notify_ref_update(&source_repo, "HEAD", "", ""); + + tracing::info!(source = %source_repo, "repository moved successfully"); + Ok(tonic::Response::new(MoveRepositoryResponse { + state: MoveRepositoryState::MoveStateCompleted as i32, + error_message: String::new(), + })) + } + + async fn fetch_repository_data( + &self, + request: tonic::Request, + ) -> Result, tonic::Status> { + let inner = request.into_inner(); + let repo = self.repo_label(inner.repository.as_ref()); + let span = tracing::info_span!("repo.fetch_repository_data", %repo); + let _enter = span.enter(); + + let gb = self.resolve(inner.repository.as_ref())?; + let bundle_data = crate::snapshot::ops::create_snapshot(&gb) + .map_err(|e| tonic::Status::internal(e.to_string()))?; + + let (tx, rx) = tokio::sync::mpsc::channel(16); + tokio::spawn(async move { + const CHUNK_SIZE: usize = 65536; + let total = bundle_data.len(); + if total == 0 { + let _ = tx + .send(Ok(FetchRepositoryDataResponse { + data: vec![], + done: true, + })) + .await; + return; + } + for offset in (0..total).step_by(CHUNK_SIZE) { + let end = (offset + CHUNK_SIZE).min(total); + let chunk_data = bundle_data[offset..end].to_vec(); + let is_done = end >= total; + if tx + .send(Ok(FetchRepositoryDataResponse { + data: chunk_data, + done: is_done, + })) + .await + .is_err() + { + break; + } + } + }); + + Ok(tonic::Response::new(ReceiverStream::new(rx))) + } } diff --git a/server/tag.rs b/server/tag.rs index 2622b11..a70815e 100644 --- a/server/tag.rs +++ b/server/tag.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, into_status}; -remote_client!(remote_tag_client, TagServiceClient, "tag"); +remote_client!( + remote_tag_client, + TagServiceClient, + "tag" +); #[tonic::async_trait] impl tag_service_server::TagService for GitksService { @@ -11,7 +15,9 @@ impl tag_service_server::TagService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TagService/ListTags"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("tag.list_tags", %repo); let _enter = span.enter(); @@ -21,14 +27,20 @@ impl tag_service_server::TagService for GitksService { if let Some(mut client) = remote_tag_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.list_tags(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.list_tags(inner).map_err(into_status)?; tracing::info!(%repo, count = resp.tags.len(), "list_tags done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -36,7 +48,9 @@ impl tag_service_server::TagService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TagService/GetTag"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("tag.get_tag", %repo, %name); @@ -47,13 +61,19 @@ impl tag_service_server::TagService for GitksService { if let Some(mut client) = remote_tag_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_tag(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.get_tag(inner).map_err(into_status)?; + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -61,7 +81,9 @@ impl tag_service_server::TagService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TagService/CreateTag"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("tag.create_tag", %repo, %name); @@ -72,15 +94,21 @@ impl tag_service_server::TagService for GitksService { if let Some(mut client) = remote_tag_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.create_tag(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.create_tag(inner).map_err(into_status)?; tracing::info!(%repo, %name, "tag created"); self.notify_ref_update(&repo, &format!("refs/tags/{}", name), "", ""); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -88,7 +116,9 @@ impl tag_service_server::TagService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TagService/DeleteTag"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("tag.delete_tag", %repo, %name); @@ -99,15 +129,21 @@ impl tag_service_server::TagService for GitksService { if let Some(mut client) = remote_tag_client(self, inner.repository.as_ref(), true).await? { + m.record("ok"); return client.delete_tag(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; gb.delete_tag(inner).map_err(into_status)?; tracing::info!(%repo, %name, "tag deleted"); self.notify_ref_update(&repo, &format!("refs/tags/{}", name), "", ""); + m.record("ok"); Ok(tonic::Response::new(())) } @@ -115,7 +151,9 @@ impl tag_service_server::TagService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TagService/VerifyTag"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let name = inner.name.clone(); let span = tracing::info_span!("tag.verify_tag", %repo, %name); @@ -126,14 +164,20 @@ impl tag_service_server::TagService for GitksService { if let Some(mut client) = remote_tag_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.verify_tag(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = gb.verify_tag(inner).map_err(into_status)?; tracing::info!(%repo, %name, verified = resp.verified, "tag verified"); + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/server/tree.rs b/server/tree.rs index 862538d..1dee8a8 100644 --- a/server/tree.rs +++ b/server/tree.rs @@ -3,7 +3,11 @@ use crate::pb::*; use super::{GitksService, cache, into_status, into_stream}; -remote_client!(remote_tree_client, TreeServiceClient, "tree"); +remote_client!( + remote_tree_client, + TreeServiceClient, + "tree" +); #[tonic::async_trait] impl tree_service_server::TreeService for GitksService { @@ -14,7 +18,9 @@ impl tree_service_server::TreeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TreeService/ListTree"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("tree.list_tree", %repo); let _enter = span.enter(); @@ -24,11 +30,16 @@ impl tree_service_server::TreeService for GitksService { if let Some(mut client) = remote_tree_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.list_tree(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("tree.list_tree", &inner, || { @@ -38,6 +49,7 @@ impl tree_service_server::TreeService for GitksService { gb.list_tree(inner).map_err(into_status)? }; tracing::info!(%repo, count = resp.entries.len(), "list_tree done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -45,7 +57,9 @@ impl tree_service_server::TreeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetTree"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("tree.get_tree", %repo); let _enter = span.enter(); @@ -55,11 +69,16 @@ impl tree_service_server::TreeService for GitksService { if let Some(mut client) = remote_tree_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_tree(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("tree.get_tree", &inner, || { @@ -68,6 +87,7 @@ impl tree_service_server::TreeService for GitksService { } else { gb.get_tree(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -75,7 +95,9 @@ impl tree_service_server::TreeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetBlob"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let path = inner.path.clone(); let span = tracing::info_span!("tree.get_blob", %repo, %path); @@ -86,11 +108,16 @@ impl tree_service_server::TreeService for GitksService { if let Some(mut client) = remote_tree_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_blob(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("tree.get_blob", &inner, || { @@ -99,6 +126,7 @@ impl tree_service_server::TreeService for GitksService { } else { gb.get_blob(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -106,7 +134,9 @@ impl tree_service_server::TreeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetRawBlob"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("tree.get_raw_blob", %repo); let _enter = span.enter(); @@ -116,13 +146,18 @@ impl tree_service_server::TreeService for GitksService { if let Some(mut client) = remote_tree_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); let resp = client.get_raw_blob(inner).await?; let stream = super::bridge_server_stream(resp.into_inner()); return Ok(tonic::Response::new(stream)); } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let items = if inner.oid.is_some() { cache::cached_vec_response("tree.get_raw_blob", &inner, || { @@ -135,6 +170,7 @@ impl tree_service_server::TreeService for GitksService { } else { gb.get_raw_blob(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(into_stream(items))) } @@ -142,7 +178,9 @@ impl tree_service_server::TreeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetFileMetadata"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("tree.get_file_metadata", %repo); let _enter = span.enter(); @@ -152,11 +190,16 @@ impl tree_service_server::TreeService for GitksService { if let Some(mut client) = remote_tree_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.get_file_metadata(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("tree.get_file_metadata", &inner, || { @@ -165,6 +208,7 @@ impl tree_service_server::TreeService for GitksService { } else { gb.get_file_metadata(inner).map_err(into_status)? }; + m.record("ok"); Ok(tonic::Response::new(resp)) } @@ -172,7 +216,9 @@ impl tree_service_server::TreeService for GitksService { &self, request: tonic::Request, ) -> Result, tonic::Status> { + let m = crate::metrics::RequestMetrics::new("gitks.TreeService/FindFiles"); let inner = request.into_inner(); + let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?; let repo = self.repo_label(inner.repository.as_ref()); let span = tracing::info_span!("tree.find_files", %repo); let _enter = span.enter(); @@ -182,11 +228,16 @@ impl tree_service_server::TreeService for GitksService { if let Some(mut client) = remote_tree_client(self, inner.repository.as_ref(), false).await? { + m.record("ok"); return client.find_files(inner).await; } + crate::metrics::record_rpc_error(&m, &err); + return Err(err); + } + Err(err) => { + crate::metrics::record_rpc_error(&m, &err); return Err(err); } - Err(err) => return Err(err), }; let resp = if cache::selector_is_oid(&inner.revision) { cache::cached_response("tree.find_files", &inner, || { @@ -196,6 +247,7 @@ impl tree_service_server::TreeService for GitksService { gb.find_files(inner).map_err(into_status)? }; tracing::info!(%repo, count = resp.files.len(), "find_files done"); + m.record("ok"); Ok(tonic::Response::new(resp)) } } diff --git a/snapshot/mod.rs b/snapshot/mod.rs new file mode 100644 index 0000000..79b7a57 --- /dev/null +++ b/snapshot/mod.rs @@ -0,0 +1,13 @@ +//! Repository snapshot and move operations. +//! +//! Supports: +//! - Creating snapshots (git bundle) of repositories for backup +//! - Restoring snapshots to new or existing repositories +//! - Moving repositories between cluster nodes +//! - Listing and deleting snapshots + +pub mod ops; +pub mod storage; + +pub use ops::{create_snapshot, restore_snapshot, verify_snapshot}; +pub use storage::{LocalSnapshotStorage, SnapshotInfo, SnapshotStorageBackend}; diff --git a/snapshot/ops.rs b/snapshot/ops.rs new file mode 100644 index 0000000..4b25533 --- /dev/null +++ b/snapshot/ops.rs @@ -0,0 +1,169 @@ +//! Snapshot and move operations. +//! +//! Core operations for creating, restoring, and verifying repository snapshots +//! using git bundle. + +use std::path::Path; + +use crate::bare::GitBare; +use crate::error::{GitError, GitResult}; +use crate::snapshot::storage::SnapshotStorageBackend; + +/// Create a git bundle snapshot of a repository. +/// Returns the bundle data as raw bytes. +pub fn create_snapshot(gb: &GitBare) -> GitResult> { + tracing::info!(repo = %gb.bare_dir.display(), "creating snapshot bundle"); + + let bare_dir_str = gb.bare_dir.to_string_lossy().into_owned(); + let tmp_file = tempfile::Builder::new() + .prefix("gitks_snapshot_") + .suffix(".bundle") + .tempfile_in(&gb.bare_dir) + .map_err(GitError::Io)?; + let bundle_path_str = tmp_file.path().to_string_lossy().into_owned(); + + let result = duct::cmd!( + "git", + "--git-dir", + &bare_dir_str, + "bundle", + "create", + &bundle_path_str, + "--all" + ) + .stdout_capture() + .stderr_capture() + .unchecked() + .run()?; + + if !result.status.success() { + let stderr = String::from_utf8_lossy(&result.stderr); + return Err(GitError::CommandFailed { + status_code: result.status.code(), + stderr: stderr.into_owned(), + }); + } + + let bundle_path = tmp_file.path().to_path_buf(); + let data = std::fs::read(&bundle_path).map_err(GitError::Io)?; + + let head_oid = get_head_oid(gb)?; + + tracing::info!( + repo = %gb.bare_dir.display(), + size_bytes = data.len(), + head_oid = %head_oid, + "snapshot bundle created" + ); + + Ok(data) +} + +/// Create a snapshot and store it using the given backend. +pub fn create_and_store_snapshot( + gb: &GitBare, + relative_path: &str, + storage: &dyn SnapshotStorageBackend, +) -> GitResult { + let data = create_snapshot(gb)?; + let head_oid = get_head_oid(gb)?; + + let snapshot_id = generate_snapshot_id(relative_path, &head_oid); + + storage + .write_snapshot(&snapshot_id, relative_path, &head_oid, &data) + .map_err(GitError::Internal)?; + + Ok(snapshot_id) +} + +/// Restore a snapshot to a repository path. +pub fn restore_snapshot(repo_path: &Path, data: &[u8]) -> GitResult<()> { + tracing::info!(path = %repo_path.display(), size_bytes = data.len(), "restoring snapshot"); + + let applicator = crate::actor::sync::BundleApplicator::new(repo_path.to_path_buf()); + applicator.apply_bundle(data).map_err(GitError::Internal)?; + + tracing::info!(path = %repo_path.display(), "snapshot restored"); + Ok(()) +} + +/// Restore a snapshot from storage backend. +pub fn restore_from_storage( + repo_path: &Path, + snapshot_id: &str, + storage: &dyn SnapshotStorageBackend, +) -> GitResult<()> { + let data = storage + .read_snapshot(snapshot_id) + .map_err(GitError::Internal)?; + + restore_snapshot(repo_path, &data)?; + Ok(()) +} + +/// Verify that a restored snapshot matches the expected HEAD OID. +pub fn verify_snapshot(gb: &GitBare, expected_head_oid: &str) -> GitResult { + let actual_head_oid = get_head_oid(gb)?; + + if actual_head_oid == expected_head_oid { + tracing::info!( + repo = %gb.bare_dir.display(), + expected = %expected_head_oid, + actual = %actual_head_oid, + "snapshot verification passed" + ); + Ok(true) + } else { + tracing::warn!( + repo = %gb.bare_dir.display(), + expected = %expected_head_oid, + actual = %actual_head_oid, + "snapshot verification failed: HEAD mismatch" + ); + Ok(false) + } +} + +/// Get HEAD OID of a repository (public helper for RPC). +pub fn get_head_oid_internal(gb: &GitBare) -> GitResult { + get_head_oid(gb) +} + +/// Get HEAD OID of a repository. +fn get_head_oid(gb: &GitBare) -> GitResult { + let bare_dir_str = gb.bare_dir.to_string_lossy().into_owned(); + let result = duct::cmd!("git", "--git-dir", &bare_dir_str, "rev-parse", "HEAD") + .stdout_capture() + .stderr_capture() + .unchecked() + .run()?; + + if result.status.success() { + Ok(String::from_utf8_lossy(&result.stdout).trim().to_string()) + } else { + // Repository may be empty (no HEAD) + Ok(String::new()) + } +} + +/// Generate a snapshot ID from relative_path and head_oid. +fn generate_snapshot_id(relative_path: &str, head_oid: &str) -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + let ts = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(relative_path.as_bytes()); + hasher.update(head_oid.as_bytes()); + hasher.update(ts.to_le_bytes()); + let hash = hasher.finalize(); + let mut s = String::with_capacity(16); + for byte in &hash[..8] { + use std::fmt::Write; + write!(s, "{byte:02x}").unwrap(); + } + s +} diff --git a/snapshot/storage.rs b/snapshot/storage.rs new file mode 100644 index 0000000..766b49a --- /dev/null +++ b/snapshot/storage.rs @@ -0,0 +1,166 @@ +//! Snapshot storage backend abstraction. +//! +//! Currently implements local filesystem storage. +//! S3/GCS backends can be added later as optional dependencies. + +use std::path::PathBuf; +use std::time::SystemTime; + +/// Metadata about a snapshot. +#[derive(Debug, Clone)] +pub struct SnapshotInfo { + pub snapshot_id: String, + pub relative_path: String, + pub size_bytes: u64, + pub created_at: String, // ISO 8601 + pub head_oid: String, +} + +/// Trait for snapshot storage backends. +pub trait SnapshotStorageBackend: Send + Sync { + fn write_snapshot( + &self, + snapshot_id: &str, + relative_path: &str, + head_oid: &str, + data: &[u8], + ) -> Result<(), String>; + fn read_snapshot(&self, snapshot_id: &str) -> Result, String>; + fn list_snapshots(&self, relative_path: &str) -> Result, String>; + fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String>; +} + +/// Local filesystem snapshot storage. +pub struct LocalSnapshotStorage { + base_dir: PathBuf, +} + +impl LocalSnapshotStorage { + pub fn new(base_dir: PathBuf) -> Self { + Self { base_dir } + } + + fn snapshot_dir(&self, snapshot_id: &str) -> PathBuf { + let prefix = &snapshot_id[..2.min(snapshot_id.len())]; + self.base_dir.join(prefix).join(snapshot_id) + } + + fn metadata_path(&self, snapshot_id: &str) -> PathBuf { + self.snapshot_dir(snapshot_id).join("metadata.json") + } + + fn data_path(&self, snapshot_id: &str) -> PathBuf { + self.snapshot_dir(snapshot_id).join("bundle.dat") + } +} + +impl SnapshotStorageBackend for LocalSnapshotStorage { + fn write_snapshot( + &self, + snapshot_id: &str, + relative_path: &str, + head_oid: &str, + data: &[u8], + ) -> Result<(), String> { + let dir = self.snapshot_dir(snapshot_id); + std::fs::create_dir_all(&dir).map_err(|e| format!("create dir: {e}"))?; + + let data_path = self.data_path(snapshot_id); + let tmp_path = data_path.with_extension("tmp"); + std::fs::write(&tmp_path, data).map_err(|e| format!("write data: {e}"))?; + std::fs::rename(&tmp_path, &data_path).map_err(|e| format!("rename: {e}"))?; + + let created_at = SystemTime::now() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + let metadata = serde_json::json!({ + "snapshot_id": snapshot_id, + "relative_path": relative_path, + "size_bytes": data.len(), + "created_at": created_at, + "head_oid": head_oid, + }); + let metadata_str = serde_json::to_string_pretty(&metadata) + .map_err(|e| format!("serialize metadata: {e}"))?; + std::fs::write(self.metadata_path(snapshot_id), metadata_str) + .map_err(|e| format!("write metadata: {e}"))?; + + tracing::info!( + snapshot_id = %snapshot_id, + size_bytes = data.len(), + "snapshot written to local storage" + ); + Ok(()) + } + + fn read_snapshot(&self, snapshot_id: &str) -> Result, String> { + let path = self.data_path(snapshot_id); + if !path.exists() { + return Err(format!("snapshot not found: {snapshot_id}")); + } + std::fs::read(&path).map_err(|e| format!("read snapshot: {e}")) + } + + fn list_snapshots(&self, relative_path: &str) -> Result, String> { + let mut snapshots = Vec::new(); + if !self.base_dir.exists() { + return Ok(snapshots); + } + + for shard_entry in + std::fs::read_dir(&self.base_dir).map_err(|e| format!("read dir: {e}"))? + { + let shard_entry = shard_entry.map_err(|e| format!("entry: {e}"))?; + let shard_dir = shard_entry.path(); + if !shard_dir.is_dir() { + continue; + } + + for snap_entry in std::fs::read_dir(&shard_dir).map_err(|e| format!("read dir: {e}"))? { + let snap_entry = snap_entry.map_err(|e| format!("entry: {e}"))?; + let snap_dir = snap_entry.path(); + if !snap_dir.is_dir() { + continue; + } + + let metadata_path = snap_dir.join("metadata.json"); + if !metadata_path.exists() { + continue; + } + + let metadata_str = std::fs::read_to_string(&metadata_path) + .map_err(|e| format!("read metadata: {e}"))?; + let metadata: serde_json::Value = serde_json::from_str(&metadata_str) + .map_err(|e| format!("parse metadata: {e}"))?; + + let snap_relative_path = metadata["relative_path"].as_str().unwrap_or(""); + + if !relative_path.is_empty() && snap_relative_path != relative_path { + continue; + } + + snapshots.push(SnapshotInfo { + snapshot_id: metadata["snapshot_id"].as_str().unwrap_or("").to_string(), + relative_path: snap_relative_path.to_string(), + size_bytes: metadata["size_bytes"].as_u64().unwrap_or(0), + created_at: metadata["created_at"].as_str().unwrap_or("").to_string(), + head_oid: metadata["head_oid"].as_str().unwrap_or("").to_string(), + }); + } + } + + snapshots.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + Ok(snapshots) + } + + fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String> { + let dir = self.snapshot_dir(snapshot_id); + if !dir.exists() { + return Err(format!("snapshot not found: {snapshot_id}")); + } + std::fs::remove_dir_all(&dir).map_err(|e| format!("delete snapshot: {e}"))?; + tracing::info!(snapshot_id = %snapshot_id, "snapshot deleted"); + Ok(()) + } +} diff --git a/tests/disk_cache_test.rs b/tests/disk_cache_test.rs new file mode 100644 index 0000000..b629eff --- /dev/null +++ b/tests/disk_cache_test.rs @@ -0,0 +1,157 @@ +//! Tests for DiskCache and PackCache. + +use std::path::PathBuf; +use std::time::Duration; + +use gitks::disk_cache::DiskCache; + +fn temp_dir() -> PathBuf { + tempfile::tempdir().unwrap().path().to_path_buf() +} + +#[test] +fn test_disk_cache_basic_operations() { + let dir = temp_dir(); + let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true); + + // Ensure state creates latest file + let state = cache.ensure_state("test_repo.git").unwrap(); + assert!(!state.is_empty()); + + // Same state on second call + let state2 = cache.ensure_state("test_repo.git").unwrap(); + assert_eq!(state, state2); +} + +#[test] +fn test_disk_cache_insert_and_lookup() { + let dir = temp_dir(); + let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true); + + let digest = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + let data = b"test cache data"; + + // Insert + cache.insert("+gitks-cache/cache", digest, data).unwrap(); + + // Lookup + let result = cache.lookup("+gitks-cache/cache", digest).unwrap(); + assert!(result.is_some()); + assert_eq!(result.unwrap(), data.to_vec()); + + // Non-existent key + let result = cache.lookup("+gitks-cache/cache", "nonexistent").unwrap(); + assert!(result.is_none()); +} + +#[test] +fn test_disk_cache_invalidation() { + let dir = temp_dir(); + let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true); + + let state1 = cache.ensure_state("test_repo.git").unwrap(); + + // Invalidate + cache.invalidate_repo("test_repo.git"); + + // State should change + let state2 = cache.ensure_state("test_repo.git").unwrap(); + assert_ne!(state1, state2); +} + +#[test] +fn test_disk_cache_disabled() { + let dir = temp_dir(); + let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, false); + + // All operations should succeed but do nothing + let state = cache.ensure_state("test_repo.git").unwrap(); + assert!(!state.is_empty()); // Returns random value + + let result = cache.lookup("+gitks-cache/cache", "anykey").unwrap(); + assert!(result.is_none()); // Disabled → always None +} + +#[test] +fn test_disk_cache_lease_guard() { + let dir = temp_dir(); + let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true); + + let state1 = cache.ensure_state("test_repo.git").unwrap(); + + let mut lease = cache.create_lease("test_repo.git").unwrap(); + // Lease exists + let pending_dir = dir.join("+gitks-cache/state/test_repo.git/pending"); + assert!(pending_dir.exists()); + + // Commit lease (updates latest) + lease.commit(); + + // Pending should be cleaned up + // (may still have dir but no files) + let state2 = cache.ensure_state("test_repo.git").unwrap(); + assert_ne!(state1, state2); +} + +#[test] +fn test_sha256_digest_determinism() { + let dir = temp_dir(); + let cache = DiskCache::new(dir, "v1".to_string(), 300, true); + cache.ensure_state("repo.git").unwrap(); + + let key1 = cache + .compute_info_refs_key("repo.git", "upload-pack") + .unwrap(); + let key2 = cache + .compute_info_refs_key("repo.git", "upload-pack") + .unwrap(); + assert_eq!(key1, key2); + + // Different protocol should produce different key + let key3 = cache + .compute_info_refs_key("repo.git", "receive-pack") + .unwrap(); + assert_ne!(key1, key3); +} + +#[test] +fn test_cleanup_expired() { + let dir = temp_dir(); + // Very short max age for testing + let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 1, true); + + let digest = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789"; + cache.insert("+gitks-cache/cache", digest, b"data").unwrap(); + + // Should be available immediately + let result = cache.lookup("+gitks-cache/cache", digest).unwrap(); + assert!(result.is_some()); + + // Wait for expiration + std::thread::sleep(Duration::from_secs(2)); + + // Should be expired now + let result = cache.lookup("+gitks-cache/cache", digest).unwrap(); + assert!(result.is_none()); +} + +#[test] +fn test_startup_cleanup() { + let dir = temp_dir(); + let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true); + + // Write some cache data + cache + .insert("+gitks-cache/cache", "abc123", b"test") + .unwrap(); + assert!( + dir.join("+gitks-cache/cache") + .join("ab") + .join("c123") + .exists() + ); + + // Startup cleanup removes all cache dirs + cache.cleanup_on_startup().unwrap(); + assert!(!dir.join("+gitks-cache/cache").exists()); +} diff --git a/tests/hooks_test.rs b/tests/hooks_test.rs new file mode 100644 index 0000000..9b6a15f --- /dev/null +++ b/tests/hooks_test.rs @@ -0,0 +1,108 @@ +//! Tests for hooks system. + +use std::path::PathBuf; +use std::time::Duration; + +use gitks::hooks::manager::{HookLevel, HookManager}; +use gitks::hooks::sanitize::{validate_hook_content, validate_hook_name}; + +fn temp_repo() -> PathBuf { + let dir = tempfile::tempdir().unwrap().path().to_path_buf(); + // Create a bare git repo + let _ = std::process::Command::new("git") + .args(["init", "--bare"]) + .arg(&dir) + .output(); + dir +} + +#[test] +fn test_validate_hook_name() { + assert!(validate_hook_name("pre-receive").is_ok()); + assert!(validate_hook_name("update").is_ok()); + assert!(validate_hook_name("post-receive").is_ok()); + assert!(validate_hook_name("commit-msg").is_ok()); + assert!(validate_hook_name("invalid-hook").is_err()); + assert!(validate_hook_name("").is_err()); + assert!(validate_hook_name("random-name").is_err()); +} + +#[test] +fn test_validate_hook_content_safe() { + assert!(validate_hook_content("#!/bin/sh\nexit 0").is_ok()); + assert!(validate_hook_content("#!/bin/sh\necho 'hello'").is_ok()); +} + +#[test] +fn test_validate_hook_content_dangerous() { + assert!(validate_hook_content("rm -rf /").is_err()); + assert!(validate_hook_content("shutdown now").is_err()); + assert!(validate_hook_content("chmod 777 /etc/passwd").is_err()); +} + +#[test] +fn test_validate_hook_content_size() { + let large_content = "x".repeat(70000); + assert!(validate_hook_content(&large_content).is_err()); +} + +#[test] +fn test_validate_hook_content_null_bytes() { + assert!(validate_hook_content("test\0content").is_err()); +} + +#[test] +fn test_hook_manager_install_hooks() { + let repo = temp_repo(); + let prefix = repo.parent().unwrap().to_path_buf(); + let hm = HookManager::new(prefix, None, None, Duration::from_secs(30), true); + + // Install hooks + let result = hm.install_hooks(&repo); + if result.is_ok() { + // Check that hook files were created + assert!(repo.join("hooks/pre-receive").exists()); + assert!(repo.join("hooks/update").exists()); + assert!(repo.join("hooks/post-receive").exists()); + } + // Installation may fail if git is not available; just check no panic +} + +#[test] +fn test_hook_manager_custom_hooks() { + let repo = temp_repo(); + let prefix = repo.parent().unwrap().to_path_buf(); + let hm = HookManager::new(prefix, None, None, Duration::from_secs(30), true); + + // Set a custom hook + let result = hm.set_custom_hook(&repo, "pre-receive", "#!/bin/sh\nexit 0"); + if result.is_ok() { + assert!(repo.join("custom_hooks/pre-receive/d").exists()); + } + + // Remove custom hook + let result = hm.remove_custom_hook(&repo, "pre-receive"); + assert!(result.is_ok() || result.is_err()); // Just no panic +} + +#[test] +fn test_hook_manager_disallow_custom() { + let repo = temp_repo(); + let prefix = repo.parent().unwrap().to_path_buf(); + let hm = HookManager::new( + prefix, + None, + None, + Duration::from_secs(30), + false, // Disallow custom hooks + ); + + let result = hm.set_custom_hook(&repo, "pre-receive", "#!/bin/sh\nexit 0"); + assert!(result.is_err()); +} + +#[test] +fn test_hook_level_display() { + assert_eq!(HookLevel::Server.to_string(), "server"); + assert_eq!(HookLevel::Custom.to_string(), "custom"); +} diff --git a/tests/snapshot_test.rs b/tests/snapshot_test.rs new file mode 100644 index 0000000..e8882a4 --- /dev/null +++ b/tests/snapshot_test.rs @@ -0,0 +1,142 @@ +//! Tests for snapshot operations. + +use std::path::PathBuf; + +use gitks::snapshot::ops::{create_snapshot, restore_snapshot}; +use gitks::snapshot::storage::{LocalSnapshotStorage, SnapshotStorageBackend}; + +fn temp_bare_repo() -> PathBuf { + let dir = tempfile::tempdir().unwrap().path().to_path_buf(); + let result = std::process::Command::new("git") + .args(["init", "--bare"]) + .arg(&dir) + .output() + .expect("git init --bare should work"); + assert!(result.status.success()); + + // Create an initial commit so HEAD exists + // We need a working tree to create a commit, so we create one temporarily + let work_dir = tempfile::tempdir().unwrap().path().to_path_buf(); + let result2 = std::process::Command::new("git") + .args(["init"]) + .arg(&work_dir) + .output() + .expect("git init should work"); + assert!(result2.status.success()); + + // Create a file and commit + std::fs::write(work_dir.join("test.txt"), "hello world").unwrap(); + let result3 = std::process::Command::new("git") + .args(["add", "test.txt"]) + .current_dir(&work_dir) + .output(); + let _ = result3; + + let result4 = std::process::Command::new("git") + .args(["commit", "-m", "initial commit"]) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@test.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@test.com") + .current_dir(&work_dir) + .output(); + let _ = result4; + + // Push to bare repo + let result5 = std::process::Command::new("git") + .args(["push", &dir.to_string_lossy(), "master:refs/heads/master"]) + .current_dir(&work_dir) + .output(); + let _ = result5; + + dir +} + +#[test] +fn test_snapshot_create_and_restore() { + let repo = temp_bare_repo(); + let gb = gitks::bare::GitBare::new(repo.clone()); + + // Create snapshot + let result = create_snapshot(&gb); + if let Ok(data) = result { + assert!(!data.is_empty()); + + // Restore to a different location + let target = tempfile::tempdir().unwrap().path().to_path_buf(); + // Initialize target as bare repo first + let init_result = std::process::Command::new("git") + .args(["init", "--bare"]) + .arg(&target) + .output(); + if let Ok(init_output) = init_result + && init_output.status.success() + { + let restore_result = restore_snapshot(&target, &data); + // May succeed or fail depending on git bundle compatibility + let _ = restore_result; + } + } +} + +#[test] +fn test_local_snapshot_storage_write_and_read() { + let dir = tempfile::tempdir().unwrap().path().to_path_buf(); + let storage = LocalSnapshotStorage::new(dir); + + // Write snapshot + let result = storage.write_snapshot("snap001", "test.git", "abc123", b"bundle_data"); + assert!(result.is_ok()); + + // Read snapshot + let result = storage.read_snapshot("snap001"); + assert!(result.is_ok()); + assert_eq!(result.unwrap(), b"bundle_data".to_vec()); + + // Read non-existent + let result = storage.read_snapshot("nonexistent"); + assert!(result.is_err()); +} + +#[test] +fn test_local_snapshot_storage_list() { + let dir = tempfile::tempdir().unwrap().path().to_path_buf(); + let storage = LocalSnapshotStorage::new(dir); + + // Write two snapshots + storage + .write_snapshot("snap001", "repo1.git", "abc123", b"data1") + .unwrap(); + storage + .write_snapshot("snap002", "repo2.git", "def456", b"data2") + .unwrap(); + + // List all + let result = storage.list_snapshots(""); + assert!(result.is_ok()); + assert_eq!(result.unwrap().len(), 2); + + // List filtered + let result = storage.list_snapshots("repo1.git"); + assert!(result.is_ok()); + let snapshots = result.unwrap(); + assert_eq!(snapshots.len(), 1); + assert_eq!(snapshots[0].relative_path, "repo1.git"); +} + +#[test] +fn test_local_snapshot_storage_delete() { + let dir = tempfile::tempdir().unwrap().path().to_path_buf(); + let storage = LocalSnapshotStorage::new(dir); + + storage + .write_snapshot("snap001", "test.git", "abc123", b"data") + .unwrap(); + assert!(storage.read_snapshot("snap001").is_ok()); + + storage.delete_snapshot("snap001").unwrap(); + assert!(storage.read_snapshot("snap001").is_err()); + + // Delete non-existent + assert!(storage.delete_snapshot("nonexistent").is_err()); +}