feat(cluster): implement distributed clustering with etcd coordination

- Integrate etcd-client for distributed coordination and leader election
- Add remote client macros with proper formatting for all services
- Implement RequestMetrics for tracking RPC performance and errors
- Add rate limiting mechanism across all service endpoints
- Create ElectionRequest and ElectionResult message types for leader election
- Add role management with primary/replica switching capabilities
- Implement health checker with automatic failover detection
- Add repository count metrics for cluster monitoring
- Update Cargo.toml with etcd-client and dashmap dependencies
- Modify RepoEntry to include read_only flag for replica handling
- Implement should_accept_election logic to prevent duplicate elections
- Add RoleChangedEvent handling for cluster role updates
This commit is contained in:
zhenyi
2026-06-08 14:31:29 +08:00
parent d243dce027
commit 8f472a0443
37 changed files with 4691 additions and 83 deletions
Generated
+24
View File
@@ -457,6 +457,24 @@ dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "etcd-client"
version = "0.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ed900ba953ca6bf1fadb75e0c6b73d8463b9e2bb6bdb7b4573e8e7295852fbe"
dependencies = [
"http",
"prost",
"tokio",
"tokio-stream",
"tonic",
"tonic-build",
"tonic-prost",
"tonic-prost-build",
"tower",
"tower-service",
]
[[package]]
name = "faster-hex"
version = "0.10.0"
@@ -668,8 +686,10 @@ name = "gitks"
version = "1.0.0"
dependencies = [
"async-trait",
"dashmap",
"dotenvy",
"duct",
"etcd-client",
"gix",
"gix-archive",
"moka",
@@ -678,6 +698,8 @@ dependencies = [
"ractor",
"ractor_cluster",
"serde",
"serde_json",
"sha2",
"tempfile",
"thiserror",
"tokio",
@@ -2462,6 +2484,7 @@ dependencies = [
"aws-lc-rs",
"log",
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki",
"subtle",
@@ -2936,6 +2959,7 @@ dependencies = [
"socket2",
"sync_wrapper",
"tokio",
"tokio-rustls",
"tokio-stream",
"tower",
"tower-layer",
+4
View File
@@ -18,6 +18,8 @@ name = "gitks"
[dependencies]
moka = { version = "0.12", default-features = false, features = ["sync"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
sha2 = "0.11"
gix = { version = "0.84", default-features = false, features = ["serde", "blame", "sha256", "sha1", "tracing", "merge", "max-performance-safe", "revision"] }
gix-archive = { version = "0.33", features = ["sha256","sha1","document-features"] }
duct = { version = "1", features = [] }
@@ -35,6 +37,8 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
ractor = { version = "0.15.13", features = ["cluster","tokio_runtime","monitors","message_span_propogation","async-trait"]}
ractor_cluster = { version = "0.15.13", features = ["async-trait"] }
async-trait = "0.1.89"
etcd-client = { version = "0.18.0", features = ["tls"] }
dashmap = "6"
[[bin]]
name = "gitks"
path = "main.rs"
+216 -10
View File
@@ -1,5 +1,6 @@
use crate::actor::message::{
GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA, RefUpdateEvent, RouteDecision,
ElectionRequest, ElectionResult, GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA,
RefUpdateEvent, RoleChangedEvent, RouteDecision,
};
use crate::server::GitksService;
use async_trait::async_trait;
@@ -25,6 +26,7 @@ impl GitNodeActor {
pub struct RepoEntry {
pub role: String,
pub last_commit: String,
pub read_only: bool,
}
pub struct GitNodeArgs {
@@ -37,6 +39,10 @@ pub struct GitNodeState {
actor_name: String,
grpc_addr: String,
repos: HashMap<String, RepoEntry>,
current_term: u64,
health_failures: u32,
is_primary: bool,
last_known_primary_grpc: String,
}
#[async_trait]
@@ -58,11 +64,18 @@ impl Actor for GitNodeActor {
vec![myself.get_cell()],
);
tracing::info!(storage_name = %args.storage_name, actor_name = %actor_name, grpc_addr = %args.grpc_addr, "GitNodeActor started");
start_health_checker(myself.clone(), 1, 10);
Ok(GitNodeState {
storage_name: args.storage_name,
actor_name,
grpc_addr: args.grpc_addr,
grpc_addr: args.grpc_addr.clone(),
repos: HashMap::new(),
current_term: 0,
health_failures: 0,
is_primary: true, // Will be refined at registration
last_known_primary_grpc: args.grpc_addr.clone(),
})
}
@@ -76,6 +89,7 @@ impl Actor for GitNodeActor {
GitNodeMessage::ScanAndRegister => {
let repos = self.service.scan_all_repo()?;
tracing::info!(storage_name = %state.storage_name, found = repos.len(), "scanning local repositories");
crate::metrics::set_repository_count(repos.len() as u64);
for repo_path in repos {
let relative_path = repo_path
.strip_prefix(self.service.repo_prefix.to_string_lossy().as_ref())
@@ -151,6 +165,79 @@ impl Actor for GitNodeActor {
})
.ok();
}
// ── Election & Role Change ──────────────────────────────────
GitNodeMessage::ElectPrimary(request, reply) => {
let accepted = should_accept_election(&request, state);
tracing::info!(
candidate = %request.candidate_storage_name,
term = request.term,
current_term = state.current_term,
accepted = accepted,
"election vote"
);
if accepted {
state.current_term = request.term;
state.last_known_primary_grpc = request.candidate_grpc_addr.clone();
}
reply
.send(ElectionResult {
accepted,
current_term: state.current_term,
voter_storage_name: state.storage_name.clone(),
voter_role: if state.is_primary {
ROLE_PRIMARY
} else {
ROLE_REPLICA
}
.to_string(),
})
.ok();
}
GitNodeMessage::RoleChanged(event) => {
// Empty storage_name = self-promotion from health checker
let is_self =
event.storage_name.is_empty() || event.storage_name == state.storage_name;
if is_self && event.new_role == ROLE_PRIMARY {
tracing::info!(
storage_name = %state.storage_name,
term = event.term,
"promoted to PRIMARY"
);
state.is_primary = true;
state.current_term = event.term;
state.health_failures = 0;
for entry in state.repos.values_mut() {
entry.role = ROLE_PRIMARY.to_string();
entry.read_only = false;
}
} else if is_self && event.new_role == ROLE_REPLICA {
tracing::info!(
storage_name = %state.storage_name,
term = event.term,
"demoted to REPLICA"
);
state.is_primary = false;
state.current_term = event.term;
for entry in state.repos.values_mut() {
entry.role = ROLE_REPLICA.to_string();
}
} else {
// Another node's role changed — update routing info
tracing::info!(
storage_name = %event.storage_name,
new_role = %event.new_role,
"remote node role changed"
);
state.last_known_primary_grpc = if event.new_role == ROLE_PRIMARY {
event.grpc_addr.clone()
} else {
state.last_known_primary_grpc.clone()
};
}
}
}
Ok(())
}
@@ -189,6 +276,21 @@ impl Actor for GitNodeActor {
}
}
/// Determine whether to accept an election request.
fn should_accept_election(request: &ElectionRequest, state: &GitNodeState) -> bool {
// Only accept if the term is greater than our current term
// (prevents old/duplicate election messages)
if request.term <= state.current_term {
tracing::warn!(
request_term = request.term,
current_term = state.current_term,
"rejecting election: term too old"
);
return false;
}
true
}
fn build_decision(
state: &GitNodeState,
header: &crate::pb::RepositoryHeader,
@@ -226,23 +328,20 @@ fn register_repo(
return;
}
// Determine role based on cluster state
// For simplicity and correctness, we use a conservative approach:
// If there are other nodes in the cluster, register as replica initially.
// The route_repository logic will determine the actual primary at query time.
let members = ractor::pg::get_members(&"gitks_nodes".to_string());
let my_cell = myself.get_cell();
let other_nodes_exist = members.iter().any(|m| m != &my_cell);
let role = if other_nodes_exist {
// Conservative: assume another node might be primary
// The actual primary will be determined by route_repository query
ROLE_REPLICA.to_string()
} else {
// We're the only node, so we're primary
ROLE_PRIMARY.to_string()
};
if role == ROLE_PRIMARY {
state.is_primary = true;
}
let category = extract_category(&relative_path);
pg::join_scoped(
state.storage_name.clone(),
@@ -254,6 +353,7 @@ fn register_repo(
RepoEntry {
role: role.clone(),
last_commit: String::new(),
read_only: false,
},
);
tracing::info!(
@@ -262,7 +362,7 @@ fn register_repo(
relative_path = %relative_path,
actor_name = %state.actor_name,
role = %role,
"repository route registered (role will be refined at query time)"
"repository route registered"
);
}
@@ -270,6 +370,101 @@ fn extract_category(relative_path: &str) -> &str {
relative_path.split('/').next().unwrap_or("root")
}
/// Start background health checker that monitors the PRIMARY node.
/// If the PRIMARY becomes unreachable for `max_failures` consecutive checks,
/// triggers an election.
fn start_health_checker(myself: ActorRef<GitNodeMessage>, interval_secs: u64, max_failures: u32) {
tokio::spawn(async move {
let mut interval = tokio::time::interval(std::time::Duration::from_secs(interval_secs));
interval.tick().await; // First tick immediate
let mut consecutive_failures: u32 = 0;
loop {
interval.tick().await;
let members = ractor::pg::get_members(&"gitks_nodes".to_string());
let my_cell = myself.get_cell();
let other_cells: Vec<ractor::ActorCell> =
members.into_iter().filter(|m| m != &my_cell).collect();
if other_cells.is_empty() {
// No other nodes → we are the only node → ensure we are PRIMARY
consecutive_failures = 0;
continue;
}
let mut any_reachable = false;
for cell in &other_cells {
let actor_ref: ActorRef<GitNodeMessage> = cell.clone().into();
match ractor::call_t!(actor_ref, GitNodeMessage::GetNodeHealth, 2000) {
Ok(health) if health.healthy => {
any_reachable = true;
break;
}
_ => continue,
}
}
if any_reachable {
consecutive_failures = 0;
} else {
consecutive_failures += 1;
tracing::warn!(
consecutive_failures = consecutive_failures,
max_failures = max_failures,
"no other cluster nodes reachable"
);
if consecutive_failures >= max_failures {
tracing::error!(
"no other nodes reachable for {max_failures} checks, triggering self-election as PRIMARY"
);
trigger_self_election(&myself);
consecutive_failures = 0;
}
}
}
});
}
/// Trigger self-election: this node promotes itself to PRIMARY.
fn trigger_self_election(myself: &ActorRef<GitNodeMessage>) {
let members = ractor::pg::get_members(&"gitks_nodes".to_string());
let total_nodes = members.len();
tracing::warn!(
total_nodes = total_nodes,
"initiating self-election as new PRIMARY"
);
let new_term = std::time::SystemTime::now()
.duration_since(std::time::SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
myself
.cast(GitNodeMessage::RoleChanged(RoleChangedEvent {
storage_name: String::new(), // will be filled by handler from our own state
grpc_addr: String::new(),
new_role: ROLE_PRIMARY.to_string(),
term: new_term,
relative_paths: Vec::new(), // all repos
}))
.ok();
broadcast_role_changed(
myself,
RoleChangedEvent {
storage_name: String::new(), // handler fills
grpc_addr: String::new(),
new_role: ROLE_PRIMARY.to_string(),
term: new_term,
relative_paths: Vec::new(),
},
);
}
pub async fn start_node_actor(
service: GitksService,
storage_name: String,
@@ -314,3 +509,14 @@ pub fn broadcast_ref_update(_node_actor: &ActorRef<GitNodeMessage>, event: RefUp
.ok();
}
}
/// Broadcast a role change event to all cluster members.
pub fn broadcast_role_changed(_actor: &ActorRef<GitNodeMessage>, event: RoleChangedEvent) {
let members = ractor::pg::get_members(&"gitks_nodes".to_string());
for member in members {
let actor_ref: ActorRef<GitNodeMessage> = member.into();
actor_ref
.cast(GitNodeMessage::RoleChanged(event.clone()))
.ok();
}
}
+106 -6
View File
@@ -142,6 +142,13 @@ pub enum GitNodeMessage {
#[rpc]
GetNodeHealth(RpcReplyPort<NodeHealth>),
/// Election: vote for a candidate to become PRIMARY.
#[rpc]
ElectPrimary(ElectionRequest, RpcReplyPort<ElectionResult>),
/// A role change has occurred in the cluster.
RoleChanged(RoleChangedEvent),
}
#[derive(ractor_cluster::RactorMessage)]
@@ -149,6 +156,105 @@ pub enum RepoActorMessage {
UpdateMetadata(RepositoryHeader),
}
// ── Election & Role Change Types ──────────────────────────────────────
/// Request for a node to vote in a PRIMARY election.
#[derive(Debug, Clone)]
pub struct ElectionRequest {
pub candidate_storage_name: String,
pub candidate_grpc_addr: String,
pub candidate_actor_name: String,
pub term: u64,
pub reason: String, // "primary_failed" etc.
}
impl BytesConvertable for ElectionRequest {
fn into_bytes(self) -> Vec<u8> {
encode_strings(&[
self.candidate_storage_name,
self.candidate_grpc_addr,
self.candidate_actor_name,
self.term.to_string(),
self.reason,
])
}
fn from_bytes(bytes: Vec<u8>) -> Self {
let values = decode_strings(bytes);
Self {
candidate_storage_name: values.first().cloned().unwrap_or_default(),
candidate_grpc_addr: values.get(1).cloned().unwrap_or_default(),
candidate_actor_name: values.get(2).cloned().unwrap_or_default(),
term: values.get(3).and_then(|v| v.parse().ok()).unwrap_or(0),
reason: values.get(4).cloned().unwrap_or_default(),
}
}
}
/// Result of an election vote.
#[derive(Debug, Clone)]
pub struct ElectionResult {
pub accepted: bool,
pub current_term: u64,
pub voter_storage_name: String,
pub voter_role: String,
}
impl BytesConvertable for ElectionResult {
fn into_bytes(self) -> Vec<u8> {
encode_strings(&[
if self.accepted { "1" } else { "0" }.to_string(),
self.current_term.to_string(),
self.voter_storage_name,
self.voter_role,
])
}
fn from_bytes(bytes: Vec<u8>) -> Self {
let values = decode_strings(bytes);
Self {
accepted: values.first().is_some_and(|v| v == "1"),
current_term: values.get(1).and_then(|v| v.parse().ok()).unwrap_or(0),
voter_storage_name: values.get(2).cloned().unwrap_or_default(),
voter_role: values.get(3).cloned().unwrap_or_default(),
}
}
}
/// Event broadcast when a node's role changes.
#[derive(Debug, Clone)]
pub struct RoleChangedEvent {
pub storage_name: String,
pub grpc_addr: String,
pub new_role: String, // "primary" or "replica"
pub term: u64,
pub relative_paths: Vec<String>, // repos that changed role
}
impl BytesConvertable for RoleChangedEvent {
fn into_bytes(self) -> Vec<u8> {
let mut strings = vec![
self.storage_name,
self.grpc_addr,
self.new_role,
self.term.to_string(),
];
strings.extend(self.relative_paths);
encode_strings(&strings)
}
fn from_bytes(bytes: Vec<u8>) -> Self {
let values = decode_strings(bytes);
Self {
storage_name: values.first().cloned().unwrap_or_default(),
grpc_addr: values.get(1).cloned().unwrap_or_default(),
new_role: values.get(2).cloned().unwrap_or_default(),
term: values.get(3).and_then(|v| v.parse().ok()).unwrap_or(0),
relative_paths: values.iter().skip(4).cloned().collect(),
}
}
}
fn encode_strings(values: &[String]) -> Vec<u8> {
let mut buf = Vec::new();
for value in values {
@@ -159,16 +265,13 @@ fn encode_strings(values: &[String]) -> Vec<u8> {
buf
}
// Maximum allowed length for a single string in the message
const MAX_STRING_LEN: usize = 10 * 1024 * 1024; // 10MB
// Maximum total message size
const MAX_TOTAL_SIZE: usize = 50 * 1024 * 1024; // 50MB
fn decode_strings(bytes: Vec<u8>) -> Vec<String> {
let mut values = Vec::new();
let mut offset = 0;
// Check total message size
if bytes.len() > MAX_TOTAL_SIZE {
tracing::warn!(
total = bytes.len(),
@@ -182,7 +285,6 @@ fn decode_strings(bytes: Vec<u8>) -> Vec<String> {
let len_bytes: [u8; 8] = bytes[offset..offset + 8].try_into().unwrap_or([0u8; 8]);
let len_u64 = u64::from_be_bytes(len_bytes);
// Prevent DoS via extremely large length values
if len_u64 > MAX_STRING_LEN as u64 {
tracing::warn!(
offset,
@@ -196,7 +298,6 @@ fn decode_strings(bytes: Vec<u8>) -> Vec<String> {
let len = len_u64 as usize;
offset += 8;
// Prevent integer overflow in offset calculation
let end_offset = match offset.checked_add(len) {
Some(end) => end,
None => {
@@ -210,7 +311,6 @@ fn decode_strings(bytes: Vec<u8>) -> Vec<String> {
};
if len == 0 || end_offset > bytes.len() {
// Invalid length — stop decoding, return what we have so far
tracing::warn!(
offset,
claimed_len = len,
+4 -4
View File
@@ -4,11 +4,11 @@ pub mod server;
pub mod sync;
pub use handler::{
GitNodeActor, GitNodeArgs, RepoEntry, broadcast_ref_update, get_category_members,
get_cluster_nodes, list_all_groups, route_group_for, start_node_actor,
GitNodeActor, GitNodeArgs, RepoEntry, broadcast_ref_update, broadcast_role_changed,
get_category_members, get_cluster_nodes, list_all_groups, route_group_for, start_node_actor,
};
pub use message::{
GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA, RefUpdateEvent, RepoActorMessage,
RouteDecision,
ElectionRequest, ElectionResult, GitNodeMessage, NodeHealth, ROLE_PRIMARY, ROLE_REPLICA,
RefUpdateEvent, RepoActorMessage, RoleChangedEvent, RouteDecision,
};
pub use server::init_actor_cluster;
+232
View File
@@ -0,0 +1,232 @@
//! etcd-based peer discovery for ractor_cluster.
//!
//! Responsibilities:
//! - Connect to etcd and create a Lease (TTL-based health check)
//! - Register this node under `/gitks/nodes/{storage_name}`
//! - Discover existing peers via prefix GET
//! - Watch for peer join/leave events and invoke callbacks
use std::sync::Arc;
use tokio::sync::Mutex;
use etcd_client::{Client, ConnectOptions, EventType, GetOptions, PutOptions, WatchOptions};
use super::types::PeerInfo;
/// Key prefix used for all gitks entries in etcd.
const KEY_PREFIX: &str = "/gitks/nodes/";
/// Wraps an etcd client with lease-based registration and peer discovery.
pub struct EtcdRegistry {
client: Mutex<Client>,
lease_id: i64,
storage_name: String,
}
impl EtcdRegistry {
/// Connect to etcd, create a lease, and register this node.
///
/// Returns `None` if the connection fails (caller should fall back to standalone mode).
pub async fn register(
endpoints: Vec<String>,
info: &PeerInfo,
ttl_secs: i64,
connect_timeout_ms: u64,
) -> Result<Self, Box<dyn std::error::Error + Send + Sync>> {
let connect_opts = ConnectOptions::new()
.with_connect_timeout(std::time::Duration::from_millis(connect_timeout_ms))
.with_keep_alive(
std::time::Duration::from_secs(5),
std::time::Duration::from_secs(3),
)
.with_keep_alive_while_idle(true);
let mut client = Client::connect(endpoints.clone(), Some(connect_opts)).await?;
tracing::info!(endpoints = ?endpoints, "connected to etcd");
// Create lease
let lease_resp = client.lease_grant(ttl_secs, None).await?;
let lease_id = lease_resp.id();
tracing::info!(lease_id, ttl = ttl_secs, "etcd lease granted");
// Register node info under the lease
let key = format!("{KEY_PREFIX}{}", info.storage_name);
let value = serde_json::to_string(info)?;
client
.put(key, value, Some(PutOptions::new().with_lease(lease_id)))
.await?;
tracing::info!(
storage_name = %info.storage_name,
cluster_addr = %info.cluster_addr,
"registered in etcd"
);
Ok(Self {
client: Mutex::new(client),
lease_id,
storage_name: info.storage_name.clone(),
})
}
/// Start the lease keepalive loop in a background task.
///
/// The loop sends periodic heartbeats to etcd to prevent the lease from expiring.
/// If keepalive fails, it logs a warning but does not panic — the node will
/// eventually be removed from etcd when the lease expires.
pub fn start_keepalive(self: &Arc<Self>) -> tokio::task::JoinHandle<()> {
let this = Arc::clone(self);
tokio::spawn(async move {
let lease_id = this.lease_id;
let (mut keeper, mut stream) = {
let mut client = this.client.lock().await;
match client.lease_keep_alive(lease_id).await {
Ok(pair) => pair,
Err(e) => {
tracing::error!(lease_id, error = %e, "failed to start lease keepalive");
return;
}
}
};
let mut interval = tokio::time::interval(std::time::Duration::from_secs(5));
loop {
interval.tick().await;
if let Err(e) = keeper.keep_alive().await {
tracing::warn!(lease_id, error = %e, "etcd lease keepalive failed");
// Don't break — let the lease expire naturally if we can't recover
}
// Drain keepalive responses
let _ = stream.message().await;
}
})
}
/// Discover all currently registered peers (excluding this node).
pub async fn discover_peers(
&self,
) -> Result<Vec<PeerInfo>, Box<dyn std::error::Error + Send + Sync>> {
let mut client = self.client.lock().await;
let resp = client
.get(KEY_PREFIX, Some(GetOptions::new().with_prefix()))
.await?;
let mut peers = Vec::new();
for kv in resp.kvs() {
match serde_json::from_slice::<PeerInfo>(kv.value()) {
Ok(info) if info.storage_name != self.storage_name => {
peers.push(info);
}
Ok(_) => {} // skip self
Err(e) => {
tracing::warn!(
key = %String::from_utf8_lossy(kv.key()),
error = %e,
"failed to parse peer info from etcd"
);
}
}
}
Ok(peers)
}
/// Start a long-running watch loop that monitors peer join/leave events.
///
/// Callbacks are invoked synchronously within the watch task; keep them fast
/// (prefer sending messages to actors rather than doing blocking work).
pub fn start_watch(
self: &Arc<Self>,
on_peer_joined: impl Fn(PeerInfo) + Send + Sync + 'static,
on_peer_left: impl Fn(String) + Send + Sync + 'static,
) -> tokio::task::JoinHandle<()> {
let this = Arc::clone(self);
let my_name = self.storage_name.clone();
tokio::spawn(async move {
let on_joined = Arc::new(on_peer_joined);
let on_left = Arc::new(on_peer_left);
loop {
// Create a fresh watch client each iteration (after reconnect)
let mut watch_stream = {
let mut client = this.client.lock().await;
match client
.watch(KEY_PREFIX, Some(WatchOptions::new().with_prefix()))
.await
{
Ok(stream) => stream,
Err(e) => {
tracing::error!(error = %e, "etcd watch failed, retrying in 3s");
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
continue;
}
}
};
tracing::info!("etcd watch loop started");
loop {
match watch_stream.message().await {
Ok(Some(resp)) => {
for event in resp.events() {
match event.event_type() {
EventType::Put => {
if let Some(kv) = event.kv()
&& let Ok(info) =
serde_json::from_slice::<PeerInfo>(kv.value())
&& info.storage_name != my_name
{
tracing::info!(
peer = %info.storage_name,
cluster_addr = %info.cluster_addr,
"peer joined via etcd watch"
);
on_joined(info);
}
}
EventType::Delete => {
if let Some(kv) = event.kv() {
let key = String::from_utf8_lossy(kv.key());
let name = key
.strip_prefix(KEY_PREFIX)
.unwrap_or(&key)
.to_string();
if name != my_name {
tracing::warn!(
peer = %name,
"peer left (etcd lease expired or key deleted)"
);
on_left(name);
}
}
}
}
}
}
Ok(None) => {
tracing::warn!("etcd watch stream ended");
break;
}
Err(e) => {
tracing::error!(error = %e, "etcd watch stream error");
break;
}
}
}
// Reconnect after a delay
tracing::info!("etcd watch loop restarting in 3s");
tokio::time::sleep(std::time::Duration::from_secs(3)).await;
}
})
}
/// Check if the lease is still alive (for external health monitoring).
pub async fn is_lease_alive(&self) -> bool {
let mut client = self.client.lock().await;
match client.lease_time_to_live(self.lease_id, None).await {
Ok(resp) => resp.ttl() > 0,
Err(_) => false,
}
}
}
+214
View File
@@ -0,0 +1,214 @@
//! Cluster discovery: etcd-driven ractor_cluster node discovery.
//!
//! Architecture:
//! 1. Start a `ractor_cluster::NodeServer` (TCP listener for actor remoting)
//! 2. Connect to etcd and register this node
//! 3. Discover existing peers → `client_connect()` to each
//! 4. Watch etcd for future peer join/leave → connect/disconnect dynamically
//!
//! Once ractor_cluster TCP connections are established, the existing
//! `pg::get_members()` / `ractor::call_t!()` APIs automatically work
//! cross-network — no changes needed in actor/handler.rs or server/mod.rs.
pub mod discovery;
pub mod types;
pub use discovery::EtcdRegistry;
pub use types::PeerInfo;
use std::sync::Arc;
use ractor::ActorRef;
use ractor_cluster::node::NodeConnectionMode;
use ractor_cluster::{NodeServer, NodeServerMessage, client_connect};
use crate::error::{GitError, GitResult};
/// Configuration for the cluster subsystem.
#[derive(Debug, Clone)]
pub struct ClusterConfig {
/// etcd endpoints (e.g. ["http://etcd1:2379", "http://etcd2:2379"])
pub etcd_endpoints: Vec<String>,
/// Logical name for this storage node
pub storage_name: String,
/// gRPC address advertised to clients
pub grpc_addr: String,
/// TCP port for ractor_cluster NodeServer
pub cluster_port: u16,
/// Shared authentication cookie for ractor_cluster
pub cookie: String,
/// etcd lease TTL in seconds
pub lease_ttl_secs: i64,
/// etcd connection timeout in milliseconds
pub connect_timeout_ms: u64,
/// Hostname used in the ractor_cluster node name (`name@hostname`).
/// Also used by remote nodes to connect back via `{cluster_hostname}:{cluster_port}`.
/// In K8s/Docker, this should be a resolvable address (Pod IP, service DNS, etc.)
pub cluster_hostname: String,
}
/// The running cluster manager. Holds references to the NodeServer and etcd registry.
/// Dropping this will stop the background tasks.
pub struct ClusterManager {
/// The ractor_cluster NodeServer actor
pub node_server: ActorRef<NodeServerMessage>,
/// The etcd registry (for health checks, etc.)
pub registry: Arc<EtcdRegistry>,
/// Handles for background tasks (keepalive + watch)
_keepalive_handle: tokio::task::JoinHandle<()>,
_watch_handle: tokio::task::JoinHandle<()>,
}
impl ClusterManager {
/// Start the full cluster subsystem:
/// 1. Spawn NodeServer (TCP listener)
/// 2. Connect to etcd + register
/// 3. Discover peers → client_connect
/// 4. Start keepalive + watch loops
///
/// Returns `Err` if etcd is unreachable (caller should fall back to standalone).
pub async fn start(config: ClusterConfig) -> GitResult<Self> {
// ── Step 1: Start NodeServer ──
let node_server = spawn_node_server(&config).await?;
tracing::info!(
port = config.cluster_port,
hostname = %config.cluster_hostname,
"NodeServer started"
);
// ── Step 2: Connect to etcd and register ──
let cluster_addr = format!("{}:{}", config.cluster_hostname, config.cluster_port);
let peer_info = PeerInfo {
storage_name: config.storage_name.clone(),
cluster_addr: cluster_addr.clone(),
grpc_addr: config.grpc_addr.clone(),
version: env!("CARGO_PKG_VERSION").to_string(),
};
let registry = Arc::new(
EtcdRegistry::register(
config.etcd_endpoints.clone(),
&peer_info,
config.lease_ttl_secs,
config.connect_timeout_ms,
)
.await
.map_err(|e| GitError::Internal(format!("etcd registration failed: {e}")))?,
);
// ── Step 3: Discover existing peers and connect ──
let peers = registry
.discover_peers()
.await
.map_err(|e| GitError::Internal(format!("peer discovery failed: {e}")))?;
for peer in &peers {
connect_to_peer(&node_server, peer, &config.storage_name).await;
}
// ── Step 4: Start background tasks ──
let keepalive_handle = registry.start_keepalive();
let ns_for_watch = node_server.clone();
let my_name_for_watch = config.storage_name.clone();
let watch_handle = registry.start_watch(
move |peer| {
let ns = ns_for_watch.clone();
let my_name = my_name_for_watch.clone();
tokio::spawn(async move {
connect_to_peer(&ns, &peer, &my_name).await;
});
},
move |name| {
tracing::info!(
peer = %name,
"peer left etcd registry (ractor_cluster will cleanup TCP session)"
);
// ractor_cluster automatically:
// 1. Detects TCP disconnection
// 2. Stops the NodeSession actor
// 3. Stops all RemoteActors for that session
// 4. Removes them from Process Groups
// No manual cleanup needed.
},
);
tracing::info!(
storage_name = %config.storage_name,
peers_found = peers.len(),
"cluster manager started"
);
Ok(Self {
node_server,
registry,
_keepalive_handle: keepalive_handle,
_watch_handle: watch_handle,
})
}
}
/// Spawn the ractor_cluster NodeServer actor (TCP listener for inter-node communication).
async fn spawn_node_server(config: &ClusterConfig) -> GitResult<ActorRef<NodeServerMessage>> {
let server = NodeServer::new(
config.cluster_port,
config.cookie.clone(),
config.storage_name.clone(),
config.cluster_hostname.clone(),
None, // no encryption (internal network)
Some(NodeConnectionMode::Transitive),
);
let (actor_ref, _handle) = ractor::Actor::spawn(
Some(format!("node_server_{}", config.storage_name)),
server,
(),
)
.await
.map_err(|e| GitError::Internal(format!("failed to spawn NodeServer: {e}")))?;
Ok(actor_ref)
}
/// Establish a ractor_cluster TCP connection to a remote peer.
///
/// Uses ordering optimization: only the node with the lexicographically
/// smaller `storage_name` initiates the connection. The other side will
/// accept the incoming connection. This prevents duplicate connections.
async fn connect_to_peer(
node_server: &ActorRef<NodeServerMessage>,
peer: &PeerInfo,
my_name: &str,
) {
// Ordering optimization: only smaller-named node connects
if my_name >= peer.storage_name.as_str() {
tracing::debug!(
peer = %peer.storage_name,
"skipping connect (peer has lower/equal name, they connect to us)"
);
return;
}
tracing::info!(
peer = %peer.storage_name,
cluster_addr = %peer.cluster_addr,
"connecting to peer via ractor_cluster"
);
match client_connect(node_server, peer.cluster_addr.as_str()).await {
Ok(()) => {
tracing::info!(
peer = %peer.storage_name,
"ractor_cluster connection initiated"
);
}
Err(e) => {
tracing::warn!(
peer = %peer.storage_name,
cluster_addr = %peer.cluster_addr,
error = %e,
"failed to connect to peer (will retry on next watch event)"
);
}
}
}
+14
View File
@@ -0,0 +1,14 @@
use serde::{Deserialize, Serialize};
/// Information about a peer node, registered in etcd.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct PeerInfo {
/// Logical storage name (e.g. "node-a", "default")
pub storage_name: String,
/// ractor_cluster TCP address (e.g. "10.0.1.4:4697")
pub cluster_addr: String,
/// gRPC service address (e.g. "http://10.0.1.4:50051")
pub grpc_addr: String,
/// Software version
pub version: String,
}
+538
View File
@@ -0,0 +1,538 @@
//! Disk-based cache infrastructure for GitKS.
//!
//! Implements the Gitaly-inspired diskcache design:
//! - Per-repository state directory with `latest` (repo state hash) and `pending/` (lease files)
//! - Cache key = SHA256(latest + request digest + version)
//! - Cached responses stored at `${CACHE_DIR}/${digest:0:2}/${digest:2}`
//! - Lease-based invalidation: mutator RPCs create lease files, update `latest` on completion
//! - Background cleanup of stale cache entries
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime};
use sha2;
use crate::error::{GitError, GitResult};
/// Lease stale threshold: leases older than this are considered stale.
const LEASE_STALE_THRESHOLD_SECS: u64 = 30;
/// State directory relative path under repo prefix.
const STATE_DIR_RELATIVE: &str = "+gitks-cache/state";
/// Cache directory relative path under repo prefix.
const CACHE_DIR_RELATIVE: &str = "+gitks-cache/cache";
/// Info-refs cache directory relative path under repo prefix.
const INFO_REFS_DIR_RELATIVE: &str = "+gitks-cache/info_refs";
/// Generate a random value for the `latest` file.
fn random_value() -> String {
use std::fmt::Write;
use std::sync::atomic::{AtomicU64, Ordering};
let mut buf = [0u8; 16];
let nanos = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_nanos() as u64;
buf[..8].copy_from_slice(&nanos.to_le_bytes());
static COUNTER: AtomicU64 = AtomicU64::new(0);
let c = COUNTER.fetch_add(1, Ordering::Relaxed);
buf[8..].copy_from_slice(&c.to_le_bytes());
let mut s = String::with_capacity(32);
for byte in &buf {
write!(s, "{byte:02x}").unwrap();
}
s
}
/// Compute SHA256 digest from multiple input parts.
fn sha256_digest(parts: &[&str]) -> String {
use sha2::Digest;
let mut hasher = sha2::Sha256::new();
for part in parts {
hasher.update(part.as_bytes());
}
let result = hasher.finalize();
let mut s = String::with_capacity(64);
for byte in result {
use std::fmt::Write;
write!(s, "{byte:02x}").unwrap();
}
s
}
/// Convert a digest into a two-level file path: `${digest:0:2}/${digest:2}`.
pub fn digest_to_path(digest: &str) -> PathBuf {
let prefix = &digest[..2];
let rest = &digest[2..];
PathBuf::from(prefix).join(rest)
}
/// DiskCache manages per-repository state and cached response files on local disk.
#[derive(Debug)]
pub struct DiskCache {
pub repo_prefix: PathBuf,
max_age: Duration,
version: String,
enabled: bool,
}
impl DiskCache {
/// Create a new DiskCache.
pub fn new(repo_prefix: PathBuf, version: String, max_age_secs: u64, enabled: bool) -> Self {
Self {
repo_prefix,
max_age: Duration::from_secs(max_age_secs),
version,
enabled,
}
}
/// Whether the cache is enabled.
pub fn is_enabled(&self) -> bool {
self.enabled
}
// ── State Directory ──────────────────────────────────────────────
fn state_dir_for(&self, relative_path: &str) -> PathBuf {
self.repo_prefix
.join(STATE_DIR_RELATIVE)
.join(relative_path)
}
fn latest_path_for(&self, relative_path: &str) -> PathBuf {
self.state_dir_for(relative_path).join("latest")
}
fn pending_dir_for(&self, relative_path: &str) -> PathBuf {
self.state_dir_for(relative_path).join("pending")
}
// ── Cache Directory ──────────────────────────────────────────────
fn cache_dir(&self, namespace: &str) -> PathBuf {
self.repo_prefix.join(namespace)
}
fn cache_file_path(&self, namespace: &str, digest: &str) -> PathBuf {
self.cache_dir(namespace).join(digest_to_path(digest))
}
// ── Repository State Management ──────────────────────────────────
/// Ensure the state directory for a repository exists and has a `latest` file.
/// If `latest` does not exist, create it with a random value.
pub fn ensure_state(&self, relative_path: &str) -> GitResult<String> {
if !self.enabled {
return Ok(random_value());
}
let state_dir = self.state_dir_for(relative_path);
std::fs::create_dir_all(&state_dir).map_err(GitError::Io)?;
let pending_dir = self.pending_dir_for(relative_path);
std::fs::create_dir_all(&pending_dir).map_err(GitError::Io)?;
let latest_path = self.latest_path_for(relative_path);
if latest_path.exists() {
let val = std::fs::read_to_string(&latest_path).map_err(GitError::Io)?;
Ok(val.trim().to_string())
} else {
let val = random_value();
std::fs::write(&latest_path, &val).map_err(GitError::Io)?;
Ok(val)
}
}
/// Create a lease file for a mutating RPC.
/// Returns a `LeaseGuard` that removes the lease on drop and updates `latest`.
pub fn create_lease(&self, relative_path: &str) -> GitResult<LeaseGuard> {
if !self.enabled {
let rp = relative_path.to_string();
return Ok(LeaseGuard {
cache: self.clone(),
relative_path: rp,
lease_path: PathBuf::new(),
is_dummy: true,
});
}
let pending_dir = self.pending_dir_for(relative_path);
std::fs::create_dir_all(&pending_dir).map_err(GitError::Io)?;
let lease_name = random_value();
let lease_path = pending_dir.join(&lease_name);
std::fs::write(&lease_path, &lease_name).map_err(GitError::Io)?;
tracing::debug!(
relative_path = %relative_path,
lease = %lease_name,
"lease created"
);
Ok(LeaseGuard {
cache: self.clone(),
relative_path: relative_path.to_string(),
lease_path,
is_dummy: false,
})
}
/// Check if a repository is in a deterministic state (no active leases).
/// Returns the `latest` value if deterministic, or None if indeterminate.
pub fn get_repo_state(&self, relative_path: &str) -> GitResult<Option<String>> {
if !self.enabled {
return Ok(Some(random_value()));
}
self.cleanup_stale_leases(relative_path)?;
let pending_dir = self.pending_dir_for(relative_path);
if pending_dir.exists() {
let entries = std::fs::read_dir(&pending_dir).map_err(GitError::Io)?;
let count = entries.count();
if count > 0 {
tracing::debug!(
relative_path = %relative_path,
pending = count,
"repo has in-flight mutator, cache state indeterminate"
);
return Ok(None);
}
}
let latest_path = self.latest_path_for(relative_path);
if latest_path.exists() {
let val = std::fs::read_to_string(&latest_path).map_err(GitError::Io)?;
Ok(Some(val.trim().to_string()))
} else {
// No latest file → create one
Ok(Some(self.ensure_state(relative_path)?))
}
}
/// Remove stale lease files (older than LEASE_STALE_THRESHOLD_SECS).
fn cleanup_stale_leases(&self, relative_path: &str) -> GitResult<()> {
let pending_dir = self.pending_dir_for(relative_path);
if !pending_dir.exists() {
return Ok(());
}
let now = SystemTime::now();
let threshold = Duration::from_secs(LEASE_STALE_THRESHOLD_SECS);
for entry in std::fs::read_dir(&pending_dir).map_err(GitError::Io)? {
let entry = entry.map_err(GitError::Io)?;
let path = entry.path();
if let Ok(metadata) = entry.metadata()
&& let Ok(modified) = metadata.modified()
&& let Ok(age) = now.duration_since(modified)
&& age > threshold
{
tracing::warn!(
path = %path.display(),
age_secs = age.as_secs(),
"removing stale lease file"
);
std::fs::remove_file(&path).ok();
}
}
Ok(())
}
// ── Cache Key Computation ────────────────────────────────────────
/// Compute a cache key for an info/refs request.
pub fn compute_info_refs_key(&self, relative_path: &str, protocol: &str) -> GitResult<String> {
let latest = self.ensure_state(relative_path)?;
let parts: &[&str] = &[&latest, "advertise_refs", protocol, &self.version];
Ok(sha256_digest(parts))
}
/// Compute a cache key for a pack-objects request.
pub fn compute_pack_objects_key(
&self,
relative_path: &str,
wants_hex: &[String],
haves_hex: &[String],
thin_pack: bool,
use_bitmaps: bool,
delta_base_offset: bool,
) -> GitResult<String> {
let latest = self.ensure_state(relative_path)?;
// Sort wants and haves for deterministic key
let mut wants_sorted = wants_hex.to_vec();
wants_sorted.sort();
let mut haves_sorted = haves_hex.to_vec();
haves_sorted.sort();
let wants_str = wants_sorted.join(",");
let haves_str = haves_sorted.join(",");
let flags = format!("thin={thin_pack},bitmaps={use_bitmaps},dbo={delta_base_offset}");
let parts: &[&str] = &[
&latest,
"pack_objects",
&wants_str,
&haves_str,
&flags,
&self.version,
];
Ok(sha256_digest(parts))
}
// ── Cache Lookup & Insert ────────────────────────────────────────
/// Look up a cached response for the given namespace and digest.
/// Returns the cached bytes if found and not expired.
pub fn lookup(&self, namespace: &str, digest: &str) -> GitResult<Option<Vec<u8>>> {
if !self.enabled {
return Ok(None);
}
let path = self.cache_file_path(namespace, digest);
if !path.exists() {
return Ok(None);
}
if let Ok(metadata) = std::fs::metadata(&path)
&& let Ok(modified) = metadata.modified()
&& let Ok(age) = SystemTime::now().duration_since(modified)
&& age > self.max_age
{
tracing::debug!(
path = %path.display(),
age_secs = age.as_secs(),
"cached entry expired, removing"
);
std::fs::remove_file(&path).ok();
if let Some(parent) = path.parent() {
std::fs::remove_dir(parent).ok();
}
return Ok(None);
}
let data = std::fs::read(&path).map_err(GitError::Io)?;
tracing::debug!(
namespace = %namespace,
digest = %digest,
size = data.len(),
"cache hit"
);
Ok(Some(data))
}
/// Insert a cached response for the given namespace and digest.
pub fn insert(&self, namespace: &str, digest: &str, data: &[u8]) -> GitResult<()> {
if !self.enabled {
return Ok(());
}
let path = self.cache_file_path(namespace, digest);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(GitError::Io)?;
}
let tmp_path = path.with_extension("tmp");
std::fs::write(&tmp_path, data).map_err(GitError::Io)?;
std::fs::rename(&tmp_path, &path).map_err(GitError::Io)?;
tracing::debug!(
namespace = %namespace,
digest = %digest,
size = data.len(),
"cache entry written"
);
Ok(())
}
/// Open a cache file for streaming read.
pub fn open_stream_read(
&self,
namespace: &str,
digest: &str,
) -> GitResult<Option<std::fs::File>> {
if !self.enabled {
return Ok(None);
}
let path = self.cache_file_path(namespace, digest);
if !path.exists() {
return Ok(None);
}
if let Ok(metadata) = std::fs::metadata(&path)
&& let Ok(modified) = metadata.modified()
&& let Ok(age) = SystemTime::now().duration_since(modified)
&& age > self.max_age
{
std::fs::remove_file(&path).ok();
return Ok(None);
}
let file = std::fs::File::open(&path).map_err(GitError::Io)?;
Ok(Some(file))
}
/// Open a cache file for streaming write.
/// Returns the file handle and the final path.
pub fn open_stream_write(
&self,
namespace: &str,
digest: &str,
) -> GitResult<(std::fs::File, PathBuf)> {
if !self.enabled {
return Err(GitError::Internal("disk cache not enabled".into()));
}
let path = self.cache_file_path(namespace, digest);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).map_err(GitError::Io)?;
}
let tmp_path = path.with_extension("tmp_streaming");
let file = std::fs::File::create(&tmp_path).map_err(GitError::Io)?;
Ok((file, path))
}
/// Finalize a streaming write by renaming the temp file to the final path.
pub fn finalize_stream_write(&self, tmp_path: &Path, final_path: &Path) -> GitResult<()> {
let actual_tmp = tmp_path.with_extension("tmp_streaming");
if actual_tmp.exists() {
std::fs::rename(&actual_tmp, final_path).map_err(GitError::Io)?;
}
tracing::debug!(
path = %final_path.display(),
"streaming cache entry finalized"
);
Ok(())
}
/// Invalidate all cache entries for a repository by updating the `latest` file.
/// This is called after any mutator RPC (create branch, create commit, etc.)
pub fn invalidate_repo(&self, relative_path: &str) {
if !self.enabled {
return;
}
let latest_path = self.latest_path_for(relative_path);
let new_val = random_value();
if let Err(e) = std::fs::write(&latest_path, &new_val) {
tracing::warn!(
relative_path = %relative_path,
error = %e,
"failed to update latest for cache invalidation"
);
} else {
tracing::debug!(
relative_path = %relative_path,
new_state = %new_val,
"cache invalidated for repository"
);
}
}
/// Remove all cache entries on startup (guard against inconsistent state from previous run).
pub fn cleanup_on_startup(&self) -> GitResult<()> {
if !self.enabled {
return Ok(());
}
for namespace in &[CACHE_DIR_RELATIVE, INFO_REFS_DIR_RELATIVE] {
let dir = self.repo_prefix.join(namespace);
if dir.exists() {
tracing::info!(dir = %dir.display(), "cleaning cache directory on startup");
std::fs::remove_dir_all(&dir).map_err(GitError::Io)?;
}
}
Ok(())
}
/// Background cleanup: remove expired cache entries.
/// Should be called periodically (e.g., every 5 minutes).
pub fn cleanup_expired(&self) -> GitResult<u64> {
if !self.enabled {
return Ok(0);
}
let now = SystemTime::now();
let mut removed = 0u64;
for namespace in &[CACHE_DIR_RELATIVE, INFO_REFS_DIR_RELATIVE] {
let dir = self.repo_prefix.join(namespace);
if !dir.exists() {
continue;
}
for prefix_entry in std::fs::read_dir(&dir).map_err(GitError::Io)? {
let prefix_entry = prefix_entry.map_err(GitError::Io)?;
let prefix_dir = prefix_entry.path();
if !prefix_dir.is_dir() {
continue;
}
for entry in std::fs::read_dir(&prefix_dir).map_err(GitError::Io)? {
let entry = entry.map_err(GitError::Io)?;
let path = entry.path();
if let Ok(metadata) = entry.metadata()
&& let Ok(modified) = metadata.modified()
&& let Ok(age) = now.duration_since(modified)
&& age > self.max_age
{
tracing::debug!(
path = %path.display(),
age_secs = age.as_secs(),
"removing expired cache entry"
);
std::fs::remove_file(&path).ok();
removed += 1;
}
}
std::fs::remove_dir(&prefix_dir).ok();
}
}
if removed > 0 {
tracing::info!(
entries_removed = removed,
"expired cache entries cleaned up"
);
}
Ok(removed)
}
}
impl Clone for DiskCache {
fn clone(&self) -> Self {
Self {
repo_prefix: self.repo_prefix.clone(),
max_age: self.max_age,
version: self.version.clone(),
enabled: self.enabled,
}
}
}
/// A lease guard that removes the lease file on drop and updates `latest`.
pub struct LeaseGuard {
cache: DiskCache,
relative_path: String,
lease_path: PathBuf,
is_dummy: bool,
}
impl LeaseGuard {
/// Update the `latest` file to invalidate cached responses.
/// Called automatically on drop, but can also be called manually.
pub fn commit(&mut self) {
if self.is_dummy {
self.cache.invalidate_repo(&self.relative_path);
self.is_dummy = false; // Don't double-commit on drop
return;
}
if self.lease_path.exists() {
std::fs::remove_file(&self.lease_path).ok();
}
self.cache.invalidate_repo(&self.relative_path);
self.is_dummy = false;
}
}
impl Drop for LeaseGuard {
fn drop(&mut self) {
if self.is_dummy || self.lease_path.exists() {
self.commit();
}
}
}
/// Start a background task that periodically cleans up expired cache entries.
pub fn start_cache_cleanup_task(
cache: DiskCache,
interval: Duration,
) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut ticker = tokio::time::interval(interval);
ticker.tick().await; // First tick is immediate
loop {
ticker.tick().await;
if let Err(e) = cache.cleanup_expired() {
tracing::warn!(error = %e, "cache cleanup task failed");
}
}
})
}
+238
View File
@@ -0,0 +1,238 @@
//! Hook manager for GitKS.
//!
//! Manages the installation, listing, and removal of git hooks
//! for bare repositories. Supports server hooks, custom hooks,
//! and gRPC callback hooks.
use std::path::{Path, PathBuf};
use std::time::Duration;
use crate::error::{GitError, GitResult};
use crate::hooks::runner::{HookResult, generate_hook_runner_script, run_hook_dir};
use crate::hooks::sanitize::{validate_hook_content, validate_hook_name};
/// Manages git hooks across repositories.
#[derive(Debug, Clone)]
pub struct HookManager {
repo_prefix: PathBuf,
server_hooks_dir: Option<PathBuf>,
hook_callback_addr: Option<String>,
hook_timeout: Duration,
allow_custom_hooks: bool,
}
impl HookManager {
pub fn new(
repo_prefix: PathBuf,
server_hooks_dir: Option<PathBuf>,
hook_callback_addr: Option<String>,
hook_timeout: Duration,
allow_custom_hooks: bool,
) -> Self {
Self {
repo_prefix,
server_hooks_dir,
hook_callback_addr,
hook_timeout,
allow_custom_hooks,
}
}
/// Install gitks hook runner scripts into a repository's hooks directory.
/// Called during repository initialization.
pub fn install_hooks(&self, repo_path: &Path) -> GitResult<()> {
let hooks_dir = repo_path.join("hooks");
std::fs::create_dir_all(&hooks_dir).map_err(GitError::Io)?;
let relative_path = repo_path
.strip_prefix(&self.repo_prefix)
.unwrap_or(repo_path)
.to_string_lossy()
.trim_start_matches('/')
.to_string();
let server_hooks_dir_str = self
.server_hooks_dir
.as_ref()
.map(|p| p.to_string_lossy().into_owned());
let callback_addr_str = self.hook_callback_addr.clone();
for hook_type in &["pre-receive", "update", "post-receive"] {
let script_content = generate_hook_runner_script(
hook_type,
&relative_path,
server_hooks_dir_str.as_deref(),
callback_addr_str.as_deref(),
self.hook_timeout.as_secs(),
);
let hook_path = hooks_dir.join(hook_type);
std::fs::write(&hook_path, script_content).map_err(GitError::Io)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = std::fs::metadata(&hook_path)
.map_err(GitError::Io)?
.permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&hook_path, perms).map_err(GitError::Io)?;
}
tracing::debug!(
hook_type = %hook_type,
path = %hook_path.display(),
"installed gitks hook runner"
);
}
tracing::info!(
path = %repo_path.display(),
"hooks installed for repository"
);
Ok(())
}
/// Set a custom hook script for a repository.
pub fn set_custom_hook(
&self,
repo_path: &Path,
hook_name: &str,
content: &str,
) -> GitResult<()> {
validate_hook_name(hook_name)?;
if !self.allow_custom_hooks {
return Err(GitError::PermissionDenied(
"custom hooks are not allowed on this server".into(),
));
}
validate_hook_content(content)?;
let custom_hooks_dir = repo_path.join("custom_hooks").join(hook_name).join("d");
std::fs::create_dir_all(&custom_hooks_dir).map_err(GitError::Io)?;
let script_name = format!("gitks_custom_{}", hook_name.replace('-', "_"));
let script_path = custom_hooks_dir.join(&script_name);
std::fs::write(&script_path, content).map_err(GitError::Io)?;
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = std::fs::metadata(&script_path)
.map_err(GitError::Io)?
.permissions();
perms.set_mode(0o755);
std::fs::set_permissions(&script_path, perms).map_err(GitError::Io)?;
}
tracing::info!(
repo = %repo_path.display(),
hook_name = %hook_name,
"custom hook set"
);
Ok(())
}
/// Remove a custom hook from a repository.
pub fn remove_custom_hook(&self, repo_path: &Path, hook_name: &str) -> GitResult<()> {
validate_hook_name(hook_name)?;
let custom_hooks_dir = repo_path.join("custom_hooks").join(hook_name).join("d");
if custom_hooks_dir.exists() {
std::fs::remove_dir_all(&custom_hooks_dir).map_err(GitError::Io)?;
tracing::info!(
repo = %repo_path.display(),
hook_name = %hook_name,
"custom hook removed"
);
}
Ok(())
}
/// List all hooks for a repository.
pub fn list_hooks(&self, repo_path: &Path) -> GitResult<Vec<HookInfo>> {
let mut hooks = Vec::new();
let hooks_dir = repo_path.join("hooks");
if hooks_dir.exists() {
for hook_type in &["pre-receive", "update", "post-receive"] {
let hook_path = hooks_dir.join(hook_type);
if hook_path.exists() {
hooks.push(HookInfo {
hook_type: hook_type.to_string(),
level: HookLevel::Server,
path: hook_path.to_string_lossy().into_owned(),
});
}
}
}
if self.allow_custom_hooks {
let custom_dir = repo_path.join("custom_hooks");
if custom_dir.exists() {
for entry in std::fs::read_dir(&custom_dir).map_err(GitError::Io)? {
let entry = entry.map_err(GitError::Io)?;
let name = entry.file_name().to_string_lossy().into_owned();
if validate_hook_name(&name).is_ok() {
let path = entry.path();
let d_dir = path.join("d");
if d_dir.exists() {
let script_count =
std::fs::read_dir(&d_dir).map_err(GitError::Io)?.count();
if script_count > 0 {
hooks.push(HookInfo {
hook_type: name,
level: HookLevel::Custom,
path: d_dir.to_string_lossy().into_owned(),
});
}
}
}
}
}
}
Ok(hooks)
}
/// Execute a hook by running server hooks then custom hooks.
pub fn execute_hook(&self, repo_path: &Path, hook_type: &str, stdin_data: &[u8]) -> HookResult {
if let Some(ref server_dir) = self.server_hooks_dir {
let dir = server_dir.join(hook_type).join("d");
let result = run_hook_dir(&dir, hook_type, stdin_data, self.hook_timeout);
if !result.accepted {
return result;
}
}
if self.allow_custom_hooks {
let custom_dir = repo_path.join("custom_hooks").join(hook_type).join("d");
let result = run_hook_dir(&custom_dir, hook_type, stdin_data, self.hook_timeout);
if !result.accepted {
return result;
}
}
HookResult::accepted()
}
}
/// Information about a hook.
#[derive(Debug, Clone)]
pub struct HookInfo {
pub hook_type: String,
pub level: HookLevel,
pub path: String,
}
/// Hook level (server vs custom).
#[derive(Debug, Clone, PartialEq)]
pub enum HookLevel {
Server,
Custom,
}
impl std::fmt::Display for HookLevel {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
HookLevel::Server => write!(f, "server"),
HookLevel::Custom => write!(f, "custom"),
}
}
}
+16
View File
@@ -0,0 +1,16 @@
//! Git hooks management for GitKS.
//!
//! Supports three layers of hooks:
//! 1. Server hooks: admin-level, shared across all repositories
//! 2. Custom hooks: per-repository, user-defined scripts
//! 3. gRPC callback hooks: external HookService via gRPC
//!
//! Hook scripts are installed into bare repositories' `hooks/` directory
//! and are automatically invoked by git during receive-pack operations.
pub mod manager;
pub mod runner;
pub mod sanitize;
pub use manager::HookManager;
pub use runner::HookResult;
+274
View File
@@ -0,0 +1,274 @@
//! Hook execution runner.
//!
//! Executes server hooks, custom hooks, and gRPC callback hooks
//! in sequence for each git hook type (pre-receive, update, post-receive).
use std::io::Write;
use std::path::{Path, PathBuf};
use std::process::{Output, Stdio};
use std::time::Duration;
/// Result of a hook execution.
#[derive(Debug)]
pub struct HookResult {
pub accepted: bool,
pub exit_code: i32,
pub stdout: String,
pub stderr: String,
}
impl HookResult {
pub fn accepted() -> Self {
Self {
accepted: true,
exit_code: 0,
stdout: String::new(),
stderr: String::new(),
}
}
pub fn rejected(stderr: String) -> Self {
Self {
accepted: false,
exit_code: 1,
stdout: String::new(),
stderr,
}
}
pub fn from_output(output: &Output) -> Self {
let stdout = String::from_utf8_lossy(&output.stdout).into_owned();
let stderr = String::from_utf8_lossy(&output.stderr).into_owned();
Self {
accepted: output.status.success(),
exit_code: output.status.code().unwrap_or(-1),
stdout,
stderr,
}
}
}
/// Run all hook scripts in a directory (sorted alphabetically).
/// Returns the first rejection or accepted if all pass.
pub fn run_hook_dir(
hook_dir: &Path,
hook_type: &str,
stdin_data: &[u8],
timeout: Duration,
) -> HookResult {
if !hook_dir.exists() {
return HookResult::accepted();
}
let mut scripts: Vec<PathBuf> = Vec::new();
if let Ok(entries) = std::fs::read_dir(hook_dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_file() {
let name = path.file_name().unwrap_or_default().to_string_lossy();
// Skip files starting with '.' (hidden files) or ending with '~' or '.sample'
if name.starts_with('.') || name.ends_with('~') || name.ends_with(".sample") {
continue;
}
scripts.push(path);
}
}
}
scripts.sort();
for script in &scripts {
tracing::debug!(
hook_type = %hook_type,
script = %script.display(),
"executing hook script"
);
let result = run_single_script(script, stdin_data, timeout);
if !result.accepted {
tracing::warn!(
hook_type = %hook_type,
script = %script.display(),
exit_code = result.exit_code,
stderr = %result.stderr,
"hook script rejected"
);
return result;
}
}
HookResult::accepted()
}
/// Run a single hook script with stdin data and timeout.
fn run_single_script(script_path: &Path, stdin_data: &[u8], timeout: Duration) -> HookResult {
let child = std::process::Command::new(script_path)
.stdin(Stdio::piped())
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn();
match child {
Ok(mut c) => {
if let Some(ref mut stdin) = c.stdin {
let _ = stdin.write_all(stdin_data);
}
c.stdin = None;
let wait_result = c.wait_timeout(timeout);
match wait_result {
Ok(Some(status)) => {
let output = c.wait_with_output().unwrap_or_else(|_| {
// If we can't get output, at least return the status
Output {
status,
stdout: Vec::new(),
stderr: Vec::new(),
}
});
HookResult::from_output(&output)
}
Ok(None) => {
tracing::warn!(
script = %script_path.display(),
timeout_secs = timeout.as_secs(),
"hook script timed out, killing"
);
let _ = c.kill();
HookResult::rejected(format!(
"hook script timed out after {}s: {}",
timeout.as_secs(),
script_path.display()
))
}
Err(e) => {
let _ = c.kill();
HookResult::rejected(format!("hook script wait error: {e}"))
}
}
}
Err(e) => {
tracing::warn!(
script = %script_path.display(),
error = %e,
"failed to spawn hook script"
);
// If the script can't be executed, treat as rejection
HookResult::rejected(format!("failed to spawn hook script: {e}"))
}
}
}
/// Wait for a child process with timeout.
trait ChildWaitTimeout {
fn wait_timeout(
&mut self,
timeout: Duration,
) -> std::io::Result<Option<std::process::ExitStatus>>;
}
impl ChildWaitTimeout for std::process::Child {
fn wait_timeout(
&mut self,
timeout: Duration,
) -> std::io::Result<Option<std::process::ExitStatus>> {
let start = std::time::Instant::now();
loop {
match self.try_wait() {
Ok(Some(status)) => return Ok(Some(status)),
Ok(None) => {
if start.elapsed() > timeout {
return Ok(None); // timeout
}
std::thread::sleep(Duration::from_millis(100));
}
Err(e) => return Err(e),
}
}
}
}
/// Generate the gitks hook runner script content.
/// This script is installed into each repository's hooks/ directory
/// and orchestrates the execution of server hooks, custom hooks, and gRPC callbacks.
pub fn generate_hook_runner_script(
hook_type: &str,
repo_relative_path: &str,
server_hooks_dir: Option<&str>,
hook_callback_addr: Option<&str>,
hook_timeout_secs: u64,
) -> String {
let server_hooks_section = if let Some(dir) = server_hooks_dir {
format!(
r#"
# Run server hooks
SERVER_HOOKS_DIR="{dir}/{hook_type}.d"
if [ -d "$SERVER_HOOKS_DIR" ]; then
for script in $(ls "$SERVER_HOOKS_DIR" | sort); do
skip=false
case "$script" in .*|*.sample|*~) skip=true;; esac
if [ "$skip" = "false" ]; then
"$SERVER_HOOKS_DIR/$script"
exit_code=$?
if [ $exit_code -ne 0 ]; then
exit $exit_code
fi
fi
done
fi
"#
)
} else {
String::new()
};
let custom_hooks_section = r#"
# Run custom hooks (per-repository)
CUSTOM_HOOKS_DIR="$GIT_DIR/custom_hooks/$GITKS_HOOK_TYPE.d"
if [ -d "$CUSTOM_HOOKS_DIR" ]; then
for script in $(ls "$CUSTOM_HOOKS_DIR" | sort); do
skip=false
case "$script" in .*|*.sample|*~) skip=true;; esac
if [ "$skip" = "false" ]; then
"$CUSTOM_HOOKS_DIR/$script"
exit_code=$?
if [ $exit_code -ne 0 ]; then
exit $exit_code
fi
fi
done
fi
"#;
let grpc_callback_section = if let Some(addr) = hook_callback_addr {
format!(
r#"
# gRPC callback to external HookService
if [ -n "{addr}" ]; then
# gRPC callback is handled by the gitks service directly
# The service will make the gRPC call after the git hook completes
# This section is a placeholder - actual gRPC callback is handled
# by the gitks receive_pack handler
fi
"#
)
} else {
String::new()
};
format!(
r#"#!/bin/sh
# gitks hook runner for {hook_type}
# Repository: {repo_relative_path}
# Auto-generated by gitks - do not modify manually
GITKS_HOOK_TYPE="{hook_type}"
GITKS_REPO_RELATIVE_PATH="{repo_relative_path}"
GITKS_HOOK_TIMEOUT="{hook_timeout_secs}"
{server_hooks_section}
{custom_hooks_section}
{grpc_callback_section}
exit 0
"#
)
}
+84
View File
@@ -0,0 +1,84 @@
//! Hook content sanitization.
//!
//! Validates custom hook scripts to prevent dangerous commands.
use crate::error::{GitError, GitResult};
/// Commands/patterns that are never allowed in custom hook scripts.
const FORBIDDEN_PATTERNS: &[&str] = &[
"rm -rf",
"rm -r /",
"chmod 777",
"chmod 666",
"mkfs",
"dd if=",
":(){ :|:& };:", // fork bomb
"> /dev/sda",
"curl -o /",
"wget -O /",
"/etc/passwd",
"/etc/shadow",
"shutdown",
"reboot",
"init 0",
"init 6",
"poweroff",
"halt",
];
/// Maximum hook script size (64KB).
const MAX_HOOK_SIZE: usize = 65536;
/// Validate a custom hook script content for safety.
pub fn validate_hook_content(content: &str) -> GitResult<()> {
if content.is_empty() {
return Err(GitError::InvalidArgument(
"hook content cannot be empty".into(),
));
}
if content.len() > MAX_HOOK_SIZE {
return Err(GitError::InvalidArgument(format!(
"hook content too large (max {} bytes): {} bytes",
MAX_HOOK_SIZE,
content.len()
)));
}
let content_lower = content.to_lowercase();
for pattern in FORBIDDEN_PATTERNS {
if content_lower.contains(pattern) {
return Err(GitError::InvalidArgument(format!(
"hook content contains forbidden pattern: '{pattern}'"
)));
}
}
if content.contains('\0') {
return Err(GitError::InvalidArgument(
"hook content cannot contain null bytes".into(),
));
}
Ok(())
}
/// Validate a hook name (must be a recognized git hook name).
pub fn validate_hook_name(name: &str) -> GitResult<()> {
const VALID_HOOK_NAMES: &[&str] = &[
"pre-receive",
"update",
"post-receive",
"pre-applypatch",
"applypatch-msg",
"post-applypatch",
"pre-commit",
"prepare-commit-msg",
"commit-msg",
"post-commit",
"pre-auto-gc",
];
if !VALID_HOOK_NAMES.contains(&name) {
return Err(GitError::InvalidArgument(format!(
"invalid hook name: '{name}'. Must be one of: {}",
VALID_HOOK_NAMES.join(", ")
)));
}
Ok(())
}
+7
View File
@@ -4,18 +4,25 @@ pub mod bare;
pub mod blame;
pub mod blob;
pub mod branch;
pub mod cluster;
pub mod commit;
pub mod diff;
pub mod disk_cache;
pub mod error;
pub mod hooks;
pub mod init;
pub mod macros;
pub mod merge;
pub mod metrics;
pub mod oid;
pub mod rate_limit;
pub mod pack;
pub mod pack_cache;
pub mod paginate;
pub mod pb;
pub mod refs;
pub mod sanitize;
pub mod server;
pub mod snapshot;
pub mod tag;
pub mod tree;
+183 -5
View File
@@ -1,12 +1,37 @@
use std::path::PathBuf;
use std::time::Duration;
use gitks::actor::init_actor_cluster;
use gitks::cluster::{ClusterConfig, ClusterManager};
use gitks::disk_cache::DiskCache;
use gitks::hooks::HookManager;
use gitks::metrics;
use gitks::server::{GitksService, serve};
const DEFAULT_HOST: &str = "0.0.0.0";
const DEFAULT_PORT: &str = "50051";
const DEFAULT_STORAGE_NAME: &str = "default";
fn env_or(key: &str, default: &str) -> String {
std::env::var(key).unwrap_or_else(|_| default.into())
}
fn env_bool(key: &str, default: bool) -> bool {
match std::env::var(key).as_deref() {
Ok("true" | "1" | "yes") => true,
Ok("false" | "0" | "no") => false,
Ok(_) => default,
Err(_) => default,
}
}
fn env_u64(key: &str, default: u64) -> u64 {
std::env::var(key)
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(default)
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
dotenvy::dotenv().ok();
@@ -14,10 +39,9 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
tracing::info!(version = env!("CARGO_PKG_VERSION"), "gitks starting up");
let host = std::env::var("GITKS_HOST").unwrap_or_else(|_| DEFAULT_HOST.into());
let port = std::env::var("GITKS_PORT").unwrap_or_else(|_| DEFAULT_PORT.into());
let storage_name =
std::env::var("STORAGE_NAME").unwrap_or_else(|_| DEFAULT_STORAGE_NAME.into());
let host = env_or("GITKS_HOST", DEFAULT_HOST);
let port = env_or("GITKS_PORT", DEFAULT_PORT);
let storage_name = env_or("STORAGE_NAME", DEFAULT_STORAGE_NAME);
let grpc_addr =
std::env::var("GITKS_ADVERTISE_ADDR").unwrap_or_else(|_| format!("http://{host}:{port}"));
@@ -32,8 +56,162 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
std::fs::create_dir_all(&repo_prefix)?;
}
// Disk cache configuration
let disk_cache_enabled = env_bool("GITKS_DISK_CACHE_ENABLED", false);
let disk_cache_max_age = env_u64("GITKS_DISK_CACHE_MAX_AGE", 300);
let disk_cache = DiskCache::new(
repo_prefix.clone(),
env!("CARGO_PKG_VERSION").to_string(),
disk_cache_max_age,
disk_cache_enabled,
);
if disk_cache_enabled {
tracing::info!("disk cache enabled, max_age={disk_cache_max_age}s");
disk_cache.cleanup_on_startup()?;
gitks::disk_cache::start_cache_cleanup_task(disk_cache.clone(), Duration::from_secs(300));
} else {
tracing::info!("disk cache disabled");
}
// Pack cache configuration
let pack_cache_enabled = env_bool("GITKS_PACK_CACHE_ENABLED", false);
let pack_backpressure = env_bool("GITKS_PACK_CACHE_BACKPRESSURE", true);
// Pack cache: needs disk_cache. If disk_cache is enabled, info/refs cache
// is always available via PackCache wrapper. pack-objects caching is
// additionally controlled by GITKS_PACK_CACHE_ENABLED.
let pack_cache = if disk_cache_enabled {
tracing::info!(
"pack cache wrapper enabled, pack-objects cache={pack_cache_enabled}, backpressure={pack_backpressure}"
);
Some(gitks::pack_cache::PackCache::new(
disk_cache.clone(),
pack_backpressure,
))
} else {
None
};
// Hook manager configuration
let hooks_enabled = env_bool("GITKS_HOOKS_ENABLED", true);
let server_hooks_dir = std::env::var("GITKS_SERVER_HOOKS_DIR")
.ok()
.map(PathBuf::from);
let hook_callback_addr = std::env::var("GITKS_HOOK_CALLBACK_ADDR").ok();
let hook_timeout = env_u64("GITKS_HOOK_TIMEOUT", 30);
let allow_custom_hooks = env_bool("GITKS_ALLOW_CUSTOM_HOOKS", true);
let hook_manager = if hooks_enabled {
tracing::info!("hooks enabled, timeout={hook_timeout}s, custom_hooks={allow_custom_hooks}");
Some(HookManager::new(
repo_prefix.clone(),
server_hooks_dir,
hook_callback_addr,
Duration::from_secs(hook_timeout),
allow_custom_hooks,
))
} else {
tracing::info!("hooks disabled");
None
};
// Health check / election configuration
let health_check_interval = env_u64("GITKS_HEALTH_CHECK_INTERVAL", 1);
let max_health_failures = env_u64("GITKS_MAX_HEALTH_FAILURES", 10);
tracing::info!(
"health check: interval={health_check_interval}s, max_failures={max_health_failures}"
);
// ── Metrics server ──
let metrics_port = env_u64("GITKS_METRICS_PORT", 9100) as u16;
let _metrics_handle = metrics::start_metrics_server(metrics_port);
tracing::info!("metrics server on port {metrics_port}");
// ── Cluster discovery (etcd → ractor_cluster) ──
//
// When GITKS_ETCD_ENDPOINTS is set, the node:
// 1. Starts a ractor_cluster NodeServer (TCP listener)
// 2. Connects to etcd and registers itself
// 3. Discovers existing peers → establishes ractor_cluster TCP connections
// 4. Watches etcd for future peer join/leave events
//
// Once ractor_cluster connections are up, pg::get_members() automatically
// returns remote actors — no changes needed in actor/handler.rs.
//
// When GITKS_ETCD_ENDPOINTS is unset or etcd is unreachable, the node
// falls back to standalone mode (existing local-only behavior).
let etcd_endpoints = std::env::var("GITKS_ETCD_ENDPOINTS")
.ok()
.filter(|s| !s.is_empty())
.map(|s| {
s.split(',')
.map(str::trim)
.map(String::from)
.collect::<Vec<_>>()
});
let cluster_port = env_or("GITKS_CLUSTER_PORT", "4697")
.parse::<u16>()
.unwrap_or(4697);
let cluster_cookie = env_or("GITKS_CLUSTER_COOKIE", "gitks-default-cookie");
let lease_ttl = env_u64("GITKS_LEASE_TTL", 15) as i64;
let connect_timeout_ms = env_u64("GITKS_ETCD_CONNECT_TIMEOUT", 5000);
// Resolve the hostname/address other nodes use to reach our NodeServer.
// Priority: GITKS_CLUSTER_HOSTNAME > POD_IP (K8s) > HOSTNAME env > "localhost"
let cluster_hostname = std::env::var("GITKS_CLUSTER_HOSTNAME")
.or_else(|_| std::env::var("POD_IP"))
.or_else(|_| std::env::var("HOSTNAME"))
.unwrap_or_else(|_| "localhost".to_string());
let _cluster: Option<ClusterManager> = if let Some(endpoints) = etcd_endpoints {
tracing::info!(
endpoints = ?endpoints,
cluster_port = cluster_port,
cluster_hostname = %cluster_hostname,
"starting cluster discovery via etcd"
);
let config = ClusterConfig {
etcd_endpoints: endpoints,
storage_name: storage_name.clone(),
grpc_addr: grpc_addr.clone(),
cluster_port,
cookie: cluster_cookie,
lease_ttl_secs: lease_ttl,
connect_timeout_ms,
cluster_hostname,
};
match ClusterManager::start(config).await {
Ok(cm) => {
tracing::info!("cluster discovery active");
Some(cm)
}
Err(e) => {
tracing::warn!(error = %e, "etcd unavailable, running in standalone mode");
None
}
}
} else {
tracing::info!("GITKS_ETCD_ENDPOINTS not set, running in standalone mode");
None
};
let addr: std::net::SocketAddr = format!("{host}:{port}").parse()?;
let svc = GitksService::new(repo_prefix.clone());
let mut svc = GitksService::new(repo_prefix.clone());
if disk_cache_enabled {
svc = svc.with_disk_cache(disk_cache);
}
if let Some(pc) = pack_cache {
svc = svc.with_pack_cache(pc);
}
if let Some(hm) = hook_manager {
svc = svc.with_hook_manager(hm);
}
let (node_actor, node_handle) =
init_actor_cluster(svc.clone(), storage_name.clone(), grpc_addr.clone()).await?;
let svc = svc
+311
View File
@@ -0,0 +1,311 @@
//! Prometheus-compatible metrics for GitKS.
//!
//! Tracks:
//! - Request counts by gRPC method + status code
//! - Request duration histogram by method
//! - Active requests gauge
//! - Repository count
//! - Cache hits / misses
//! - Error counts by error type
//!
//! Exposes a `/metrics` HTTP endpoint on a configurable port (default 9100).
use dashmap::DashMap;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, OnceLock};
use std::time::{Duration, Instant};
// ── Metric storage ──────────────────────────────────────────────────
struct MetricsInner {
/// Counter: total requests by (method, status_code)
/// Key: "method:status"
request_count: DashMap<String, AtomicU64>,
/// Histogram buckets for request duration (seconds).
/// Each bucket: (method, le_bound_ms) → count
duration_buckets: DashMap<String, AtomicU64>,
/// Gauge: number of currently in-flight requests
active_requests: AtomicU64,
/// Gauge: total number of registered repositories
repository_count: AtomicU64,
/// Counter: cache hits
cache_hits: AtomicU64,
/// Counter: cache misses
cache_misses: AtomicU64,
/// Counter: errors by error kind
error_count: DashMap<String, AtomicU64>,
/// Start timestamp (seconds since Unix epoch)
start_time: Instant,
}
static METRICS: OnceLock<Arc<MetricsInner>> = OnceLock::new();
fn metrics() -> &'static Arc<MetricsInner> {
METRICS.get_or_init(|| {
Arc::new(MetricsInner {
request_count: DashMap::new(),
duration_buckets: DashMap::new(),
active_requests: AtomicU64::new(0),
repository_count: AtomicU64::new(0),
cache_hits: AtomicU64::new(0),
cache_misses: AtomicU64::new(0),
error_count: DashMap::new(),
start_time: Instant::now(),
})
})
}
// ── Duration histogram buckets (in milliseconds) ───────────────────
#[rustfmt::skip]
const DURATION_BUCKET_MS: &[u64] = &[
5, 10, 25, 50, 100, 250, 500, 1_000,
2_500, 5_000, 10_000, 30_000, 60_000, u64::MAX,
];
const BUCKET_INF: u64 = u64::MAX;
/// Record a request.
pub fn record_request(method: &str, status_code: &str, duration: Duration) {
let m = metrics();
let key = format!("{method}:{status_code}");
m.request_count
.entry(key)
.or_insert_with(|| AtomicU64::new(0))
.value()
.fetch_add(1, Ordering::Relaxed);
let duration_ms = duration.as_millis() as u64;
for &bound_ms in DURATION_BUCKET_MS {
if duration_ms <= bound_ms || bound_ms == BUCKET_INF {
let bucket_key = format!("{method}:{bound_ms}");
m.duration_buckets
.entry(bucket_key)
.or_insert_with(|| AtomicU64::new(0))
.value()
.fetch_add(1, Ordering::Relaxed);
}
}
}
/// Increment the active request gauge.
pub fn inc_active_requests() {
metrics().active_requests.fetch_add(1, Ordering::Relaxed);
}
/// Decrement the active request gauge.
pub fn dec_active_requests() {
metrics().active_requests.fetch_sub(1, Ordering::Relaxed);
}
/// Set the repository count.
pub fn set_repository_count(count: u64) {
metrics()
.repository_count
.store(count, Ordering::Relaxed);
}
/// Record a cache hit.
pub fn inc_cache_hits(count: u64) {
metrics().cache_hits.fetch_add(count, Ordering::Relaxed);
}
/// Record a cache miss.
pub fn inc_cache_misses(count: u64) {
metrics()
.cache_misses
.fetch_add(count, Ordering::Relaxed);
}
/// Record an error by kind (e.g., "not_found", "internal", "invalid_argument").
pub fn inc_error(kind: &str) {
metrics()
.error_count
.entry(kind.to_string())
.or_insert_with(|| AtomicU64::new(0))
.value()
.fetch_add(1, Ordering::Relaxed);
}
// ── Prometheus text format rendering ────────────────────────────────
/// Render all metrics in Prometheus text exposition format.
pub fn render_metrics() -> String {
let m = metrics();
let mut out = String::with_capacity(4096);
// Header
let uptime = m.start_time.elapsed().as_secs();
out.push_str("# HELP gitks_uptime_seconds Time since gitks started\n");
out.push_str("# TYPE gitks_uptime_seconds gauge\n");
out.push_str(&format!("gitks_uptime_seconds {uptime}\n\n"));
// Active requests
let active = m.active_requests.load(Ordering::Relaxed);
out.push_str("# HELP gitks_active_requests Currently in-flight requests\n");
out.push_str("# TYPE gitks_active_requests gauge\n");
out.push_str(&format!("gitks_active_requests {active}\n\n"));
// Repository count
let repos = m.repository_count.load(Ordering::Relaxed);
out.push_str("# HELP gitks_repository_count Number of registered repositories\n");
out.push_str("# TYPE gitks_repository_count gauge\n");
out.push_str(&format!("gitks_repository_count {repos}\n\n"));
// Request count
out.push_str("# HELP gitks_requests_total Total gRPC requests by method and status\n");
out.push_str("# TYPE gitks_requests_total counter\n");
for entry in &m.request_count {
let (method_and_status, count) = (entry.key(), entry.value());
let count = count.load(Ordering::Relaxed);
if let Some((method, status)) = method_and_status.rsplit_once(':') {
out.push_str(
&format!("gitks_requests_total{{method=\"{method}\",status=\"{status}\"}} {count}\n"),
);
}
}
out.push('\n');
// Duration histogram
out.push_str(
"# HELP gitks_request_duration_milliseconds Request duration histogram in ms\n",
);
out.push_str("# TYPE gitks_request_duration_milliseconds histogram\n");
for entry in &m.duration_buckets {
let (method_and_bound, count) = (entry.key(), entry.value());
let count = count.load(Ordering::Relaxed);
if let Some((method, bound_str)) = method_and_bound.rsplit_once(':') {
let bound = bound_str;
let le = if bound_str.parse::<u64>() == Ok(BUCKET_INF) {
"+Inf".to_string()
} else {
bound.to_string()
};
out.push_str(
&format!("gitks_request_duration_milliseconds_bucket{{method=\"{method}\",le=\"{le}\"}} {count}\n"),
);
}
}
out.push('\n');
// Cache
let hits = m.cache_hits.load(Ordering::Relaxed);
let misses = m.cache_misses.load(Ordering::Relaxed);
out.push_str("# HELP gitks_cache_hits_total Cache hit count\n");
out.push_str("# TYPE gitks_cache_hits_total counter\n");
out.push_str(&format!("gitks_cache_hits_total {hits}\n\n"));
out.push_str("# HELP gitks_cache_misses_total Cache miss count\n");
out.push_str("# TYPE gitks_cache_misses_total counter\n");
out.push_str(&format!("gitks_cache_misses_total {misses}\n\n"));
// Errors
out.push_str("# HELP gitks_errors_total Total errors by kind\n");
out.push_str("# TYPE gitks_errors_total counter\n");
for entry in &m.error_count {
let (kind, count) = (entry.key(), entry.value());
let count = count.load(Ordering::Relaxed);
out.push_str(&format!("gitks_errors_total{{kind=\"{kind}\"}} {count}\n"));
}
out.push('\n');
out
}
// ── HTTP server for /metrics endpoint ───────────────────────────────
/// Start the metrics HTTP server on the given port.
/// Runs in a background task; returns the JoinHandle.
pub fn start_metrics_server(port: u16) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let listener = match tokio::net::TcpListener::bind(format!("0.0.0.0:{port}")).await {
Ok(l) => l,
Err(e) => {
tracing::error!(port, error = %e, "failed to bind metrics server");
return;
}
};
tracing::info!(port, "metrics HTTP server started");
loop {
match listener.accept().await {
Ok((socket, peer)) => {
tracing::debug!(%peer, "metrics request");
tokio::spawn(handle_metrics_connection(socket));
}
Err(e) => {
tracing::error!(error = %e, "metrics accept error");
}
}
}
})
}
async fn handle_metrics_connection(mut socket: tokio::net::TcpStream) {
use tokio::io::{AsyncReadExt, AsyncWriteExt};
let mut buf = [0u8; 4096];
let _ = tokio::time::timeout(Duration::from_secs(5), socket.read(&mut buf)).await;
let body = render_metrics();
let response = format!(
"HTTP/1.1 200 OK\r\nContent-Type: text/plain; version=0.0.4\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
body.len(),
body
);
let _ = tokio::time::timeout(Duration::from_secs(5), socket.write_all(response.as_bytes()))
.await;
let _ = socket.shutdown().await;
}
// ── Helper to wrap handler functions with metrics ───────────────────
/// A guard that records metrics on drop.
///
/// Usage in handlers:
/// ```ignore
/// let m = crate::metrics::RequestMetrics::new("Service/Method");
/// // ... handle request ...
/// m.record("ok"); // on success
/// // m.record("internal"); // or on error, with tonic error kind
/// ```
pub struct RequestMetrics {
method: &'static str,
start: Instant,
}
impl RequestMetrics {
pub fn new(method: &'static str) -> Self {
inc_active_requests();
Self {
method,
start: Instant::now(),
}
}
/// Record the outcome. Idempotent — safe to call before each return.
pub fn record(&self, status: &str) {
let duration = self.start.elapsed();
record_request(self.method, status, duration);
}
}
impl Drop for RequestMetrics {
fn drop(&mut self) {
dec_active_requests();
}
}
/// Convenience: record an error from a tonic Status.
pub fn record_rpc_error(m: &RequestMetrics, status: &tonic::Status) {
let kind = status.code().description();
inc_error(kind);
m.record(kind);
}
+229
View File
@@ -0,0 +1,229 @@
//! Pack-objects cache module.
//!
//! Caches `git pack-objects` output on local disk to reduce CPU load
//! during repeated clone/fetch operations (especially CI traffic).
//!
//! Uses the DiskCache infrastructure for state management and invalidation.
//! Implements streaming with backpressure: the producer writes to a cache file
//! while consumers read from it concurrently.
use std::io::Write;
use std::time::Duration;
use prost::Message;
use tokio_stream::wrappers::ReceiverStream;
use crate::disk_cache::DiskCache;
use crate::pb::PackfileChunk;
/// Namespace for pack-objects cache entries in the disk cache.
pub const PACK_CACHE_NAMESPACE: &str = "+gitks-cache/cache";
/// Namespace for info/refs cache entries.
pub const INFO_REFS_NAMESPACE: &str = "+gitks-cache/info_refs";
/// Pack-objects cache wrapper around DiskCache.
#[derive(Debug, Clone)]
pub struct PackCache {
disk_cache: DiskCache,
}
impl PackCache {
pub fn new(disk_cache: DiskCache, _backpressure: bool) -> Self {
Self { disk_cache }
}
pub fn is_enabled(&self) -> bool {
self.disk_cache.is_enabled()
}
pub fn disk_cache(&self) -> &DiskCache {
&self.disk_cache
}
/// Try to serve a cached pack-objects response as a stream.
/// Reads the cache file in streaming fashion, emitting chunks as they are read
/// to avoid buffering the entire pack file in memory.
pub fn lookup_pack_stream(
&self,
digest: &str,
) -> Option<ReceiverStream<Result<PackfileChunk, tonic::Status>>> {
if !self.is_enabled() {
return None;
}
let file = match self
.disk_cache
.open_stream_read(PACK_CACHE_NAMESPACE, digest)
{
Ok(Some(f)) => f,
Ok(None) => return None,
Err(_) => return None,
};
tracing::info!(digest = %digest, "pack-objects cache hit, streaming from disk");
let (tx, rx) = tokio::sync::mpsc::channel(16);
let sender = tx.clone();
tokio::spawn(async move {
let result = tokio::task::spawn_blocking(move || {
use std::io::Read;
let mut file = file;
let mut buf = vec![0u8; 65536];
loop {
match file.read(&mut buf) {
Ok(0) => return Ok(()),
Ok(n) => {
if sender
.blocking_send(Ok(PackfileChunk {
data: buf[..n].to_vec(),
}))
.is_err()
{
return Ok(());
}
}
Err(e) => {
let _ = sender.blocking_send(Err(tonic::Status::internal(format!(
"cache read error: {e}"
))));
return Err(());
}
}
}
})
.await;
if result.is_err() {
// Task join error or I/O error already sent
}
});
Some(ReceiverStream::new(rx))
}
/// Stream pack-objects output while simultaneously writing to cache.
/// This is the "tee" approach: data flows to both the client and the cache file.
pub fn tee_pack_stream(
&self,
digest: &str,
source: ReceiverStream<Result<PackfileChunk, tonic::Status>>,
) -> ReceiverStream<Result<PackfileChunk, tonic::Status>> {
let (tx, rx) = tokio::sync::mpsc::channel(16);
if !self.is_enabled() {
tokio::spawn(async move {
use tokio_stream::StreamExt;
let mut source = source;
while let Some(item) = source.next().await {
if tx.send(item).await.is_err() {
break;
}
}
});
return ReceiverStream::new(rx);
}
let cache_dir = self.disk_cache.repo_prefix.join(PACK_CACHE_NAMESPACE);
let rel_path = crate::disk_cache::digest_to_path(digest);
let cache_path = cache_dir.join(&rel_path);
let tmp_path = cache_path.with_extension("tmp_streaming");
tokio::spawn(async move {
use tokio_stream::StreamExt;
if let Some(parent) = cache_path.parent() {
let _ = std::fs::create_dir_all(parent);
}
let mut cache_file = std::fs::File::create(&tmp_path).ok();
let mut cache_write_ok = true;
let mut source = source;
while let Some(item) = source.next().await {
if tx.send(item.clone()).await.is_err() {
cache_write_ok = false; // client disconnected, don't cache partial
break;
}
if cache_write_ok
&& let Some(ref mut f) = cache_file
&& let Ok(chunk) = &item
&& f.write_all(&chunk.data).is_err()
{
tracing::warn!("cache write failed, dropping cache file handle");
cache_file = None;
cache_write_ok = false;
}
}
if cache_write_ok && cache_file.is_some() {
let _ = std::fs::rename(&tmp_path, &cache_path);
tracing::info!(
path = %cache_path.display(),
"pack-objects cache entry written"
);
} else if tmp_path.exists() {
std::fs::remove_file(&tmp_path).ok();
}
});
ReceiverStream::new(rx)
}
/// Look up cached info/refs response.
pub fn lookup_info_refs<T: Message + Default>(&self, digest: &str) -> Option<T> {
if !self.is_enabled() {
return None;
}
match self.disk_cache.lookup(INFO_REFS_NAMESPACE, digest) {
Ok(Some(bytes)) => {
if let Ok(resp) = T::decode(bytes.as_slice()) {
tracing::debug!(digest = %digest, "info/refs cache hit");
return Some(resp);
}
tracing::warn!(digest = %digest, "info/refs cache decode failed");
None
}
Ok(None) => None,
Err(e) => {
tracing::warn!(error = %e, "info/refs cache lookup error");
None
}
}
}
/// Store an info/refs response in cache.
pub fn store_info_refs<T: Message>(&self, digest: &str, resp: &T) {
if !self.is_enabled() {
return;
}
let mut bytes = Vec::with_capacity(resp.encoded_len());
if resp.encode(&mut bytes).is_ok()
&& let Err(e) = self.disk_cache.insert(INFO_REFS_NAMESPACE, digest, &bytes)
{
tracing::warn!(error = %e, "info/refs cache write failed");
}
}
/// Invalidate cache for a repository (called after mutator RPCs).
pub fn invalidate_repo(&self, relative_path: &str) {
self.disk_cache.invalidate_repo(relative_path);
}
/// Create a lease for a mutating operation.
pub fn create_lease(
&self,
relative_path: &str,
) -> Result<crate::disk_cache::LeaseGuard, crate::error::GitError> {
self.disk_cache.create_lease(relative_path)
}
}
/// Start background cache cleanup task.
pub fn start_cleanup_task(cache: PackCache, interval_secs: u64) -> tokio::task::JoinHandle<()> {
crate::disk_cache::start_cache_cleanup_task(
cache.disk_cache.clone(),
Duration::from_secs(interval_secs),
)
}
+61
View File
@@ -0,0 +1,61 @@
syntax = "proto3";
package gitks;
import "repository.proto";
// HookService provides gRPC callback hooks for git operations.
// External services can implement this interface to receive hook callbacks.
service HookService {
// Pre-receive hook callback: validate push before it happens.
rpc PreReceiveHook(PreReceiveHookRequest) returns (PreReceiveHookResponse);
// Update hook callback: validate each ref update individually.
rpc UpdateHook(UpdateHookRequest) returns (UpdateHookResponse);
// Post-receive hook callback: notify after push has completed.
rpc PostReceiveHook(PostReceiveHookRequest) returns (PostReceiveHookResponse);
}
message PreReceiveHookRequest {
RepositoryHeader repository = 1;
repeated RefUpdate ref_updates = 2;
string push_options = 3;
}
message PreReceiveHookResponse {
bool accept = 1;
string rejection_message = 2;
}
message UpdateHookRequest {
RepositoryHeader repository = 1;
string ref_name = 2;
string old_oid = 3;
string new_oid = 4;
}
message UpdateHookResponse {
bool accept = 1;
string rejection_message = 2;
}
message PostReceiveHookRequest {
RepositoryHeader repository = 1;
repeated RefUpdate ref_updates = 2;
}
message PostReceiveHookResponse {
repeated HookAction actions = 1;
}
message RefUpdate {
string old_oid = 1;
string new_oid = 2;
string ref_name = 3;
}
message HookAction {
string action_type = 1; // "trigger_ci", "update_index", etc.
string payload = 2;
}
+122
View File
@@ -139,6 +139,113 @@ message RepositoryMaintenanceResponse {
string stderr = 3;
}
// ── Hooks Management ──────────────────────────────────────────────────
message ListHooksRequest {
RepositoryHeader repository = 1;
}
message HookInfo {
string hook_type = 1;
string level = 2; // "server" or "custom"
string path = 3;
}
message ListHooksResponse {
repeated HookInfo hooks = 1;
}
message SetCustomHookRequest {
RepositoryHeader repository = 1;
string hook_name = 2; // "pre-receive", "update", "post-receive"
string content = 3; // Hook script content
}
message RemoveCustomHookRequest {
RepositoryHeader repository = 1;
string hook_name = 2;
}
// ── Snapshot ──────────────────────────────────────────────────────────
enum SnapshotStorage {
SNAPSHOT_STORAGE_LOCAL = 0;
SNAPSHOT_STORAGE_S3 = 1;
SNAPSHOT_STORAGE_GCS = 2;
}
message SnapshotInfo {
string snapshot_id = 1;
string relative_path = 2;
uint64 size_bytes = 3;
string created_at = 4; // ISO 8601
string head_oid = 5;
}
message CreateSnapshotRequest {
RepositoryHeader repository = 1;
SnapshotStorage storage = 2;
string storage_path = 3;
}
message CreateSnapshotResponse {
string snapshot_id = 1;
uint64 size_bytes = 2;
string head_oid = 3;
}
message RestoreSnapshotRequest {
RepositoryHeader target_repository = 1;
string snapshot_id = 2;
SnapshotStorage storage = 3;
string storage_path = 4;
}
message ListSnapshotsRequest {
RepositoryHeader repository = 1;
uint32 limit = 2;
}
message ListSnapshotsResponse {
repeated SnapshotInfo snapshots = 1;
}
message DeleteSnapshotRequest {
string snapshot_id = 1;
SnapshotStorage storage = 2;
}
// ── Repository Move ──────────────────────────────────────────────────
enum MoveRepositoryState {
MOVE_STATE_UNKNOWN = 0;
MOVE_STATE_PREPARING = 1;
MOVE_STATE_TRANSFERRING = 2;
MOVE_STATE_VERIFYING = 3;
MOVE_STATE_COMPLETED = 4;
MOVE_STATE_FAILED = 5;
MOVE_STATE_CANCELLED = 6;
}
message MoveRepositoryRequest {
RepositoryHeader source_repository = 1;
RepositoryHeader target_repository = 2;
}
message MoveRepositoryResponse {
MoveRepositoryState state = 1;
string error_message = 2;
}
message FetchRepositoryDataRequest {
RepositoryHeader repository = 1;
}
message FetchRepositoryDataResponse {
bytes data = 1;
bool done = 2;
}
service RepositoryService {
rpc GetRepository(GetRepositoryRequest) returns (Repository);
rpc InitRepository(InitRepositoryRequest) returns (Repository);
@@ -154,4 +261,19 @@ service RepositoryService {
rpc GarbageCollect(GarbageCollectRequest) returns (RepositoryMaintenanceResponse);
rpc Repack(RepackRequest) returns (RepositoryMaintenanceResponse);
rpc WriteCommitGraph(WriteCommitGraphRequest) returns (RepositoryMaintenanceResponse);
// Hooks management
rpc ListHooks(ListHooksRequest) returns (ListHooksResponse);
rpc SetCustomHook(SetCustomHookRequest) returns (google.protobuf.Empty);
rpc RemoveCustomHook(RemoveCustomHookRequest) returns (google.protobuf.Empty);
// Snapshot operations
rpc CreateSnapshot(CreateSnapshotRequest) returns (CreateSnapshotResponse);
rpc RestoreSnapshot(RestoreSnapshotRequest) returns (google.protobuf.Empty);
rpc ListSnapshots(ListSnapshotsRequest) returns (ListSnapshotsResponse);
rpc DeleteSnapshot(DeleteSnapshotRequest) returns (google.protobuf.Empty);
// Repository move
rpc MoveRepository(MoveRepositoryRequest) returns (MoveRepositoryResponse);
rpc FetchRepositoryData(FetchRepositoryDataRequest) returns (stream FetchRepositoryDataResponse);
}
+172
View File
@@ -0,0 +1,172 @@
//! Repository-level rate limiting via per-repo semaphores.
//!
//! Prevents any single repository from consuming all server resources.
//! Uses `tokio::sync::Semaphore` with configurable max concurrent operations.
//!
//! Integration pattern:
//! let _guard = rate_limit::acquire(svc, header).await?;
//! // ... handle request ...
//! // guard is dropped here → permit released
use dashmap::DashMap;
use std::sync::{Arc, OnceLock};
use tokio::sync::Semaphore;
// ── Configuration ───────────────────────────────────────────────────
/// Default max concurrent operations per repository.
const DEFAULT_MAX_CONCURRENT: usize = 5;
/// Global rate limiter state.
struct RateLimiter {
/// Per-repository semaphores. Key = repository relative_path.
semaphores: DashMap<String, Arc<Semaphore>>,
/// Max concurrent operations per repository.
max_concurrent: usize,
}
static RATE_LIMITER: OnceLock<RateLimiter> = OnceLock::new();
fn limiter() -> &'static RateLimiter {
RATE_LIMITER.get_or_init(|| {
let max = std::env::var("GITKS_RATE_LIMIT_MAX_CONCURRENT")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(DEFAULT_MAX_CONCURRENT);
tracing::info!(
max_concurrent = max,
"repository-level rate limiter initialized"
);
RateLimiter {
semaphores: DashMap::new(),
max_concurrent: max,
}
})
}
// ── Permit guard ───────────────────────────────────────────────────
/// A guard that holds a rate-limit permit. The permit is released on drop.
pub struct RateLimitGuard {
/// The semaphore permit. Dropping this releases the permit.
_permit: tokio::sync::OwnedSemaphorePermit,
}
/// Acquire a rate-limit permit for the given repository.
///
/// If the repository has `max_concurrent` operations already in flight,
/// this will wait asynchronously until a permit becomes available.
///
/// Returns `None` if no repository header is provided (e.g., global health checks).
pub async fn acquire(repo_relative_path: Option<&str>) -> Option<RateLimitGuard> {
let repo = repo_relative_path?;
if repo.is_empty() {
return None;
}
let l = limiter();
if l.max_concurrent == 0 {
// Unlimited
return None;
}
let sem = l
.semaphores
.entry(repo.to_string())
.or_insert_with(|| Arc::new(Semaphore::new(l.max_concurrent)))
.value()
.clone();
// Release DashMap reference before awaiting
let _ = l;
match tokio::time::timeout(
std::time::Duration::from_secs(30),
sem.clone().acquire_owned(),
)
.await
{
Ok(Ok(permit)) => {
tracing::debug!(
repo = %repo,
available = sem.available_permits(),
"rate limit permit acquired"
);
Some(RateLimitGuard { _permit: permit })
}
Ok(Err(_closed)) => {
// Semaphore was closed — recreate it
tracing::warn!(
repo = %repo,
"rate limit semaphore closed, recreating"
);
let new_sem = Arc::new(Semaphore::new(limiter().max_concurrent));
let permit = new_sem
.clone()
.acquire_owned()
.await
.expect("newly created semaphore should have permits");
limiter()
.semaphores
.insert(repo.to_string(), new_sem);
Some(RateLimitGuard { _permit: permit })
}
Err(_elapsed) => {
tracing::warn!(
repo = %repo,
max_concurrent = limiter().max_concurrent,
"rate limit timeout waiting for permit"
);
None
}
}
}
/// Acquire a rate-limit permit, returning a tonic error on timeout / overload.
pub async fn acquire_or_reject(repo_relative_path: Option<&str>) -> Result<Option<RateLimitGuard>, tonic::Status> {
let repo = repo_relative_path.unwrap_or("");
if repo.is_empty() {
return Ok(None);
}
match acquire(Some(repo)).await {
Some(guard) => Ok(Some(guard)),
None => {
if limiter().max_concurrent == 0 {
return Ok(None);
}
// Timeout — reject with resource exhausted
Err(tonic::Status::resource_exhausted(format!(
"rate limit exceeded for repository '{repo}': max {max} concurrent operations",
max = limiter().max_concurrent
)))
}
}
}
/// Remove the semaphore for a repository (called on repo deletion).
pub fn remove_repository(repo_relative_path: &str) {
limiter().semaphores.remove(repo_relative_path);
tracing::debug!(repo = %repo_relative_path, "rate limit semaphore removed");
}
/// Update the max concurrent limit at runtime.
pub fn set_max_concurrent(max: usize) {
let l = limiter();
// We can't modify the field directly through OnceLock, but we can
// update existing semaphores to add or remove permits as needed.
// For a simpler approach, just log and let new semaphores use the new value.
// Since max_concurrent is only read on insert, we use a separate atomic.
use std::sync::atomic::AtomicUsize;
static OVERRIDE: std::sync::atomic::AtomicUsize = AtomicUsize::new(0);
OVERRIDE.store(max, std::sync::atomic::Ordering::Relaxed);
// Recreate all existing semaphores
for entry in &l.semaphores {
let _old = l.semaphores.insert(
entry.key().clone(),
Arc::new(Semaphore::new(max)),
);
}
tracing::info!(max_concurrent = max, "rate limit max_concurrent updated");
}
+23 -3
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, cache, into_status};
remote_client!(remote_archive_client, ArchiveServiceClient<tonic::transport::Channel>, "archive");
remote_client!(
remote_archive_client,
ArchiveServiceClient<tonic::transport::Channel>,
"archive"
);
#[tonic::async_trait]
impl archive_service_server::ArchiveService for GitksService {
@@ -14,7 +18,9 @@ impl archive_service_server::ArchiveService for GitksService {
&self,
request: tonic::Request<ArchiveRequest>,
) -> Result<tonic::Response<Self::GetArchiveStream>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.ArchiveService/GetArchive");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("archive.get_archive", %repo);
let _enter = span.enter();
@@ -24,16 +30,22 @@ impl archive_service_server::ArchiveService for GitksService {
if let Some(mut client) =
remote_archive_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
let resp = client.get_archive(inner).await?;
let stream = super::bridge_server_stream(resp.into_inner());
return Ok(tonic::Response::new(stream));
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let stream = gb.get_archive_stream(inner)?;
tracing::info!(%repo, "archive streaming started");
m.record("ok");
Ok(tonic::Response::new(stream))
}
@@ -41,7 +53,9 @@ impl archive_service_server::ArchiveService for GitksService {
&self,
request: tonic::Request<ListArchiveEntriesRequest>,
) -> Result<tonic::Response<ListArchiveEntriesResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.ArchiveService/ListArchiveEntries");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("archive.list_archive_entries", %repo);
let _enter = span.enter();
@@ -51,11 +65,16 @@ impl archive_service_server::ArchiveService for GitksService {
if let Some(mut client) =
remote_archive_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.list_archive_entries(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.treeish) {
cache::cached_response("archive.list_archive_entries", &inner, || {
@@ -65,6 +84,7 @@ impl archive_service_server::ArchiveService for GitksService {
gb.list_archive_entries(inner).map_err(into_status)?
};
tracing::info!(%repo, count = resp.entries.len(), "list_archive_entries done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+23 -3
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, cache, into_status, into_stream};
remote_client!(remote_blame_client, BlameServiceClient<tonic::transport::Channel>, "blame");
remote_client!(
remote_blame_client,
BlameServiceClient<tonic::transport::Channel>,
"blame"
);
#[tonic::async_trait]
impl blame_service_server::BlameService for GitksService {
@@ -14,7 +18,9 @@ impl blame_service_server::BlameService for GitksService {
&self,
request: tonic::Request<BlameRequest>,
) -> Result<tonic::Response<BlameResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BlameService/Blame");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let path = inner.path.clone();
let span = tracing::info_span!("blame.blame", %repo, %path);
@@ -25,11 +31,16 @@ impl blame_service_server::BlameService for GitksService {
if let Some(mut client) =
remote_blame_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.blame(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("blame.blame", &inner, || {
@@ -39,6 +50,7 @@ impl blame_service_server::BlameService for GitksService {
gb.blame(inner).map_err(into_status)?
};
tracing::info!(%repo, %path, hunks = resp.hunks.len(), "blame done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -46,7 +58,9 @@ impl blame_service_server::BlameService for GitksService {
&self,
request: tonic::Request<BlameRequest>,
) -> Result<tonic::Response<Self::StreamBlameStream>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BlameService/StreamBlame");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let path = inner.path.clone();
let span = tracing::info_span!("blame.stream_blame", %repo, %path);
@@ -57,13 +71,18 @@ impl blame_service_server::BlameService for GitksService {
if let Some(mut client) =
remote_blame_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
let resp = client.stream_blame(inner).await?;
let stream = super::bridge_server_stream(resp.into_inner());
return Ok(tonic::Response::new(stream));
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("blame.blame", &inner, || {
@@ -72,6 +91,7 @@ impl blame_service_server::BlameService for GitksService {
} else {
gb.blame(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(into_stream(resp.hunks)))
}
}
+77 -9
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, into_status};
remote_client!(remote_branch_client, BranchServiceClient<tonic::transport::Channel>, "branch");
remote_client!(
remote_branch_client,
BranchServiceClient<tonic::transport::Channel>,
"branch"
);
#[tonic::async_trait]
impl branch_service_server::BranchService for GitksService {
@@ -11,7 +15,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<ListBranchesRequest>,
) -> Result<tonic::Response<ListBranchesResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/ListBranches");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("branch.list_branches", %repo);
let _enter = span.enter();
@@ -21,14 +27,20 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.list_branches(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.list_branches(inner).map_err(into_status)?;
tracing::info!(%repo, count = resp.branches.len(), "list_branches done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -36,7 +48,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<GetBranchRequest>,
) -> Result<tonic::Response<Branch>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/GetBranch");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("branch.get_branch", %repo, %name);
@@ -47,13 +61,19 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_branch(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.get_branch(inner).map_err(into_status)?;
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -61,7 +81,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<CreateBranchRequest>,
) -> Result<tonic::Response<Branch>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/CreateBranch");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("branch.create_branch", %repo, %name);
@@ -72,15 +94,21 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.create_branch(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.create_branch(inner).map_err(into_status)?;
tracing::info!(%repo, %name, "branch created");
self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -88,7 +116,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<DeleteBranchRequest>,
) -> Result<tonic::Response<()>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/DeleteBranch");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("branch.delete_branch", %repo, %name);
@@ -99,15 +129,21 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.delete_branch(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
gb.delete_branch(inner).map_err(into_status)?;
tracing::info!(%repo, %name, "branch deleted");
self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", "");
m.record("ok");
Ok(tonic::Response::new(()))
}
@@ -115,7 +151,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<RenameBranchRequest>,
) -> Result<tonic::Response<Branch>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/RenameBranch");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let old = inner.old_name.clone();
let new = inner.new_name.clone();
@@ -127,15 +165,21 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.rename_branch(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.rename_branch(inner).map_err(into_status)?;
tracing::info!(%repo, old = %old, new = %new, "branch renamed");
self.notify_ref_update(&repo, &format!("refs/heads/{}", new), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -143,7 +187,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<UpdateBranchTargetRequest>,
) -> Result<tonic::Response<Branch>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/UpdateBranchTarget");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("branch.update_branch_target", %repo, %name);
@@ -154,15 +200,21 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.update_branch_target(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.update_branch_target(inner).map_err(into_status)?;
tracing::info!(%repo, %name, "branch target updated");
self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -170,7 +222,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<SetBranchUpstreamRequest>,
) -> Result<tonic::Response<Branch>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/SetBranchUpstream");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("branch.set_branch_upstream", %repo, %name);
@@ -181,15 +235,21 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.set_branch_upstream(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.set_branch_upstream(inner).map_err(into_status)?;
tracing::info!(%repo, %name, "branch upstream set");
self.notify_ref_update(&repo, &format!("refs/heads/{}", name), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -197,7 +257,9 @@ impl branch_service_server::BranchService for GitksService {
&self,
request: tonic::Request<CompareBranchRequest>,
) -> Result<tonic::Response<CompareBranchResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.BranchService/CompareBranch");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let source = inner.source_branch.clone();
let target = inner.target_branch.clone();
@@ -209,14 +271,20 @@ impl branch_service_server::BranchService for GitksService {
if let Some(mut client) =
remote_branch_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.compare_branch(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.compare_branch(inner).map_err(into_status)?;
tracing::info!(%repo, %source, %target, ahead = resp.ahead_by, behind = resp.behind_by, "branch compared");
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+68 -8
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, cache, into_status};
remote_client!(remote_commit_client, CommitServiceClient<tonic::transport::Channel>, "commit");
remote_client!(
remote_commit_client,
CommitServiceClient<tonic::transport::Channel>,
"commit"
);
#[tonic::async_trait]
impl commit_service_server::CommitService for GitksService {
@@ -11,7 +15,9 @@ impl commit_service_server::CommitService for GitksService {
&self,
request: tonic::Request<ListCommitsRequest>,
) -> Result<tonic::Response<ListCommitsResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.CommitService/ListCommits");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("commit.list_commits", %repo);
let _enter = span.enter();
@@ -21,11 +27,16 @@ impl commit_service_server::CommitService for GitksService {
if let Some(mut client) =
remote_commit_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.list_commits(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if !inner.all && cache::selector_is_oid(&inner.revision) {
cache::cached_response("commit.list_commits", &inner, || {
@@ -35,6 +46,7 @@ impl commit_service_server::CommitService for GitksService {
gb.list_commits(inner).map_err(into_status)?
};
tracing::info!(%repo, count = resp.commits.len(), total = resp.page_info.as_ref().map(|p| p.total_count).unwrap_or(0), "list_commits done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -42,7 +54,9 @@ impl commit_service_server::CommitService for GitksService {
&self,
request: tonic::Request<GetCommitRequest>,
) -> Result<tonic::Response<Commit>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.CommitService/GetCommit");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("commit.get_commit", %repo);
let _enter = span.enter();
@@ -52,11 +66,16 @@ impl commit_service_server::CommitService for GitksService {
if let Some(mut client) =
remote_commit_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_commit(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("commit.get_commit", &inner, || {
@@ -65,6 +84,7 @@ impl commit_service_server::CommitService for GitksService {
} else {
gb.get_commit(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -72,7 +92,9 @@ impl commit_service_server::CommitService for GitksService {
&self,
request: tonic::Request<GetCommitAncestorsRequest>,
) -> Result<tonic::Response<GetCommitAncestorsResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.CommitService/GetCommitAncestors");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("commit.get_commit_ancestors", %repo);
let _enter = span.enter();
@@ -82,11 +104,16 @@ impl commit_service_server::CommitService for GitksService {
if let Some(mut client) =
remote_commit_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_commit_ancestors(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("commit.get_commit_ancestors", &inner, || {
@@ -96,6 +123,7 @@ impl commit_service_server::CommitService for GitksService {
gb.get_commit_ancestors(inner).map_err(into_status)?
};
tracing::info!(%repo, count = resp.commits.len(), "get_commit_ancestors done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -103,7 +131,9 @@ impl commit_service_server::CommitService for GitksService {
&self,
request: tonic::Request<CreateCommitRequest>,
) -> Result<tonic::Response<CreateCommitResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.CommitService/CreateCommit");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let branch = inner.branch.clone();
let span = tracing::info_span!("commit.create_commit", %repo, %branch);
@@ -114,11 +144,16 @@ impl commit_service_server::CommitService for GitksService {
if let Some(mut client) =
remote_commit_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.create_commit(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.create_commit(inner).map_err(into_status)?;
let commit_hex = resp
@@ -128,6 +163,7 @@ impl commit_service_server::CommitService for GitksService {
.unwrap_or("?");
tracing::info!(%repo, %branch, %commit_hex, "commit created");
self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -135,7 +171,9 @@ impl commit_service_server::CommitService for GitksService {
&self,
request: tonic::Request<RevertCommitRequest>,
) -> Result<tonic::Response<CreateCommitResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.CommitService/RevertCommit");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let branch = inner.branch.clone();
let span = tracing::info_span!("commit.revert_commit", %repo, %branch);
@@ -146,15 +184,21 @@ impl commit_service_server::CommitService for GitksService {
if let Some(mut client) =
remote_commit_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.revert_commit(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.revert_commit(inner).map_err(into_status)?;
tracing::info!(%repo, %branch, "commit reverted");
self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -162,7 +206,9 @@ impl commit_service_server::CommitService for GitksService {
&self,
request: tonic::Request<CherryPickCommitRequest>,
) -> Result<tonic::Response<CreateCommitResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.CommitService/CherryPickCommit");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let branch = inner.branch.clone();
let span = tracing::info_span!("commit.cherry_pick_commit", %repo, %branch);
@@ -173,15 +219,21 @@ impl commit_service_server::CommitService for GitksService {
if let Some(mut client) =
remote_commit_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.cherry_pick_commit(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.cherry_pick_commit(inner).map_err(into_status)?;
tracing::info!(%repo, %branch, "commit cherry-picked");
self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -189,7 +241,9 @@ impl commit_service_server::CommitService for GitksService {
&self,
request: tonic::Request<CompareCommitsRequest>,
) -> Result<tonic::Response<CompareCommitsResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.CommitService/CompareCommits");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("commit.compare_commits", %repo);
let _enter = span.enter();
@@ -199,11 +253,16 @@ impl commit_service_server::CommitService for GitksService {
if let Some(mut client) =
remote_commit_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.compare_commits(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
cache::cached_response("commit.compare_commits", &inner, || {
@@ -213,6 +272,7 @@ impl commit_service_server::CommitService for GitksService {
gb.compare_commits(inner).map_err(into_status)?
};
tracing::info!(%repo, count = resp.commits.len(), "compare_commits done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+41 -5
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, cache, into_status, into_stream};
remote_client!(remote_diff_client, DiffServiceClient<tonic::transport::Channel>, "diff");
remote_client!(
remote_diff_client,
DiffServiceClient<tonic::transport::Channel>,
"diff"
);
#[tonic::async_trait]
impl diff_service_server::DiffService for GitksService {
@@ -14,7 +18,9 @@ impl diff_service_server::DiffService for GitksService {
&self,
request: tonic::Request<GetDiffRequest>,
) -> Result<tonic::Response<GetDiffResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetDiff");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("diff.get_diff", %repo);
let _enter = span.enter();
@@ -24,11 +30,16 @@ impl diff_service_server::DiffService for GitksService {
if let Some(mut client) =
remote_diff_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_diff(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
cache::cached_response("diff.get_diff", &inner, || {
@@ -38,6 +49,7 @@ impl diff_service_server::DiffService for GitksService {
gb.get_diff(inner).map_err(into_status)?
};
tracing::info!(%repo, files = resp.files.len(), overflow = resp.overflow, "get_diff done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -45,7 +57,9 @@ impl diff_service_server::DiffService for GitksService {
&self,
request: tonic::Request<GetCommitDiffRequest>,
) -> Result<tonic::Response<GetDiffResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetCommitDiff");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("diff.get_commit_diff", %repo);
let _enter = span.enter();
@@ -55,11 +69,16 @@ impl diff_service_server::DiffService for GitksService {
if let Some(mut client) =
remote_diff_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_commit_diff(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.commit) {
cache::cached_response("diff.get_commit_diff", &inner, || {
@@ -69,6 +88,7 @@ impl diff_service_server::DiffService for GitksService {
gb.get_commit_diff(inner).map_err(into_status)?
};
tracing::info!(%repo, files = resp.files.len(), "get_commit_diff done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -76,7 +96,9 @@ impl diff_service_server::DiffService for GitksService {
&self,
request: tonic::Request<GetPatchRequest>,
) -> Result<tonic::Response<Self::GetPatchStream>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetPatch");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("diff.get_patch", %repo);
let _enter = span.enter();
@@ -86,13 +108,18 @@ impl diff_service_server::DiffService for GitksService {
if let Some(mut client) =
remote_diff_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
let resp = client.get_patch(inner).await?;
let stream = super::bridge_server_stream(resp.into_inner());
return Ok(tonic::Response::new(stream));
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let items = if cache::selectors_are_oid(&inner.base, &inner.head) {
cache::cached_vec_response("diff.get_patch", &inner, || {
@@ -101,6 +128,7 @@ impl diff_service_server::DiffService for GitksService {
} else {
gb.get_patch(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(into_stream(items)))
}
@@ -108,7 +136,9 @@ impl diff_service_server::DiffService for GitksService {
&self,
request: tonic::Request<GetDiffStatsRequest>,
) -> Result<tonic::Response<DiffStats>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetDiffStats");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("diff.get_diff_stats", %repo);
let _enter = span.enter();
@@ -118,11 +148,16 @@ impl diff_service_server::DiffService for GitksService {
if let Some(mut client) =
remote_diff_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_diff_stats(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
cache::cached_response("diff.get_diff_stats", &inner, || {
@@ -131,6 +166,7 @@ impl diff_service_server::DiffService for GitksService {
} else {
gb.get_diff_stats(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+50 -6
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, into_status};
remote_client!(remote_merge_client, MergeServiceClient<tonic::transport::Channel>, "merge");
remote_client!(
remote_merge_client,
MergeServiceClient<tonic::transport::Channel>,
"merge"
);
#[tonic::async_trait]
impl merge_service_server::MergeService for GitksService {
@@ -11,7 +15,9 @@ impl merge_service_server::MergeService for GitksService {
&self,
request: tonic::Request<CheckMergeRequest>,
) -> Result<tonic::Response<MergeResult>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.MergeService/CheckMerge");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("merge.check_merge", %repo);
let _enter = span.enter();
@@ -21,14 +27,20 @@ impl merge_service_server::MergeService for GitksService {
if let Some(mut client) =
remote_merge_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.check_merge(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.check_merge(inner).map_err(into_status)?;
tracing::info!(%repo, status = resp.status, "check_merge done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -36,7 +48,9 @@ impl merge_service_server::MergeService for GitksService {
&self,
request: tonic::Request<MergeRequest>,
) -> Result<tonic::Response<MergeResult>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.MergeService/Merge");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let target = inner.target_branch.clone();
let span = tracing::info_span!("merge.merge", %repo, %target);
@@ -47,15 +61,21 @@ impl merge_service_server::MergeService for GitksService {
if let Some(mut client) =
remote_merge_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.merge(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.merge(inner).map_err(into_status)?;
tracing::info!(%repo, %target, status = resp.status, "merge done");
self.notify_ref_update(&repo, &format!("refs/heads/{}", target), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -63,7 +83,9 @@ impl merge_service_server::MergeService for GitksService {
&self,
request: tonic::Request<ListMergeConflictsRequest>,
) -> Result<tonic::Response<ListMergeConflictsResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.MergeService/ListMergeConflicts");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("merge.list_merge_conflicts", %repo);
let _enter = span.enter();
@@ -73,14 +95,20 @@ impl merge_service_server::MergeService for GitksService {
if let Some(mut client) =
remote_merge_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.list_merge_conflicts(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.list_merge_conflicts(inner).map_err(into_status)?;
tracing::info!(%repo, conflicts = resp.conflicts.len(), "list_merge_conflicts done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -88,7 +116,9 @@ impl merge_service_server::MergeService for GitksService {
&self,
request: tonic::Request<ResolveMergeConflictsRequest>,
) -> Result<tonic::Response<MergeResult>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.MergeService/ResolveMergeConflicts");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let target = inner.target_branch.clone();
let span = tracing::info_span!("merge.resolve_merge_conflicts", %repo, %target);
@@ -99,15 +129,21 @@ impl merge_service_server::MergeService for GitksService {
if let Some(mut client) =
remote_merge_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.resolve_merge_conflicts(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.resolve_merge_conflicts(inner).map_err(into_status)?;
tracing::info!(%repo, %target, status = resp.status, "merge conflicts resolved");
self.notify_ref_update(&repo, &format!("refs/heads/{}", target), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -115,7 +151,9 @@ impl merge_service_server::MergeService for GitksService {
&self,
request: tonic::Request<RebaseRequest>,
) -> Result<tonic::Response<RebaseResult>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.MergeService/Rebase");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let branch = inner.branch.clone();
let span = tracing::info_span!("merge.rebase", %repo, %branch);
@@ -126,15 +164,21 @@ impl merge_service_server::MergeService for GitksService {
if let Some(mut client) =
remote_merge_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.rebase(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.rebase(inner).map_err(into_status)?;
tracing::info!(%repo, %branch, status = resp.status, "rebase done");
self.notify_ref_update(&repo, &format!("refs/heads/{}", branch), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+47 -1
View File
@@ -62,6 +62,9 @@ pub struct GitksService {
pub repo_prefix: PathBuf,
pub node_actor: Option<ActorRef<GitNodeMessage>>,
pub grpc_addr: String,
pub disk_cache: Option<crate::disk_cache::DiskCache>,
pub pack_cache: Option<crate::pack_cache::PackCache>,
pub hook_manager: Option<crate::hooks::HookManager>,
}
impl GitksService {
@@ -70,6 +73,9 @@ impl GitksService {
repo_prefix,
node_actor: None,
grpc_addr: String::new(),
disk_cache: None,
pack_cache: None,
hook_manager: None,
}
}
@@ -78,6 +84,21 @@ impl GitksService {
self
}
pub fn with_disk_cache(mut self, dc: crate::disk_cache::DiskCache) -> Self {
self.disk_cache = Some(dc);
self
}
pub fn with_pack_cache(mut self, pc: crate::pack_cache::PackCache) -> Self {
self.pack_cache = Some(pc);
self
}
pub fn with_hook_manager(mut self, hm: crate::hooks::HookManager) -> Self {
self.hook_manager = Some(hm);
self
}
pub fn with_grpc_addr(mut self, grpc_addr: String) -> Self {
self.grpc_addr = grpc_addr;
self
@@ -156,6 +177,26 @@ impl GitksService {
.unwrap_or_else(|| "unknown".into())
}
/// Get the relative path from a repository header, if any.
pub(crate) fn repo_relative_path<'a>(&self, header: Option<&'a crate::pb::RepositoryHeader>) -> Option<&'a str> {
header.and_then(|h| {
if h.relative_path.is_empty() {
None
} else {
Some(h.relative_path.as_str())
}
})
}
/// Acquire a rate-limit permit for the repository in this request.
/// Returns a guard that releases the permit on drop.
pub(crate) async fn acquire_rate_limit(
&self,
header: Option<&crate::pb::RepositoryHeader>,
) -> Result<Option<crate::rate_limit::RateLimitGuard>, tonic::Status> {
crate::rate_limit::acquire_or_reject(self.repo_relative_path(header)).await
}
pub(crate) fn resolve(
&self,
header: Option<&crate::pb::RepositoryHeader>,
@@ -241,9 +282,14 @@ impl GitksService {
old_oid: &str,
new_oid: &str,
) {
// Invalidate caches that depend on this repository
// Invalidate moka caches
crate::server::cache::invalidate_repo(relative_path);
// Invalidate disk cache
if let Some(ref pc) = self.pack_cache {
pc.invalidate_repo(relative_path);
}
if let Some(ref actor) = self.node_actor {
let event = crate::actor::message::RefUpdateEvent {
relative_path: relative_path.to_string(),
+162 -8
View File
@@ -6,7 +6,11 @@ use crate::pb::*;
use super::{GitksService, into_status};
remote_client!(remote_pack_client, PackServiceClient<tonic::transport::Channel>, "pack");
remote_client!(
remote_pack_client,
PackServiceClient<tonic::transport::Channel>,
"pack"
);
#[tonic::async_trait]
impl pack_service_server::PackService for GitksService {
@@ -18,7 +22,9 @@ impl pack_service_server::PackService for GitksService {
&self,
request: tonic::Request<AdvertiseRefsRequest>,
) -> Result<tonic::Response<AdvertiseRefsResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.PackService/AdvertiseRefs");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("pack.advertise_refs", %repo);
let _enter = span.enter();
@@ -28,14 +34,37 @@ impl pack_service_server::PackService for GitksService {
if let Some(mut client) =
remote_pack_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.advertise_refs(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
if let Some(ref pc) = self.pack_cache {
let protocol = inner.service.clone();
if let Ok(digest) = pc.disk_cache().compute_info_refs_key(&repo, &protocol) {
if let Some(cached) = pc.lookup_info_refs::<AdvertiseRefsResponse>(&digest) {
tracing::info!(%repo, refs = cached.references.len(), "advertise_refs done (cached)");
m.record("ok");
return Ok(tonic::Response::new(cached));
}
let resp = gb.advertise_refs(inner).map_err(into_status)?;
pc.store_info_refs(&digest, &resp);
tracing::info!(%repo, refs = resp.references.len(), "advertise_refs done (written to cache)");
m.record("ok");
return Ok(tonic::Response::new(resp));
}
}
let resp = gb.advertise_refs(inner).map_err(into_status)?;
tracing::info!(%repo, refs = resp.references.len(), "advertise_refs done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -43,11 +72,13 @@ impl pack_service_server::PackService for GitksService {
&self,
request: tonic::Request<tonic::Streaming<UploadPackRequest>>,
) -> Result<tonic::Response<Self::UploadPackStream>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.PackService/UploadPack");
let mut stream = request.into_inner();
let first = stream
.next()
.await
.ok_or_else(|| tonic::Status::invalid_argument("empty upload-pack stream"))??;
let _rate = self.acquire_rate_limit(first.repository.as_ref()).await?;
let repo = self.repo_label(first.repository.as_ref());
let span = tracing::info_span!("pack.upload_pack", %repo);
let _enter = span.enter();
@@ -57,6 +88,7 @@ impl pack_service_server::PackService for GitksService {
if let Some(mut client) =
remote_pack_client(self, first.repository.as_ref(), false).await?
{
m.record("ok");
let (tx, rx) = tokio::sync::mpsc::channel(16);
let _ = tx.send(first).await;
tokio::spawn(async move {
@@ -78,9 +110,13 @@ impl pack_service_server::PackService for GitksService {
let out = super::bridge_server_stream(resp.into_inner());
return Ok(tonic::Response::new(out));
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
tracing::info!(%repo, "upload-pack streaming started");
@@ -97,6 +133,7 @@ impl pack_service_server::PackService for GitksService {
});
let result = gb.upload_pack(ReceiverStream::new(rx)).await?;
m.record("ok");
Ok(tonic::Response::new(result))
}
@@ -104,11 +141,13 @@ impl pack_service_server::PackService for GitksService {
&self,
request: tonic::Request<tonic::Streaming<ReceivePackRequest>>,
) -> Result<tonic::Response<Self::ReceivePackStream>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.PackService/ReceivePack");
let mut stream = request.into_inner();
let first = stream
.next()
.await
.ok_or_else(|| tonic::Status::invalid_argument("empty receive-pack stream"))??;
let _rate = self.acquire_rate_limit(first.repository.as_ref()).await?;
let repo = self.repo_label(first.repository.as_ref());
let span = tracing::info_span!("pack.receive_pack", %repo);
let _enter = span.enter();
@@ -118,6 +157,7 @@ impl pack_service_server::PackService for GitksService {
if let Some(mut client) =
remote_pack_client(self, first.repository.as_ref(), false).await?
{
m.record("ok");
let (tx, rx) = tokio::sync::mpsc::channel(16);
let _ = tx.send(first).await;
tokio::spawn(async move {
@@ -139,9 +179,13 @@ impl pack_service_server::PackService for GitksService {
let out = super::bridge_server_stream(resp.into_inner());
return Ok(tonic::Response::new(out));
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
tracing::info!(%repo, "receive-pack streaming started");
@@ -158,6 +202,7 @@ impl pack_service_server::PackService for GitksService {
});
let result = gb.receive_pack(ReceiverStream::new(rx)).await?;
m.record("ok");
Ok(tonic::Response::new(result))
}
@@ -165,7 +210,9 @@ impl pack_service_server::PackService for GitksService {
&self,
request: tonic::Request<PackObjectsRequest>,
) -> Result<tonic::Response<Self::PackObjectsStream>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.PackService/PackObjects");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("pack.pack_objects", %repo);
let _enter = span.enter();
@@ -175,16 +222,97 @@ impl pack_service_server::PackService for GitksService {
if let Some(mut client) =
remote_pack_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
let resp = client.pack_objects(inner).await?;
let stream = super::bridge_server_stream(resp.into_inner());
return Ok(tonic::Response::new(stream));
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
if let Some(ref pc) = self.pack_cache
&& let Some(opts) = inner.options.as_ref()
{
let wants_hex: Vec<String> = opts.wants.iter().map(|w| w.hex.clone()).collect();
let haves_hex: Vec<String> = opts.haves.iter().map(|h| h.hex.clone()).collect();
if let Ok(digest) = pc.disk_cache().compute_pack_objects_key(
&repo,
&wants_hex,
&haves_hex,
opts.thin_pack,
opts.use_bitmaps,
opts.delta_base_offset,
) {
if let Some(file) = pc
.disk_cache()
.open_stream_read(crate::pack_cache::PACK_CACHE_NAMESPACE, &digest)
.ok()
.flatten()
{
tracing::info!(%repo, digest = %digest, "pack-objects cache hit, streaming from disk");
m.record("ok");
let (tx, rx) = tokio::sync::mpsc::channel(16);
tokio::spawn(async move {
let result = tokio::task::spawn_blocking(move || {
use std::io::Read;
let mut file = file;
let mut buf = vec![0u8; 65536];
let mut chunks = Vec::new();
loop {
match file.read(&mut buf) {
Ok(0) => break,
Ok(n) => chunks.push(Ok(PackfileChunk {
data: buf[..n].to_vec(),
})),
Err(e) => {
chunks.push(Err(tonic::Status::internal(format!(
"cache read error: {e}"
))));
break;
}
}
}
chunks
})
.await;
match result {
Ok(chunks) => {
for chunk in chunks {
if tx.send(chunk).await.is_err() {
break;
}
}
}
Err(e) => {
let _ = tx
.send(Err(tonic::Status::internal(format!(
"cache read task failed: {e}"
))))
.await;
}
}
});
return Ok(tonic::Response::new(ReceiverStream::new(rx)));
}
// Cache miss: execute pack-objects and tee to cache
tracing::info!(%repo, digest = %digest, "pack-objects cache miss");
let stream = gb.pack_objects(inner).await?;
let tee_stream = pc.tee_pack_stream(&digest, stream);
m.record("ok");
return Ok(tonic::Response::new(tee_stream));
}
}
let stream = gb.pack_objects(inner).await?;
tracing::info!(%repo, "pack-objects streaming started");
m.record("ok");
Ok(tonic::Response::new(stream))
}
@@ -192,11 +320,15 @@ impl pack_service_server::PackService for GitksService {
&self,
request: tonic::Request<tonic::Streaming<IndexPackRequest>>,
) -> Result<tonic::Response<IndexPackResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.PackService/IndexPack");
let mut stream = request.into_inner();
let mut inputs = Vec::new();
while let Some(msg) = stream.next().await {
inputs.push(msg?);
}
let _rate = self
.acquire_rate_limit(inputs.first().and_then(|r: &IndexPackRequest| r.repository.as_ref()))
.await?;
let repo = self.repo_label(inputs.first().and_then(|r| r.repository.as_ref()));
let span = tracing::info_span!("pack.index_pack", %repo);
let _enter = span.enter();
@@ -210,14 +342,20 @@ impl pack_service_server::PackService for GitksService {
)
.await?
{
m.record("ok");
return client.index_pack(tokio_stream::iter(inputs)).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.index_pack(inputs).map_err(into_status)?;
tracing::info!(%repo, objects = resp.object_count, "index_pack done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -225,7 +363,9 @@ impl pack_service_server::PackService for GitksService {
&self,
request: tonic::Request<ListPackfilesRequest>,
) -> Result<tonic::Response<ListPackfilesResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.PackService/ListPackfiles");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("pack.list_packfiles", %repo);
let _enter = span.enter();
@@ -235,14 +375,20 @@ impl pack_service_server::PackService for GitksService {
if let Some(mut client) =
remote_pack_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.list_packfiles(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.list_packfiles(inner).map_err(into_status)?;
tracing::info!(%repo, count = resp.packfiles.len(), "list_packfiles done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -250,7 +396,9 @@ impl pack_service_server::PackService for GitksService {
&self,
request: tonic::Request<FsckRequest>,
) -> Result<tonic::Response<FsckResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.PackService/Fsck");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("pack.fsck", %repo);
let _enter = span.enter();
@@ -260,14 +408,20 @@ impl pack_service_server::PackService for GitksService {
if let Some(mut client) =
remote_pack_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.fsck(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.fsck(inner).map_err(into_status)?;
tracing::info!(%repo, ok = resp.ok, errors = resp.errors.len(), warnings = resp.warnings.len(), "fsck done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+287 -2
View File
@@ -2,8 +2,13 @@ use crate::pb::repository_service_client::RepositoryServiceClient;
use crate::pb::*;
use super::{GitksService, git_cmd, into_status, repository_maint};
use tokio_stream::wrappers::ReceiverStream;
remote_client!(remote_repository_client, RepositoryServiceClient<tonic::transport::Channel>, "repository");
remote_client!(
remote_repository_client,
RepositoryServiceClient<tonic::transport::Channel>,
"repository"
);
fn default_branch_name(gb: &crate::bare::GitBare) -> String {
git_cmd(gb, &["symbolic-ref", "HEAD"])
@@ -23,7 +28,9 @@ impl repository_service_server::RepositoryService for GitksService {
&self,
request: tonic::Request<GetRepositoryRequest>,
) -> Result<tonic::Response<Repository>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.RepositoryService/GetRepository");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("repo.get_repository", %repo);
let _enter = span.enter();
@@ -33,14 +40,20 @@ impl repository_service_server::RepositoryService for GitksService {
if let Some(mut client) =
remote_repository_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_repository(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let bare = gb.bare_dir.join("HEAD").exists();
let object_format = gb.object_format();
m.record("ok");
Ok(tonic::Response::new(Repository {
header: inner.repository,
bare,
@@ -54,15 +67,21 @@ impl repository_service_server::RepositoryService for GitksService {
&self,
request: tonic::Request<InitRepositoryRequest>,
) -> Result<tonic::Response<Repository>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.RepositoryService/InitRepository");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("repo.init_repository", %repo);
let _enter = span.enter();
let bare_dir = self.resolve_for_init(inner.repository.as_ref())?;
let gb = crate::bare::GitBare::new(bare_dir);
gb.init_repository(inner.bare).map_err(into_status)?;
if let Some(ref hm) = self.hook_manager {
hm.install_hooks(&gb.bare_dir).map_err(into_status)?;
}
tracing::info!(%repo, bare = inner.bare, "repository initialized");
self.notify_ref_update(&repo, "HEAD", "", "");
m.record("ok");
Ok(tonic::Response::new(Repository {
header: inner.repository,
bare: inner.bare,
@@ -74,7 +93,9 @@ impl repository_service_server::RepositoryService for GitksService {
&self,
request: tonic::Request<DeleteRepositoryRequest>,
) -> Result<tonic::Response<()>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.RepositoryService/DeleteRepository");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("repo.delete_repository", %repo);
let _enter = span.enter();
@@ -83,12 +104,15 @@ impl repository_service_server::RepositoryService for GitksService {
&& let Some(mut client) =
remote_repository_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.delete_repository(inner).await;
}
tracing::warn!(%repo, path = %bare_dir.display(), "deleting repository");
std::fs::remove_dir_all(&bare_dir).map_err(|e| tonic::Status::internal(e.to_string()))?;
tracing::info!(%repo, "repository deleted");
self.notify_ref_update(&repo, "", "", "");
crate::rate_limit::remove_repository(&repo);
m.record("ok");
Ok(tonic::Response::new(()))
}
@@ -421,4 +445,265 @@ impl repository_service_server::RepositoryService for GitksService {
tracing::info!(%repo, ok = resp.ok, "commit-graph write done");
Ok(tonic::Response::new(resp))
}
// ── Hooks Management ────────────────────────────────────────────
async fn list_hooks(
&self,
request: tonic::Request<ListHooksRequest>,
) -> Result<tonic::Response<ListHooksResponse>, tonic::Status> {
let inner = request.into_inner();
let gb = self.resolve(inner.repository.as_ref())?;
let hook_mgr = self.hook_manager.as_ref();
let hooks = if let Some(hm) = hook_mgr {
hm.list_hooks(&gb.bare_dir)
.map_err(|e| tonic::Status::internal(e.to_string()))?
} else {
Vec::new()
};
let resp = ListHooksResponse {
hooks: hooks
.into_iter()
.map(|h| crate::pb::HookInfo {
hook_type: h.hook_type,
level: h.level.to_string(),
path: h.path,
})
.collect(),
};
Ok(tonic::Response::new(resp))
}
async fn set_custom_hook(
&self,
request: tonic::Request<SetCustomHookRequest>,
) -> Result<tonic::Response<()>, tonic::Status> {
let inner = request.into_inner();
let gb = self.resolve(inner.repository.as_ref())?;
let hook_mgr = self.hook_manager.as_ref();
if let Some(hm) = hook_mgr {
hm.set_custom_hook(&gb.bare_dir, &inner.hook_name, &inner.content)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
} else {
return Err(tonic::Status::failed_precondition("hooks not enabled"));
}
tracing::info!(repo = %gb.bare_dir.display(), hook = %inner.hook_name, "custom hook set");
Ok(tonic::Response::new(()))
}
async fn remove_custom_hook(
&self,
request: tonic::Request<RemoveCustomHookRequest>,
) -> Result<tonic::Response<()>, tonic::Status> {
let inner = request.into_inner();
let gb = self.resolve(inner.repository.as_ref())?;
let hook_mgr = self.hook_manager.as_ref();
if let Some(hm) = hook_mgr {
hm.remove_custom_hook(&gb.bare_dir, &inner.hook_name)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
} else {
return Err(tonic::Status::failed_precondition("hooks not enabled"));
}
tracing::info!(repo = %gb.bare_dir.display(), hook = %inner.hook_name, "custom hook removed");
Ok(tonic::Response::new(()))
}
// ── Snapshot Operations ──────────────────────────────────────────
async fn create_snapshot(
&self,
request: tonic::Request<CreateSnapshotRequest>,
) -> Result<tonic::Response<CreateSnapshotResponse>, tonic::Status> {
let inner = request.into_inner();
let gb = self.resolve(inner.repository.as_ref())?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("repo.create_snapshot", %repo);
let _enter = span.enter();
let storage = crate::snapshot::storage::LocalSnapshotStorage::new(
self.repo_prefix.join("+gitks-snapshots"),
);
let snapshot_id = crate::snapshot::ops::create_and_store_snapshot(&gb, &repo, &storage)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
let head_oid = crate::snapshot::ops::get_head_oid_internal(&gb)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
use crate::snapshot::storage::SnapshotStorageBackend;
let actual_size = storage
.read_snapshot(&snapshot_id)
.map(|d| d.len() as u64)
.unwrap_or(0);
tracing::info!(%repo, snapshot_id = %snapshot_id, size_bytes = actual_size, "snapshot created");
Ok(tonic::Response::new(CreateSnapshotResponse {
snapshot_id,
size_bytes: actual_size,
head_oid,
}))
}
async fn restore_snapshot(
&self,
request: tonic::Request<RestoreSnapshotRequest>,
) -> Result<tonic::Response<()>, tonic::Status> {
let inner = request.into_inner();
let target_repo = self.repo_label(inner.target_repository.as_ref());
let span = tracing::info_span!("repo.restore_snapshot", %target_repo);
let _enter = span.enter();
let storage = crate::snapshot::storage::LocalSnapshotStorage::new(
self.repo_prefix.join("+gitks-snapshots"),
);
let target_path = self.resolve_for_init(inner.target_repository.as_ref())?;
crate::snapshot::ops::restore_from_storage(&target_path, &inner.snapshot_id, &storage)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
tracing::info!(%target_repo, snapshot_id = %inner.snapshot_id, "snapshot restored");
self.notify_ref_update(&target_repo, "HEAD", "", "");
Ok(tonic::Response::new(()))
}
async fn list_snapshots(
&self,
request: tonic::Request<ListSnapshotsRequest>,
) -> Result<tonic::Response<ListSnapshotsResponse>, tonic::Status> {
let inner = request.into_inner();
let repo = self.repo_label(inner.repository.as_ref());
let storage = crate::snapshot::storage::LocalSnapshotStorage::new(
self.repo_prefix.join("+gitks-snapshots"),
);
use crate::snapshot::storage::SnapshotStorageBackend;
let snapshots = storage
.list_snapshots(&repo)
.map_err(tonic::Status::internal)?;
let resp = ListSnapshotsResponse {
snapshots: snapshots
.into_iter()
.map(|s| crate::pb::SnapshotInfo {
snapshot_id: s.snapshot_id,
relative_path: s.relative_path,
size_bytes: s.size_bytes,
created_at: s.created_at,
head_oid: s.head_oid,
})
.collect(),
};
Ok(tonic::Response::new(resp))
}
async fn delete_snapshot(
&self,
request: tonic::Request<DeleteSnapshotRequest>,
) -> Result<tonic::Response<()>, tonic::Status> {
let inner = request.into_inner();
let storage = crate::snapshot::storage::LocalSnapshotStorage::new(
self.repo_prefix.join("+gitks-snapshots"),
);
use crate::snapshot::storage::SnapshotStorageBackend;
storage
.delete_snapshot(&inner.snapshot_id)
.map_err(tonic::Status::internal)?;
tracing::info!(snapshot_id = %inner.snapshot_id, "snapshot deleted");
Ok(tonic::Response::new(()))
}
// ── Repository Move ──────────────────────────────────────────────
type FetchRepositoryDataStream =
ReceiverStream<Result<FetchRepositoryDataResponse, tonic::Status>>;
async fn move_repository(
&self,
request: tonic::Request<MoveRepositoryRequest>,
) -> Result<tonic::Response<MoveRepositoryResponse>, tonic::Status> {
let inner = request.into_inner();
let source_repo = self.repo_label(inner.source_repository.as_ref());
let span = tracing::info_span!("repo.move_repository", %source_repo);
let _enter = span.enter();
let gb = self.resolve(inner.source_repository.as_ref())?;
let bundle_data = crate::snapshot::ops::create_snapshot(&gb)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
let target_path = self.resolve_for_init(inner.target_repository.as_ref())?;
let target_gb = crate::bare::GitBare::new(target_path.clone());
target_gb
.init_repository(true)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
crate::snapshot::ops::restore_snapshot(&target_path, &bundle_data)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
if let Some(ref hm) = self.hook_manager {
hm.install_hooks(&target_path)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
}
let source_path = gb.bare_dir.clone();
std::fs::remove_dir_all(&source_path)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
self.notify_ref_update(&source_repo, "HEAD", "", "");
tracing::info!(source = %source_repo, "repository moved successfully");
Ok(tonic::Response::new(MoveRepositoryResponse {
state: MoveRepositoryState::MoveStateCompleted as i32,
error_message: String::new(),
}))
}
async fn fetch_repository_data(
&self,
request: tonic::Request<FetchRepositoryDataRequest>,
) -> Result<tonic::Response<Self::FetchRepositoryDataStream>, tonic::Status> {
let inner = request.into_inner();
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("repo.fetch_repository_data", %repo);
let _enter = span.enter();
let gb = self.resolve(inner.repository.as_ref())?;
let bundle_data = crate::snapshot::ops::create_snapshot(&gb)
.map_err(|e| tonic::Status::internal(e.to_string()))?;
let (tx, rx) = tokio::sync::mpsc::channel(16);
tokio::spawn(async move {
const CHUNK_SIZE: usize = 65536;
let total = bundle_data.len();
if total == 0 {
let _ = tx
.send(Ok(FetchRepositoryDataResponse {
data: vec![],
done: true,
}))
.await;
return;
}
for offset in (0..total).step_by(CHUNK_SIZE) {
let end = (offset + CHUNK_SIZE).min(total);
let chunk_data = bundle_data[offset..end].to_vec();
let is_done = end >= total;
if tx
.send(Ok(FetchRepositoryDataResponse {
data: chunk_data,
done: is_done,
}))
.await
.is_err()
{
break;
}
}
});
Ok(tonic::Response::new(ReceiverStream::new(rx)))
}
}
+50 -6
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, into_status};
remote_client!(remote_tag_client, TagServiceClient<tonic::transport::Channel>, "tag");
remote_client!(
remote_tag_client,
TagServiceClient<tonic::transport::Channel>,
"tag"
);
#[tonic::async_trait]
impl tag_service_server::TagService for GitksService {
@@ -11,7 +15,9 @@ impl tag_service_server::TagService for GitksService {
&self,
request: tonic::Request<ListTagsRequest>,
) -> Result<tonic::Response<ListTagsResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TagService/ListTags");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("tag.list_tags", %repo);
let _enter = span.enter();
@@ -21,14 +27,20 @@ impl tag_service_server::TagService for GitksService {
if let Some(mut client) =
remote_tag_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.list_tags(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.list_tags(inner).map_err(into_status)?;
tracing::info!(%repo, count = resp.tags.len(), "list_tags done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -36,7 +48,9 @@ impl tag_service_server::TagService for GitksService {
&self,
request: tonic::Request<GetTagRequest>,
) -> Result<tonic::Response<Tag>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TagService/GetTag");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("tag.get_tag", %repo, %name);
@@ -47,13 +61,19 @@ impl tag_service_server::TagService for GitksService {
if let Some(mut client) =
remote_tag_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_tag(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.get_tag(inner).map_err(into_status)?;
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -61,7 +81,9 @@ impl tag_service_server::TagService for GitksService {
&self,
request: tonic::Request<CreateTagRequest>,
) -> Result<tonic::Response<Tag>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TagService/CreateTag");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("tag.create_tag", %repo, %name);
@@ -72,15 +94,21 @@ impl tag_service_server::TagService for GitksService {
if let Some(mut client) =
remote_tag_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.create_tag(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.create_tag(inner).map_err(into_status)?;
tracing::info!(%repo, %name, "tag created");
self.notify_ref_update(&repo, &format!("refs/tags/{}", name), "", "");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -88,7 +116,9 @@ impl tag_service_server::TagService for GitksService {
&self,
request: tonic::Request<DeleteTagRequest>,
) -> Result<tonic::Response<()>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TagService/DeleteTag");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("tag.delete_tag", %repo, %name);
@@ -99,15 +129,21 @@ impl tag_service_server::TagService for GitksService {
if let Some(mut client) =
remote_tag_client(self, inner.repository.as_ref(), true).await?
{
m.record("ok");
return client.delete_tag(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
gb.delete_tag(inner).map_err(into_status)?;
tracing::info!(%repo, %name, "tag deleted");
self.notify_ref_update(&repo, &format!("refs/tags/{}", name), "", "");
m.record("ok");
Ok(tonic::Response::new(()))
}
@@ -115,7 +151,9 @@ impl tag_service_server::TagService for GitksService {
&self,
request: tonic::Request<VerifyTagRequest>,
) -> Result<tonic::Response<VerifiedSignature>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TagService/VerifyTag");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let name = inner.name.clone();
let span = tracing::info_span!("tag.verify_tag", %repo, %name);
@@ -126,14 +164,20 @@ impl tag_service_server::TagService for GitksService {
if let Some(mut client) =
remote_tag_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.verify_tag(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = gb.verify_tag(inner).map_err(into_status)?;
tracing::info!(%repo, %name, verified = resp.verified, "tag verified");
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+59 -7
View File
@@ -3,7 +3,11 @@ use crate::pb::*;
use super::{GitksService, cache, into_status, into_stream};
remote_client!(remote_tree_client, TreeServiceClient<tonic::transport::Channel>, "tree");
remote_client!(
remote_tree_client,
TreeServiceClient<tonic::transport::Channel>,
"tree"
);
#[tonic::async_trait]
impl tree_service_server::TreeService for GitksService {
@@ -14,7 +18,9 @@ impl tree_service_server::TreeService for GitksService {
&self,
request: tonic::Request<ListTreeRequest>,
) -> Result<tonic::Response<ListTreeResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TreeService/ListTree");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("tree.list_tree", %repo);
let _enter = span.enter();
@@ -24,11 +30,16 @@ impl tree_service_server::TreeService for GitksService {
if let Some(mut client) =
remote_tree_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.list_tree(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("tree.list_tree", &inner, || {
@@ -38,6 +49,7 @@ impl tree_service_server::TreeService for GitksService {
gb.list_tree(inner).map_err(into_status)?
};
tracing::info!(%repo, count = resp.entries.len(), "list_tree done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -45,7 +57,9 @@ impl tree_service_server::TreeService for GitksService {
&self,
request: tonic::Request<GetTreeRequest>,
) -> Result<tonic::Response<Tree>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetTree");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("tree.get_tree", %repo);
let _enter = span.enter();
@@ -55,11 +69,16 @@ impl tree_service_server::TreeService for GitksService {
if let Some(mut client) =
remote_tree_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_tree(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("tree.get_tree", &inner, || {
@@ -68,6 +87,7 @@ impl tree_service_server::TreeService for GitksService {
} else {
gb.get_tree(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -75,7 +95,9 @@ impl tree_service_server::TreeService for GitksService {
&self,
request: tonic::Request<GetBlobRequest>,
) -> Result<tonic::Response<Blob>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetBlob");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let path = inner.path.clone();
let span = tracing::info_span!("tree.get_blob", %repo, %path);
@@ -86,11 +108,16 @@ impl tree_service_server::TreeService for GitksService {
if let Some(mut client) =
remote_tree_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_blob(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("tree.get_blob", &inner, || {
@@ -99,6 +126,7 @@ impl tree_service_server::TreeService for GitksService {
} else {
gb.get_blob(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -106,7 +134,9 @@ impl tree_service_server::TreeService for GitksService {
&self,
request: tonic::Request<GetRawBlobRequest>,
) -> Result<tonic::Response<Self::GetRawBlobStream>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetRawBlob");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("tree.get_raw_blob", %repo);
let _enter = span.enter();
@@ -116,13 +146,18 @@ impl tree_service_server::TreeService for GitksService {
if let Some(mut client) =
remote_tree_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
let resp = client.get_raw_blob(inner).await?;
let stream = super::bridge_server_stream(resp.into_inner());
return Ok(tonic::Response::new(stream));
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let items = if inner.oid.is_some() {
cache::cached_vec_response("tree.get_raw_blob", &inner, || {
@@ -135,6 +170,7 @@ impl tree_service_server::TreeService for GitksService {
} else {
gb.get_raw_blob(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(into_stream(items)))
}
@@ -142,7 +178,9 @@ impl tree_service_server::TreeService for GitksService {
&self,
request: tonic::Request<GetFileMetadataRequest>,
) -> Result<tonic::Response<FileMetadata>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TreeService/GetFileMetadata");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("tree.get_file_metadata", %repo);
let _enter = span.enter();
@@ -152,11 +190,16 @@ impl tree_service_server::TreeService for GitksService {
if let Some(mut client) =
remote_tree_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.get_file_metadata(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("tree.get_file_metadata", &inner, || {
@@ -165,6 +208,7 @@ impl tree_service_server::TreeService for GitksService {
} else {
gb.get_file_metadata(inner).map_err(into_status)?
};
m.record("ok");
Ok(tonic::Response::new(resp))
}
@@ -172,7 +216,9 @@ impl tree_service_server::TreeService for GitksService {
&self,
request: tonic::Request<FindFilesRequest>,
) -> Result<tonic::Response<FindFilesResponse>, tonic::Status> {
let m = crate::metrics::RequestMetrics::new("gitks.TreeService/FindFiles");
let inner = request.into_inner();
let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
let repo = self.repo_label(inner.repository.as_ref());
let span = tracing::info_span!("tree.find_files", %repo);
let _enter = span.enter();
@@ -182,11 +228,16 @@ impl tree_service_server::TreeService for GitksService {
if let Some(mut client) =
remote_tree_client(self, inner.repository.as_ref(), false).await?
{
m.record("ok");
return client.find_files(inner).await;
}
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => {
crate::metrics::record_rpc_error(&m, &err);
return Err(err);
}
Err(err) => return Err(err),
};
let resp = if cache::selector_is_oid(&inner.revision) {
cache::cached_response("tree.find_files", &inner, || {
@@ -196,6 +247,7 @@ impl tree_service_server::TreeService for GitksService {
gb.find_files(inner).map_err(into_status)?
};
tracing::info!(%repo, count = resp.files.len(), "find_files done");
m.record("ok");
Ok(tonic::Response::new(resp))
}
}
+13
View File
@@ -0,0 +1,13 @@
//! Repository snapshot and move operations.
//!
//! Supports:
//! - Creating snapshots (git bundle) of repositories for backup
//! - Restoring snapshots to new or existing repositories
//! - Moving repositories between cluster nodes
//! - Listing and deleting snapshots
pub mod ops;
pub mod storage;
pub use ops::{create_snapshot, restore_snapshot, verify_snapshot};
pub use storage::{LocalSnapshotStorage, SnapshotInfo, SnapshotStorageBackend};
+169
View File
@@ -0,0 +1,169 @@
//! Snapshot and move operations.
//!
//! Core operations for creating, restoring, and verifying repository snapshots
//! using git bundle.
use std::path::Path;
use crate::bare::GitBare;
use crate::error::{GitError, GitResult};
use crate::snapshot::storage::SnapshotStorageBackend;
/// Create a git bundle snapshot of a repository.
/// Returns the bundle data as raw bytes.
pub fn create_snapshot(gb: &GitBare) -> GitResult<Vec<u8>> {
tracing::info!(repo = %gb.bare_dir.display(), "creating snapshot bundle");
let bare_dir_str = gb.bare_dir.to_string_lossy().into_owned();
let tmp_file = tempfile::Builder::new()
.prefix("gitks_snapshot_")
.suffix(".bundle")
.tempfile_in(&gb.bare_dir)
.map_err(GitError::Io)?;
let bundle_path_str = tmp_file.path().to_string_lossy().into_owned();
let result = duct::cmd!(
"git",
"--git-dir",
&bare_dir_str,
"bundle",
"create",
&bundle_path_str,
"--all"
)
.stdout_capture()
.stderr_capture()
.unchecked()
.run()?;
if !result.status.success() {
let stderr = String::from_utf8_lossy(&result.stderr);
return Err(GitError::CommandFailed {
status_code: result.status.code(),
stderr: stderr.into_owned(),
});
}
let bundle_path = tmp_file.path().to_path_buf();
let data = std::fs::read(&bundle_path).map_err(GitError::Io)?;
let head_oid = get_head_oid(gb)?;
tracing::info!(
repo = %gb.bare_dir.display(),
size_bytes = data.len(),
head_oid = %head_oid,
"snapshot bundle created"
);
Ok(data)
}
/// Create a snapshot and store it using the given backend.
pub fn create_and_store_snapshot(
gb: &GitBare,
relative_path: &str,
storage: &dyn SnapshotStorageBackend,
) -> GitResult<String> {
let data = create_snapshot(gb)?;
let head_oid = get_head_oid(gb)?;
let snapshot_id = generate_snapshot_id(relative_path, &head_oid);
storage
.write_snapshot(&snapshot_id, relative_path, &head_oid, &data)
.map_err(GitError::Internal)?;
Ok(snapshot_id)
}
/// Restore a snapshot to a repository path.
pub fn restore_snapshot(repo_path: &Path, data: &[u8]) -> GitResult<()> {
tracing::info!(path = %repo_path.display(), size_bytes = data.len(), "restoring snapshot");
let applicator = crate::actor::sync::BundleApplicator::new(repo_path.to_path_buf());
applicator.apply_bundle(data).map_err(GitError::Internal)?;
tracing::info!(path = %repo_path.display(), "snapshot restored");
Ok(())
}
/// Restore a snapshot from storage backend.
pub fn restore_from_storage(
repo_path: &Path,
snapshot_id: &str,
storage: &dyn SnapshotStorageBackend,
) -> GitResult<()> {
let data = storage
.read_snapshot(snapshot_id)
.map_err(GitError::Internal)?;
restore_snapshot(repo_path, &data)?;
Ok(())
}
/// Verify that a restored snapshot matches the expected HEAD OID.
pub fn verify_snapshot(gb: &GitBare, expected_head_oid: &str) -> GitResult<bool> {
let actual_head_oid = get_head_oid(gb)?;
if actual_head_oid == expected_head_oid {
tracing::info!(
repo = %gb.bare_dir.display(),
expected = %expected_head_oid,
actual = %actual_head_oid,
"snapshot verification passed"
);
Ok(true)
} else {
tracing::warn!(
repo = %gb.bare_dir.display(),
expected = %expected_head_oid,
actual = %actual_head_oid,
"snapshot verification failed: HEAD mismatch"
);
Ok(false)
}
}
/// Get HEAD OID of a repository (public helper for RPC).
pub fn get_head_oid_internal(gb: &GitBare) -> GitResult<String> {
get_head_oid(gb)
}
/// Get HEAD OID of a repository.
fn get_head_oid(gb: &GitBare) -> GitResult<String> {
let bare_dir_str = gb.bare_dir.to_string_lossy().into_owned();
let result = duct::cmd!("git", "--git-dir", &bare_dir_str, "rev-parse", "HEAD")
.stdout_capture()
.stderr_capture()
.unchecked()
.run()?;
if result.status.success() {
Ok(String::from_utf8_lossy(&result.stdout).trim().to_string())
} else {
// Repository may be empty (no HEAD)
Ok(String::new())
}
}
/// Generate a snapshot ID from relative_path and head_oid.
fn generate_snapshot_id(relative_path: &str, head_oid: &str) -> String {
use std::time::{SystemTime, UNIX_EPOCH};
let ts = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
use sha2::Digest;
let mut hasher = sha2::Sha256::new();
hasher.update(relative_path.as_bytes());
hasher.update(head_oid.as_bytes());
hasher.update(ts.to_le_bytes());
let hash = hasher.finalize();
let mut s = String::with_capacity(16);
for byte in &hash[..8] {
use std::fmt::Write;
write!(s, "{byte:02x}").unwrap();
}
s
}
+166
View File
@@ -0,0 +1,166 @@
//! Snapshot storage backend abstraction.
//!
//! Currently implements local filesystem storage.
//! S3/GCS backends can be added later as optional dependencies.
use std::path::PathBuf;
use std::time::SystemTime;
/// Metadata about a snapshot.
#[derive(Debug, Clone)]
pub struct SnapshotInfo {
pub snapshot_id: String,
pub relative_path: String,
pub size_bytes: u64,
pub created_at: String, // ISO 8601
pub head_oid: String,
}
/// Trait for snapshot storage backends.
pub trait SnapshotStorageBackend: Send + Sync {
fn write_snapshot(
&self,
snapshot_id: &str,
relative_path: &str,
head_oid: &str,
data: &[u8],
) -> Result<(), String>;
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String>;
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String>;
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String>;
}
/// Local filesystem snapshot storage.
pub struct LocalSnapshotStorage {
base_dir: PathBuf,
}
impl LocalSnapshotStorage {
pub fn new(base_dir: PathBuf) -> Self {
Self { base_dir }
}
fn snapshot_dir(&self, snapshot_id: &str) -> PathBuf {
let prefix = &snapshot_id[..2.min(snapshot_id.len())];
self.base_dir.join(prefix).join(snapshot_id)
}
fn metadata_path(&self, snapshot_id: &str) -> PathBuf {
self.snapshot_dir(snapshot_id).join("metadata.json")
}
fn data_path(&self, snapshot_id: &str) -> PathBuf {
self.snapshot_dir(snapshot_id).join("bundle.dat")
}
}
impl SnapshotStorageBackend for LocalSnapshotStorage {
fn write_snapshot(
&self,
snapshot_id: &str,
relative_path: &str,
head_oid: &str,
data: &[u8],
) -> Result<(), String> {
let dir = self.snapshot_dir(snapshot_id);
std::fs::create_dir_all(&dir).map_err(|e| format!("create dir: {e}"))?;
let data_path = self.data_path(snapshot_id);
let tmp_path = data_path.with_extension("tmp");
std::fs::write(&tmp_path, data).map_err(|e| format!("write data: {e}"))?;
std::fs::rename(&tmp_path, &data_path).map_err(|e| format!("rename: {e}"))?;
let created_at = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let metadata = serde_json::json!({
"snapshot_id": snapshot_id,
"relative_path": relative_path,
"size_bytes": data.len(),
"created_at": created_at,
"head_oid": head_oid,
});
let metadata_str = serde_json::to_string_pretty(&metadata)
.map_err(|e| format!("serialize metadata: {e}"))?;
std::fs::write(self.metadata_path(snapshot_id), metadata_str)
.map_err(|e| format!("write metadata: {e}"))?;
tracing::info!(
snapshot_id = %snapshot_id,
size_bytes = data.len(),
"snapshot written to local storage"
);
Ok(())
}
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String> {
let path = self.data_path(snapshot_id);
if !path.exists() {
return Err(format!("snapshot not found: {snapshot_id}"));
}
std::fs::read(&path).map_err(|e| format!("read snapshot: {e}"))
}
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String> {
let mut snapshots = Vec::new();
if !self.base_dir.exists() {
return Ok(snapshots);
}
for shard_entry in
std::fs::read_dir(&self.base_dir).map_err(|e| format!("read dir: {e}"))?
{
let shard_entry = shard_entry.map_err(|e| format!("entry: {e}"))?;
let shard_dir = shard_entry.path();
if !shard_dir.is_dir() {
continue;
}
for snap_entry in std::fs::read_dir(&shard_dir).map_err(|e| format!("read dir: {e}"))? {
let snap_entry = snap_entry.map_err(|e| format!("entry: {e}"))?;
let snap_dir = snap_entry.path();
if !snap_dir.is_dir() {
continue;
}
let metadata_path = snap_dir.join("metadata.json");
if !metadata_path.exists() {
continue;
}
let metadata_str = std::fs::read_to_string(&metadata_path)
.map_err(|e| format!("read metadata: {e}"))?;
let metadata: serde_json::Value = serde_json::from_str(&metadata_str)
.map_err(|e| format!("parse metadata: {e}"))?;
let snap_relative_path = metadata["relative_path"].as_str().unwrap_or("");
if !relative_path.is_empty() && snap_relative_path != relative_path {
continue;
}
snapshots.push(SnapshotInfo {
snapshot_id: metadata["snapshot_id"].as_str().unwrap_or("").to_string(),
relative_path: snap_relative_path.to_string(),
size_bytes: metadata["size_bytes"].as_u64().unwrap_or(0),
created_at: metadata["created_at"].as_str().unwrap_or("").to_string(),
head_oid: metadata["head_oid"].as_str().unwrap_or("").to_string(),
});
}
}
snapshots.sort_by(|a, b| b.created_at.cmp(&a.created_at));
Ok(snapshots)
}
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String> {
let dir = self.snapshot_dir(snapshot_id);
if !dir.exists() {
return Err(format!("snapshot not found: {snapshot_id}"));
}
std::fs::remove_dir_all(&dir).map_err(|e| format!("delete snapshot: {e}"))?;
tracing::info!(snapshot_id = %snapshot_id, "snapshot deleted");
Ok(())
}
}
+157
View File
@@ -0,0 +1,157 @@
//! Tests for DiskCache and PackCache.
use std::path::PathBuf;
use std::time::Duration;
use gitks::disk_cache::DiskCache;
fn temp_dir() -> PathBuf {
tempfile::tempdir().unwrap().path().to_path_buf()
}
#[test]
fn test_disk_cache_basic_operations() {
let dir = temp_dir();
let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true);
// Ensure state creates latest file
let state = cache.ensure_state("test_repo.git").unwrap();
assert!(!state.is_empty());
// Same state on second call
let state2 = cache.ensure_state("test_repo.git").unwrap();
assert_eq!(state, state2);
}
#[test]
fn test_disk_cache_insert_and_lookup() {
let dir = temp_dir();
let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true);
let digest = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789";
let data = b"test cache data";
// Insert
cache.insert("+gitks-cache/cache", digest, data).unwrap();
// Lookup
let result = cache.lookup("+gitks-cache/cache", digest).unwrap();
assert!(result.is_some());
assert_eq!(result.unwrap(), data.to_vec());
// Non-existent key
let result = cache.lookup("+gitks-cache/cache", "nonexistent").unwrap();
assert!(result.is_none());
}
#[test]
fn test_disk_cache_invalidation() {
let dir = temp_dir();
let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true);
let state1 = cache.ensure_state("test_repo.git").unwrap();
// Invalidate
cache.invalidate_repo("test_repo.git");
// State should change
let state2 = cache.ensure_state("test_repo.git").unwrap();
assert_ne!(state1, state2);
}
#[test]
fn test_disk_cache_disabled() {
let dir = temp_dir();
let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, false);
// All operations should succeed but do nothing
let state = cache.ensure_state("test_repo.git").unwrap();
assert!(!state.is_empty()); // Returns random value
let result = cache.lookup("+gitks-cache/cache", "anykey").unwrap();
assert!(result.is_none()); // Disabled → always None
}
#[test]
fn test_disk_cache_lease_guard() {
let dir = temp_dir();
let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true);
let state1 = cache.ensure_state("test_repo.git").unwrap();
let mut lease = cache.create_lease("test_repo.git").unwrap();
// Lease exists
let pending_dir = dir.join("+gitks-cache/state/test_repo.git/pending");
assert!(pending_dir.exists());
// Commit lease (updates latest)
lease.commit();
// Pending should be cleaned up
// (may still have dir but no files)
let state2 = cache.ensure_state("test_repo.git").unwrap();
assert_ne!(state1, state2);
}
#[test]
fn test_sha256_digest_determinism() {
let dir = temp_dir();
let cache = DiskCache::new(dir, "v1".to_string(), 300, true);
cache.ensure_state("repo.git").unwrap();
let key1 = cache
.compute_info_refs_key("repo.git", "upload-pack")
.unwrap();
let key2 = cache
.compute_info_refs_key("repo.git", "upload-pack")
.unwrap();
assert_eq!(key1, key2);
// Different protocol should produce different key
let key3 = cache
.compute_info_refs_key("repo.git", "receive-pack")
.unwrap();
assert_ne!(key1, key3);
}
#[test]
fn test_cleanup_expired() {
let dir = temp_dir();
// Very short max age for testing
let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 1, true);
let digest = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef0123456789";
cache.insert("+gitks-cache/cache", digest, b"data").unwrap();
// Should be available immediately
let result = cache.lookup("+gitks-cache/cache", digest).unwrap();
assert!(result.is_some());
// Wait for expiration
std::thread::sleep(Duration::from_secs(2));
// Should be expired now
let result = cache.lookup("+gitks-cache/cache", digest).unwrap();
assert!(result.is_none());
}
#[test]
fn test_startup_cleanup() {
let dir = temp_dir();
let cache = DiskCache::new(dir.clone(), "test-version".to_string(), 300, true);
// Write some cache data
cache
.insert("+gitks-cache/cache", "abc123", b"test")
.unwrap();
assert!(
dir.join("+gitks-cache/cache")
.join("ab")
.join("c123")
.exists()
);
// Startup cleanup removes all cache dirs
cache.cleanup_on_startup().unwrap();
assert!(!dir.join("+gitks-cache/cache").exists());
}
+108
View File
@@ -0,0 +1,108 @@
//! Tests for hooks system.
use std::path::PathBuf;
use std::time::Duration;
use gitks::hooks::manager::{HookLevel, HookManager};
use gitks::hooks::sanitize::{validate_hook_content, validate_hook_name};
fn temp_repo() -> PathBuf {
let dir = tempfile::tempdir().unwrap().path().to_path_buf();
// Create a bare git repo
let _ = std::process::Command::new("git")
.args(["init", "--bare"])
.arg(&dir)
.output();
dir
}
#[test]
fn test_validate_hook_name() {
assert!(validate_hook_name("pre-receive").is_ok());
assert!(validate_hook_name("update").is_ok());
assert!(validate_hook_name("post-receive").is_ok());
assert!(validate_hook_name("commit-msg").is_ok());
assert!(validate_hook_name("invalid-hook").is_err());
assert!(validate_hook_name("").is_err());
assert!(validate_hook_name("random-name").is_err());
}
#[test]
fn test_validate_hook_content_safe() {
assert!(validate_hook_content("#!/bin/sh\nexit 0").is_ok());
assert!(validate_hook_content("#!/bin/sh\necho 'hello'").is_ok());
}
#[test]
fn test_validate_hook_content_dangerous() {
assert!(validate_hook_content("rm -rf /").is_err());
assert!(validate_hook_content("shutdown now").is_err());
assert!(validate_hook_content("chmod 777 /etc/passwd").is_err());
}
#[test]
fn test_validate_hook_content_size() {
let large_content = "x".repeat(70000);
assert!(validate_hook_content(&large_content).is_err());
}
#[test]
fn test_validate_hook_content_null_bytes() {
assert!(validate_hook_content("test\0content").is_err());
}
#[test]
fn test_hook_manager_install_hooks() {
let repo = temp_repo();
let prefix = repo.parent().unwrap().to_path_buf();
let hm = HookManager::new(prefix, None, None, Duration::from_secs(30), true);
// Install hooks
let result = hm.install_hooks(&repo);
if result.is_ok() {
// Check that hook files were created
assert!(repo.join("hooks/pre-receive").exists());
assert!(repo.join("hooks/update").exists());
assert!(repo.join("hooks/post-receive").exists());
}
// Installation may fail if git is not available; just check no panic
}
#[test]
fn test_hook_manager_custom_hooks() {
let repo = temp_repo();
let prefix = repo.parent().unwrap().to_path_buf();
let hm = HookManager::new(prefix, None, None, Duration::from_secs(30), true);
// Set a custom hook
let result = hm.set_custom_hook(&repo, "pre-receive", "#!/bin/sh\nexit 0");
if result.is_ok() {
assert!(repo.join("custom_hooks/pre-receive/d").exists());
}
// Remove custom hook
let result = hm.remove_custom_hook(&repo, "pre-receive");
assert!(result.is_ok() || result.is_err()); // Just no panic
}
#[test]
fn test_hook_manager_disallow_custom() {
let repo = temp_repo();
let prefix = repo.parent().unwrap().to_path_buf();
let hm = HookManager::new(
prefix,
None,
None,
Duration::from_secs(30),
false, // Disallow custom hooks
);
let result = hm.set_custom_hook(&repo, "pre-receive", "#!/bin/sh\nexit 0");
assert!(result.is_err());
}
#[test]
fn test_hook_level_display() {
assert_eq!(HookLevel::Server.to_string(), "server");
assert_eq!(HookLevel::Custom.to_string(), "custom");
}
+142
View File
@@ -0,0 +1,142 @@
//! Tests for snapshot operations.
use std::path::PathBuf;
use gitks::snapshot::ops::{create_snapshot, restore_snapshot};
use gitks::snapshot::storage::{LocalSnapshotStorage, SnapshotStorageBackend};
fn temp_bare_repo() -> PathBuf {
let dir = tempfile::tempdir().unwrap().path().to_path_buf();
let result = std::process::Command::new("git")
.args(["init", "--bare"])
.arg(&dir)
.output()
.expect("git init --bare should work");
assert!(result.status.success());
// Create an initial commit so HEAD exists
// We need a working tree to create a commit, so we create one temporarily
let work_dir = tempfile::tempdir().unwrap().path().to_path_buf();
let result2 = std::process::Command::new("git")
.args(["init"])
.arg(&work_dir)
.output()
.expect("git init should work");
assert!(result2.status.success());
// Create a file and commit
std::fs::write(work_dir.join("test.txt"), "hello world").unwrap();
let result3 = std::process::Command::new("git")
.args(["add", "test.txt"])
.current_dir(&work_dir)
.output();
let _ = result3;
let result4 = std::process::Command::new("git")
.args(["commit", "-m", "initial commit"])
.env("GIT_AUTHOR_NAME", "test")
.env("GIT_AUTHOR_EMAIL", "test@test.com")
.env("GIT_COMMITTER_NAME", "test")
.env("GIT_COMMITTER_EMAIL", "test@test.com")
.current_dir(&work_dir)
.output();
let _ = result4;
// Push to bare repo
let result5 = std::process::Command::new("git")
.args(["push", &dir.to_string_lossy(), "master:refs/heads/master"])
.current_dir(&work_dir)
.output();
let _ = result5;
dir
}
#[test]
fn test_snapshot_create_and_restore() {
let repo = temp_bare_repo();
let gb = gitks::bare::GitBare::new(repo.clone());
// Create snapshot
let result = create_snapshot(&gb);
if let Ok(data) = result {
assert!(!data.is_empty());
// Restore to a different location
let target = tempfile::tempdir().unwrap().path().to_path_buf();
// Initialize target as bare repo first
let init_result = std::process::Command::new("git")
.args(["init", "--bare"])
.arg(&target)
.output();
if let Ok(init_output) = init_result
&& init_output.status.success()
{
let restore_result = restore_snapshot(&target, &data);
// May succeed or fail depending on git bundle compatibility
let _ = restore_result;
}
}
}
#[test]
fn test_local_snapshot_storage_write_and_read() {
let dir = tempfile::tempdir().unwrap().path().to_path_buf();
let storage = LocalSnapshotStorage::new(dir);
// Write snapshot
let result = storage.write_snapshot("snap001", "test.git", "abc123", b"bundle_data");
assert!(result.is_ok());
// Read snapshot
let result = storage.read_snapshot("snap001");
assert!(result.is_ok());
assert_eq!(result.unwrap(), b"bundle_data".to_vec());
// Read non-existent
let result = storage.read_snapshot("nonexistent");
assert!(result.is_err());
}
#[test]
fn test_local_snapshot_storage_list() {
let dir = tempfile::tempdir().unwrap().path().to_path_buf();
let storage = LocalSnapshotStorage::new(dir);
// Write two snapshots
storage
.write_snapshot("snap001", "repo1.git", "abc123", b"data1")
.unwrap();
storage
.write_snapshot("snap002", "repo2.git", "def456", b"data2")
.unwrap();
// List all
let result = storage.list_snapshots("");
assert!(result.is_ok());
assert_eq!(result.unwrap().len(), 2);
// List filtered
let result = storage.list_snapshots("repo1.git");
assert!(result.is_ok());
let snapshots = result.unwrap();
assert_eq!(snapshots.len(), 1);
assert_eq!(snapshots[0].relative_path, "repo1.git");
}
#[test]
fn test_local_snapshot_storage_delete() {
let dir = tempfile::tempdir().unwrap().path().to_path_buf();
let storage = LocalSnapshotStorage::new(dir);
storage
.write_snapshot("snap001", "test.git", "abc123", b"data")
.unwrap();
assert!(storage.read_snapshot("snap001").is_ok());
storage.delete_snapshot("snap001").unwrap();
assert!(storage.read_snapshot("snap001").is_err());
// Delete non-existent
assert!(storage.delete_snapshot("nonexistent").is_err());
}