feat(cluster): implement distributed clustering with etcd coordination

- Integrate etcd-client for distributed coordination and leader election
- Add remote client macros with proper formatting for all services
- Implement RequestMetrics for tracking RPC performance and errors
- Add rate limiting mechanism across all service endpoints
- Create ElectionRequest and ElectionResult message types for leader election
- Add role management with primary/replica switching capabilities
- Implement health checker with automatic failover detection
- Add repository count metrics for cluster monitoring
- Update Cargo.toml with etcd-client and dashmap dependencies
- Modify RepoEntry to include read_only flag for replica handling
- Implement should_accept_election logic to prevent duplicate elections
- Add RoleChangedEvent handling for cluster role updates

snapshot/storage.rs

@@ -0,0 +1,166 @@
+//! Snapshot storage backend abstraction.
+//!
+//! Currently implements local filesystem storage.
+//! S3/GCS backends can be added later as optional dependencies.
+
+use std::path::PathBuf;
+use std::time::SystemTime;
+
+/// Metadata about a snapshot.
+#[derive(Debug, Clone)]
+pub struct SnapshotInfo {
+    pub snapshot_id: String,
+    pub relative_path: String,
+    pub size_bytes: u64,
+    pub created_at: String, // ISO 8601
+    pub head_oid: String,
+}
+
+/// Trait for snapshot storage backends.
+pub trait SnapshotStorageBackend: Send + Sync {
+    fn write_snapshot(
+        &self,
+        snapshot_id: &str,
+        relative_path: &str,
+        head_oid: &str,
+        data: &[u8],
+    ) -> Result<(), String>;
+    fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String>;
+    fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String>;
+    fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String>;
+}
+
+/// Local filesystem snapshot storage.
+pub struct LocalSnapshotStorage {
+    base_dir: PathBuf,
+}
+
+impl LocalSnapshotStorage {
+    pub fn new(base_dir: PathBuf) -> Self {
+        Self { base_dir }
+    }
+
+    fn snapshot_dir(&self, snapshot_id: &str) -> PathBuf {
+        let prefix = &snapshot_id[..2.min(snapshot_id.len())];
+        self.base_dir.join(prefix).join(snapshot_id)
+    }
+
+    fn metadata_path(&self, snapshot_id: &str) -> PathBuf {
+        self.snapshot_dir(snapshot_id).join("metadata.json")
+    }
+
+    fn data_path(&self, snapshot_id: &str) -> PathBuf {
+        self.snapshot_dir(snapshot_id).join("bundle.dat")
+    }
+}
+
+impl SnapshotStorageBackend for LocalSnapshotStorage {
+    fn write_snapshot(
+        &self,
+        snapshot_id: &str,
+        relative_path: &str,
+        head_oid: &str,
+        data: &[u8],
+    ) -> Result<(), String> {
+        let dir = self.snapshot_dir(snapshot_id);
+        std::fs::create_dir_all(&dir).map_err(|e| format!("create dir: {e}"))?;
+
+        let data_path = self.data_path(snapshot_id);
+        let tmp_path = data_path.with_extension("tmp");
+        std::fs::write(&tmp_path, data).map_err(|e| format!("write data: {e}"))?;
+        std::fs::rename(&tmp_path, &data_path).map_err(|e| format!("rename: {e}"))?;
+
+        let created_at = SystemTime::now()
+            .duration_since(SystemTime::UNIX_EPOCH)
+            .unwrap_or_default()
+            .as_secs();
+        let metadata = serde_json::json!({
+            "snapshot_id": snapshot_id,
+            "relative_path": relative_path,
+            "size_bytes": data.len(),
+            "created_at": created_at,
+            "head_oid": head_oid,
+        });
+        let metadata_str = serde_json::to_string_pretty(&metadata)
+            .map_err(|e| format!("serialize metadata: {e}"))?;
+        std::fs::write(self.metadata_path(snapshot_id), metadata_str)
+            .map_err(|e| format!("write metadata: {e}"))?;
+
+        tracing::info!(
+            snapshot_id = %snapshot_id,
+            size_bytes = data.len(),
+            "snapshot written to local storage"
+        );
+        Ok(())
+    }
+
+    fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String> {
+        let path = self.data_path(snapshot_id);
+        if !path.exists() {
+            return Err(format!("snapshot not found: {snapshot_id}"));
+        }
+        std::fs::read(&path).map_err(|e| format!("read snapshot: {e}"))
+    }
+
+    fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String> {
+        let mut snapshots = Vec::new();
+        if !self.base_dir.exists() {
+            return Ok(snapshots);
+        }
+
+        for shard_entry in
+            std::fs::read_dir(&self.base_dir).map_err(|e| format!("read dir: {e}"))?
+        {
+            let shard_entry = shard_entry.map_err(|e| format!("entry: {e}"))?;
+            let shard_dir = shard_entry.path();
+            if !shard_dir.is_dir() {
+                continue;
+            }
+
+            for snap_entry in std::fs::read_dir(&shard_dir).map_err(|e| format!("read dir: {e}"))? {
+                let snap_entry = snap_entry.map_err(|e| format!("entry: {e}"))?;
+                let snap_dir = snap_entry.path();
+                if !snap_dir.is_dir() {
+                    continue;
+                }
+
+                let metadata_path = snap_dir.join("metadata.json");
+                if !metadata_path.exists() {
+                    continue;
+                }
+
+                let metadata_str = std::fs::read_to_string(&metadata_path)
+                    .map_err(|e| format!("read metadata: {e}"))?;
+                let metadata: serde_json::Value = serde_json::from_str(&metadata_str)
+                    .map_err(|e| format!("parse metadata: {e}"))?;
+
+                let snap_relative_path = metadata["relative_path"].as_str().unwrap_or("");
+
+                if !relative_path.is_empty() && snap_relative_path != relative_path {
+                    continue;
+                }
+
+                snapshots.push(SnapshotInfo {
+                    snapshot_id: metadata["snapshot_id"].as_str().unwrap_or("").to_string(),
+                    relative_path: snap_relative_path.to_string(),
+                    size_bytes: metadata["size_bytes"].as_u64().unwrap_or(0),
+                    created_at: metadata["created_at"].as_str().unwrap_or("").to_string(),
+                    head_oid: metadata["head_oid"].as_str().unwrap_or("").to_string(),
+                });
+            }
+        }
+
+        snapshots.sort_by(|a, b| b.created_at.cmp(&a.created_at));
+        Ok(snapshots)
+    }
+
+    fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String> {
+        let dir = self.snapshot_dir(snapshot_id);
+        if !dir.exists() {
+            return Err(format!("snapshot not found: {snapshot_id}"));
+        }
+        std::fs::remove_dir_all(&dir).map_err(|e| format!("delete snapshot: {e}"))?;
+        tracing::info!(snapshot_id = %snapshot_id, "snapshot deleted");
+        Ok(())
+    }
+}