Files
gitks/snapshot/storage.rs
T
zhenyi 8f472a0443 feat(cluster): implement distributed clustering with etcd coordination
- Integrate etcd-client for distributed coordination and leader election
- Add remote client macros with proper formatting for all services
- Implement RequestMetrics for tracking RPC performance and errors
- Add rate limiting mechanism across all service endpoints
- Create ElectionRequest and ElectionResult message types for leader election
- Add role management with primary/replica switching capabilities
- Implement health checker with automatic failover detection
- Add repository count metrics for cluster monitoring
- Update Cargo.toml with etcd-client and dashmap dependencies
- Modify RepoEntry to include read_only flag for replica handling
- Implement should_accept_election logic to prevent duplicate elections
- Add RoleChangedEvent handling for cluster role updates
2026-06-08 14:31:29 +08:00

167 lines
5.8 KiB
Rust

//! Snapshot storage backend abstraction.
//!
//! Currently implements local filesystem storage.
//! S3/GCS backends can be added later as optional dependencies.
use std::path::PathBuf;
use std::time::SystemTime;
/// Metadata about a snapshot.
#[derive(Debug, Clone)]
pub struct SnapshotInfo {
pub snapshot_id: String,
pub relative_path: String,
pub size_bytes: u64,
pub created_at: String, // ISO 8601
pub head_oid: String,
}
/// Trait for snapshot storage backends.
pub trait SnapshotStorageBackend: Send + Sync {
fn write_snapshot(
&self,
snapshot_id: &str,
relative_path: &str,
head_oid: &str,
data: &[u8],
) -> Result<(), String>;
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String>;
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String>;
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String>;
}
/// Local filesystem snapshot storage.
pub struct LocalSnapshotStorage {
base_dir: PathBuf,
}
impl LocalSnapshotStorage {
pub fn new(base_dir: PathBuf) -> Self {
Self { base_dir }
}
fn snapshot_dir(&self, snapshot_id: &str) -> PathBuf {
let prefix = &snapshot_id[..2.min(snapshot_id.len())];
self.base_dir.join(prefix).join(snapshot_id)
}
fn metadata_path(&self, snapshot_id: &str) -> PathBuf {
self.snapshot_dir(snapshot_id).join("metadata.json")
}
fn data_path(&self, snapshot_id: &str) -> PathBuf {
self.snapshot_dir(snapshot_id).join("bundle.dat")
}
}
impl SnapshotStorageBackend for LocalSnapshotStorage {
fn write_snapshot(
&self,
snapshot_id: &str,
relative_path: &str,
head_oid: &str,
data: &[u8],
) -> Result<(), String> {
let dir = self.snapshot_dir(snapshot_id);
std::fs::create_dir_all(&dir).map_err(|e| format!("create dir: {e}"))?;
let data_path = self.data_path(snapshot_id);
let tmp_path = data_path.with_extension("tmp");
std::fs::write(&tmp_path, data).map_err(|e| format!("write data: {e}"))?;
std::fs::rename(&tmp_path, &data_path).map_err(|e| format!("rename: {e}"))?;
let created_at = SystemTime::now()
.duration_since(SystemTime::UNIX_EPOCH)
.unwrap_or_default()
.as_secs();
let metadata = serde_json::json!({
"snapshot_id": snapshot_id,
"relative_path": relative_path,
"size_bytes": data.len(),
"created_at": created_at,
"head_oid": head_oid,
});
let metadata_str = serde_json::to_string_pretty(&metadata)
.map_err(|e| format!("serialize metadata: {e}"))?;
std::fs::write(self.metadata_path(snapshot_id), metadata_str)
.map_err(|e| format!("write metadata: {e}"))?;
tracing::info!(
snapshot_id = %snapshot_id,
size_bytes = data.len(),
"snapshot written to local storage"
);
Ok(())
}
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String> {
let path = self.data_path(snapshot_id);
if !path.exists() {
return Err(format!("snapshot not found: {snapshot_id}"));
}
std::fs::read(&path).map_err(|e| format!("read snapshot: {e}"))
}
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String> {
let mut snapshots = Vec::new();
if !self.base_dir.exists() {
return Ok(snapshots);
}
for shard_entry in
std::fs::read_dir(&self.base_dir).map_err(|e| format!("read dir: {e}"))?
{
let shard_entry = shard_entry.map_err(|e| format!("entry: {e}"))?;
let shard_dir = shard_entry.path();
if !shard_dir.is_dir() {
continue;
}
for snap_entry in std::fs::read_dir(&shard_dir).map_err(|e| format!("read dir: {e}"))? {
let snap_entry = snap_entry.map_err(|e| format!("entry: {e}"))?;
let snap_dir = snap_entry.path();
if !snap_dir.is_dir() {
continue;
}
let metadata_path = snap_dir.join("metadata.json");
if !metadata_path.exists() {
continue;
}
let metadata_str = std::fs::read_to_string(&metadata_path)
.map_err(|e| format!("read metadata: {e}"))?;
let metadata: serde_json::Value = serde_json::from_str(&metadata_str)
.map_err(|e| format!("parse metadata: {e}"))?;
let snap_relative_path = metadata["relative_path"].as_str().unwrap_or("");
if !relative_path.is_empty() && snap_relative_path != relative_path {
continue;
}
snapshots.push(SnapshotInfo {
snapshot_id: metadata["snapshot_id"].as_str().unwrap_or("").to_string(),
relative_path: snap_relative_path.to_string(),
size_bytes: metadata["size_bytes"].as_u64().unwrap_or(0),
created_at: metadata["created_at"].as_str().unwrap_or("").to_string(),
head_oid: metadata["head_oid"].as_str().unwrap_or("").to_string(),
});
}
}
snapshots.sort_by(|a, b| b.created_at.cmp(&a.created_at));
Ok(snapshots)
}
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String> {
let dir = self.snapshot_dir(snapshot_id);
if !dir.exists() {
return Err(format!("snapshot not found: {snapshot_id}"));
}
std::fs::remove_dir_all(&dir).map_err(|e| format!("delete snapshot: {e}"))?;
tracing::info!(snapshot_id = %snapshot_id, "snapshot deleted");
Ok(())
}
}