8f472a0443
- Integrate etcd-client for distributed coordination and leader election - Add remote client macros with proper formatting for all services - Implement RequestMetrics for tracking RPC performance and errors - Add rate limiting mechanism across all service endpoints - Create ElectionRequest and ElectionResult message types for leader election - Add role management with primary/replica switching capabilities - Implement health checker with automatic failover detection - Add repository count metrics for cluster monitoring - Update Cargo.toml with etcd-client and dashmap dependencies - Modify RepoEntry to include read_only flag for replica handling - Implement should_accept_election logic to prevent duplicate elections - Add RoleChangedEvent handling for cluster role updates
167 lines
5.8 KiB
Rust
167 lines
5.8 KiB
Rust
//! Snapshot storage backend abstraction.
|
|
//!
|
|
//! Currently implements local filesystem storage.
|
|
//! S3/GCS backends can be added later as optional dependencies.
|
|
|
|
use std::path::PathBuf;
|
|
use std::time::SystemTime;
|
|
|
|
/// Metadata about a snapshot.
|
|
#[derive(Debug, Clone)]
|
|
pub struct SnapshotInfo {
|
|
pub snapshot_id: String,
|
|
pub relative_path: String,
|
|
pub size_bytes: u64,
|
|
pub created_at: String, // ISO 8601
|
|
pub head_oid: String,
|
|
}
|
|
|
|
/// Trait for snapshot storage backends.
|
|
pub trait SnapshotStorageBackend: Send + Sync {
|
|
fn write_snapshot(
|
|
&self,
|
|
snapshot_id: &str,
|
|
relative_path: &str,
|
|
head_oid: &str,
|
|
data: &[u8],
|
|
) -> Result<(), String>;
|
|
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String>;
|
|
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String>;
|
|
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String>;
|
|
}
|
|
|
|
/// Local filesystem snapshot storage.
|
|
pub struct LocalSnapshotStorage {
|
|
base_dir: PathBuf,
|
|
}
|
|
|
|
impl LocalSnapshotStorage {
|
|
pub fn new(base_dir: PathBuf) -> Self {
|
|
Self { base_dir }
|
|
}
|
|
|
|
fn snapshot_dir(&self, snapshot_id: &str) -> PathBuf {
|
|
let prefix = &snapshot_id[..2.min(snapshot_id.len())];
|
|
self.base_dir.join(prefix).join(snapshot_id)
|
|
}
|
|
|
|
fn metadata_path(&self, snapshot_id: &str) -> PathBuf {
|
|
self.snapshot_dir(snapshot_id).join("metadata.json")
|
|
}
|
|
|
|
fn data_path(&self, snapshot_id: &str) -> PathBuf {
|
|
self.snapshot_dir(snapshot_id).join("bundle.dat")
|
|
}
|
|
}
|
|
|
|
impl SnapshotStorageBackend for LocalSnapshotStorage {
|
|
fn write_snapshot(
|
|
&self,
|
|
snapshot_id: &str,
|
|
relative_path: &str,
|
|
head_oid: &str,
|
|
data: &[u8],
|
|
) -> Result<(), String> {
|
|
let dir = self.snapshot_dir(snapshot_id);
|
|
std::fs::create_dir_all(&dir).map_err(|e| format!("create dir: {e}"))?;
|
|
|
|
let data_path = self.data_path(snapshot_id);
|
|
let tmp_path = data_path.with_extension("tmp");
|
|
std::fs::write(&tmp_path, data).map_err(|e| format!("write data: {e}"))?;
|
|
std::fs::rename(&tmp_path, &data_path).map_err(|e| format!("rename: {e}"))?;
|
|
|
|
let created_at = SystemTime::now()
|
|
.duration_since(SystemTime::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs();
|
|
let metadata = serde_json::json!({
|
|
"snapshot_id": snapshot_id,
|
|
"relative_path": relative_path,
|
|
"size_bytes": data.len(),
|
|
"created_at": created_at,
|
|
"head_oid": head_oid,
|
|
});
|
|
let metadata_str = serde_json::to_string_pretty(&metadata)
|
|
.map_err(|e| format!("serialize metadata: {e}"))?;
|
|
std::fs::write(self.metadata_path(snapshot_id), metadata_str)
|
|
.map_err(|e| format!("write metadata: {e}"))?;
|
|
|
|
tracing::info!(
|
|
snapshot_id = %snapshot_id,
|
|
size_bytes = data.len(),
|
|
"snapshot written to local storage"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String> {
|
|
let path = self.data_path(snapshot_id);
|
|
if !path.exists() {
|
|
return Err(format!("snapshot not found: {snapshot_id}"));
|
|
}
|
|
std::fs::read(&path).map_err(|e| format!("read snapshot: {e}"))
|
|
}
|
|
|
|
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String> {
|
|
let mut snapshots = Vec::new();
|
|
if !self.base_dir.exists() {
|
|
return Ok(snapshots);
|
|
}
|
|
|
|
for shard_entry in
|
|
std::fs::read_dir(&self.base_dir).map_err(|e| format!("read dir: {e}"))?
|
|
{
|
|
let shard_entry = shard_entry.map_err(|e| format!("entry: {e}"))?;
|
|
let shard_dir = shard_entry.path();
|
|
if !shard_dir.is_dir() {
|
|
continue;
|
|
}
|
|
|
|
for snap_entry in std::fs::read_dir(&shard_dir).map_err(|e| format!("read dir: {e}"))? {
|
|
let snap_entry = snap_entry.map_err(|e| format!("entry: {e}"))?;
|
|
let snap_dir = snap_entry.path();
|
|
if !snap_dir.is_dir() {
|
|
continue;
|
|
}
|
|
|
|
let metadata_path = snap_dir.join("metadata.json");
|
|
if !metadata_path.exists() {
|
|
continue;
|
|
}
|
|
|
|
let metadata_str = std::fs::read_to_string(&metadata_path)
|
|
.map_err(|e| format!("read metadata: {e}"))?;
|
|
let metadata: serde_json::Value = serde_json::from_str(&metadata_str)
|
|
.map_err(|e| format!("parse metadata: {e}"))?;
|
|
|
|
let snap_relative_path = metadata["relative_path"].as_str().unwrap_or("");
|
|
|
|
if !relative_path.is_empty() && snap_relative_path != relative_path {
|
|
continue;
|
|
}
|
|
|
|
snapshots.push(SnapshotInfo {
|
|
snapshot_id: metadata["snapshot_id"].as_str().unwrap_or("").to_string(),
|
|
relative_path: snap_relative_path.to_string(),
|
|
size_bytes: metadata["size_bytes"].as_u64().unwrap_or(0),
|
|
created_at: metadata["created_at"].as_str().unwrap_or("").to_string(),
|
|
head_oid: metadata["head_oid"].as_str().unwrap_or("").to_string(),
|
|
});
|
|
}
|
|
}
|
|
|
|
snapshots.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
|
Ok(snapshots)
|
|
}
|
|
|
|
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String> {
|
|
let dir = self.snapshot_dir(snapshot_id);
|
|
if !dir.exists() {
|
|
return Err(format!("snapshot not found: {snapshot_id}"));
|
|
}
|
|
std::fs::remove_dir_all(&dir).map_err(|e| format!("delete snapshot: {e}"))?;
|
|
tracing::info!(snapshot_id = %snapshot_id, "snapshot deleted");
|
|
Ok(())
|
|
}
|
|
}
|