934858bebf
- Add repo_path parameter to cached_response and cached_vec_response functions - Implement structured cache key format with namespace, repo_path, and request proto - Replace global cache with Moka in-memory cache using weight-based eviction - Set 256MB memory cap with 10-minute TTL and 2-minute TTI policy - Add metrics collection for cache operations and evictions - Implement efficient repo-scoped invalidation using key structure - Add detailed documentation comments explaining cache architecture - Remove outdated dependencies and update dependency versions - Add error handling for encoding failures in cache operations - Optimize Vec responses with length-delimited encoding and pre-allocation
184 lines
6.4 KiB
Rust
184 lines
6.4 KiB
Rust
//! Snapshot storage backend abstraction.
|
|
//!
|
|
//! Currently implements local filesystem storage.
|
|
//! S3/GCS backends can be added later as optional dependencies.
|
|
|
|
use std::path::PathBuf;
|
|
use std::time::SystemTime;
|
|
|
|
/// Metadata about a snapshot.
|
|
#[derive(Debug, Clone)]
|
|
pub struct SnapshotInfo {
|
|
pub snapshot_id: String,
|
|
pub relative_path: String,
|
|
pub size_bytes: u64,
|
|
pub created_at: String, // ISO 8601
|
|
pub head_oid: String,
|
|
}
|
|
|
|
/// Trait for snapshot storage backends.
|
|
pub trait SnapshotStorageBackend: Send + Sync {
|
|
fn write_snapshot(
|
|
&self,
|
|
snapshot_id: &str,
|
|
relative_path: &str,
|
|
head_oid: &str,
|
|
data: &[u8],
|
|
) -> Result<(), String>;
|
|
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String>;
|
|
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String>;
|
|
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String>;
|
|
}
|
|
|
|
/// Local filesystem snapshot storage.
|
|
pub struct LocalSnapshotStorage {
|
|
base_dir: PathBuf,
|
|
}
|
|
|
|
impl LocalSnapshotStorage {
|
|
pub fn new(base_dir: PathBuf) -> Self {
|
|
Self { base_dir }
|
|
}
|
|
|
|
fn validate_snapshot_id(snapshot_id: &str) -> Result<(), String> {
|
|
if snapshot_id.is_empty() {
|
|
return Err("snapshot_id cannot be empty".into());
|
|
}
|
|
if snapshot_id.len() > 64
|
|
|| !snapshot_id
|
|
.chars()
|
|
.all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_')
|
|
{
|
|
return Err(format!("invalid snapshot_id: {snapshot_id}"));
|
|
}
|
|
Ok(())
|
|
}
|
|
|
|
fn snapshot_dir(&self, snapshot_id: &str) -> PathBuf {
|
|
let prefix = snapshot_id.get(..2).unwrap_or(snapshot_id);
|
|
self.base_dir.join(prefix).join(snapshot_id)
|
|
}
|
|
|
|
fn metadata_path(&self, snapshot_id: &str) -> PathBuf {
|
|
self.snapshot_dir(snapshot_id).join("metadata.json")
|
|
}
|
|
|
|
fn data_path(&self, snapshot_id: &str) -> PathBuf {
|
|
self.snapshot_dir(snapshot_id).join("bundle.dat")
|
|
}
|
|
}
|
|
|
|
impl SnapshotStorageBackend for LocalSnapshotStorage {
|
|
fn write_snapshot(
|
|
&self,
|
|
snapshot_id: &str,
|
|
relative_path: &str,
|
|
head_oid: &str,
|
|
data: &[u8],
|
|
) -> Result<(), String> {
|
|
Self::validate_snapshot_id(snapshot_id)?;
|
|
let dir = self.snapshot_dir(snapshot_id);
|
|
std::fs::create_dir_all(&dir).map_err(|e| format!("create dir: {e}"))?;
|
|
|
|
let data_path = self.data_path(snapshot_id);
|
|
let tmp_path = data_path.with_extension("tmp");
|
|
std::fs::write(&tmp_path, data).map_err(|e| format!("write data: {e}"))?;
|
|
std::fs::rename(&tmp_path, &data_path).map_err(|e| format!("rename: {e}"))?;
|
|
|
|
let created_at = SystemTime::now()
|
|
.duration_since(SystemTime::UNIX_EPOCH)
|
|
.unwrap_or_default()
|
|
.as_secs();
|
|
let metadata = serde_json::json!({
|
|
"snapshot_id": snapshot_id,
|
|
"relative_path": relative_path,
|
|
"size_bytes": data.len(),
|
|
"created_at": created_at,
|
|
"head_oid": head_oid,
|
|
});
|
|
let metadata_str = serde_json::to_string_pretty(&metadata)
|
|
.map_err(|e| format!("serialize metadata: {e}"))?;
|
|
std::fs::write(self.metadata_path(snapshot_id), metadata_str)
|
|
.map_err(|e| format!("write metadata: {e}"))?;
|
|
|
|
tracing::info!(
|
|
snapshot_id = %snapshot_id,
|
|
size_bytes = data.len(),
|
|
"snapshot written to local storage"
|
|
);
|
|
Ok(())
|
|
}
|
|
|
|
fn read_snapshot(&self, snapshot_id: &str) -> Result<Vec<u8>, String> {
|
|
Self::validate_snapshot_id(snapshot_id)?;
|
|
let path = self.data_path(snapshot_id);
|
|
if !path.exists() {
|
|
return Err(format!("snapshot not found: {snapshot_id}"));
|
|
}
|
|
std::fs::read(&path).map_err(|e| format!("read snapshot: {e}"))
|
|
}
|
|
|
|
fn list_snapshots(&self, relative_path: &str) -> Result<Vec<SnapshotInfo>, String> {
|
|
let mut snapshots = Vec::new();
|
|
if !self.base_dir.exists() {
|
|
return Ok(snapshots);
|
|
}
|
|
|
|
for shard_entry in
|
|
std::fs::read_dir(&self.base_dir).map_err(|e| format!("read dir: {e}"))?
|
|
{
|
|
let shard_entry = shard_entry.map_err(|e| format!("entry: {e}"))?;
|
|
let shard_dir = shard_entry.path();
|
|
if !shard_dir.is_dir() {
|
|
continue;
|
|
}
|
|
|
|
for snap_entry in std::fs::read_dir(&shard_dir).map_err(|e| format!("read dir: {e}"))? {
|
|
let snap_entry = snap_entry.map_err(|e| format!("entry: {e}"))?;
|
|
let snap_dir = snap_entry.path();
|
|
if !snap_dir.is_dir() {
|
|
continue;
|
|
}
|
|
|
|
let metadata_path = snap_dir.join("metadata.json");
|
|
if !metadata_path.exists() {
|
|
continue;
|
|
}
|
|
|
|
let metadata_str = std::fs::read_to_string(&metadata_path)
|
|
.map_err(|e| format!("read metadata: {e}"))?;
|
|
let metadata: serde_json::Value = serde_json::from_str(&metadata_str)
|
|
.map_err(|e| format!("parse metadata: {e}"))?;
|
|
|
|
let snap_relative_path = metadata["relative_path"].as_str().unwrap_or("");
|
|
|
|
if !relative_path.is_empty() && snap_relative_path != relative_path {
|
|
continue;
|
|
}
|
|
|
|
snapshots.push(SnapshotInfo {
|
|
snapshot_id: metadata["snapshot_id"].as_str().unwrap_or("").to_string(),
|
|
relative_path: snap_relative_path.to_string(),
|
|
size_bytes: metadata["size_bytes"].as_u64().unwrap_or(0),
|
|
created_at: metadata["created_at"].as_str().unwrap_or("").to_string(),
|
|
head_oid: metadata["head_oid"].as_str().unwrap_or("").to_string(),
|
|
});
|
|
}
|
|
}
|
|
|
|
snapshots.sort_by(|a, b| b.created_at.cmp(&a.created_at));
|
|
Ok(snapshots)
|
|
}
|
|
|
|
fn delete_snapshot(&self, snapshot_id: &str) -> Result<(), String> {
|
|
Self::validate_snapshot_id(snapshot_id)?;
|
|
let dir = self.snapshot_dir(snapshot_id);
|
|
if !dir.exists() {
|
|
return Err(format!("snapshot not found: {snapshot_id}"));
|
|
}
|
|
std::fs::remove_dir_all(&dir).map_err(|e| format!("delete snapshot: {e}"))?;
|
|
tracing::info!(snapshot_id = %snapshot_id, "snapshot deleted");
|
|
Ok(())
|
|
}
|
|
}
|