feat(cluster): implement distributed clustering with etcd coordination

- Integrate etcd-client for distributed coordination and leader election
- Add remote client macros with proper formatting for all services
- Implement RequestMetrics for tracking RPC performance and errors
- Add rate limiting mechanism across all service endpoints
- Create ElectionRequest and ElectionResult message types for leader election
- Add role management with primary/replica switching capabilities
- Implement health checker with automatic failover detection
- Add repository count metrics for cluster monitoring
- Update Cargo.toml with etcd-client and dashmap dependencies
- Modify RepoEntry to include read_only flag for replica handling
- Implement should_accept_election logic to prevent duplicate elections
- Add RoleChangedEvent handling for cluster role updates

server/diff.rs

@@ -3,7 +3,11 @@ use crate::pb::*;
 
 use super::{GitksService, cache, into_status, into_stream};
 
-remote_client!(remote_diff_client, DiffServiceClient<tonic::transport::Channel>, "diff");
+remote_client!(
+    remote_diff_client,
+    DiffServiceClient<tonic::transport::Channel>,
+    "diff"
+);
 
 #[tonic::async_trait]
 impl diff_service_server::DiffService for GitksService {
@@ -14,7 +18,9 @@ impl diff_service_server::DiffService for GitksService {
         &self,
         request: tonic::Request<GetDiffRequest>,
     ) -> Result<tonic::Response<GetDiffResponse>, tonic::Status> {
+        let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetDiff");
         let inner = request.into_inner();
+        let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
         let repo = self.repo_label(inner.repository.as_ref());
         let span = tracing::info_span!("diff.get_diff", %repo);
         let _enter = span.enter();
@@ -24,11 +30,16 @@ impl diff_service_server::DiffService for GitksService {
                 if let Some(mut client) =
                     remote_diff_client(self, inner.repository.as_ref(), false).await?
                 {
+                    m.record("ok");
                     return client.get_diff(inner).await;
                 }
+                crate::metrics::record_rpc_error(&m, &err);
+                return Err(err);
+            }
+            Err(err) => {
+                crate::metrics::record_rpc_error(&m, &err);
                 return Err(err);
             }
-            Err(err) => return Err(err),
         };
         let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
             cache::cached_response("diff.get_diff", &inner, || {
@@ -38,6 +49,7 @@ impl diff_service_server::DiffService for GitksService {
             gb.get_diff(inner).map_err(into_status)?
         };
         tracing::info!(%repo, files = resp.files.len(), overflow = resp.overflow, "get_diff done");
+        m.record("ok");
         Ok(tonic::Response::new(resp))
     }
 
@@ -45,7 +57,9 @@ impl diff_service_server::DiffService for GitksService {
         &self,
         request: tonic::Request<GetCommitDiffRequest>,
     ) -> Result<tonic::Response<GetDiffResponse>, tonic::Status> {
+        let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetCommitDiff");
         let inner = request.into_inner();
+        let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
         let repo = self.repo_label(inner.repository.as_ref());
         let span = tracing::info_span!("diff.get_commit_diff", %repo);
         let _enter = span.enter();
@@ -55,11 +69,16 @@ impl diff_service_server::DiffService for GitksService {
                 if let Some(mut client) =
                     remote_diff_client(self, inner.repository.as_ref(), false).await?
                 {
+                    m.record("ok");
                     return client.get_commit_diff(inner).await;
                 }
+                crate::metrics::record_rpc_error(&m, &err);
+                return Err(err);
+            }
+            Err(err) => {
+                crate::metrics::record_rpc_error(&m, &err);
                 return Err(err);
             }
-            Err(err) => return Err(err),
         };
         let resp = if cache::selector_is_oid(&inner.commit) {
             cache::cached_response("diff.get_commit_diff", &inner, || {
@@ -69,6 +88,7 @@ impl diff_service_server::DiffService for GitksService {
             gb.get_commit_diff(inner).map_err(into_status)?
         };
         tracing::info!(%repo, files = resp.files.len(), "get_commit_diff done");
+        m.record("ok");
         Ok(tonic::Response::new(resp))
     }
 
@@ -76,7 +96,9 @@ impl diff_service_server::DiffService for GitksService {
         &self,
         request: tonic::Request<GetPatchRequest>,
     ) -> Result<tonic::Response<Self::GetPatchStream>, tonic::Status> {
+        let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetPatch");
         let inner = request.into_inner();
+        let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
         let repo = self.repo_label(inner.repository.as_ref());
         let span = tracing::info_span!("diff.get_patch", %repo);
         let _enter = span.enter();
@@ -86,13 +108,18 @@ impl diff_service_server::DiffService for GitksService {
                 if let Some(mut client) =
                     remote_diff_client(self, inner.repository.as_ref(), false).await?
                 {
+                    m.record("ok");
                     let resp = client.get_patch(inner).await?;
                     let stream = super::bridge_server_stream(resp.into_inner());
                     return Ok(tonic::Response::new(stream));
                 }
+                crate::metrics::record_rpc_error(&m, &err);
+                return Err(err);
+            }
+            Err(err) => {
+                crate::metrics::record_rpc_error(&m, &err);
                 return Err(err);
             }
-            Err(err) => return Err(err),
         };
         let items = if cache::selectors_are_oid(&inner.base, &inner.head) {
             cache::cached_vec_response("diff.get_patch", &inner, || {
@@ -101,6 +128,7 @@ impl diff_service_server::DiffService for GitksService {
         } else {
             gb.get_patch(inner).map_err(into_status)?
         };
+        m.record("ok");
         Ok(tonic::Response::new(into_stream(items)))
     }
 
@@ -108,7 +136,9 @@ impl diff_service_server::DiffService for GitksService {
         &self,
         request: tonic::Request<GetDiffStatsRequest>,
     ) -> Result<tonic::Response<DiffStats>, tonic::Status> {
+        let m = crate::metrics::RequestMetrics::new("gitks.DiffService/GetDiffStats");
         let inner = request.into_inner();
+        let _rate = self.acquire_rate_limit(inner.repository.as_ref()).await?;
         let repo = self.repo_label(inner.repository.as_ref());
         let span = tracing::info_span!("diff.get_diff_stats", %repo);
         let _enter = span.enter();
@@ -118,11 +148,16 @@ impl diff_service_server::DiffService for GitksService {
                 if let Some(mut client) =
                     remote_diff_client(self, inner.repository.as_ref(), false).await?
                 {
+                    m.record("ok");
                     return client.get_diff_stats(inner).await;
                 }
+                crate::metrics::record_rpc_error(&m, &err);
+                return Err(err);
+            }
+            Err(err) => {
+                crate::metrics::record_rpc_error(&m, &err);
                 return Err(err);
             }
-            Err(err) => return Err(err),
         };
         let resp = if cache::selectors_are_oid(&inner.base, &inner.head) {
             cache::cached_response("diff.get_diff_stats", &inner, || {
@@ -131,6 +166,7 @@ impl diff_service_server::DiffService for GitksService {
         } else {
             gb.get_diff_stats(inner).map_err(into_status)?
         };
+        m.record("ok");
         Ok(tonic::Response::new(resp))
     }
 }