feat(cluster): implement distributed clustering with etcd coordination
- Integrate etcd-client for distributed coordination and leader election - Add remote client macros with proper formatting for all services - Implement RequestMetrics for tracking RPC performance and errors - Add rate limiting mechanism across all service endpoints - Create ElectionRequest and ElectionResult message types for leader election - Add role management with primary/replica switching capabilities - Implement health checker with automatic failover detection - Add repository count metrics for cluster monitoring - Update Cargo.toml with etcd-client and dashmap dependencies - Modify RepoEntry to include read_only flag for replica handling - Implement should_accept_election logic to prevent duplicate elections - Add RoleChangedEvent handling for cluster role updates
This commit is contained in:
zhenyi
@@ -0,0 +1,238 @@
|
||||
//! Hook manager for GitKS.
|
||||
//!
|
||||
//! Manages the installation, listing, and removal of git hooks
|
||||
//! for bare repositories. Supports server hooks, custom hooks,
|
||||
//! and gRPC callback hooks.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use crate::error::{GitError, GitResult};
|
||||
use crate::hooks::runner::{HookResult, generate_hook_runner_script, run_hook_dir};
|
||||
use crate::hooks::sanitize::{validate_hook_content, validate_hook_name};
|
||||
|
||||
/// Manages git hooks across repositories.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HookManager {
|
||||
repo_prefix: PathBuf,
|
||||
server_hooks_dir: Option<PathBuf>,
|
||||
hook_callback_addr: Option<String>,
|
||||
hook_timeout: Duration,
|
||||
allow_custom_hooks: bool,
|
||||
}
|
||||
|
||||
impl HookManager {
|
||||
pub fn new(
|
||||
repo_prefix: PathBuf,
|
||||
server_hooks_dir: Option<PathBuf>,
|
||||
hook_callback_addr: Option<String>,
|
||||
hook_timeout: Duration,
|
||||
allow_custom_hooks: bool,
|
||||
) -> Self {
|
||||
Self {
|
||||
repo_prefix,
|
||||
server_hooks_dir,
|
||||
hook_callback_addr,
|
||||
hook_timeout,
|
||||
allow_custom_hooks,
|
||||
}
|
||||
}
|
||||
|
||||
/// Install gitks hook runner scripts into a repository's hooks directory.
|
||||
/// Called during repository initialization.
|
||||
pub fn install_hooks(&self, repo_path: &Path) -> GitResult<()> {
|
||||
let hooks_dir = repo_path.join("hooks");
|
||||
std::fs::create_dir_all(&hooks_dir).map_err(GitError::Io)?;
|
||||
|
||||
let relative_path = repo_path
|
||||
.strip_prefix(&self.repo_prefix)
|
||||
.unwrap_or(repo_path)
|
||||
.to_string_lossy()
|
||||
.trim_start_matches('/')
|
||||
.to_string();
|
||||
|
||||
let server_hooks_dir_str = self
|
||||
.server_hooks_dir
|
||||
.as_ref()
|
||||
.map(|p| p.to_string_lossy().into_owned());
|
||||
let callback_addr_str = self.hook_callback_addr.clone();
|
||||
|
||||
for hook_type in &["pre-receive", "update", "post-receive"] {
|
||||
let script_content = generate_hook_runner_script(
|
||||
hook_type,
|
||||
&relative_path,
|
||||
server_hooks_dir_str.as_deref(),
|
||||
callback_addr_str.as_deref(),
|
||||
self.hook_timeout.as_secs(),
|
||||
);
|
||||
let hook_path = hooks_dir.join(hook_type);
|
||||
std::fs::write(&hook_path, script_content).map_err(GitError::Io)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut perms = std::fs::metadata(&hook_path)
|
||||
.map_err(GitError::Io)?
|
||||
.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&hook_path, perms).map_err(GitError::Io)?;
|
||||
}
|
||||
tracing::debug!(
|
||||
hook_type = %hook_type,
|
||||
path = %hook_path.display(),
|
||||
"installed gitks hook runner"
|
||||
);
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
path = %repo_path.display(),
|
||||
"hooks installed for repository"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set a custom hook script for a repository.
|
||||
pub fn set_custom_hook(
|
||||
&self,
|
||||
repo_path: &Path,
|
||||
hook_name: &str,
|
||||
content: &str,
|
||||
) -> GitResult<()> {
|
||||
validate_hook_name(hook_name)?;
|
||||
if !self.allow_custom_hooks {
|
||||
return Err(GitError::PermissionDenied(
|
||||
"custom hooks are not allowed on this server".into(),
|
||||
));
|
||||
}
|
||||
validate_hook_content(content)?;
|
||||
|
||||
let custom_hooks_dir = repo_path.join("custom_hooks").join(hook_name).join("d");
|
||||
std::fs::create_dir_all(&custom_hooks_dir).map_err(GitError::Io)?;
|
||||
|
||||
let script_name = format!("gitks_custom_{}", hook_name.replace('-', "_"));
|
||||
let script_path = custom_hooks_dir.join(&script_name);
|
||||
std::fs::write(&script_path, content).map_err(GitError::Io)?;
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut perms = std::fs::metadata(&script_path)
|
||||
.map_err(GitError::Io)?
|
||||
.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&script_path, perms).map_err(GitError::Io)?;
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
repo = %repo_path.display(),
|
||||
hook_name = %hook_name,
|
||||
"custom hook set"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Remove a custom hook from a repository.
|
||||
pub fn remove_custom_hook(&self, repo_path: &Path, hook_name: &str) -> GitResult<()> {
|
||||
validate_hook_name(hook_name)?;
|
||||
let custom_hooks_dir = repo_path.join("custom_hooks").join(hook_name).join("d");
|
||||
if custom_hooks_dir.exists() {
|
||||
std::fs::remove_dir_all(&custom_hooks_dir).map_err(GitError::Io)?;
|
||||
tracing::info!(
|
||||
repo = %repo_path.display(),
|
||||
hook_name = %hook_name,
|
||||
"custom hook removed"
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// List all hooks for a repository.
|
||||
pub fn list_hooks(&self, repo_path: &Path) -> GitResult<Vec<HookInfo>> {
|
||||
let mut hooks = Vec::new();
|
||||
|
||||
let hooks_dir = repo_path.join("hooks");
|
||||
if hooks_dir.exists() {
|
||||
for hook_type in &["pre-receive", "update", "post-receive"] {
|
||||
let hook_path = hooks_dir.join(hook_type);
|
||||
if hook_path.exists() {
|
||||
hooks.push(HookInfo {
|
||||
hook_type: hook_type.to_string(),
|
||||
level: HookLevel::Server,
|
||||
path: hook_path.to_string_lossy().into_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if self.allow_custom_hooks {
|
||||
let custom_dir = repo_path.join("custom_hooks");
|
||||
if custom_dir.exists() {
|
||||
for entry in std::fs::read_dir(&custom_dir).map_err(GitError::Io)? {
|
||||
let entry = entry.map_err(GitError::Io)?;
|
||||
let name = entry.file_name().to_string_lossy().into_owned();
|
||||
if validate_hook_name(&name).is_ok() {
|
||||
let path = entry.path();
|
||||
let d_dir = path.join("d");
|
||||
if d_dir.exists() {
|
||||
let script_count =
|
||||
std::fs::read_dir(&d_dir).map_err(GitError::Io)?.count();
|
||||
if script_count > 0 {
|
||||
hooks.push(HookInfo {
|
||||
hook_type: name,
|
||||
level: HookLevel::Custom,
|
||||
path: d_dir.to_string_lossy().into_owned(),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hooks)
|
||||
}
|
||||
|
||||
/// Execute a hook by running server hooks then custom hooks.
|
||||
pub fn execute_hook(&self, repo_path: &Path, hook_type: &str, stdin_data: &[u8]) -> HookResult {
|
||||
if let Some(ref server_dir) = self.server_hooks_dir {
|
||||
let dir = server_dir.join(hook_type).join("d");
|
||||
let result = run_hook_dir(&dir, hook_type, stdin_data, self.hook_timeout);
|
||||
if !result.accepted {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
if self.allow_custom_hooks {
|
||||
let custom_dir = repo_path.join("custom_hooks").join(hook_type).join("d");
|
||||
let result = run_hook_dir(&custom_dir, hook_type, stdin_data, self.hook_timeout);
|
||||
if !result.accepted {
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
HookResult::accepted()
|
||||
}
|
||||
}
|
||||
|
||||
/// Information about a hook.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HookInfo {
|
||||
pub hook_type: String,
|
||||
pub level: HookLevel,
|
||||
pub path: String,
|
||||
}
|
||||
|
||||
/// Hook level (server vs custom).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum HookLevel {
|
||||
Server,
|
||||
Custom,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for HookLevel {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
HookLevel::Server => write!(f, "server"),
|
||||
HookLevel::Custom => write!(f, "custom"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
//! Git hooks management for GitKS.
|
||||
//!
|
||||
//! Supports three layers of hooks:
|
||||
//! 1. Server hooks: admin-level, shared across all repositories
|
||||
//! 2. Custom hooks: per-repository, user-defined scripts
|
||||
//! 3. gRPC callback hooks: external HookService via gRPC
|
||||
//!
|
||||
//! Hook scripts are installed into bare repositories' `hooks/` directory
|
||||
//! and are automatically invoked by git during receive-pack operations.
|
||||
|
||||
pub mod manager;
|
||||
pub mod runner;
|
||||
pub mod sanitize;
|
||||
|
||||
pub use manager::HookManager;
|
||||
pub use runner::HookResult;
|
||||
+274
@@ -0,0 +1,274 @@
|
||||
//! Hook execution runner.
|
||||
//!
|
||||
//! Executes server hooks, custom hooks, and gRPC callback hooks
|
||||
//! in sequence for each git hook type (pre-receive, update, post-receive).
|
||||
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Output, Stdio};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Result of a hook execution.
|
||||
#[derive(Debug)]
|
||||
pub struct HookResult {
|
||||
pub accepted: bool,
|
||||
pub exit_code: i32,
|
||||
pub stdout: String,
|
||||
pub stderr: String,
|
||||
}
|
||||
|
||||
impl HookResult {
|
||||
pub fn accepted() -> Self {
|
||||
Self {
|
||||
accepted: true,
|
||||
exit_code: 0,
|
||||
stdout: String::new(),
|
||||
stderr: String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn rejected(stderr: String) -> Self {
|
||||
Self {
|
||||
accepted: false,
|
||||
exit_code: 1,
|
||||
stdout: String::new(),
|
||||
stderr,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_output(output: &Output) -> Self {
|
||||
let stdout = String::from_utf8_lossy(&output.stdout).into_owned();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).into_owned();
|
||||
Self {
|
||||
accepted: output.status.success(),
|
||||
exit_code: output.status.code().unwrap_or(-1),
|
||||
stdout,
|
||||
stderr,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run all hook scripts in a directory (sorted alphabetically).
|
||||
/// Returns the first rejection or accepted if all pass.
|
||||
pub fn run_hook_dir(
|
||||
hook_dir: &Path,
|
||||
hook_type: &str,
|
||||
stdin_data: &[u8],
|
||||
timeout: Duration,
|
||||
) -> HookResult {
|
||||
if !hook_dir.exists() {
|
||||
return HookResult::accepted();
|
||||
}
|
||||
|
||||
let mut scripts: Vec<PathBuf> = Vec::new();
|
||||
if let Ok(entries) = std::fs::read_dir(hook_dir) {
|
||||
for entry in entries.flatten() {
|
||||
let path = entry.path();
|
||||
if path.is_file() {
|
||||
let name = path.file_name().unwrap_or_default().to_string_lossy();
|
||||
// Skip files starting with '.' (hidden files) or ending with '~' or '.sample'
|
||||
if name.starts_with('.') || name.ends_with('~') || name.ends_with(".sample") {
|
||||
continue;
|
||||
}
|
||||
scripts.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
scripts.sort();
|
||||
|
||||
for script in &scripts {
|
||||
tracing::debug!(
|
||||
hook_type = %hook_type,
|
||||
script = %script.display(),
|
||||
"executing hook script"
|
||||
);
|
||||
let result = run_single_script(script, stdin_data, timeout);
|
||||
if !result.accepted {
|
||||
tracing::warn!(
|
||||
hook_type = %hook_type,
|
||||
script = %script.display(),
|
||||
exit_code = result.exit_code,
|
||||
stderr = %result.stderr,
|
||||
"hook script rejected"
|
||||
);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
HookResult::accepted()
|
||||
}
|
||||
|
||||
/// Run a single hook script with stdin data and timeout.
|
||||
fn run_single_script(script_path: &Path, stdin_data: &[u8], timeout: Duration) -> HookResult {
|
||||
let child = std::process::Command::new(script_path)
|
||||
.stdin(Stdio::piped())
|
||||
.stdout(Stdio::piped())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn();
|
||||
|
||||
match child {
|
||||
Ok(mut c) => {
|
||||
if let Some(ref mut stdin) = c.stdin {
|
||||
let _ = stdin.write_all(stdin_data);
|
||||
}
|
||||
c.stdin = None;
|
||||
|
||||
let wait_result = c.wait_timeout(timeout);
|
||||
match wait_result {
|
||||
Ok(Some(status)) => {
|
||||
let output = c.wait_with_output().unwrap_or_else(|_| {
|
||||
// If we can't get output, at least return the status
|
||||
Output {
|
||||
status,
|
||||
stdout: Vec::new(),
|
||||
stderr: Vec::new(),
|
||||
}
|
||||
});
|
||||
HookResult::from_output(&output)
|
||||
}
|
||||
Ok(None) => {
|
||||
tracing::warn!(
|
||||
script = %script_path.display(),
|
||||
timeout_secs = timeout.as_secs(),
|
||||
"hook script timed out, killing"
|
||||
);
|
||||
let _ = c.kill();
|
||||
HookResult::rejected(format!(
|
||||
"hook script timed out after {}s: {}",
|
||||
timeout.as_secs(),
|
||||
script_path.display()
|
||||
))
|
||||
}
|
||||
Err(e) => {
|
||||
let _ = c.kill();
|
||||
HookResult::rejected(format!("hook script wait error: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!(
|
||||
script = %script_path.display(),
|
||||
error = %e,
|
||||
"failed to spawn hook script"
|
||||
);
|
||||
// If the script can't be executed, treat as rejection
|
||||
HookResult::rejected(format!("failed to spawn hook script: {e}"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Wait for a child process with timeout.
|
||||
trait ChildWaitTimeout {
|
||||
fn wait_timeout(
|
||||
&mut self,
|
||||
timeout: Duration,
|
||||
) -> std::io::Result<Option<std::process::ExitStatus>>;
|
||||
}
|
||||
|
||||
impl ChildWaitTimeout for std::process::Child {
|
||||
fn wait_timeout(
|
||||
&mut self,
|
||||
timeout: Duration,
|
||||
) -> std::io::Result<Option<std::process::ExitStatus>> {
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
match self.try_wait() {
|
||||
Ok(Some(status)) => return Ok(Some(status)),
|
||||
Ok(None) => {
|
||||
if start.elapsed() > timeout {
|
||||
return Ok(None); // timeout
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Generate the gitks hook runner script content.
|
||||
/// This script is installed into each repository's hooks/ directory
|
||||
/// and orchestrates the execution of server hooks, custom hooks, and gRPC callbacks.
|
||||
pub fn generate_hook_runner_script(
|
||||
hook_type: &str,
|
||||
repo_relative_path: &str,
|
||||
server_hooks_dir: Option<&str>,
|
||||
hook_callback_addr: Option<&str>,
|
||||
hook_timeout_secs: u64,
|
||||
) -> String {
|
||||
let server_hooks_section = if let Some(dir) = server_hooks_dir {
|
||||
format!(
|
||||
r#"
|
||||
# Run server hooks
|
||||
SERVER_HOOKS_DIR="{dir}/{hook_type}.d"
|
||||
if [ -d "$SERVER_HOOKS_DIR" ]; then
|
||||
for script in $(ls "$SERVER_HOOKS_DIR" | sort); do
|
||||
skip=false
|
||||
case "$script" in .*|*.sample|*~) skip=true;; esac
|
||||
if [ "$skip" = "false" ]; then
|
||||
"$SERVER_HOOKS_DIR/$script"
|
||||
exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
exit $exit_code
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
"#
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
let custom_hooks_section = r#"
|
||||
# Run custom hooks (per-repository)
|
||||
CUSTOM_HOOKS_DIR="$GIT_DIR/custom_hooks/$GITKS_HOOK_TYPE.d"
|
||||
if [ -d "$CUSTOM_HOOKS_DIR" ]; then
|
||||
for script in $(ls "$CUSTOM_HOOKS_DIR" | sort); do
|
||||
skip=false
|
||||
case "$script" in .*|*.sample|*~) skip=true;; esac
|
||||
if [ "$skip" = "false" ]; then
|
||||
"$CUSTOM_HOOKS_DIR/$script"
|
||||
exit_code=$?
|
||||
if [ $exit_code -ne 0 ]; then
|
||||
exit $exit_code
|
||||
fi
|
||||
fi
|
||||
done
|
||||
fi
|
||||
"#;
|
||||
|
||||
let grpc_callback_section = if let Some(addr) = hook_callback_addr {
|
||||
format!(
|
||||
r#"
|
||||
# gRPC callback to external HookService
|
||||
if [ -n "{addr}" ]; then
|
||||
# gRPC callback is handled by the gitks service directly
|
||||
# The service will make the gRPC call after the git hook completes
|
||||
# This section is a placeholder - actual gRPC callback is handled
|
||||
# by the gitks receive_pack handler
|
||||
fi
|
||||
"#
|
||||
)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
format!(
|
||||
r#"#!/bin/sh
|
||||
# gitks hook runner for {hook_type}
|
||||
# Repository: {repo_relative_path}
|
||||
# Auto-generated by gitks - do not modify manually
|
||||
|
||||
GITKS_HOOK_TYPE="{hook_type}"
|
||||
GITKS_REPO_RELATIVE_PATH="{repo_relative_path}"
|
||||
GITKS_HOOK_TIMEOUT="{hook_timeout_secs}"
|
||||
|
||||
{server_hooks_section}
|
||||
{custom_hooks_section}
|
||||
{grpc_callback_section}
|
||||
|
||||
exit 0
|
||||
"#
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
//! Hook content sanitization.
|
||||
//!
|
||||
//! Validates custom hook scripts to prevent dangerous commands.
|
||||
|
||||
use crate::error::{GitError, GitResult};
|
||||
|
||||
/// Commands/patterns that are never allowed in custom hook scripts.
|
||||
const FORBIDDEN_PATTERNS: &[&str] = &[
|
||||
"rm -rf",
|
||||
"rm -r /",
|
||||
"chmod 777",
|
||||
"chmod 666",
|
||||
"mkfs",
|
||||
"dd if=",
|
||||
":(){ :|:& };:", // fork bomb
|
||||
"> /dev/sda",
|
||||
"curl -o /",
|
||||
"wget -O /",
|
||||
"/etc/passwd",
|
||||
"/etc/shadow",
|
||||
"shutdown",
|
||||
"reboot",
|
||||
"init 0",
|
||||
"init 6",
|
||||
"poweroff",
|
||||
"halt",
|
||||
];
|
||||
|
||||
/// Maximum hook script size (64KB).
|
||||
const MAX_HOOK_SIZE: usize = 65536;
|
||||
|
||||
/// Validate a custom hook script content for safety.
|
||||
pub fn validate_hook_content(content: &str) -> GitResult<()> {
|
||||
if content.is_empty() {
|
||||
return Err(GitError::InvalidArgument(
|
||||
"hook content cannot be empty".into(),
|
||||
));
|
||||
}
|
||||
if content.len() > MAX_HOOK_SIZE {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"hook content too large (max {} bytes): {} bytes",
|
||||
MAX_HOOK_SIZE,
|
||||
content.len()
|
||||
)));
|
||||
}
|
||||
let content_lower = content.to_lowercase();
|
||||
for pattern in FORBIDDEN_PATTERNS {
|
||||
if content_lower.contains(pattern) {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"hook content contains forbidden pattern: '{pattern}'"
|
||||
)));
|
||||
}
|
||||
}
|
||||
if content.contains('\0') {
|
||||
return Err(GitError::InvalidArgument(
|
||||
"hook content cannot contain null bytes".into(),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate a hook name (must be a recognized git hook name).
|
||||
pub fn validate_hook_name(name: &str) -> GitResult<()> {
|
||||
const VALID_HOOK_NAMES: &[&str] = &[
|
||||
"pre-receive",
|
||||
"update",
|
||||
"post-receive",
|
||||
"pre-applypatch",
|
||||
"applypatch-msg",
|
||||
"post-applypatch",
|
||||
"pre-commit",
|
||||
"prepare-commit-msg",
|
||||
"commit-msg",
|
||||
"post-commit",
|
||||
"pre-auto-gc",
|
||||
];
|
||||
if !VALID_HOOK_NAMES.contains(&name) {
|
||||
return Err(GitError::InvalidArgument(format!(
|
||||
"invalid hook name: '{name}'. Must be one of: {}",
|
||||
VALID_HOOK_NAMES.join(", ")
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user