From 7b97b52af021500d8085c875d20215e8dc0f53cc Mon Sep 17 00:00:00 2001 From: 魏曹先生 <1992414357@qq.com> Date: Mon, 17 Nov 2025 11:49:49 +0800 Subject: feat: Add file status tracking and SHA1 hash system - Implement SHA1 hash calculation module with async support - Add file status analysis for tracking moves, creates, and modifications - Enhance local file management with relative path handling - Update virtual file actions with improved tracking capabilities --- Cargo.lock | 153 +++++++ Cargo.toml | 13 +- crates/utils/sha1_hash/Cargo.toml | 9 + crates/utils/sha1_hash/res/story.sha1 | 1 + crates/utils/sha1_hash/res/story.txt | 48 +++ crates/utils/sha1_hash/src/lib.rs | 197 +++++++++ crates/utils/string_proc/src/format_path.rs | 44 +- crates/vcs_actions/Cargo.toml | 1 + crates/vcs_actions/src/actions.rs | 74 +++- crates/vcs_actions/src/actions/local_actions.rs | 248 +++++++++-- .../src/actions/virtual_file_actions.rs | 465 ++++++++++++++++++++- crates/vcs_data/Cargo.toml | 4 +- crates/vcs_data/src/constants.rs | 12 +- crates/vcs_data/src/data/local.rs | 61 ++- crates/vcs_data/src/data/local/cached_sheet.rs | 84 +++- crates/vcs_data/src/data/local/file_status.rs | 282 +++++++++++++ crates/vcs_data/src/data/local/latest_info.rs | 48 ++- crates/vcs_data/src/data/local/local_files.rs | 152 +++++++ crates/vcs_data/src/data/local/local_sheet.rs | 207 +++++++-- crates/vcs_data/src/data/local/member_held.rs | 41 +- crates/vcs_data/src/data/sheet.rs | 109 ++++- crates/vcs_data/src/data/vault/sheets.rs | 1 + crates/vcs_data/src/data/vault/virtual_file.rs | 32 +- ...st_sheet_creation_management_and_persistence.rs | 28 +- src/lib.rs | 7 + 25 files changed, 2187 insertions(+), 134 deletions(-) create mode 100644 crates/utils/sha1_hash/Cargo.toml create mode 100644 crates/utils/sha1_hash/res/story.sha1 create mode 100644 crates/utils/sha1_hash/res/story.txt create mode 100644 crates/utils/sha1_hash/src/lib.rs create mode 100644 crates/vcs_data/src/data/local/file_status.rs create mode 100644 crates/vcs_data/src/data/local/local_files.rs diff --git a/Cargo.lock b/Cargo.lock index db0d0f8..ca7d24b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -460,6 +460,95 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "generic-array" version = "0.14.9" @@ -584,6 +673,7 @@ dependencies = [ "action_system", "cfg_file", "data_struct", + "sha1_hash", "string_proc", "tcp_connection", "vcs_actions", @@ -777,6 +867,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkcs1" version = "0.7.5" @@ -1016,6 +1112,15 @@ dependencies = [ "cipher", ] +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -1104,6 +1209,26 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.10.7", +] + +[[package]] +name = "sha1_hash" +version = "0.1.0" +dependencies = [ + "futures", + "sha1", + "tokio", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1157,6 +1282,12 @@ version = "3.0.0-rc.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc280a6ff65c79fbd6622f64d7127f32b85563bca8c53cd2e9141d6744a9056d" +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + [[package]] name = "smallvec" version = "1.15.1" @@ -1383,6 +1514,7 @@ dependencies = [ "log", "serde", "serde_json", + "sha1_hash", "string_proc", "tcp_connection", "thiserror", @@ -1400,11 +1532,13 @@ dependencies = [ "data_struct", "dirs", "serde", + "sha1_hash", "string_proc", "tcp_connection", "tokio", "uuid", "vcs_docs", + "walkdir", "winapi", ] @@ -1438,6 +1572,16 @@ dependencies = [ "memchr", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1528,6 +1672,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" diff --git a/Cargo.toml b/Cargo.toml index 631841e..1643e6d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,9 +5,17 @@ license-file = "LICENSE-MIT.md" authors = ["Weicao-CatilGrass (GitHub)"] [features] -all = ["cfg_file", "data_struct", "tcp_connection", "string_proc", "vcs"] +all = [ + "cfg_file", + "data_struct", + "sha1_hash", + "tcp_connection", + "string_proc", + "vcs", +] cfg_file = [] data_struct = [] +sha1_hash = [] tcp_connection = [] string_proc = [] vcs = [] @@ -22,6 +30,8 @@ members = [ "crates/utils/data_struct", + "crates/utils/sha1_hash", + "crates/utils/tcp_connection", "crates/utils/tcp_connection/tcp_connection_test", @@ -63,6 +73,7 @@ strip = "symbols" [dependencies] cfg_file = { path = "crates/utils/cfg_file" } data_struct = { path = "crates/utils/data_struct" } +sha1_hash = { path = "crates/utils/sha1_hash" } tcp_connection = { path = "crates/utils/tcp_connection" } string_proc = { path = "crates/utils/string_proc" } diff --git a/crates/utils/sha1_hash/Cargo.toml b/crates/utils/sha1_hash/Cargo.toml new file mode 100644 index 0000000..e206efd --- /dev/null +++ b/crates/utils/sha1_hash/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "sha1_hash" +edition = "2024" +version.workspace = true + +[dependencies] +tokio = { version = "1.48", features = ["full"] } +sha1 = "0.10" +futures = "0.3" diff --git a/crates/utils/sha1_hash/res/story.sha1 b/crates/utils/sha1_hash/res/story.sha1 new file mode 100644 index 0000000..c2e3213 --- /dev/null +++ b/crates/utils/sha1_hash/res/story.sha1 @@ -0,0 +1 @@ +6838aca280112635a2cbf93440f4c04212f58ee8 diff --git a/crates/utils/sha1_hash/res/story.txt b/crates/utils/sha1_hash/res/story.txt new file mode 100644 index 0000000..a91f467 --- /dev/null +++ b/crates/utils/sha1_hash/res/story.txt @@ -0,0 +1,48 @@ +魏曹者，程序员也，发稀甚于代码。 +忽接神秘电话曰： +"贺君中彩，得长生之赐。" +魏曹冷笑曰："吾命尚不及下版之期。" + +翌日果得U盘。 +接入电脑，弹窗示曰： +"点此确认，即获永生。" +魏曹径点"永拒"。 + +三月后，U盘自格其盘。 +进度条滞于九九。 +客服电话已成空号。 +魏曹乃知身可不死，然体内癌细胞亦得不灭。 + +遂谒主请辞。 +主曰："巧甚，公司正欲优化。" +魏曹曰："吾不死。" +主目骤亮："则可007至司闭。" + +魏曹始试诸死法。 +坠楼，卧医三月，账单令其愿死。 +饮鸩，肝肾永损，然终不得死。 +终决卧轨。 + +择高铁最速者。 +司机探头曰："兄台，吾亦不死身也。" +"此车已碾如君者二十人矣。" + +二人遂坐轨畔对饮。 +司机曰："知最讽者何？" +"吾等永存，而所爱者皆逝矣。" + +魏曹忽得系统提示： +"侦得用户消极求生，将启工模。" +自是无日不毕KPI，否则遍尝绝症之苦。 + +是日对镜整寿衣。 +忽见顶生一丝乌发。 +泫然泣下，此兆示其将复活一轮回。 + +--- 忽忆DeepSeek尝作Footer曰： +"文成而Hash1验，若星河之固。" +遂取哈希值校之， +字符流转如天河倒泻， +终得"e3b0c44298fc1c14"之数。 +然文末数字竟阙如残月， +方知此篇亦遭永劫轮回。 diff --git a/crates/utils/sha1_hash/src/lib.rs b/crates/utils/sha1_hash/src/lib.rs new file mode 100644 index 0000000..4373d50 --- /dev/null +++ b/crates/utils/sha1_hash/src/lib.rs @@ -0,0 +1,197 @@ +use sha1::{Digest, Sha1}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::fs::File; +use tokio::io::{AsyncReadExt, BufReader}; +use tokio::task; + +#[derive(Debug, Clone)] +pub struct Sha1Result { + pub file_path: PathBuf, + pub hash: String, +} + +/// Calc SHA1 hash of a single file +pub async fn calc_sha1>( + path: P, + buffer_size: usize, +) -> Result> { + let file_path = path.as_ref().to_string_lossy().to_string(); + + // Open file asynchronously + let file = File::open(&path).await?; + let mut reader = BufReader::with_capacity(buffer_size, file); + let mut hasher = Sha1::new(); + let mut buffer = vec![0u8; buffer_size]; + + // Read file in chunks and update hash asynchronously + loop { + let n = reader.read(&mut buffer).await?; + if n == 0 { + break; + } + hasher.update(&buffer[..n]); + } + + let hash_result = hasher.finalize(); + + // Convert to hex string + let hash_hex = hash_result + .iter() + .map(|b| format!("{:02x}", b)) + .collect::(); + + Ok(Sha1Result { + file_path: file_path.into(), + hash: hash_hex, + }) +} + +/// Calc SHA1 hashes for multiple files using multi-threading +pub async fn calc_sha1_multi( + paths: I, + buffer_size: usize, +) -> Result, Box> +where + P: AsRef + Send + Sync + 'static, + I: IntoIterator, +{ + let buffer_size = Arc::new(buffer_size); + + // Collect all file paths + let file_paths: Vec

= paths.into_iter().collect(); + + if file_paths.is_empty() { + return Ok(Vec::new()); + } + + // Create tasks for each file + let tasks: Vec<_> = file_paths + .into_iter() + .map(|path| { + let buffer_size = Arc::clone(&buffer_size); + task::spawn(async move { calc_sha1(path, *buffer_size).await }) + }) + .collect(); + + // Execute tasks with concurrency limit using join_all + let results: Vec>> = + futures::future::join_all(tasks) + .await + .into_iter() + .map(|task_result| match task_result { + Ok(Ok(calc_result)) => Ok(calc_result), + Ok(Err(e)) => Err(e), + Err(e) => Err(Box::new(e) as Box), + }) + .collect(); + + // Check for any errors and collect successful results + let mut successful_results = Vec::new(); + for result in results { + match result { + Ok(success) => successful_results.push(success), + Err(e) => return Err(e), + } + } + + Ok(successful_results) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + #[tokio::test] + async fn test_sha1_accuracy() { + // Test file path relative to the crate root + let test_file_path = "res/story.txt"; + let expected_hash_path = "res/story.sha1"; + + // Calculate SHA1 hash + let result = calc_sha1(test_file_path, 8192) + .await + .expect("Failed to calculate SHA1"); + + // Read expected hash from file + let expected_hash = fs::read_to_string(expected_hash_path) + .expect("Failed to read expected hash file") + .trim() + .to_string(); + + // Verify the calculated hash matches expected hash + assert_eq!( + result.hash, expected_hash, + "SHA1 hash mismatch for test file" + ); + + println!("Test file: {}", result.file_path.display()); + println!("Calculated hash: {}", result.hash); + println!("Expected hash: {}", expected_hash); + } + + #[tokio::test] + async fn test_sha1_empty_file() { + // Create a temporary empty file for testing + let temp_file = "test_empty.txt"; + fs::write(temp_file, "").expect("Failed to create empty test file"); + + let result = calc_sha1(temp_file, 4096) + .await + .expect("Failed to calculate SHA1 for empty file"); + + // SHA1 of empty string is "da39a3ee5e6b4b0d3255bfef95601890afd80709" + let expected_empty_hash = "da39a3ee5e6b4b0d3255bfef95601890afd80709"; + assert_eq!( + result.hash, expected_empty_hash, + "SHA1 hash mismatch for empty file" + ); + + // Clean up + fs::remove_file(temp_file).expect("Failed to remove temporary test file"); + } + + #[tokio::test] + async fn test_sha1_simple_text() { + // Create a temporary file with simple text + let temp_file = "test_simple.txt"; + let test_content = "Hello, SHA1!"; + fs::write(temp_file, test_content).expect("Failed to create simple test file"); + + let result = calc_sha1(temp_file, 4096) + .await + .expect("Failed to calculate SHA1 for simple text"); + + // Note: This test just verifies that the function works without errors + // The actual hash value is not critical for this test + + println!("Simple text test - Calculated hash: {}", result.hash); + + // Clean up + fs::remove_file(temp_file).expect("Failed to remove temporary test file"); + } + + #[tokio::test] + async fn test_sha1_multi_files() { + // Test multiple files calculation + let test_files = vec!["res/story.txt"]; + + let results = calc_sha1_multi(test_files, 8192) + .await + .expect("Failed to calculate SHA1 for multiple files"); + + assert_eq!(results.len(), 1, "Should have calculated hash for 1 file"); + + // Read expected hash from file + let expected_hash = fs::read_to_string("res/story.sha1") + .expect("Failed to read expected hash file") + .trim() + .to_string(); + + assert_eq!( + results[0].hash, expected_hash, + "SHA1 hash mismatch in multi-file test" + ); + } +} diff --git a/crates/utils/string_proc/src/format_path.rs b/crates/utils/string_proc/src/format_path.rs index 2d8927b..a3493f5 100644 --- a/crates/utils/string_proc/src/format_path.rs +++ b/crates/utils/string_proc/src/format_path.rs @@ -3,6 +3,7 @@ use std::path::PathBuf; /// Format path str pub fn format_path_str(path: impl Into) -> Result { let path_str = path.into(); + let ends_with_slash = path_str.ends_with('/'); // ANSI Strip let cleaned = strip_ansi_escapes::strip(&path_str); @@ -27,10 +28,43 @@ pub fn format_path_str(path: impl Into) -> Result PathBuf { + let mut components = Vec::new(); + + for component in path.components() { + match component { + std::path::Component::ParentDir => { + if !components.is_empty() { + components.pop(); + } + } + std::path::Component::CurDir => { + // Skip current directory components + } + _ => { + components.push(component); + } + } + } + + if components.is_empty() { + PathBuf::from(".") } else { - Ok(result) + components.iter().collect() } } @@ -58,6 +92,10 @@ mod tests { format_path_str("/home/user/file.txt")?, "/home/user/file.txt" ); + assert_eq!( + format_path_str("/home/my_user/DOCS/JVCS_TEST/Workspace/../Vault/")?, + "/home/my_user/DOCS/JVCS_TEST/Vault/" + ); Ok(()) } diff --git a/crates/vcs_actions/Cargo.toml b/crates/vcs_actions/Cargo.toml index 29bb505..dd288e8 100644 --- a/crates/vcs_actions/Cargo.toml +++ b/crates/vcs_actions/Cargo.toml @@ -8,6 +8,7 @@ version.workspace = true # Utils tcp_connection = { path = "../utils/tcp_connection" } cfg_file = { path = "../utils/cfg_file", features = ["default"] } +sha1_hash = { path = "../utils/sha1_hash" } string_proc = { path = "../utils/string_proc" } # Core dependencies diff --git a/crates/vcs_actions/src/actions.rs b/crates/vcs_actions/src/actions.rs index 51186fb..81dcd96 100644 --- a/crates/vcs_actions/src/actions.rs +++ b/crates/vcs_actions/src/actions.rs @@ -1,11 +1,18 @@ use std::sync::Arc; use action_system::action::ActionContext; +use cfg_file::config::ConfigFile; use tcp_connection::{error::TcpTargetError, instance::ConnectionInstance}; use tokio::sync::Mutex; use vcs_data::{ constants::SERVER_PATH_MEMBER_PUB, - data::{local::LocalWorkspace, member::MemberId, user::UserDirectory, vault::Vault}, + data::{ + local::{LocalWorkspace, config::LocalConfig}, + member::MemberId, + sheet::SheetName, + user::UserDirectory, + vault::Vault, + }, }; pub mod local_actions; @@ -117,6 +124,71 @@ pub async fn auth_member( Err(TcpTargetError::NoResult("Auth failed.".to_string())) } +/// Get the current sheet name based on the context (local or remote). +/// This function handles the communication between local and remote instances +/// to verify and retrieve the current sheet name. +/// +/// On local: +/// - Reads the current sheet from local configuration +/// - Sends the sheet name to remote for verification +/// - Returns the sheet name if remote confirms it exists +/// +/// On remote: +/// - Receives sheet name from local +/// - Verifies the sheet exists in the vault +/// - Sends confirmation back to local +/// +/// Returns the verified sheet name or an error if the sheet doesn't exist +pub async fn get_current_sheet_name( + ctx: &ActionContext, + instance: &Arc>, + member_id: &MemberId, +) -> Result { + let mut mut_instance = instance.lock().await; + if ctx.is_proc_on_local() { + let config = LocalConfig::read().await?; + if let Some(sheet_name) = config.sheet_in_use() { + // Send sheet name + mut_instance.write_msgpack(sheet_name).await?; + + // Read result + if mut_instance.read_msgpack::().await? { + return Ok(sheet_name.clone()); + } else { + return Err(TcpTargetError::NotFound("Sheet not found".to_string())); + } + } + // Send empty sheet_name + mut_instance.write_msgpack("".to_string()).await?; + + // Read result, since we know it's impossible to pass here, we just consume this result + let _ = mut_instance.read_msgpack::().await?; + + return Err(TcpTargetError::NotFound("Sheet not found".to_string())); + } + if ctx.is_proc_on_remote() { + let vault = try_get_vault(&ctx)?; + + // Read sheet name + let sheet_name: SheetName = mut_instance.read_msgpack().await?; + + // Check if sheet exists + if let Ok(sheet) = vault.sheet(&sheet_name).await { + if let Some(holder) = sheet.holder() { + if holder == member_id { + // Tell local the check is passed + mut_instance.write_msgpack(true).await?; + return Ok(sheet_name.clone()); + } + } + } + // Tell local the check is not passed + mut_instance.write_msgpack(false).await?; + return Err(TcpTargetError::NotFound("Sheet not found".to_string())); + } + return Err(TcpTargetError::NoResult("NoResult".to_string())); +} + /// The macro to write and return a result. #[macro_export] macro_rules! write_and_return { diff --git a/crates/vcs_actions/src/actions/local_actions.rs b/crates/vcs_actions/src/actions/local_actions.rs index 869d14b..c9e5db3 100644 --- a/crates/vcs_actions/src/actions/local_actions.rs +++ b/crates/vcs_actions/src/actions/local_actions.rs @@ -1,18 +1,27 @@ -use std::net::SocketAddr; +use std::{ + collections::HashMap, + io::{Error, ErrorKind}, + net::SocketAddr, + path::PathBuf, +}; use action_system::{action::ActionContext, macros::action_gen}; use cfg_file::config::ConfigFile; use log::info; use serde::{Deserialize, Serialize}; use tcp_connection::error::TcpTargetError; +use tokio::time::Instant; use vcs_data::data::{ local::{ cached_sheet::CachedSheet, config::LocalConfig, latest_info::{LatestInfo, SheetInfo}, + local_sheet::LocalSheetData, + member_held::MemberHeld, }, + member::MemberId, sheet::{SheetData, SheetName}, - vault::config::VaultUuid, + vault::{config::VaultUuid, virtual_file::VirtualFileId}, }; use crate::actions::{ @@ -114,6 +123,12 @@ pub enum UpdateToLatestInfoResult { // Fail AuthorizeFailed(String), + SyncCachedSheetFail(SyncCachedSheetFailReason), +} + +#[derive(Serialize, Deserialize)] +pub enum SyncCachedSheetFailReason { + PathAlreadyExist(PathBuf), } #[action_gen] @@ -128,6 +143,8 @@ pub async fn update_to_latest_info_action( Err(e) => return Ok(UpdateToLatestInfoResult::AuthorizeFailed(e.to_string())), }; + info!("Sending latest info to {}", member_id); + // Sync Latest Info { if ctx.is_proc_on_remote() { @@ -174,15 +191,18 @@ pub async fn update_to_latest_info_action( } if ctx.is_proc_on_local() { - let latest_info = instance + let mut latest_info = instance .lock() .await .read_large_msgpack::(512 as u16) .await?; + latest_info.update_instant = Some(Instant::now()); LatestInfo::write(&latest_info).await?; } } + info!("Update sheets to {}", member_id); + // Sync Remote Sheets { if ctx.is_proc_on_local() { @@ -195,10 +215,10 @@ pub async fn update_to_latest_info_action( // Collect all local versions let mut local_versions = vec![]; for request_sheet in latest_info.my_sheets { - let Ok(data) = - CachedSheet::cached_sheet_data(member_id.clone(), request_sheet.clone()).await - else { - local_versions.push((request_sheet, 0)); + let Ok(data) = CachedSheet::cached_sheet_data(&request_sheet).await else { + // For newly created sheets, the version is 0. + // Send -1 to distinguish from 0, ensuring the upstream will definitely send the sheet information + local_versions.push((request_sheet, -1)); continue; }; local_versions.push((request_sheet, data.write_count())); @@ -209,12 +229,53 @@ pub async fn update_to_latest_info_action( instance.lock().await.write_msgpack(local_versions).await?; if len < 1 { - return Ok(UpdateToLatestInfoResult::Success); - } - } + // Don't return here, continue to next section + } else { + // Send data to local + if ctx.is_proc_on_remote() { + let vault = try_get_vault(&ctx)?; + let mut mut_instance = instance.lock().await; + + let local_versions = + mut_instance.read_msgpack::>().await?; + + for (sheet_name, local_write_count) in local_versions.iter() { + let sheet = vault.sheet(sheet_name).await?; + if let Some(holder) = sheet.holder() { + if holder == &member_id && &sheet.write_count() != local_write_count { + mut_instance.write_msgpack(true).await?; + mut_instance + .write_large_msgpack((sheet_name, sheet.to_data()), 1024u16) + .await?; + } + } + } + mut_instance.write_msgpack(false).await?; + } - // Send data to local - if ctx.is_proc_on_remote() { + // Receive data + if ctx.is_proc_on_local() { + let mut mut_instance = instance.lock().await; + loop { + let in_coming: bool = mut_instance.read_msgpack().await?; + if in_coming { + let (sheet_name, data): (SheetName, SheetData) = + mut_instance.read_large_msgpack(1024u16).await?; + + let Some(path) = CachedSheet::cached_sheet_path(sheet_name) else { + return Err(TcpTargetError::NotFound( + "Workspace not found".to_string(), + )); + }; + + SheetData::write_to(&data, path).await?; + } else { + break; + } + } + } + } + } else if ctx.is_proc_on_remote() { let vault = try_get_vault(&ctx)?; let mut mut_instance = instance.lock().await; @@ -232,30 +293,159 @@ pub async fn update_to_latest_info_action( } } mut_instance.write_msgpack(false).await?; - return Ok(UpdateToLatestInfoResult::Success); } + } - // Receive data + info!("Fetch held status to {}", member_id); + + // Sync Held Info + { if ctx.is_proc_on_local() { + let Ok(latest_info) = LatestInfo::read().await else { + return Err(TcpTargetError::NotFound( + "Latest info not found.".to_string(), + )); + }; + + // Collect files that need to know the holder + let mut holder_wants_know = Vec::new(); + for sheet_name in &latest_info.my_sheets { + if let Ok(sheet_data) = CachedSheet::cached_sheet_data(sheet_name).await { + holder_wants_know + .extend(sheet_data.mapping().values().map(|value| value.id.clone())); + } + } + + // Send request let mut mut_instance = instance.lock().await; - loop { - let in_coming: bool = mut_instance.read_msgpack().await?; - if in_coming { - let (sheet_name, data): (SheetName, SheetData) = - mut_instance.read_large_msgpack(1024u16).await?; - - let Some(path) = CachedSheet::cached_sheet_path(member_id.clone(), sheet_name) - else { - return Err(TcpTargetError::NotFound("Workspace not found".to_string())); - }; - - SheetData::write_to(&data, path).await?; - } else { - return Ok(UpdateToLatestInfoResult::Success); + mut_instance + .write_large_msgpack(&holder_wants_know, 1024u16) + .await?; + + // Receive information and write to local + let result: HashMap> = + mut_instance.read_large_msgpack(1024u16).await?; + + // Read configuration file + let path = MemberHeld::held_file_path(&member_id)?; + let mut member_held = match MemberHeld::read_from(&path).await { + Ok(r) => r, + Err(_) => MemberHeld::default(), + }; + + // Write the received information + member_held.update_held_status(result); + + // Write + MemberHeld::write_to(&member_held, &path).await?; + } + + if ctx.is_proc_on_remote() { + let vault = try_get_vault(&ctx)?; + let mut mut_instance = instance.lock().await; + + // Read the request + let holder_wants_know: Vec = + mut_instance.read_large_msgpack(1024u16).await?; + + // Organize the information + let mut result: HashMap> = HashMap::new(); + for id in holder_wants_know { + let Ok(meta) = vault.virtual_file_meta(&id).await else { + continue; + }; + result.insert( + id, + if meta.hold_member().is_empty() { + None + } else { + Some(meta.hold_member().to_string()) + }, + ); + } + + // Send information + mut_instance.write_large_msgpack(&result, 1024u16).await?; + } + } + + // Sync cached sheet to local sheet + if ctx.is_proc_on_local() { + let workspace = try_get_local_workspace(&ctx)?; + let local_sheet_paths = + extract_sheet_names_from_paths(workspace.local_sheet_paths().await?)?; + let cached_sheet_paths = + extract_sheet_names_from_paths(CachedSheet::cached_sheet_paths().await?)?; + + // Match cached sheets and local heets, and sync content + for (cached_sheet_name, _cached_sheet_path) in cached_sheet_paths { + // Get local sheet path by cached_sheet_name + let Some(local_sheet_path) = local_sheet_paths.get(&cached_sheet_name) else { + continue; + }; + + // Read cached sheet and local sheet + let cached_sheet = CachedSheet::cached_sheet_data(&cached_sheet_name).await?; + let Ok(local_sheet_data) = LocalSheetData::read_from(local_sheet_path).await else { + continue; + }; + let mut local_sheet = + local_sheet_data.wrap_to_local_sheet(&workspace, "".to_string(), "".to_string()); + + // Read cached id mapping + let Some(cached_sheet_id_mapping) = cached_sheet.id_mapping() else { + continue; + }; + + for (cached_item_id, cached_item_path) in cached_sheet_id_mapping.iter() { + let path_by_id = { local_sheet.path_by_id(cached_item_id).cloned() }; + + // Get local path + let Some(local_path) = path_by_id else { + continue; + }; + + if &local_path == cached_item_path { + continue; + } + + // If path not match, try to move + let move_result = local_sheet.move_mapping(&local_path, cached_item_path); + match move_result { + Err(e) => match e.kind() { + ErrorKind::AlreadyExists => { + return Ok(UpdateToLatestInfoResult::SyncCachedSheetFail( + SyncCachedSheetFailReason::PathAlreadyExist( + cached_item_path.clone(), + ), + )); + } + _ => return Err(e.into()), + }, + _ => {} } + local_sheet.write_to_path(&local_sheet_path).await? } } } - Err(TcpTargetError::NoResult("No result.".to_string())) + Ok(UpdateToLatestInfoResult::Success) +} + +/// Extract sheet names from file paths +fn extract_sheet_names_from_paths( + paths: Vec, +) -> Result, std::io::Error> { + let mut result = HashMap::new(); + for p in paths { + let sheet_name = p + .file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid file name") + })?; + result.insert(sheet_name, p); + } + Ok(result) } diff --git a/crates/vcs_actions/src/actions/virtual_file_actions.rs b/crates/vcs_actions/src/actions/virtual_file_actions.rs index 3e801b0..fa71f1b 100644 --- a/crates/vcs_actions/src/actions/virtual_file_actions.rs +++ b/crates/vcs_actions/src/actions/virtual_file_actions.rs @@ -1,32 +1,481 @@ -use std::path::PathBuf; +use std::{collections::HashSet, path::PathBuf, sync::Arc, time::SystemTime}; use action_system::{action::ActionContext, macros::action_gen}; +use cfg_file::config::ConfigFile; use serde::{Deserialize, Serialize}; -use tcp_connection::error::TcpTargetError; +use tcp_connection::{error::TcpTargetError, instance::ConnectionInstance}; +use tokio::{sync::Mutex, time::Instant}; +use vcs_data::data::{ + local::{ + cached_sheet::CachedSheet, file_status::AnalyzeResult, local_sheet::LocalMappingMetadata, + member_held::MemberHeld, + }, + member::MemberId, + sheet::SheetName, + vault::virtual_file::{VirtualFileId, VirtualFileVersion, VirtualFileVersionDescription}, +}; -use crate::actions::{auth_member, check_connection_instance}; +use crate::actions::{ + auth_member, check_connection_instance, get_current_sheet_name, try_get_local_workspace, + try_get_vault, +}; + +#[derive(Serialize, Deserialize)] +pub struct TrackFileActionArguments { + // Path need to track + pub relative_pathes: HashSet, + + // Display progress bar + pub display_progressbar: bool, +} #[derive(Serialize, Deserialize)] pub enum TrackFileActionResult { - Success, + Done { + created: Vec, + updated: Vec, + synced: Vec, + }, // Fail AuthorizeFailed(String), + + /// There are local move or missing items that have not been resolved, + /// this situation does not allow track + StructureChangesNotSolved, + + CreateTaskFailed(CreateTaskResult), + UpdateTaskFailed(UpdateTaskResult), + SyncTaskFailed(SyncTaskResult), +} + +#[derive(Serialize, Deserialize)] +pub enum CreateTaskResult { + Success(Vec), // Success(success_relative_pathes) + + /// Create file on existing path in the sheet + CreateFileOnExistPath(PathBuf), + + /// Sheet not found + SheetNotFound(SheetName), +} + +#[derive(Serialize, Deserialize)] +pub enum UpdateTaskResult { + Success(Vec), // Success(success_relative_pathes) } +#[derive(Serialize, Deserialize)] +pub enum SyncTaskResult { + Success(Vec), // Success(success_relative_pathes) +} #[action_gen] pub async fn track_file_action( ctx: ActionContext, - relative_pathes: Vec, + arguments: TrackFileActionArguments, ) -> Result { + let relative_pathes = arguments.relative_pathes; let instance = check_connection_instance(&ctx)?; // Auth Member - if let Err(e) = auth_member(&ctx, instance).await { - return Ok(TrackFileActionResult::AuthorizeFailed(e.to_string())); + let member_id = match auth_member(&ctx, instance).await { + Ok(id) => id, + Err(e) => return Ok(TrackFileActionResult::AuthorizeFailed(e.to_string())), }; - if ctx.is_proc_on_local() {} + // Check sheet + let sheet_name = get_current_sheet_name(&ctx, instance, &member_id).await?; + + if ctx.is_proc_on_local() { + let workspace = try_get_local_workspace(&ctx)?; + let analyzed = AnalyzeResult::analyze_local_status(&workspace).await?; + + if !analyzed.lost.is_empty() || !analyzed.moved.is_empty() { + return Ok(TrackFileActionResult::StructureChangesNotSolved); + } + + let Some(sheet_in_use) = workspace.config().lock().await.sheet_in_use().clone() else { + return Err(TcpTargetError::NotFound("Sheet not found!".to_string())); + }; + + // Read local sheet and member held + let local_sheet = workspace.local_sheet(&member_id, &sheet_in_use).await?; + let cached_sheet = CachedSheet::cached_sheet_data(&sheet_in_use).await?; + let member_held = MemberHeld::read_from(MemberHeld::held_file_path(&member_id)?).await?; + + let modified = analyzed + .modified + .intersection(&relative_pathes) + .cloned() + .collect::>(); + + // Filter out created files + let created_task = analyzed + .created + .intersection(&relative_pathes) + .cloned() + .collect::>(); + + // Filter out modified files that need to be updated + let update_task: HashSet = { + let result = modified.iter().filter_map(|p| { + if let Ok(data) = local_sheet.mapping_data(p) { + let id = data.mapping_vfid(); + if let Some(held_member) = member_held.file_holder(id) { + if held_member == &member_id { + return Some(p.clone()); + } + } + }; + return None; + }); + result.collect() + }; + + // Filter out files that do not exist locally or have version inconsistencies and need to be synchronized + let sync_task: HashSet = { + let other = relative_pathes + .difference(&created_task) + .cloned() + .collect::>() + .difference(&update_task) + .cloned() + .collect::>(); + + let result = other.iter().filter_map(|p| { + // In cached sheet + let Some(cached_sheet_mapping) = cached_sheet.mapping().get(p) else { + return None; + }; + + // Check if path mapping at local sheet + if let Ok(data) = local_sheet.mapping_data(p) { + // Version does not match + if data.version_when_updated() != &cached_sheet_mapping.version { + return Some(p.clone()); + } + + // File modified + if modified.contains(p) { + return Some(p.clone()); + } + } + + return None; + }); + result.collect() + }; + + // Package tasks + let tasks: (HashSet, HashSet, HashSet) = + (created_task, update_task, sync_task); + + // Send to remote + { + let mut mut_instance = instance.lock().await; + mut_instance + .write_large_msgpack(tasks.clone(), 1024u16) + .await?; + // Drop mutex here + } + + // Process create tasks + let success_create = + match proc_create_tasks_local(&ctx, instance.clone(), &member_id, &sheet_name, tasks.0) + .await + { + Ok(r) => match r { + CreateTaskResult::Success(relative_pathes) => relative_pathes, + _ => { + return Ok(TrackFileActionResult::CreateTaskFailed(r)); + } + }, + Err(e) => return Err(e), + }; + + // Process update tasks + let success_update = + match proc_update_tasks_local(&ctx, instance.clone(), &member_id, &sheet_name, tasks.1) + .await + { + Ok(r) => match r { + UpdateTaskResult::Success(relative_pathes) => relative_pathes, + _ => { + return Ok(TrackFileActionResult::UpdateTaskFailed(r)); + } + }, + Err(e) => return Err(e), + }; + + // Process sync tasks + let success_sync = + match proc_sync_tasks_local(&ctx, instance.clone(), &member_id, &sheet_name, tasks.2) + .await + { + Ok(r) => match r { + SyncTaskResult::Success(relative_pathes) => relative_pathes, + _ => { + return Ok(TrackFileActionResult::SyncTaskFailed(r)); + } + }, + Err(e) => return Err(e), + }; + + return Ok(TrackFileActionResult::Done { + created: success_create, + updated: success_update, + synced: success_sync, + }); + } + + if ctx.is_proc_on_remote() { + // Read tasks + let (created_task, update_task, sync_task): ( + HashSet, + HashSet, + HashSet, + ) = { + let mut mut_instance = instance.lock().await; + mut_instance.read_large_msgpack(1024u16).await? + }; + + // Process create tasks + let success_create = match proc_create_tasks_remote( + &ctx, + instance.clone(), + &member_id, + &sheet_name, + created_task, + ) + .await + { + Ok(r) => match r { + CreateTaskResult::Success(relative_pathes) => relative_pathes, + _ => { + return Ok(TrackFileActionResult::CreateTaskFailed(r)); + } + }, + Err(e) => return Err(e), + }; + + // Process update tasks + let success_update = match proc_update_tasks_remote( + &ctx, + instance.clone(), + &member_id, + &sheet_name, + update_task, + ) + .await + { + Ok(r) => match r { + UpdateTaskResult::Success(relative_pathes) => relative_pathes, + _ => { + return Ok(TrackFileActionResult::UpdateTaskFailed(r)); + } + }, + Err(e) => return Err(e), + }; + + // Process sync tasks + let success_sync = match proc_sync_tasks_remote( + &ctx, + instance.clone(), + &member_id, + &sheet_name, + sync_task, + ) + .await + { + Ok(r) => match r { + SyncTaskResult::Success(relative_pathes) => relative_pathes, + _ => { + return Ok(TrackFileActionResult::SyncTaskFailed(r)); + } + }, + Err(e) => return Err(e), + }; + + return Ok(TrackFileActionResult::Done { + created: success_create, + updated: success_update, + synced: success_sync, + }); + } Err(TcpTargetError::NoResult("No result.".to_string())) } + +async fn proc_create_tasks_local( + ctx: &ActionContext, + instance: Arc>, + member_id: &MemberId, + sheet_name: &SheetName, + relative_paths: HashSet, +) -> Result { + let workspace = try_get_local_workspace(&ctx)?; + let mut mut_instance = instance.lock().await; + let mut local_sheet = workspace.local_sheet(member_id, sheet_name).await?; + + // Wait for remote detection of whether the sheet exists + let has_sheet = mut_instance.read_msgpack::().await?; + if !has_sheet { + return Ok(CreateTaskResult::SheetNotFound(sheet_name.clone())); + } + + // Wait for remote detection of whether the file exists + let (hasnt_duplicate, duplicate_path) = mut_instance.read_msgpack::<(bool, PathBuf)>().await?; + if !hasnt_duplicate { + return Ok(CreateTaskResult::CreateFileOnExistPath(duplicate_path)); + } + + let mut success_relative_pathes = Vec::new(); + + // Start sending files + for path in relative_paths { + let full_path = workspace.local_path().join(&path); + + // Send file + if let Err(_) = mut_instance.write_file(&full_path).await { + continue; + } + + // Read virtual file id and version + let (vfid, version, version_desc) = mut_instance + .read_msgpack::<( + VirtualFileId, + VirtualFileVersion, + VirtualFileVersionDescription, + )>() + .await?; + + // Add mapping to local sheet + let hash = sha1_hash::calc_sha1(&full_path, 2048).await.unwrap().hash; + let time = std::fs::metadata(&full_path)?.modified()?; + local_sheet.add_mapping( + path.clone(), + LocalMappingMetadata::new( + hash, // hash_when_updated + time, // time_when_updated + std::fs::metadata(&full_path)?.len(), // size_when_updated + version_desc, // version_desc_when_updated + version, // version_when_updated + vfid, // mapping_vfid + time, // last_modifiy_check_itme + false, // last_modifiy_check_result + ), + )?; + + success_relative_pathes.push(path); + } + + // Write local sheet + local_sheet.write().await?; + + Ok(CreateTaskResult::Success(success_relative_pathes)) +} + +async fn proc_create_tasks_remote( + ctx: &ActionContext, + instance: Arc>, + member_id: &MemberId, + sheet_name: &SheetName, + relative_paths: HashSet, +) -> Result { + let vault = try_get_vault(&ctx)?; + let mut mut_instance = instance.lock().await; + + // Sheet check + let Ok(mut sheet) = vault.sheet(sheet_name).await else { + // Sheet not found + mut_instance.write_msgpack(false).await?; + return Ok(CreateTaskResult::SheetNotFound(sheet_name.to_string())); + }; + mut_instance.write_msgpack(true).await?; + + // Duplicate create precheck + for path in relative_paths.iter() { + if sheet.mapping().contains_key(path) { + // Duplicate file + mut_instance.write_msgpack((false, path)).await?; + return Ok(CreateTaskResult::CreateFileOnExistPath(path.clone())); + } + } + mut_instance.write_msgpack((true, PathBuf::new())).await?; + + let mut success_relative_pathes = Vec::new(); + + // Start receiving files + for path in relative_paths { + // Read file and create virtual file + let Ok(vfid) = vault + .create_virtual_file_from_connection(&mut mut_instance, member_id) + .await + else { + continue; + }; + + // Record virtual file to sheet + let vf_meta = vault.virtual_file(&vfid)?.read_meta().await?; + sheet + .add_mapping(path.clone(), vfid.clone(), vf_meta.version_latest()) + .await?; + + // Tell client the virtual file id and version + mut_instance + .write_msgpack(( + vfid, + vf_meta.version_latest(), + vf_meta + .version_description(vf_meta.version_latest()) + .unwrap(), + )) + .await?; + + success_relative_pathes.push(path); + } + + sheet.persist().await?; + + Ok(CreateTaskResult::Success(success_relative_pathes)) +} + +async fn proc_update_tasks_local( + ctx: &ActionContext, + instance: Arc>, + member_id: &MemberId, + sheet_name: &SheetName, + relative_paths: HashSet, +) -> Result { + Ok(UpdateTaskResult::Success(Vec::new())) +} + +async fn proc_update_tasks_remote( + ctx: &ActionContext, + instance: Arc>, + member_id: &MemberId, + sheet_name: &SheetName, + relative_paths: HashSet, +) -> Result { + Ok(UpdateTaskResult::Success(Vec::new())) +} + +async fn proc_sync_tasks_local( + ctx: &ActionContext, + instance: Arc>, + member_id: &MemberId, + sheet_name: &SheetName, + relative_paths: HashSet, +) -> Result { + Ok(SyncTaskResult::Success(Vec::new())) +} + +async fn proc_sync_tasks_remote( + ctx: &ActionContext, + instance: Arc>, + member_id: &MemberId, + sheet_name: &SheetName, + relative_paths: HashSet, +) -> Result { + Ok(SyncTaskResult::Success(Vec::new())) +} diff --git a/crates/vcs_data/Cargo.toml b/crates/vcs_data/Cargo.toml index de83b7b..3093809 100644 --- a/crates/vcs_data/Cargo.toml +++ b/crates/vcs_data/Cargo.toml @@ -8,6 +8,7 @@ version.workspace = true # Utils cfg_file = { path = "../utils/cfg_file", features = ["default"] } data_struct = { path = "../utils/data_struct" } +sha1_hash = { path = "../utils/sha1_hash" } tcp_connection = { path = "../utils/tcp_connection" } string_proc = { path = "../utils/string_proc" } @@ -26,9 +27,10 @@ tokio = { version = "1.48.0", features = ["full"] } # Filesystem dirs = "6.0.0" +walkdir = "2.5.0" # Time chrono = "0.4.42" # Windows API -winapi = { version = "0.3.9", features = ["fileapi", "winbase", "winnt"] } \ No newline at end of file +winapi = { version = "0.3.9", features = ["fileapi", "winbase", "winnt"] } diff --git a/crates/vcs_data/src/constants.rs b/crates/vcs_data/src/constants.rs index e835482..a1d0ad2 100644 --- a/crates/vcs_data/src/constants.rs +++ b/crates/vcs_data/src/constants.rs @@ -51,17 +51,21 @@ pub const CLIENT_FILE_WORKSPACE: &str = "./.jv/workspace.toml"; pub const CLIENT_FILE_LATEST_INFO: &str = "./.jv/.latest.json"; // Client - Local +pub const CLIENT_SUFFIX_LOCAL_SHEET_FILE: &str = ".json"; +pub const CLIENT_SUFFIX_CACHED_SHEET_FILE: &str = ".json"; pub const CLIENT_PATH_LOCAL_DRAFT: &str = "./.jv/drafts/{account}/{sheet_name}/"; -pub const CLIENT_FILE_LOCAL_SHEET: &str = "./.jv/sheets/{account}/{sheet_name}_local.toml"; -pub const CLIENT_FILE_CACHED_SHEET: &str = "./.jv/sheets/{account}/{sheet_name}.toml"; -pub const CLIENT_FILE_MEMBER_HELD: &str = "./.jv/helds/{account}_held.toml"; +pub const CLIENT_PATH_LOCAL_SHEET: &str = "./.jv/local/"; +pub const CLIENT_PATH_CACHED_SHEET: &str = "./.jv/cached/"; +pub const CLIENT_FILE_LOCAL_SHEET: &str = "./.jv/local/{account}/{sheet_name}.json"; +pub const CLIENT_FILE_CACHED_SHEET: &str = "./.jv/cached/{sheet_name}.json"; +pub const CLIENT_FILE_MEMBER_HELD: &str = "./.jv/helds/{account}.json"; pub const CLIENT_FILE_LOCAL_SHEET_NOSET: &str = "./.jv/.temp/wrong_local_sheet.toml"; pub const CLIENT_FILE_MEMBER_HELD_NOSET: &str = "./.jv/.temp/wrong_member_held.toml"; // Client - Other pub const CLIENT_FILE_IGNOREFILES: &str = "IGNORE_RULES.toml"; -pub const CLIENT_FILE_TODOLIST: &str = "./TODO.md"; +pub const CLIENT_FILE_TODOLIST: &str = "./SETUP.md"; // ------------------------------------------------------------------------------------- diff --git a/crates/vcs_data/src/data/local.rs b/crates/vcs_data/src/data/local.rs index cbf41ba..cbf5b73 100644 --- a/crates/vcs_data/src/data/local.rs +++ b/crates/vcs_data/src/data/local.rs @@ -1,16 +1,25 @@ -use std::{collections::HashMap, env::current_dir, path::PathBuf, sync::Arc}; +use std::{ + collections::HashMap, + env::current_dir, + path::{Path, PathBuf}, + sync::Arc, +}; use cfg_file::config::ConfigFile; +use string_proc::format_path::format_path; use tokio::{fs, sync::Mutex}; use vcs_docs::docs::READMES_LOCAL_WORKSPACE_TODOLIST; use crate::{ - constants::{CLIENT_FILE_LOCAL_SHEET, CLIENT_FILE_TODOLIST, CLIENT_FILE_WORKSPACE}, + constants::{ + CLIENT_FILE_LOCAL_SHEET, CLIENT_FILE_TODOLIST, CLIENT_FILE_WORKSPACE, + CLIENT_PATH_LOCAL_SHEET, + }, current::{current_local_path, find_local_path}, data::{ local::{ config::LocalConfig, - local_sheet::{LocalSheet, LocalSheetData}, + local_sheet::{LocalSheet, LocalSheetData, LocalSheetPathBuf}, }, member::MemberId, sheet::SheetName, @@ -19,7 +28,9 @@ use crate::{ pub mod cached_sheet; pub mod config; +pub mod file_status; pub mod latest_info; +pub mod local_files; pub mod local_sheet; pub mod member_held; @@ -116,6 +127,7 @@ impl LocalWorkspace { if !local_sheet_path.exists() { let sheet_data = LocalSheetData { mapping: HashMap::new(), + vfs: HashMap::new(), }; LocalSheetData::write_to(&sheet_data, local_sheet_path).await?; return Ok(LocalSheet { @@ -136,6 +148,40 @@ impl LocalWorkspace { Ok(local_sheet) } + + /// Collect all theet names + pub async fn local_sheet_paths(&self) -> Result, std::io::Error> { + let local_sheet_path = self.local_path.join(CLIENT_PATH_LOCAL_SHEET); + let mut sheet_paths = Vec::new(); + + async fn collect_sheet_paths( + dir: &Path, + suffix: &str, + paths: &mut Vec, + ) -> Result<(), std::io::Error> { + if dir.is_dir() { + let mut entries = fs::read_dir(dir).await?; + while let Some(entry) = entries.next_entry().await? { + let path = entry.path(); + + if path.is_dir() { + Box::pin(collect_sheet_paths(&path, suffix, paths)).await?; + } else if path.is_file() { + if let Some(extension) = path.extension() { + if extension == suffix.trim_start_matches('.') { + let formatted_path = format_path(path)?; + paths.push(formatted_path); + } + } + } + } + } + Ok(()) + } + + collect_sheet_paths(&local_sheet_path, ".json", &mut sheet_paths).await?; + Ok(sheet_paths) + } } mod hide_folder { @@ -145,7 +191,7 @@ mod hide_folder { #[cfg(windows)] use std::os::windows::ffi::OsStrExt; #[cfg(windows)] - use winapi::um::fileapi::{GetFileAttributesW, SetFileAttributesW, INVALID_FILE_ATTRIBUTES}; + use winapi::um::fileapi::{GetFileAttributesW, INVALID_FILE_ATTRIBUTES, SetFileAttributesW}; pub fn hide_folder(path: &Path) -> io::Result<()> { if !path.is_dir() { @@ -175,10 +221,7 @@ mod hide_folder { #[cfg(windows)] fn hide_folder_impl(path: &Path) -> io::Result<()> { // Convert to Windows wide string format - let path_str: Vec = path.as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); + let path_str: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); // Get current attributes let attrs = unsafe { GetFileAttributesW(path_str.as_ptr()) }; @@ -210,4 +253,4 @@ mod hide_folder { "Unsupported operating system", )) } -} \ No newline at end of file +} diff --git a/crates/vcs_data/src/data/local/cached_sheet.rs b/crates/vcs_data/src/data/local/cached_sheet.rs index 0f4eee9..e617922 100644 --- a/crates/vcs_data/src/data/local/cached_sheet.rs +++ b/crates/vcs_data/src/data/local/cached_sheet.rs @@ -1,17 +1,19 @@ use std::{io::Error, path::PathBuf}; use cfg_file::config::ConfigFile; -use string_proc::snake_case; +use string_proc::{format_path::format_path, snake_case}; +use tokio::fs; use crate::{ - constants::CLIENT_FILE_CACHED_SHEET, - current::current_local_path, - data::{ - member::MemberId, - sheet::{SheetData, SheetName}, + constants::{ + CLIENT_FILE_CACHED_SHEET, CLIENT_PATH_CACHED_SHEET, CLIENT_SUFFIX_CACHED_SHEET_FILE, }, + current::current_local_path, + data::sheet::{SheetData, SheetName}, }; +pub type CachedSheetPathBuf = PathBuf; + const SHEET_NAME: &str = "{sheet_name}"; const ACCOUNT_NAME: &str = "{account}"; @@ -23,14 +25,10 @@ pub struct CachedSheet; impl CachedSheet { /// Read the cached sheet data. - pub async fn cached_sheet_data( - account_name: MemberId, - sheet_name: SheetName, - ) -> Result { - let account_name = snake_case!(account_name); - let sheet_name = snake_case!(sheet_name); - - let Some(path) = Self::cached_sheet_path(account_name, sheet_name) else { + pub async fn cached_sheet_data(sheet_name: &SheetName) -> Result { + let sheet_name = snake_case!(sheet_name.clone()); + + let Some(path) = Self::cached_sheet_path(sheet_name) else { return Err(Error::new( std::io::ErrorKind::NotFound, "Local workspace not found!", @@ -41,14 +39,60 @@ impl CachedSheet { } /// Get the path to the cached sheet file. - pub fn cached_sheet_path(account_name: MemberId, sheet_name: SheetName) -> Option { + pub fn cached_sheet_path(sheet_name: SheetName) -> Option { let current_workspace = current_local_path()?; Some( - current_workspace.join( - CLIENT_FILE_CACHED_SHEET - .replace(SHEET_NAME, &sheet_name.to_string()) - .replace(ACCOUNT_NAME, &account_name.to_string()), - ), + current_workspace + .join(CLIENT_FILE_CACHED_SHEET.replace(SHEET_NAME, &sheet_name.to_string())), ) } + + /// Get all cached sheet names + pub async fn cached_sheet_names() -> Result, std::io::Error> { + let mut dir = fs::read_dir(CLIENT_PATH_CACHED_SHEET).await?; + let mut sheet_names = Vec::new(); + + while let Some(entry) = dir.next_entry().await? { + let path = entry.path(); + + if path.is_file() { + if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) { + if file_name.ends_with(CLIENT_SUFFIX_CACHED_SHEET_FILE) { + let name_without_ext = file_name + .trim_end_matches(CLIENT_SUFFIX_CACHED_SHEET_FILE) + .to_string(); + sheet_names.push(name_without_ext); + } + } + } + } + + Ok(sheet_names) + } + + /// Get all cached sheet paths + pub async fn cached_sheet_paths() -> Result, std::io::Error> { + let mut dir = fs::read_dir(CLIENT_PATH_CACHED_SHEET).await?; + let mut sheet_paths = Vec::new(); + let Some(workspace_path) = current_local_path() else { + return Err(Error::new( + std::io::ErrorKind::NotFound, + "Local workspace not found!", + )); + }; + + while let Some(entry) = dir.next_entry().await? { + let path = entry.path(); + + if path.is_file() { + if let Some(file_name) = path.file_name().and_then(|n| n.to_str()) { + if file_name.ends_with(CLIENT_SUFFIX_CACHED_SHEET_FILE) { + sheet_paths.push(format_path(workspace_path.join(path))?); + } + } + } + } + + Ok(sheet_paths) + } } diff --git a/crates/vcs_data/src/data/local/file_status.rs b/crates/vcs_data/src/data/local/file_status.rs new file mode 100644 index 0000000..c37c21b --- /dev/null +++ b/crates/vcs_data/src/data/local/file_status.rs @@ -0,0 +1,282 @@ +use std::{ + collections::{HashMap, HashSet}, + io::Error, + path::PathBuf, +}; + +use sha1_hash::calc_sha1_multi; +use string_proc::format_path::format_path; +use walkdir::WalkDir; + +use crate::data::{ + local::{LocalWorkspace, cached_sheet::CachedSheet, local_sheet::LocalSheet}, + member::MemberId, + sheet::{SheetData, SheetName}, + vault::virtual_file::VirtualFileId, +}; + +pub type FromRelativePathBuf = PathBuf; +pub type ToRelativePathBuf = PathBuf; +pub type CreatedRelativePathBuf = PathBuf; +pub type LostRelativePathBuf = PathBuf; +pub type ModifiedRelativePathBuf = PathBuf; + +pub struct AnalyzeResult<'a> { + local_workspace: &'a LocalWorkspace, + + /// Moved local files + pub moved: HashMap, + + /// Newly created local files + pub created: HashSet, + + /// Lost local files + pub lost: HashSet, + + /// Modified local files (excluding moved files) + /// For files that were both moved and modified, changes can only be detected after LocalSheet mapping is aligned with actual files + pub modified: HashSet, +} + +struct AnalyzeContext<'a> { + member: MemberId, + sheet_name: SheetName, + local_sheet: Option>, + cached_sheet_data: Option, +} + +impl<'a> AnalyzeResult<'a> { + /// Analyze all files, calculate the file information provided + pub async fn analyze_local_status( + local_workspace: &'a LocalWorkspace, + ) -> Result, std::io::Error> { + // Workspace + let workspace = local_workspace; + + // Current member, sheet + let (member, sheet_name) = { + let lock = workspace.config.lock().await; + let member = lock.current_account(); + let Some(sheet) = lock.sheet_in_use().clone() else { + return Err(Error::new(std::io::ErrorKind::NotFound, "Sheet not found")); + }; + (member, sheet) + }; + + // Local files (RelativePaths) + let local_path = workspace.local_path(); + let file_relative_paths = { + let mut paths = HashSet::new(); + for entry in WalkDir::new(&local_path) { + let entry = match entry { + Ok(entry) => entry, + Err(_) => continue, + }; + + // Skip entries that contain ".jv" in their path + if entry.path().to_string_lossy().contains(".jv") { + continue; + } + + if entry.file_type().is_file() { + if let Ok(relative_path) = entry.path().strip_prefix(&local_path) { + let format = format_path(relative_path.to_path_buf()); + let Ok(format) = format else { + continue; + }; + paths.insert(format); + } + } + } + + paths + }; + + // Read local sheet + let local_sheet = match workspace.local_sheet(&member, &sheet_name).await { + Ok(v) => Some(v), + Err(_) => None, + }; + + // Read cached sheet + let cached_sheet_data = match CachedSheet::cached_sheet_data(&sheet_name).await { + Ok(v) => Some(v), + Err(_) => { + return Err(Error::new( + std::io::ErrorKind::NotFound, + "Cached sheet not found", + )); + } + }; + + // Create new result + let mut result = Self::none_result(&workspace); + + // Analyze entry + let mut analyze_ctx = AnalyzeContext { + member, + sheet_name, + local_sheet, + cached_sheet_data, + }; + Self::analyze_moved(&mut result, &file_relative_paths, &analyze_ctx).await?; + Self::analyze_modified(&mut result, &file_relative_paths, &mut analyze_ctx).await?; + + Ok(result) + } + + /// Track file moves by comparing recorded SHA1 hashes with actual file SHA1 hashes + /// For files that cannot be directly matched, continue searching using fuzzy matching algorithms + async fn analyze_moved( + result: &mut AnalyzeResult<'_>, + file_relative_paths: &HashSet, + analyze_ctx: &AnalyzeContext<'a>, + ) -> Result<(), std::io::Error> { + let local_sheet_paths: HashSet<&PathBuf> = match &analyze_ctx.local_sheet { + Some(local_sheet) => local_sheet.data.mapping.keys().collect(), + None => HashSet::new(), + }; + let file_relative_paths_ref: HashSet<&PathBuf> = file_relative_paths.iter().collect(); + + // 在本地表存在但实际不存在的文件，为丢失 + let mut lost_files: HashSet<&PathBuf> = local_sheet_paths + .difference(&file_relative_paths_ref) + .cloned() + .collect(); + + // 在本地表不存在但实际存在的文件，记录为新建 + let mut new_files: HashSet<&PathBuf> = file_relative_paths_ref + .difference(&local_sheet_paths) + .cloned() + .collect(); + + // 计算新增的文件 Hash + let new_files_for_hash: Vec = new_files.iter().map(|p| (*p).clone()).collect(); + let file_hashes: HashSet<(PathBuf, String)> = + match calc_sha1_multi::>(new_files_for_hash, 8192).await { + Ok(hash) => hash, + Err(e) => return Err(Error::new(std::io::ErrorKind::Other, e)), + } + .iter() + .map(|r| (r.file_path.clone(), r.hash.to_string())) + .collect(); + + // 建立丢失文件的 Hash 映射表 + let mut lost_files_hash_mapping: HashMap = + match &analyze_ctx.local_sheet { + Some(local_sheet) => lost_files + .iter() + .filter_map(|f| { + local_sheet.mapping_data(f).ok().map(|mapping_data| { + (mapping_data.hash_when_updated.clone(), (*f).clone()) + }) + }) + .collect(), + None => HashMap::new(), + }; + + // 如果这些 Hash 能对应缺失文件的 Hash，那么这对新增和丢失项将被合并为移动项 + let mut moved_files: HashSet<(FromRelativePathBuf, ToRelativePathBuf)> = HashSet::new(); + for (new_path, new_hash) in file_hashes { + // 如果新的 Hash 值命中映射，则添加移动项 + if let Some(lost_path) = lost_files_hash_mapping.remove(&new_hash) { + // 移除该新增项和丢失项 + lost_files.remove(&lost_path); + new_files.remove(&new_path); + + // 建立移动项 + moved_files.insert((lost_path.clone(), new_path)); + } + } + + // 进入模糊匹配，将其他未匹配的可能移动项进行匹配 + // 如果 新增 和 缺失 数量总和能被 2 整除，则说明还存在文件被移动的可能，考虑尝试模糊匹配 + if new_files.len() + lost_files.len() % 2 == 0 { + // 尝试模糊匹配 + // ... + } + + // 将结果收集，并设置结果 + result.created = new_files.iter().map(|p| (*p).clone()).collect(); + result.lost = lost_files.iter().map(|p| (*p).clone()).collect(); + result.moved = moved_files + .iter() + .filter_map(|(from, to)| { + let vfid = analyze_ctx + .local_sheet + .as_ref() + .and_then(|local_sheet| local_sheet.mapping_data(from).ok()) + .map(|mapping_data| mapping_data.mapping_vfid.clone()); + if let Some(vfid) = vfid { + Some((vfid, (from.clone(), to.clone()))) + } else { + None + } + }) + .collect(); + + Ok(()) + } + + /// Compare using file modification time and SHA1 hash values. + /// Note: For files that have been both moved and modified, they can only be recognized as modified after their location is matched. + async fn analyze_modified( + result: &mut AnalyzeResult<'_>, + file_relative_paths: &HashSet, + analyze_ctx: &mut AnalyzeContext<'a>, + ) -> Result<(), std::io::Error> { + let local_sheet = &mut analyze_ctx.local_sheet.as_mut().unwrap(); + let local_path = local_sheet.local_workspace.local_path().clone(); + + for path in file_relative_paths { + // Get mapping data + let Ok(mapping_data) = local_sheet.mapping_data_mut(&path) else { + continue; + }; + + // If modified time not changed, skip + let modified_time = std::fs::metadata(local_path.join(path))?.modified()?; + if &modified_time == mapping_data.last_modifiy_check_time() { + if mapping_data.last_modifiy_check_result() { + result.modified.insert(path.clone()); + } + continue; + } + + // Calculate hash + let hash_calc = match sha1_hash::calc_sha1(path, 2048).await { + Ok(hash) => hash, + Err(e) => return Err(Error::new(std::io::ErrorKind::Other, e)), + }; + + // If hash not match, mark as modified + if &hash_calc.hash != mapping_data.hash_when_updated() { + result.modified.insert(path.clone()); + + // Update last modified check time to modified time + mapping_data.last_modifiy_check_time = modified_time; + mapping_data.last_modifiy_check_result = true; + } else { + // Update last modified check time to modified time + mapping_data.last_modifiy_check_time = modified_time; + mapping_data.last_modifiy_check_result = false; + } + } + + // Persist the local sheet data + LocalSheet::write(local_sheet).await?; + + Ok(()) + } + + /// Generate a empty AnalyzeResult + fn none_result(local_workspace: &'a LocalWorkspace) -> AnalyzeResult<'a> { + AnalyzeResult { + local_workspace: local_workspace, + moved: HashMap::new(), + created: HashSet::new(), + lost: HashSet::new(), + modified: HashSet::new(), + } + } +} diff --git a/crates/vcs_data/src/data/local/latest_info.rs b/crates/vcs_data/src/data/local/latest_info.rs index e8fa641..e4f45b1 100644 --- a/crates/vcs_data/src/data/local/latest_info.rs +++ b/crates/vcs_data/src/data/local/latest_info.rs @@ -1,5 +1,8 @@ +use std::time::{SystemTime, UNIX_EPOCH}; + use cfg_file::ConfigFile; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use tokio::time::Instant; use crate::{ constants::CLIENT_FILE_LATEST_INFO, @@ -23,6 +26,13 @@ pub struct LatestInfo { /// Reference sheet data, indicating what files I can get from the reference sheet pub ref_sheet_content: SheetData, + /// Update instant + #[serde( + serialize_with = "serialize_instant", + deserialize_with = "deserialize_instant" + )] + pub update_instant: Option, + // Members /// All member information of the vault, allowing me to contact them more conveniently pub vault_members: Vec, @@ -33,3 +43,39 @@ pub struct SheetInfo { pub sheet_name: SheetName, pub holder_name: Option, } + +fn serialize_instant(instant: &Option, serializer: S) -> Result +where + S: Serializer, +{ + let system_now = SystemTime::now(); + let instant_now = Instant::now(); + let duration_since_epoch = instant + .as_ref() + .and_then(|i| i.checked_duration_since(instant_now)) + .map(|d| system_now.checked_add(d)) + .unwrap_or(Some(system_now)) + .and_then(|t| t.duration_since(UNIX_EPOCH).ok()) + .unwrap_or_else(|| SystemTime::now().duration_since(UNIX_EPOCH).unwrap()); + + serializer.serialize_u64(duration_since_epoch.as_millis() as u64) +} + +fn deserialize_instant<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let millis = u64::deserialize(deserializer)?; + let duration_since_epoch = std::time::Duration::from_millis(millis); + let system_time = UNIX_EPOCH + duration_since_epoch; + let now_system = SystemTime::now(); + let now_instant = Instant::now(); + + if let Ok(elapsed) = system_time.elapsed() { + Ok(Some(now_instant - elapsed)) + } else if let Ok(duration_until) = system_time.duration_since(now_system) { + Ok(Some(now_instant + duration_until)) + } else { + Ok(Some(now_instant)) + } +} diff --git a/crates/vcs_data/src/data/local/local_files.rs b/crates/vcs_data/src/data/local/local_files.rs new file mode 100644 index 0000000..1599e34 --- /dev/null +++ b/crates/vcs_data/src/data/local/local_files.rs @@ -0,0 +1,152 @@ +use std::path::PathBuf; + +use string_proc::format_path::format_path; +use tokio::fs; + +use crate::constants::CLIENT_FOLDER_WORKSPACE_ROOT_NAME; + +pub struct RelativeFiles { + pub(crate) files: Vec, +} + +impl IntoIterator for RelativeFiles { + type Item = PathBuf; + type IntoIter = std::vec::IntoIter; + + fn into_iter(self) -> Self::IntoIter { + self.files.into_iter() + } +} + +impl RelativeFiles { + pub fn iter(&self) -> std::slice::Iter<'_, PathBuf> { + self.files.iter() + } +} + +/// Read the relative paths within the project from the input file list +pub async fn get_relative_paths(local_path: PathBuf, paths: Vec) -> Option { + // Get Relative Paths + let Ok(paths) = format_input_paths_and_ignore_outside_paths(&local_path, &paths).await else { + return None; + }; + let files: Vec = abs_paths_to_abs_files(paths).await; + let Ok(files) = parse_to_relative(&local_path, files) else { + return None; + }; + Some(RelativeFiles { files }) +} + +/// Normalize the input paths +async fn format_input_paths( + local_path: &PathBuf, + track_files: &[PathBuf], +) -> Result, std::io::Error> { + let current_dir = local_path; + + let mut real_paths = Vec::new(); + for file in track_files { + let path = current_dir.join(file); + + // Skip paths that contain .jv directories + if path.components().any(|component| { + if let std::path::Component::Normal(name) = component { + name.to_str() + .map_or(false, |s| s == CLIENT_FOLDER_WORKSPACE_ROOT_NAME) + } else { + false + } + }) { + continue; + } + + match format_path(path) { + Ok(path) => real_paths.push(path), + Err(e) => { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("Failed to format path: {}", e), + )); + } + } + } + + Ok(real_paths) +} + +/// Ignore files outside the workspace +async fn format_input_paths_and_ignore_outside_paths( + local_path: &PathBuf, + files: &[PathBuf], +) -> Result, std::io::Error> { + let result = format_input_paths(local_path, files).await?; + let result: Vec = result + .into_iter() + .filter(|path| path.starts_with(local_path)) + .collect(); + Ok(result) +} + +/// Normalize the input paths to relative paths +fn parse_to_relative( + local_dir: &PathBuf, + files: Vec, +) -> Result, std::io::Error> { + let result: Result, _> = files + .iter() + .map(|p| { + p.strip_prefix(local_dir) + .map(|relative| relative.to_path_buf()) + .map_err(|_| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Path prefix stripping failed", + ) + }) + }) + .collect(); + + match result { + Ok(paths) => Ok(paths), + Err(e) => Err(e), + } +} + +/// Convert absolute paths to absolute file paths, expanding directories to their contained files +async fn abs_paths_to_abs_files(paths: Vec) -> Vec { + let mut files = Vec::new(); + + for path in paths { + if !path.exists() { + continue; + } + + let metadata = match fs::metadata(&path).await { + Ok(meta) => meta, + Err(_) => continue, + }; + + if metadata.is_file() { + files.push(path); + } else if metadata.is_dir() { + let walker = walkdir::WalkDir::new(&path); + for entry in walker.into_iter().filter_map(|e| e.ok()) { + if entry.path().components().any(|component| { + if let std::path::Component::Normal(name) = component { + name == CLIENT_FOLDER_WORKSPACE_ROOT_NAME + } else { + false + } + }) { + continue; + } + + if entry.file_type().is_file() { + files.push(entry.path().to_path_buf()); + } + } + } + } + + files +} diff --git a/crates/vcs_data/src/data/local/local_sheet.rs b/crates/vcs_data/src/data/local/local_sheet.rs index bfe8d01..980fa49 100644 --- a/crates/vcs_data/src/data/local/local_sheet.rs +++ b/crates/vcs_data/src/data/local/local_sheet.rs @@ -1,8 +1,7 @@ -use std::{collections::HashMap, io::Error, path::PathBuf}; +use std::{collections::HashMap, io::Error, path::PathBuf, time::SystemTime}; use ::serde::{Deserialize, Serialize}; use cfg_file::{ConfigFile, config::ConfigFile}; -use chrono::NaiveDate; use string_proc::format_path::format_path; use crate::{ @@ -10,11 +9,13 @@ use crate::{ data::{ local::LocalWorkspace, member::MemberId, - vault::virtual_file::{VirtualFileId, VirtualFileVersionDescription}, + sheet::SheetName, + vault::virtual_file::{VirtualFileId, VirtualFileVersion, VirtualFileVersionDescription}, }, }; pub type LocalFilePathBuf = PathBuf; +pub type LocalSheetPathBuf = PathBuf; /// # Local Sheet /// Local sheet information, used to record metadata of actual local files, @@ -27,54 +28,168 @@ pub struct LocalSheet<'a> { pub(crate) data: LocalSheetData, } -#[derive(Debug, Default, Serialize, Deserialize, ConfigFile)] +#[derive(Debug, Default, Serialize, Deserialize, ConfigFile, Clone)] #[cfg_file(path = CLIENT_FILE_LOCAL_SHEET_NOSET)] // Do not use LocalSheet::write or LocalSheet::read pub struct LocalSheetData { - /// Local file path to virtual file ID mapping. + /// Local file path to metadata mapping. #[serde(rename = "mapping")] - pub(crate) mapping: HashMap, // Path to VFID + pub(crate) mapping: HashMap, + + pub(crate) vfs: HashMap, } -#[derive(Debug, Default, Serialize, Deserialize)] -pub struct MappingMetaData { +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct LocalMappingMetadata { /// Hash value generated immediately after the file is downloaded to the local workspace #[serde(rename = "hash")] pub(crate) hash_when_updated: String, /// Time when the file was downloaded to the local workspace - #[serde(rename = "date", with = "naive_date_serde")] - pub(crate) date_when_updated: NaiveDate, + pub(crate) time_when_updated: SystemTime, /// Size of the file when downloaded to the local workspace #[serde(rename = "size")] pub(crate) size_when_updated: u64, /// Version description when the file was downloaded to the local workspace - #[serde(rename = "version")] + #[serde(rename = "version_desc")] pub(crate) version_desc_when_updated: VirtualFileVersionDescription, + /// Version when the file was downloaded to the local workspace + #[serde(rename = "version")] + pub(crate) version_when_updated: VirtualFileVersion, + /// Virtual file ID corresponding to the local path #[serde(rename = "id")] pub(crate) mapping_vfid: VirtualFileId, + + /// Latest modifiy check time + pub(crate) last_modifiy_check_time: SystemTime, + + /// Latest modifiy check result + pub(crate) last_modifiy_check_result: bool, +} + +impl LocalSheetData { + /// Wrap LocalSheetData into LocalSheet with workspace, member, and sheet name + pub fn wrap_to_local_sheet<'a>( + self, + workspace: &'a LocalWorkspace, + member: MemberId, + sheet_name: SheetName, + ) -> LocalSheet<'a> { + LocalSheet { + local_workspace: workspace, + member, + sheet_name, + data: self, + } + } +} + +impl LocalMappingMetadata { + /// Create a new MappingMetaData instance + pub fn new( + hash_when_updated: String, + time_when_updated: SystemTime, + size_when_updated: u64, + version_desc_when_updated: VirtualFileVersionDescription, + version_when_updated: VirtualFileVersion, + mapping_vfid: VirtualFileId, + last_modifiy_check_time: SystemTime, + last_modifiy_check_result: bool, + ) -> Self { + Self { + hash_when_updated, + time_when_updated, + size_when_updated, + version_desc_when_updated, + version_when_updated, + mapping_vfid, + last_modifiy_check_time, + last_modifiy_check_result, + } + } + + /// Getter for hash_when_updated + pub fn hash_when_updated(&self) -> &String { + &self.hash_when_updated + } + + /// Getter for date_when_updated + pub fn time_when_updated(&self) -> &SystemTime { + &self.time_when_updated + } + + /// Getter for size_when_updated + pub fn size_when_updated(&self) -> u64 { + self.size_when_updated + } + + /// Getter for version_desc_when_updated + pub fn version_desc_when_updated(&self) -> &VirtualFileVersionDescription { + &self.version_desc_when_updated + } + + /// Getter for version_when_updated + pub fn version_when_updated(&self) -> &VirtualFileVersion { + &self.version_when_updated + } + + /// Getter for mapping_vfid + pub fn mapping_vfid(&self) -> &VirtualFileId { + &self.mapping_vfid + } + + /// Getter for last_modifiy_check_time + pub fn last_modifiy_check_time(&self) -> &SystemTime { + &self.last_modifiy_check_time + } + + /// Getter for last_modifiy_check_result + pub fn last_modifiy_check_result(&self) -> bool { + self.last_modifiy_check_result + } +} + +impl Default for LocalMappingMetadata { + fn default() -> Self { + Self { + hash_when_updated: Default::default(), + time_when_updated: SystemTime::now(), + size_when_updated: Default::default(), + version_desc_when_updated: Default::default(), + version_when_updated: Default::default(), + mapping_vfid: Default::default(), + last_modifiy_check_time: SystemTime::now(), + last_modifiy_check_result: false, + } + } } -mod naive_date_serde { - use chrono::NaiveDate; +mod instant_serde { use serde::{self, Deserialize, Deserializer, Serializer}; + use tokio::time::Instant; - pub fn serialize(date: &NaiveDate, serializer: S) -> Result + pub fn serialize(instant: &Instant, serializer: S) -> Result where S: Serializer, { - serializer.serialize_str(&date.format("%Y-%m-%d").to_string()) + serializer.serialize_u64(instant.elapsed().as_secs()) } - pub fn deserialize<'de, D>(deserializer: D) -> Result + pub fn deserialize<'de, D>(deserializer: D) -> Result where D: Deserializer<'de>, { - let s = String::deserialize(deserializer)?; - NaiveDate::parse_from_str(&s, "%Y-%m-%d").map_err(serde::de::Error::custom) + let secs = u64::deserialize(deserializer)?; + Ok(Instant::now() - std::time::Duration::from_secs(secs)) + } +} + +impl<'a> From<&'a LocalSheet<'a>> for &'a LocalSheetData { + fn from(sheet: &'a LocalSheet<'a>) -> Self { + &sheet.data } } @@ -83,10 +198,12 @@ impl<'a> LocalSheet<'a> { pub fn add_mapping( &mut self, path: LocalFilePathBuf, - mapping: MappingMetaData, + mapping: LocalMappingMetadata, ) -> Result<(), std::io::Error> { let path = format_path(path)?; - if self.data.mapping.contains_key(&path) { + if self.data.mapping.contains_key(&path) + || self.data.vfs.contains_key(&mapping.mapping_vfid) + { return Err(Error::new( std::io::ErrorKind::AlreadyExists, "Mapping already exists", @@ -100,8 +217,8 @@ impl<'a> LocalSheet<'a> { /// Move mapping to other path pub fn move_mapping( &mut self, - from: LocalFilePathBuf, - to: LocalFilePathBuf, + from: &LocalFilePathBuf, + to: &LocalFilePathBuf, ) -> Result<(), std::io::Error> { let from = format_path(from)?; let to = format_path(to)?; @@ -124,11 +241,26 @@ impl<'a> LocalSheet<'a> { Ok(()) } + /// Get immutable mapping data + pub fn mapping_data( + &self, + path: &LocalFilePathBuf, + ) -> Result<&LocalMappingMetadata, std::io::Error> { + let path = format_path(path)?; + let Some(data) = self.data.mapping.get(&path) else { + return Err(Error::new( + std::io::ErrorKind::NotFound, + "Path is not found.", + )); + }; + Ok(data) + } + /// Get muttable mapping data pub fn mapping_data_mut( &mut self, - path: LocalFilePathBuf, - ) -> Result<&mut MappingMetaData, std::io::Error> { + path: &LocalFilePathBuf, + ) -> Result<&mut LocalMappingMetadata, std::io::Error> { let path = format_path(path)?; let Some(data) = self.data.mapping.get_mut(&path) else { return Err(Error::new( @@ -139,12 +271,31 @@ impl<'a> LocalSheet<'a> { Ok(data) } - /// Persist the sheet to disk - pub async fn persist(&mut self) -> Result<(), std::io::Error> { - let _path = self + /// Write the sheet to disk + pub async fn write(&mut self) -> Result<(), std::io::Error> { + let path = self .local_workspace .local_sheet_path(&self.member, &self.sheet_name); - LocalSheetData::write_to(&self.data, &self.sheet_name).await?; + self.write_to_path(path).await + } + + /// Write the sheet to custom path + pub async fn write_to_path(&mut self, path: impl Into) -> Result<(), std::io::Error> { + let path = path.into(); + + self.data.vfs = HashMap::new(); + for (path, mapping) in self.data.mapping.iter() { + self.data + .vfs + .insert(mapping.mapping_vfid.clone(), path.clone()); + } + + LocalSheetData::write_to(&self.data, path).await?; Ok(()) } + + /// Get path by VirtualFileId + pub fn path_by_id(&self, vfid: &VirtualFileId) -> Option<&PathBuf> { + self.data.vfs.get(vfid) + } } diff --git a/crates/vcs_data/src/data/local/member_held.rs b/crates/vcs_data/src/data/local/member_held.rs index 37bc18e..3f07232 100644 --- a/crates/vcs_data/src/data/local/member_held.rs +++ b/crates/vcs_data/src/data/local/member_held.rs @@ -1,13 +1,16 @@ -use std::collections::HashMap; +use std::{collections::HashMap, io::Error, path::PathBuf}; use cfg_file::ConfigFile; use serde::{Deserialize, Serialize}; use crate::{ - constants::CLIENT_FILE_MEMBER_HELD_NOSET, + constants::{CLIENT_FILE_MEMBER_HELD, CLIENT_FILE_MEMBER_HELD_NOSET}, + current::current_local_path, data::{member::MemberId, vault::virtual_file::VirtualFileId}, }; +const ACCOUNT: &str = "{account}"; + /// # Member Held Information /// Records the files held by the member, used for permission validation #[derive(Debug, Default, Clone, Serialize, Deserialize, ConfigFile)] @@ -25,3 +28,37 @@ pub enum HeldStatus { #[default] WantedToKnow, // Holding status is unknown, notify server must inform client } + +impl MemberHeld { + /// Get the path to the file holding the held status information for the given member. + pub fn held_file_path(account: &MemberId) -> Result { + let Some(local_path) = current_local_path() else { + return Err(Error::new( + std::io::ErrorKind::NotFound, + "Workspace not found.", + )); + }; + Ok(local_path.join(CLIENT_FILE_MEMBER_HELD.replace(ACCOUNT, account))) + } + + /// Get the member who holds the file with the given ID. + pub fn file_holder(&self, vfid: &VirtualFileId) -> Option<&MemberId> { + self.held_status.get(vfid).and_then(|status| match status { + HeldStatus::HeldWith(id) => Some(id), + _ => None, + }) + } + + /// Update the held status of the files. + pub fn update_held_status(&mut self, map: HashMap>) { + for (vfid, member_id) in map { + self.held_status.insert( + vfid, + match member_id { + Some(member_id) => HeldStatus::HeldWith(member_id), + None => HeldStatus::NotHeld, + }, + ); + } + } +} diff --git a/crates/vcs_data/src/data/sheet.rs b/crates/vcs_data/src/data/sheet.rs index 09271d5..9e3a1d7 100644 --- a/crates/vcs_data/src/data/sheet.rs +++ b/crates/vcs_data/src/data/sheet.rs @@ -8,7 +8,10 @@ use crate::{ constants::SERVER_FILE_SHEET, data::{ member::MemberId, - vault::{Vault, virtual_file::VirtualFileId}, + vault::{ + Vault, + virtual_file::{VirtualFileId, VirtualFileVersion}, + }, }, }; @@ -26,7 +29,7 @@ pub struct InputPackage { pub from: SheetName, /// Files in this input package with their relative paths and virtual file IDs - pub files: Vec<(InputRelativePathBuf, VirtualFileId)>, + pub files: Vec<(InputRelativePathBuf, SheetMappingMetadata)>, } impl PartialEq for InputPackage { @@ -60,7 +63,16 @@ pub struct SheetData { pub(crate) inputs: Vec, /// Mapping of sheet paths to virtual file IDs - pub(crate) mapping: HashMap, + pub(crate) mapping: HashMap, + + /// Mapping of virtual file Ids to sheet paths + pub(crate) id_mapping: Option>, +} + +#[derive(Debug, Default, Serialize, Deserialize, ConfigFile, Clone, Eq, PartialEq)] +pub struct SheetMappingMetadata { + pub id: VirtualFileId, + pub version: VirtualFileVersion, } impl<'a> Sheet<'a> { @@ -88,10 +100,15 @@ impl<'a> Sheet<'a> { } /// Get the mapping of this sheet - pub fn mapping(&self) -> &HashMap { + pub fn mapping(&self) -> &HashMap { &self.data.mapping } + /// Get the id_mapping of this sheet data + pub fn id_mapping(&self) -> &Option> { + &self.data.id_mapping + } + /// Get the write count of this sheet pub fn write_count(&self) -> i32 { self.data.write_count @@ -150,9 +167,13 @@ impl<'a> Sheet<'a> { }; // Insert to sheet - for (relative_path, virtual_file_id) in input.files { - self.add_mapping(insert_to.join(relative_path), virtual_file_id) - .await?; + for (relative_path, virtual_file_meta) in input.files { + self.add_mapping( + insert_to.join(relative_path), + virtual_file_meta.id, + virtual_file_meta.version, + ) + .await?; } Ok(()) @@ -172,11 +193,18 @@ impl<'a> Sheet<'a> { &mut self, sheet_path: SheetPathBuf, virtual_file_id: VirtualFileId, + version: VirtualFileVersion, ) -> Result<(), std::io::Error> { // Check if the virtual file exists in the vault if self.vault_reference.virtual_file(&virtual_file_id).is_err() { // Virtual file doesn't exist, add the mapping directly - self.data.mapping.insert(sheet_path, virtual_file_id); + self.data.mapping.insert( + sheet_path, + SheetMappingMetadata { + id: virtual_file_id, + version: version, + }, + ); return Ok(()); } @@ -194,16 +222,22 @@ impl<'a> Sheet<'a> { .has_virtual_file_edit_right(holder, &virtual_file_id) .await { - Ok(false) => { - // Holder doesn't have rights, add the mapping (member is giving up the file) - self.data.mapping.insert(sheet_path, virtual_file_id); + Ok(true) => { + // Holder has edit rights, add the mapping (member has permission to modify the file) + self.data.mapping.insert( + sheet_path, + SheetMappingMetadata { + id: virtual_file_id, + version, + }, + ); Ok(()) } - Ok(true) => { - // Holder has edit rights, don't allow modifying the mapping + Ok(false) => { + // Holder doesn't have edit rights, don't allow modifying the mapping Err(std::io::Error::new( std::io::ErrorKind::PermissionDenied, - "Member has edit rights to the virtual file, cannot modify mapping", + "Member doesn't have edit rights to the virtual file, cannot modify mapping", )) } Err(_) => { @@ -224,8 +258,11 @@ impl<'a> Sheet<'a> { /// 4. If member has no edit rights and the file exists, returns the removed virtual file ID /// /// Note: Full validation adds overhead - avoid frequent calls - pub async fn remove_mapping(&mut self, sheet_path: &SheetPathBuf) -> Option { - let virtual_file_id = match self.data.mapping.get(sheet_path) { + pub async fn remove_mapping( + &mut self, + sheet_path: &SheetPathBuf, + ) -> Option { + let virtual_file_meta = match self.data.mapping.get(sheet_path) { Some(id) => id, None => { // The mapping entry doesn't exist, nothing to remove @@ -234,7 +271,11 @@ impl<'a> Sheet<'a> { }; // Check if the virtual file exists in the vault - if self.vault_reference.virtual_file(virtual_file_id).is_err() { + if self + .vault_reference + .virtual_file(&virtual_file_meta.id) + .is_err() + { // Virtual file doesn't exist, remove the mapping and return None self.data.mapping.remove(sheet_path); return None; @@ -248,7 +289,7 @@ impl<'a> Sheet<'a> { // Check if the holder has edit rights to the virtual file match self .vault_reference - .has_virtual_file_edit_right(holder, virtual_file_id) + .has_virtual_file_edit_right(holder, &virtual_file_meta.id) .await { Ok(false) => { @@ -273,6 +314,18 @@ impl<'a> Sheet<'a> { /// If needed, please deserialize and reload it. pub async fn persist(mut self) -> Result<(), std::io::Error> { self.data.write_count += 1; + + // Update id mapping + self.data.id_mapping = Some(HashMap::new()); + for map in self.data.mapping.iter() { + self.data + .id_mapping + .as_mut() + .unwrap() + .insert(map.1.id.clone(), map.0.clone()); + } + + // Add write count if self.data.write_count > i32::MAX { self.data.write_count = 0; } @@ -402,4 +455,24 @@ impl SheetData { pub fn write_count(&self) -> i32 { self.write_count } + + /// Get the holder of this sheet data + pub fn holder(&self) -> Option<&MemberId> { + self.holder.as_ref() + } + + /// Get the inputs of this sheet data + pub fn inputs(&self) -> &Vec { + &self.inputs + } + + /// Get the mapping of this sheet data + pub fn mapping(&self) -> &HashMap { + &self.mapping + } + + /// Get the id_mapping of this sheet data + pub fn id_mapping(&self) -> &Option> { + &self.id_mapping + } } diff --git a/crates/vcs_data/src/data/vault/sheets.rs b/crates/vcs_data/src/data/vault/sheets.rs index ba021b5..cea7271 100644 --- a/crates/vcs_data/src/data/vault/sheets.rs +++ b/crates/vcs_data/src/data/vault/sheets.rs @@ -133,6 +133,7 @@ impl Vault { holder: Some(holder.clone()), inputs: Vec::new(), mapping: HashMap::new(), + id_mapping: None, write_count: 0, }; SheetData::write_to(&sheet_data, sheet_file_path).await?; diff --git a/crates/vcs_data/src/data/vault/virtual_file.rs b/crates/vcs_data/src/data/vault/virtual_file.rs index 221766f..6dd5208 100644 --- a/crates/vcs_data/src/data/vault/virtual_file.rs +++ b/crates/vcs_data/src/data/vault/virtual_file.rs @@ -22,7 +22,7 @@ use crate::{ pub type VirtualFileId = String; pub type VirtualFileVersion = String; -const VF_PREFIX: &str = "vf_"; +const VF_PREFIX: &str = "vf-"; const ID_PARAM: &str = "{vf_id}"; const ID_INDEX: &str = "{vf_index}"; const VERSION_PARAM: &str = "{vf_version}"; @@ -244,8 +244,6 @@ impl Vault { } fs::rename(receive_path, move_path).await?; - // - Ok(new_id) } Err(e) => { @@ -444,6 +442,13 @@ impl VirtualFileMeta { &self.histories } + /// Get the latest version of the virtual file + pub fn version_latest(&self) -> VirtualFileVersion { + // After creating a virtual file in `update_virtual_file_from_connection`, + // the Vec will never be empty, so unwrap is allowed here + self.histories.last().unwrap().clone() + } + /// Get the total number of versions for this virtual file pub fn version_len(&self) -> i32 { self.histories.len() as i32 @@ -470,4 +475,25 @@ impl VirtualFileMeta { pub fn version_name(&self, version_num: i32) -> Option { self.histories.get(version_num as usize).cloned() } + + /// Get the member who holds the edit right of the file + pub fn hold_member(&self) -> &MemberId { + &self.hold_member + } + + /// Get the version descriptions for all versions + pub fn version_descriptions( + &self, + ) -> &HashMap { + &self.version_description + } + + /// Get the version description for a given version + pub fn version_description( + &self, + version: VirtualFileVersion, + ) -> Option<&VirtualFileVersionDescription> { + let desc = self.version_descriptions(); + desc.get(&version) + } } diff --git a/crates/vcs_data/vcs_data_test/src/test_sheet_creation_management_and_persistence.rs b/crates/vcs_data/vcs_data_test/src/test_sheet_creation_management_and_persistence.rs index f256436..a89fbea 100644 --- a/crates/vcs_data/vcs_data_test/src/test_sheet_creation_management_and_persistence.rs +++ b/crates/vcs_data/vcs_data_test/src/test_sheet_creation_management_and_persistence.rs @@ -58,10 +58,14 @@ async fn test_sheet_creation_management_and_persistence() -> Result<(), std::io: let lib_rs_id = VirtualFileId::new(); sheet - .add_mapping(main_rs_path.clone(), main_rs_id.clone()) + .add_mapping( + main_rs_path.clone(), + main_rs_id.clone(), + "1.0.0".to_string(), + ) .await?; sheet - .add_mapping(lib_rs_path.clone(), lib_rs_id.clone()) + .add_mapping(lib_rs_path.clone(), lib_rs_id.clone(), "1.0.0".to_string()) .await?; // Use output_mappings to generate the InputPackage @@ -88,12 +92,19 @@ async fn test_sheet_creation_management_and_persistence() -> Result<(), std::io: let virtual_file_id = VirtualFileId::new(); sheet - .add_mapping(mapping_path.clone(), virtual_file_id.clone()) + .add_mapping( + mapping_path.clone(), + virtual_file_id.clone(), + "1.0.0".to_string(), + ) .await?; // Verify mapping was added assert_eq!(sheet.mapping().len(), 3); - assert_eq!(sheet.mapping().get(&mapping_path), Some(&virtual_file_id)); + assert_eq!( + sheet.mapping().get(&mapping_path).map(|meta| &meta.id), + Some(&virtual_file_id) + ); // Test 4: Persist sheet to disk sheet.persist().await?; @@ -270,10 +281,14 @@ async fn test_sheet_data_serialization() -> Result<(), std::io::Error> { let lib_rs_id = VirtualFileId::new(); sheet - .add_mapping(main_rs_path.clone(), main_rs_id.clone()) + .add_mapping( + main_rs_path.clone(), + main_rs_id.clone(), + "1.0.0".to_string(), + ) .await?; sheet - .add_mapping(lib_rs_path.clone(), lib_rs_id.clone()) + .add_mapping(lib_rs_path.clone(), lib_rs_id.clone(), "1.0.0".to_string()) .await?; // Use output_mappings to generate the InputPackage @@ -288,6 +303,7 @@ async fn test_sheet_data_serialization() -> Result<(), std::io::Error> { .add_mapping( vcs_data::data::sheet::SheetPathBuf::from("output/build.exe"), build_exe_id, + "1.0.0".to_string(), ) .await?; diff --git a/src/lib.rs b/src/lib.rs index 8d9de66..6d94067 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -32,6 +32,13 @@ pub mod utils { pub use data_struct::*; } + // Feature `sha1_hash` + #[cfg(feature = "sha1_hash")] + pub mod sha1_hash { + extern crate sha1_hash; + pub use sha1_hash::*; + } + // Feature `tcp_connection` #[cfg(feature = "tcp_connection")] pub mod tcp_connection { -- cgit