Initialize Butchunker project structure and policy system

7 files changed, 293 insertions, 0 deletions

diff --git a/policy/_policies/Cargo.lock b/policy/_policies/Cargo.lock
new file mode 100644
index 0000000..8f7bc05
--- /dev/null
+++ b/policy/_policies/Cargo.lock
@@ -0,0 +1,7 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 4
+
+[[package]]
+name = "butck_policies"
+version = "0.1.0"
diff --git a/policy/_policies/Cargo.toml b/policy/_policies/Cargo.toml
new file mode 100644
index 0000000..d939dd2
--- /dev/null
+++ b/policy/_policies/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+name = "butck_policies"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+thiserror = "2"
+tokio = { version = "1", features = ["fs"] }
+
+# Auto generated dependencies
+# If you find issues with the dependencies, please
+#   1. Delete all code after this comment
+#   2. Clear the file `policy/_policies/src/lib.rs`
+#   3. Run `cargo run --bin butckrepo-refresh` in the Butchunker root directory
+butck_fixed_size = { path = "../../policy/butck/butck_fixed_size" }
\ No newline at end of file
diff --git a/policy/_policies/Cargo.toml.t b/policy/_policies/Cargo.toml.t
new file mode 100644
index 0000000..aab90b9
--- /dev/null
+++ b/policy/_policies/Cargo.toml.t
@@ -0,0 +1,19 @@
+[package]
+name = "butck_policies"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+thiserror = "2"
+tokio = { version = "1", features = ["fs"] }
+
+# Auto generated dependencies
+# If you find issues with the dependencies, please
+#   1. Delete all code after this comment
+#   2. Clear the file `policy/_policies/src/lib.rs`
+#   3. Run `cargo run --bin butckrepo-refresh` in the Butchunker root directory
+>>>>>>>>>> deps
+
+@@@ >>> deps
+<<<crate_name>>> = { path = "../../<<<path>>>" }
+@@@ <<<
diff --git a/policy/_policies/src/error.rs b/policy/_policies/src/error.rs
new file mode 100644
index 0000000..975749d
--- /dev/null
+++ b/policy/_policies/src/error.rs
@@ -0,0 +1,14 @@
+#[derive(Debug, thiserror::Error)]
+pub enum ChunkFailed {
+    #[error("IO error: {0}")]
+    Io(#[from] std::io::Error),
+
+    #[error("Target policy not found")]
+    PolicyNotFound,
+
+    #[error("File read failed: {0}")]
+    FileReadFailed(std::path::PathBuf),
+
+    #[error("File open failed: {0}")]
+    FileOpenFailed(std::path::PathBuf),
+}
diff --git a/policy/_policies/src/lib.rs b/policy/_policies/src/lib.rs
new file mode 100644
index 0000000..397579a
--- /dev/null
+++ b/policy/_policies/src/lib.rs
@@ -0,0 +1,75 @@
+// Auto generated dependencies
+// If you find issues with the dependencies, please
+//   1. Delete all code after this comment
+//   2. Clear the auto generated part in `policy/_policies/Cargo.toml`
+//   3. Run `cargo run --bin butckrepo-refresh` in the Butchunker root directory
+pub mod error;
+pub mod stream_read;
+
+use error::ChunkFailed;
+use std::{collections::HashMap, path::Path};
+
+use crate::stream_read::chunk_stream_process;
+
+/// Chunks the specified raw data using the specified chunking policy
+///
+/// # Parameters
+/// - `policy_name`: Chunking policy name, currently supports 1 policies
+/// - `raw_data`: Raw data byte slice
+/// - `params`: Hashmap of parameters required by the chunking policy
+pub async fn chunk_with(
+    policy_name: &str,
+    raw_data: &[u8],
+    params: &HashMap<&str, &str>,
+) -> Result<Vec<u32>, ChunkFailed> {
+    match policy_name {
+        "butck_fixed_size" => Ok(butck_fixed_size::chunk(raw_data, params).await),
+        _ => Err(ChunkFailed::PolicyNotFound),
+    }
+}
+
+pub async fn chunk_stream_with(
+    policy_name: &str,
+    size: u32,
+    path: &Path,
+    params: &HashMap<&str, &str>,
+) -> Result<Vec<u32>, ChunkFailed> {
+    match policy_name {
+        "butck_fixed_size" => {
+            let mut stream = butck_fixed_size::FixedSizeStream::default();
+            chunk_stream_process(
+                path, &mut stream, size, params,
+                async |current_data, len, stream, params| {
+                    butck_fixed_size::chunk_stream(current_data, len, stream, params).await
+                },
+            )
+            .await
+        }
+        _ => Err(ChunkFailed::PolicyNotFound),
+    }
+}
+
+pub fn policies() -> Vec<&'static str> {
+    vec![
+        // butck_fixed_size
+        "butck_fixed_size",
+    ]
+}
+
+pub mod butck_fixed_size {
+    pub use butck_fixed_size::FixedSizeStream;
+    use std::collections::HashMap;
+
+    pub async fn chunk(raw_data: &[u8], params: &HashMap<&str, &str>) -> Vec<u32> {
+        butck_fixed_size::chunk_fixed_size(raw_data, params).await
+    }
+
+    pub async fn chunk_stream(
+        current_data: &[u8],
+        len: u32,
+        stream: &mut butck_fixed_size::FixedSizeStream,
+        params: &HashMap<&str, &str>,
+    ) -> Option<u32> {
+        butck_fixed_size::chunk_fixed_size_stream(current_data, len, stream, params).await
+    }
+}
\ No newline at end of file
diff --git a/policy/_policies/src/lib.rs.t b/policy/_policies/src/lib.rs.t
new file mode 100644
index 0000000..873a4cd
--- /dev/null
+++ b/policy/_policies/src/lib.rs.t
@@ -0,0 +1,117 @@
+// Auto generated dependencies
+// If you find issues with the dependencies, please
+//   1. Delete all code after this comment
+//   2. Clear the auto generated part in `policy/_policies/Cargo.toml`
+//   3. Run `cargo run --bin butckrepo-refresh` in the Butchunker root directory
+pub mod error;
+pub mod stream_read;
+
+use error::ChunkFailed;
+use std::{collections::HashMap, path::Path};
+
+use crate::stream_read::chunk_stream_process;
+
+/// Chunks the specified raw data using the specified chunking policy
+///
+/// # Parameters
+/// - `policy_name`: Chunking policy name, currently supports <<<policy_count>>> policies
+/// - `raw_data`: Raw data byte slice
+/// - `params`: Hashmap of parameters required by the chunking policy
+pub async fn chunk_with(
+    policy_name: &str,
+    raw_data: &[u8],
+    params: &HashMap<&str, &str>,
+) -> Result<Vec<u32>, ChunkFailed> {
+    match policy_name {
+>>>>>>>>>> match_arms
+        _ => Err(ChunkFailed::PolicyNotFound),
+    }
+}
+
+pub async fn chunk_stream_with(
+    policy_name: &str,
+    size: u32,
+    path: &Path,
+    params: &HashMap<&str, &str>,
+) -> Result<Vec<u32>, ChunkFailed> {
+    match policy_name {
+>>>>>>>>>> match_arms_stream
+        _ => Err(ChunkFailed::PolicyNotFound),
+    }
+}
+
+pub fn policies() -> Vec<&'static str> {
+    vec![
+>>>>>>>>>> policy_names
+    ]
+}
+
+>>>>>>>>>> exports_simple
+>>>>>>>>>> exports_stream
+>>>>>>>>>> exports_both
+
+@@@ >>> match_arms
+        "<<<crate_name>>>" => Ok(<<<crate_name>>>::chunk(raw_data, params).await),
+@@@ <<<
+
+@@@ >>> match_arms_stream
+        "<<<crate_name>>>" => {
+            let mut stream = <<<stream_struct_id>>>::default();
+            chunk_stream_process(
+                path, &mut stream, size, params,
+                async |current_data, len, stream, params| {
+                    <<<crate_name>>>::chunk_stream(current_data, len, stream, params).await
+                },
+            )
+            .await
+        }
+@@@ <<<
+
+@@@ >>> policy_names
+        // <<<name>>>
+        "<<<name>>>",
+@@@ <<<
+
+@@@ >>> exports_simple
+pub mod <<<crate_name>>> {
+    use std::collections::HashMap;
+    pub async fn chunk(raw_data: &[u8], params: &HashMap<&str, &str>) -> Vec<u32> {
+        <<<crate_name>>>::<<<matched_func>>>(raw_data, params)<<<has_await>>>
+    }
+}
+@@@ <<<
+
+@@@ >>> exports_stream
+pub mod <<<crate_name>>> {
+    pub use <<<stream_struct_id>>>;
+
+    pub async fn chunk_stream(
+        current_data: &[u8],
+        len: u32,
+        stream: &mut <<<stream_struct_id>>>,
+        params: &std::collections::HashMap<&str, &str>,
+    ) -> Option<u32> {
+        <<<crate_name>>>::<<<matched_func_stream>>>(current_data, len, stream, params)<<<has_await_stream>>>
+    }
+}
+@@@ <<<
+
+@@@ >>> exports_both
+pub mod <<<crate_name>>> {
+    pub use <<<stream_struct_id>>>;
+    use std::collections::HashMap;
+
+    pub async fn chunk(raw_data: &[u8], params: &HashMap<&str, &str>) -> Vec<u32> {
+        <<<crate_name>>>::<<<matched_func>>>(raw_data, params)<<<has_await>>>
+    }
+
+    pub async fn chunk_stream(
+        current_data: &[u8],
+        len: u32,
+        stream: &mut <<<stream_struct_id>>>,
+        params: &HashMap<&str, &str>,
+    ) -> Option<u32> {
+        <<<crate_name>>>::<<<matched_func_stream>>>(current_data, len, stream, params)<<<has_await_stream>>>
+    }
+}
+@@@ <<<
diff --git a/policy/_policies/src/stream_read.rs b/policy/_policies/src/stream_read.rs
new file mode 100644
index 0000000..5cf7791
--- /dev/null
+++ b/policy/_policies/src/stream_read.rs
@@ -0,0 +1,46 @@
+use crate::error::ChunkFailed;
+use std::{collections::HashMap, path::Path};
+
+pub async fn chunk_stream_process<T, F>(
+    path: &Path,
+    stream_data: &mut T,
+    size: u32,
+    params: &HashMap<&str, &str>,
+    chunk_func: F,
+) -> Result<Vec<u32>, ChunkFailed>
+where
+    T: Default,
+    F: AsyncFn(&[u8], u32, &mut T, &HashMap<&str, &str>) -> Option<u32>,
+{
+    let mut file = tokio::fs::File::open(path)
+        .await
+        .map_err(|_| ChunkFailed::FileOpenFailed(path.to_path_buf()))?;
+    let mut buffer = vec![0u8; size as usize];
+    let mut splits = Vec::new();
+    let mut total_read = 0;
+
+    loop {
+        let bytes_read = tokio::io::AsyncReadExt::read(&mut file, &mut buffer)
+            .await
+            .map_err(|_| ChunkFailed::FileReadFailed(path.to_path_buf()))?;
+
+        if bytes_read == 0 {
+            break Ok(splits);
+        }
+
+        // Process chunking on the buffer slice
+        let chunk_result = chunk_func(
+            &buffer[..bytes_read],
+            bytes_read as u32,
+            stream_data,
+            params,
+        )
+        .await;
+
+        if let Some(offset) = chunk_result {
+            splits.push(total_read + offset);
+        }
+
+        total_read += bytes_read as u32;
+    }
+}