1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
|
// Auto generated dependencies
// If you find issues with the dependencies, please
// 1. Delete all code after this comment
// 2. Clear the auto generated part in `policy/_policies/Cargo.toml`
// 3. Run `cargo run --bin butckrepo-refresh` in the Butchunker root directory
pub mod error;
pub mod stream_read;
use error::ChunkFailed;
use std::{collections::HashMap, path::Path};
use crate::stream_read::chunk_stream_process;
/// Chunks the specified raw data using the specified chunking policy
///
/// # Parameters
/// - `policy_name`: Chunking policy name, currently supports 1 policies
/// - `raw_data`: Raw data byte slice
/// - `params`: Hashmap of parameters required by the chunking policy
pub async fn chunk_with(
policy_name: &str,
raw_data: &[u8],
params: &HashMap<&str, &str>,
) -> Result<Vec<u32>, ChunkFailed> {
match policy_name {
"butck_fixed_size" => Ok(butck_fixed_size::chunk(raw_data, params).await),
_ => Err(ChunkFailed::PolicyNotFound),
}
}
pub async fn chunk_stream_with<F>(
policy_name: &str,
size: u32,
path: &Path,
callback: F,
params: &HashMap<&str, &str>,
) -> Result<(), ChunkFailed>
where
F: FnMut(Vec<u8>) -> std::pin::Pin<Box<dyn std::future::Future<Output = Result<(), std::io::Error>> + Send>>,
{
match policy_name {
"butck_fixed_size" => {
let mut stream = butck_fixed_size::FixedSizeStream::default();
chunk_stream_process(
path, &mut stream, size, callback, params,
async |current_data, len, stream, params| {
butck_fixed_size::chunk_stream(current_data, len, stream, params).await
},
)
.await
}
_ => Err(ChunkFailed::PolicyNotFound),
}
}
pub async fn chunk_stream_display_boundaries(
policy_name: &str,
size: u32,
path: &Path,
params: &HashMap<&str, &str>,
) -> Result<Vec<u32>, ChunkFailed> {
match policy_name {
"butck_fixed_size" => {
let mut stream = butck_fixed_size::FixedSizeStream::default();
crate::stream_read::chunk_stream_display_boundaries(
path, &mut stream, size, params,
async |current_data, len, stream, params| {
butck_fixed_size::chunk_stream(current_data, len, stream, params).await
},
)
.await
}
_ => Err(ChunkFailed::PolicyNotFound),
}
}
pub fn policies() -> Vec<&'static str> {
vec![
// butck_fixed_size
"butck_fixed_size",
]
}
pub fn stream_policies() -> Vec<&'static str> {
vec![
// butck_fixed_size
"butck_fixed_size",
]
}
pub mod butck_fixed_size {
pub use butck_fixed_size::FixedSizeStream;
use std::collections::HashMap;
pub async fn chunk(raw_data: &[u8], params: &HashMap<&str, &str>) -> Vec<u32> {
butck_fixed_size::chunk_fixed_size(raw_data, params).await
}
pub async fn chunk_stream(
current_data: &[u8],
len: u32,
stream: &mut butck_fixed_size::FixedSizeStream,
params: &HashMap<&str, &str>,
) -> Option<u32> {
butck_fixed_size::chunk_fixed_size_stream(current_data, len, stream, params).await
}
}
|
