jeang3nie/haggis-rs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add par_create_archive function

Browse Source
This commit is contained in:
Nathan Fisher 2023-07-05 14:27:40 -04:00
parent 8a3bd19185
commit 797aa16a0d
3 changed files with 146 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
Cargo.lock generated
View File

@ -2,6 +2,12 @@
# It is not intended for manual editing.
version = 3
[[package]]
name = "autocfg"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
[[package]]
name = "block-buffer"
version = "0.10.4"
@ -26,6 +32,49 @@ dependencies = [
"libc",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
dependencies = [
"cfg-if",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-deque"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
dependencies = [
"cfg-if",
"crossbeam-epoch",
"crossbeam-utils",
]
[[package]]
name = "crossbeam-epoch"
version = "0.9.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae211234986c545741a7dc064309f67ee1e5ad243d0e48335adc0484d960bcc7"
dependencies = [
"autocfg",
"cfg-if",
"crossbeam-utils",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a22b2d63d4d1dc0b7f1b6b2747dd0088008a9be28b6ddf0b1e7d335e3037294"
dependencies = [
"cfg-if",
]
[[package]]
name = "crypto-common"
version = "0.1.6"
@ -46,6 +95,12 @@ dependencies = [
"crypto-common",
]
[[package]]
name = "either"
version = "1.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91"
[[package]]
name = "generic-array"
version = "0.14.7"
@ -62,10 +117,17 @@ version = "0.1.0"
dependencies = [
"libc",
"md-5",
"rayon",
"sha1",
"sha2",
]
[[package]]
name = "hermit-abi"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b"
[[package]]
name = "libc"
version = "0.2.147"
@ -81,6 +143,53 @@ dependencies = [
"digest",
]
[[package]]
name = "memoffset"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c"
dependencies = [
"autocfg",
]
[[package]]
name = "num_cpus"
version = "1.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43"
dependencies = [
"hermit-abi",
"libc",
]
[[package]]
name = "rayon"
version = "1.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
dependencies = [
"either",
"rayon-core",
]
[[package]]
name = "rayon-core"
version = "1.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
dependencies = [
"crossbeam-channel",
"crossbeam-deque",
"crossbeam-utils",
"num_cpus",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
[[package]]
name = "sha1"
version = "0.10.5"

2
Cargo.toml
View File

@ -5,9 +5,11 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[features]
parallel = ["dep:rayon"]
[dependencies]
libc = "0.2"
md-5 = "0.10"
rayon = { version = "1.7", optional = true }
sha1 = "0.10"
sha2 = "0.10"

39
src/lib.rs
View File

@ -1,5 +1,7 @@
#![doc = include_str!("../README.md")]
use std::{fs, collections::HashMap, io::BufWriter, sync::Mutex};
#[cfg(feature = "parallel")]
use rayon::prelude::{IntoParallelRefIterator, ParallelIterator};
use std::{collections::HashMap, fs, io::BufWriter, sync::Mutex};
mod checksum;
mod error;
@ -11,7 +13,12 @@ mod special;
mod stream;
pub use {
checksum::{Algorithm, Checksum}, error::Error, file::File, filetype::FileType, node::Node, special::Special,
checksum::{Algorithm, Checksum},
error::Error,
file::File,
filetype::FileType,
node::Node,
special::Special,
stream::Stream,
};
@ -21,10 +28,34 @@ pub fn create_archive(path: &str, files: Vec<&str>, algorithm: Algorithm) -> Res
.truncate(true)
.open(path)?;
let mut writer = BufWriter::new(fd);
let mut links = Mutex::new(HashMap::new());
let links = Mutex::new(HashMap::new());
for f in files.iter() {
let node = Node::from_path(f, algorithm, &mut links)?;
let node = Node::from_path(f, algorithm, &links)?;
node.write(&mut writer)?;
}
Ok(())
}
#[cfg(feature = "parallel")]
pub fn par_create_archive(
path: &str,
files: Vec<String>,
algorithm: Algorithm,
) -> Result<(), Error> {
let links = Mutex::new(HashMap::<u64, String>::new());
{
let _fd = fs::File::create(path)?;
}
files.par_iter().try_for_each(|f| {
let node = Node::from_path(f, algorithm, &links)?;
let fd = fs::OpenOptions::new()
.create(false)
.truncate(false)
.append(true)
.open(path)?;
let mut writer = BufWriter::new(fd);
node.write(&mut writer)?;
Ok::<(), Error>(())
})?;
Ok(())
}