From 39b8664c9e4396d27ae8e2cba8e7a8e1faa399df Mon Sep 17 00:00:00 2001 From: Nathan Fisher Date: Mon, 27 Mar 2023 03:18:20 -0400 Subject: [PATCH] Added package `Creator` struct, with `create` method to process package entries fully multithreaded including compressing into the archive, as well as passing messages through an mpsc::channel() interface for use in a background thread in various frontends. Much better suited to task than previous `create_package` function. Untested. --- bootstrap/src/lib.rs | 9 +++- src/bootstrap.rs | 8 ++- src/creator/mod.rs | 115 +++++++++++++++++++++++++++++++++++++++++++ src/db/mod.rs | 2 +- src/lib.rs | 2 + src/package/mod.rs | 2 +- 6 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 src/creator/mod.rs diff --git a/bootstrap/src/lib.rs b/bootstrap/src/lib.rs index e1c1ec8..2a4ffbb 100644 --- a/bootstrap/src/lib.rs +++ b/bootstrap/src/lib.rs @@ -1,12 +1,17 @@ use { clap_complete::{generate_to, shells}, clap_complete_nushell::Nushell, - std::{error::Error, fs, ops::Deref, path::{Path, PathBuf}}, + std::{ + error::Error, + fs, + ops::Deref, + path::{Path, PathBuf}, + }, }; static PROGNAME: &str = "hpk"; -fn gencomp(outdir: PathBuf, gen:&str) -> Result<(), Box> { +fn gencomp(outdir: PathBuf, gen: &str) -> Result<(), Box> { let mut cmd = cli::cli(); let path = match gen { "bash" => generate_to(shells::Bash, &mut cmd, PROGNAME, outdir)?, diff --git a/src/bootstrap.rs b/src/bootstrap.rs index 2393a27..330c202 100644 --- a/src/bootstrap.rs +++ b/src/bootstrap.rs @@ -17,10 +17,14 @@ fn main() -> Result<(), Box> { Arg::new("output") .help("the output directory for the installation") .required(true) - .num_args(1) + .num_args(1), ]) .get_matches(); - let outdir = matches.get_one::("output").unwrap().to_string().into(); + let outdir = matches + .get_one::("output") + .unwrap() + .to_string() + .into(); let arch = matches.get_one::("arch").map(|x| x.to_string()); bootstrap::install(outdir, arch)?; Ok(()) diff --git a/src/creator/mod.rs b/src/creator/mod.rs new file mode 100644 index 0000000..b9303b4 --- /dev/null +++ b/src/creator/mod.rs @@ -0,0 +1,115 @@ +use std::{ + borrow::BorrowMut, + fs::File, + io::Write, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc::{self, Receiver}, + Mutex, + }, +}; + +use rayon::prelude::{IntoParallelRefIterator, ParallelIterator}; +use walkdir::WalkDir; +use zstd::Encoder; + +use crate::{Entry, Item, Plist, Package}; + +use { + crate::Specs, + std::{env, error::Error, io, path::Path, sync::mpsc::Sender}, + walkdir::DirEntry, +}; + +pub enum Message { + Success(String), + Failure(String), +} + +pub struct Creator { + entries: Vec>, + specs: Specs, + sender: Sender, + receiver: Receiver, +} + +impl Creator { + pub fn new(path: &Path, specs: Specs) -> Result { + env::set_current_dir(path)?; + let entries = WalkDir::new(".").into_iter().collect::>(); + let (sender, receiver) = mpsc::channel(); + Ok(Self { + entries, + specs, + sender, + receiver, + }) + } + + pub fn create(self, outdir: &Path) -> Result<(), Box> { + let plist = Mutex::new(Plist::default()); + let totalsize: AtomicUsize = 0.into(); + let fullname = format!( + "{}-{}_{}", + &self.specs.name, self.specs.version, self.specs.release + ); + let mut archive = outdir.to_path_buf(); + archive.push(&fullname); + archive.set_extension("tar.zst"); + let fd = File::create(&archive)?; + let writer = Mutex::new(Encoder::new(fd, 0)?); + let sender = Mutex::new(self.sender.clone()); + self.entries + .par_iter() + .filter(|x| x.is_ok()) + .map(|x| x.as_ref().unwrap()) + .for_each(|x| { + let sender = sender.lock().unwrap().clone(); + if let Ok(item) = Item::try_create(x.path().to_path_buf().as_path()) { + if let Entry::File { + path: _, + sha256sum: _, + mode: _, + size, + } = &item.entry + { + totalsize.fetch_add(*size, Ordering::Release); + } + let path = match item.entry.clone() { + Entry::File { + path, + sha256sum: _, + mode: _, + size: _, + } => path.clone(), + Entry::Link { path, target: _ } => path.clone(), + Entry::Directory { path, mode: _ } => path.clone(), + }; + plist.lock().unwrap().borrow_mut().entries.push(item.entry); + match writer.lock().unwrap().borrow_mut().write_all(&item.data) { + Ok(_) => sender + .send(Message::Success(format!( + "{} added to archive", + path.display() + ))) + .expect("couldn't send message"), + Err(e) => sender + .send(Message::Failure(format!("{e}"))) + .expect("couldn't send message"), + } + } else { + sender + .send(Message::Failure("Could not process DirEntry".to_string())) + .expect("could not send message"); + } + }); + let mut package: Package = self.specs.into(); + package.size = totalsize.into_inner(); + let node = package.save_ron_and_create_tar_node(outdir)?; + let mut writer = writer.into_inner()?; + writer.write_all(&node.to_vec()?)?; + let _fd = writer.finish()?; + self.sender.send(Message::Success(format!("{} saved", archive.display())))?; + Ok(()) + } +} diff --git a/src/db/mod.rs b/src/db/mod.rs index 73e995f..23797fc 100644 --- a/src/db/mod.rs +++ b/src/db/mod.rs @@ -1,5 +1,5 @@ use { - crate::{Repository, Package, Version}, + crate::{Package, Repository, Version}, serde::{Deserialize, Serialize}, std::{collections::HashMap, error::Error}, url::Url, diff --git a/src/lib.rs b/src/lib.rs index f0a18ce..eaa0a95 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,4 +1,5 @@ #![warn(clippy::all, clippy::pedantic)] +mod creator; mod db; mod hooks; mod item; @@ -8,6 +9,7 @@ mod repository; mod version; pub use { + creator::Creator, db::Database, hooks::Hooks, item::Item, diff --git a/src/package/mod.rs b/src/package/mod.rs index 13877ce..23e2227 100644 --- a/src/package/mod.rs +++ b/src/package/mod.rs @@ -92,7 +92,7 @@ impl Package { to_string_pretty(self, cfg) } - fn save_ron_and_create_tar_node(&self, outdir: &Path) -> Result> { + pub(crate) fn save_ron_and_create_tar_node(&self, outdir: &Path) -> Result> { if !outdir.exists() { fs::create_dir_all(outdir)?; }