diff --git a/addonscript/src/manifest/mod.rs b/addonscript/src/manifest/mod.rs index a6c0b80..6e04ffb 100644 --- a/addonscript/src/manifest/mod.rs +++ b/addonscript/src/manifest/mod.rs @@ -52,7 +52,7 @@ pub struct Relation { pub options: HashSet, } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum RelationType { Mod, diff --git a/mpt/Cargo.toml b/mpt/Cargo.toml index 80278ab..ac86ed6 100644 --- a/mpt/Cargo.toml +++ b/mpt/Cargo.toml @@ -12,10 +12,12 @@ path = "src/main.rs" [dependencies] addonscript = { path = "../addonscript"} anyhow = "1.0.43" +async-trait = "0.1.51" crossterm = "0.21.0" futures = "0.3.16" indicatif = "0.16.2" json5 = "0.3.0" +percent-encoding = "2.1.0" reqwest = { version = "0.11.4", features = ["stream"] } serde = { version = "1.0.129", features = ["derive"] } serde_json = "1.0.67" diff --git a/mpt/src/commands/downloadmods.rs b/mpt/src/commands/downloadmods.rs index 7be016c..5e9e6b2 100644 --- a/mpt/src/commands/downloadmods.rs +++ b/mpt/src/commands/downloadmods.rs @@ -1,29 +1,41 @@ -use crate::{config::Config, util::mvn_artifact_to_url}; +use crate::{ + config::Config, + downloader::{ + Callback, + CallbackStatus, + DownloadError, + DownloadInfo, + Downloader, + FileToDownload, + }, + util::{mvn_artifact_to_url, progress_style, CliStyle}, +}; use addonscript::manifest::{ installer::Installer, link::Link, File, - FileOpt, Manifest, - Relation, - Repository, + RelationType, RepositoryType, }; -use anyhow::{bail, Context}; -use crossterm::style::{Attribute, Color, Stylize}; -use futures::stream::{self, StreamExt}; +use anyhow::Context; +use async_trait::async_trait; +use crossterm::style::Stylize; use indicatif::ProgressBar; -use reqwest::{Client, StatusCode}; +use percent_encoding::percent_decode; +use reqwest::Client; use std::{ + borrow::Borrow, + collections::HashMap, path::{Path, PathBuf}, sync::Arc, }; -use tokio::{fs::File as TokioFile, io::AsyncWriteExt}; + use url::Url; pub async fn run( (config, manifest): (Config, Manifest), - dir: PathBuf, + target: PathBuf, all: bool, ) -> anyhow::Result<()> { let http = Arc::new(Client::new()); @@ -32,104 +44,72 @@ pub async fn run( repositories, .. } = manifest; - let repositories = Arc::new(repositories); + let version = versions.pop().context("Manifest has no Versions!")?; - let futs = versions - .pop() - .context("Manifest has no versions!")? - .relations - .into_iter() - .filter(|rel| { - rel.options.contains(&FileOpt::Client) - && rel - .file - .as_ref() - .map(|f| { - matches!(f.installer(), Installer::Dir(dir) if all || dir == Path::new("mods")) - }) - .unwrap_or(false) - }) - .map(|rel| download_file(Arc::clone(&http), Arc::clone(&repositories), dir.clone(), all, rel)) - .collect::>(); + let mut repos = HashMap::new(); - let pb = ProgressBar::new(futs.len() as u64); - let mut futs = stream::iter(futs).buffer_unordered(config.downloads.max_threads as usize); - - while let Some(res) = futs.next().await { - match res { - Ok(i) => match i { - DownloadInfo::Local { from, to } => pb.println(format!( - "{} {} {} {}", - "Copied local file".with(Color::Green), - from.with(Color::Cyan).attribute(Attribute::Bold), - "to".with(Color::Green), - to.with(Color::Cyan).attribute(Attribute::Bold), - )), - DownloadInfo::Http { status, from, to } => pb.println(format!( - "{} {} {} {} {} {}", - "Downloaded file".with(Color::Green), - from.with(Color::Cyan).attribute(Attribute::Bold), - "to".with(Color::Green), - to.with(Color::Cyan).attribute(Attribute::Bold), - "with status".with(Color::Green), - status - .to_string() - .with(Color::Cyan) - .attribute(Attribute::Bold), - )), - }, - Err(e) => { - pb.println(format!( - "{} {:?}", - "Error downloading file:" - .with(Color::Red) - .attribute(Attribute::Bold), - e, - )); - }, - } - pb.inc(1); + for repo in repositories { + repos.insert(repo.id.clone(), repo); } - Ok(()) -} + let mut links = vec![]; -enum DownloadInfo { - Local { - from: String, - to: String, - }, - Http { - status: StatusCode, - from: String, - to: String, - }, -} + let pb = ProgressBar::new(version.relations.len() as u64) + .with_prefix("Resolving") + .with_style(progress_style()); -async fn download_file( - http: Arc, - repos: Arc>, - mut target_dir: PathBuf, - all: bool, - rel: Relation, -) -> anyhow::Result { - let link; - let link = match &rel.file { - Some(File::Link { link, .. }) => link, - Some(File::Maven { - artifact, - repository, - .. - }) => { - if let Some(repo) = repos.iter().find(|r| &r.id == repository) { - match repo.repo_type { + for rel in version.relations { + pb.inc(1); + + // Only mods + if rel.relation_type != RelationType::Mod { + pb.println( + format!("Skipping non-mod relation `{}`", &rel.id) + .info() + .to_string(), + ); + continue; + } + + let rel_id = rel.id; + let file = rel + .file + .with_context(|| format!("Relation `{}` has no file!", rel_id))?; + + if !matches!(file.installer(), Installer::Dir(d) if all || d == Path::new("mods")) { + pb.println( + format!("Skipping excluded file `{:?}`", &file) + .info() + .to_string(), + ); + continue; + } + + let rels = match file { + File::Link { + link, installer, .. + } => (installer, link), + File::Maven { + installer, + artifact, + repository, + } => { + let repo = repos.get(&repository).with_context(|| { + format!("File references non-existant repository `{}`", repository) + })?; + + let url = match repo.repo_type { RepositoryType::Maven => { - link = Link::Http(mvn_artifact_to_url(&artifact, &repo)?); - &link + let url = mvn_artifact_to_url(&artifact, &repo)?; + pb.println(format!( + "{} {}", + "Resolved maven artifact with url".green(), + url.as_str().cyan().bold() + )); + url }, RepositoryType::Curseforge => { let mut splits = artifact.split(':').skip(1); - let p_id = splits .next() .context("Couldn't parse curseforge artifact!")?; @@ -142,60 +122,114 @@ async fn download_file( p_id, f_id ); - link = Link::Http( - Url::parse(http.get(url).send().await?.text().await?.trim()) - .context("failed to parse curseforge URL")?, - ); - &link + let url = Url::parse(http.get(url).send().await?.text().await?.trim()) + .context("failed to parse curseforge URL")?; + + pb.println(format!( + "{} {}", + "Resolved curseforge artifact with url".green(), + url.as_str().cyan().bold() + )); + + url }, - } + }; + + (installer, Link::Http(url)) + }, + }; + + links.push(rels); + } + pb.finish(); + + let mut files = vec![]; + for (installer, link) in links { + let mut to = target.clone(); + if all { + let dir = if let Installer::Dir(dir) = installer { + dir } else { - bail!("Rel {:?} references non-existant repository!") - } - }, - None => bail!("Rel {:?} has no file!", rel), - }; + // we checked that we have a dir installer earlier + unreachable!() + }; - if let (true, Installer::Dir(d)) = (all, rel.file.as_ref().unwrap().installer()) { - target_dir.push(d); + to.push(dir); + } + + match link { + Link::File(p) => { + to.push(p.file_name().context("Local file has no file name!")?); + tokio::fs::copy(&p, to).await?; + println!( + "{} {}", + "Copied local file".green(), + p.to_string_lossy().cyan().bold() + ); + }, + Link::Http(url) => { + let file = url + .path_segments() + .context("File uses base URL without path!")? + .last() + .context("File uses empty URL!")?; + let file = percent_decode(file.as_bytes()).decode_utf8_lossy(); + to.push(Borrow::::borrow(&file)); + files.push(FileToDownload { url, target: to }); + }, + } } - tokio::fs::create_dir_all(&target_dir).await?; - - match link { - Link::File(f) => { - let to = target_dir.join(f.file_name().context("File to copy is not a file!")?); - tokio::fs::copy( - f, - target_dir.join(f.file_name().context("File to copy is not a file!")?), - ) - .await - .context("Failed to install file with file link.")?; - - Ok(DownloadInfo::Local { - from: f.to_string_lossy().to_string(), - to: to.to_string_lossy().to_string(), - }) - }, - Link::Http(l) => { - let res = http.get(l.clone()).send().await?; - let p = target_dir.join( - Path::new(l.path()) - .file_name() - .context("HTTP File has no file name!")?, - ); - let mut file = TokioFile::create(&p).await?; - let status = res.status(); - let mut stream = res.bytes_stream(); - while let Some(b) = stream.next().await { - file.write_all_buf(&mut b?).await?; - } - - Ok(DownloadInfo::Http { - status, - from: l.to_string(), - to: p.to_string_lossy().to_string(), - }) - }, + struct Cb { + pb: ProgressBar, } + + #[async_trait] + impl Callback for Cb { + type EndRes = Result<(), DownloadError>; + type StopInfo = DownloadError; + + async fn on_download_complete( + &mut self, + res: Result, + ) -> CallbackStatus { + self.pb.inc(1); + match res { + Ok(i) => { + self.pb.println(format!( + "{} {} => {}", + i.status, + i.from.as_str().cyan().bold(), + i.to.to_string_lossy().cyan().bold() + )); + + CallbackStatus::Continue + }, + + Err(e) => CallbackStatus::Stop(e), + } + } + + async fn on_completed(self, stop_info: Option) -> Self::EndRes { + self.pb.finish(); + match stop_info { + Some(e) => Err(e), + None => Ok(()), + } + } + } + + let pb = ProgressBar::new(files.len() as u64) + .with_prefix("Downloading") + .with_style(progress_style()); + Downloader { + files, + callback: Cb { pb }, + parellel_count: config.downloads.max_threads, + client: http, + } + .download() + .await?; + + Ok(()) } diff --git a/mpt/src/config.rs b/mpt/src/config.rs index ad81e79..95765dd 100644 --- a/mpt/src/config.rs +++ b/mpt/src/config.rs @@ -17,6 +17,6 @@ pub struct Locations { #[derive(Deserialize)] #[serde(rename_all = "camelCase")] pub struct Downloads { - pub max_threads: u16, + pub max_threads: usize, pub http_timeout: u32, } diff --git a/mpt/src/downloader.rs b/mpt/src/downloader.rs new file mode 100644 index 0000000..626ad24 --- /dev/null +++ b/mpt/src/downloader.rs @@ -0,0 +1,107 @@ +use async_trait::async_trait; +use futures::{stream, StreamExt}; +use reqwest::{Client, StatusCode}; +use std::{path::PathBuf, sync::Arc}; +use thiserror::Error; +use tokio::{fs::File, io::AsyncWriteExt}; +use url::Url; + +pub struct Downloader { + pub callback: C, + pub files: Vec, + pub parellel_count: usize, + pub client: Arc, +} + +impl Downloader { + pub async fn download(self) -> C::EndRes { + let Self { + mut callback, + files, + parellel_count: parallel_count, + client, + } = self; + + let it = files + .into_iter() + .map(|f| Self::download_one(Arc::clone(&client), f.url, f.target)); + + let mut stream = stream::iter(it).buffer_unordered(parallel_count); + + let mut stop_info = None; + while let Some(res) = stream.next().await { + match callback.on_download_complete(res).await { + CallbackStatus::Stop(i) => { + stop_info = Some(i); + break; + }, + CallbackStatus::Continue => {}, + } + } + + callback.on_completed(stop_info).await + } + + async fn download_one( + client: Arc, + url: Url, + target: PathBuf, + ) -> Result { + if let Some(parent) = target.parent() { + tokio::fs::create_dir_all(parent).await?; + } + + let mut file = File::create(&target).await?; + + let res = client.get(url.clone()).send().await?; + let status = res.status(); + let mut stream = res.bytes_stream(); + + if let Some(b) = stream.next().await { + file.write_all_buf(&mut b?).await?; + } + + Ok(DownloadInfo { + from: url, + to: target, + status, + }) + } +} + +pub struct FileToDownload { + pub url: Url, + pub target: PathBuf, +} + +pub struct DownloadInfo { + pub from: Url, + pub to: PathBuf, + pub status: StatusCode, +} + +#[derive(Debug, Error)] +pub enum DownloadError { + #[error("HTTP Error: {0}")] + HttpError(#[from] reqwest::Error), + #[error("Filesystem error: {0}")] + FilesystemError(#[from] std::io::Error), +} + +pub enum CallbackStatus { + Stop(I), + Continue, +} + +#[async_trait] +pub trait Callback { + type EndRes; + type StopInfo; + + async fn on_download_complete( + &mut self, + res: Result, + ) -> CallbackStatus; + + async fn on_completed(self, stop_info: Option) -> Self::EndRes; +} diff --git a/mpt/src/main.rs b/mpt/src/main.rs index 75836d1..7a932a2 100644 --- a/mpt/src/main.rs +++ b/mpt/src/main.rs @@ -3,6 +3,7 @@ use structopt::StructOpt; mod commands; mod config; +mod downloader; mod forge; mod util; diff --git a/mpt/src/util.rs b/mpt/src/util.rs index eecacea..4f42485 100644 --- a/mpt/src/util.rs +++ b/mpt/src/util.rs @@ -1,4 +1,6 @@ use addonscript::manifest::{Manifest, Repository}; +use crossterm::style::{Attribute, Attributes, Color, ContentStyle, Stylize}; +use indicatif::ProgressStyle; use std::path::Path; use thiserror::Error; use url::Url; @@ -80,3 +82,25 @@ mod tests { ); } } + +pub trait CliStyle: Stylize { + fn info(self) -> Self::Styled; +} + +impl CliStyle for T +where + T::Styled: AsRef + AsMut, +{ + fn info(self) -> T::Styled { + let mut s = self.stylize(); + s.as_mut().foreground_color = Some(Color::Cyan); + s.as_mut().attributes = Attributes::from(Attribute::Italic); + s + } +} + +pub fn progress_style() -> ProgressStyle { + ProgressStyle::default_bar() + .template("{prefix:.bold} [{wide_bar:.green}] {pos}/{len}") + .progress_chars("█▇▆▅▄▃▂▁ ") +}