LordMZTE/legacympt-rs
1
0
Fork 0
Code Issues Pull requests Projects Releases 2 Wiki Activity

abstract file downloader and change downloadmods

Browse source
This commit is contained in:
LordMZTE 2021-08-30 19:56:24 +02:00
parent af3c217e88
commit 52d21fba63
7 changed files with 318 additions and 150 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
addonscript/src/manifest/mod.rs
View file

@ -52,7 +52,7 @@ pub struct Relation {
pub options: HashSet<FileOpt>,
}
#[derive(Deserialize, Serialize, Debug)]
#[derive(Deserialize, Serialize, Debug, Eq, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum RelationType {
Mod,

2
mpt/Cargo.toml
View file

@ -12,10 +12,12 @@ path = "src/main.rs"
[dependencies]
addonscript = { path = "../addonscript"}
anyhow = "1.0.43"
async-trait = "0.1.51"
crossterm = "0.21.0"
futures = "0.3.16"
indicatif = "0.16.2"
json5 = "0.3.0"
percent-encoding = "2.1.0"
reqwest = { version = "0.11.4", features = ["stream"] }
serde = { version = "1.0.129", features = ["derive"] }
serde_json = "1.0.67"

330
mpt/src/commands/downloadmods.rs
View file

@ -1,29 +1,41 @@
use crate::{config::Config, util::mvn_artifact_to_url};
use crate::{
config::Config,
downloader::{
Callback,
CallbackStatus,
DownloadError,
DownloadInfo,
Downloader,
FileToDownload,
},
util::{mvn_artifact_to_url, progress_style, CliStyle},
};
use addonscript::manifest::{
installer::Installer,
link::Link,
File,
FileOpt,
Manifest,
Relation,
Repository,
RelationType,
RepositoryType,
};
use anyhow::{bail, Context};
use crossterm::style::{Attribute, Color, Stylize};
use futures::stream::{self, StreamExt};
use anyhow::Context;
use async_trait::async_trait;
use crossterm::style::Stylize;
use indicatif::ProgressBar;
use reqwest::{Client, StatusCode};
use percent_encoding::percent_decode;
use reqwest::Client;
use std::{
borrow::Borrow,
collections::HashMap,
path::{Path, PathBuf},
sync::Arc,
};
use tokio::{fs::File as TokioFile, io::AsyncWriteExt};
use url::Url;
pub async fn run(
(config, manifest): (Config, Manifest),
dir: PathBuf,
target: PathBuf,
all: bool,
) -> anyhow::Result<()> {
let http = Arc::new(Client::new());
@ -32,104 +44,72 @@ pub async fn run(
repositories,
..
} = manifest;
let repositories = Arc::new(repositories);
let version = versions.pop().context("Manifest has no Versions!")?;
let futs = versions
.pop()
.context("Manifest has no versions!")?
.relations
.into_iter()
.filter(|rel| {
rel.options.contains(&FileOpt::Client)
&& rel
.file
.as_ref()
.map(|f| {
matches!(f.installer(), Installer::Dir(dir) if all || dir == Path::new("mods"))
})
.unwrap_or(false)
})
.map(|rel| download_file(Arc::clone(&http), Arc::clone(&repositories), dir.clone(), all, rel))
.collect::<Vec<_>>();
let mut repos = HashMap::new();
let pb = ProgressBar::new(futs.len() as u64);
let mut futs = stream::iter(futs).buffer_unordered(config.downloads.max_threads as usize);
while let Some(res) = futs.next().await {
match res {
Ok(i) => match i {
DownloadInfo::Local { from, to } => pb.println(format!(
"{} {} {} {}",
"Copied local file".with(Color::Green),
from.with(Color::Cyan).attribute(Attribute::Bold),
"to".with(Color::Green),
to.with(Color::Cyan).attribute(Attribute::Bold),
)),
DownloadInfo::Http { status, from, to } => pb.println(format!(
"{} {} {} {} {} {}",
"Downloaded file".with(Color::Green),
from.with(Color::Cyan).attribute(Attribute::Bold),
"to".with(Color::Green),
to.with(Color::Cyan).attribute(Attribute::Bold),
"with status".with(Color::Green),
status
.to_string()
.with(Color::Cyan)
.attribute(Attribute::Bold),
)),
},
Err(e) => {
pb.println(format!(
"{} {:?}",
"Error downloading file:"
.with(Color::Red)
.attribute(Attribute::Bold),
e,
));
},
}
pb.inc(1);
for repo in repositories {
repos.insert(repo.id.clone(), repo);
}
Ok(())
}
let mut links = vec![];
enum DownloadInfo {
Local {
from: String,
to: String,
},
Http {
status: StatusCode,
from: String,
to: String,
},
}
let pb = ProgressBar::new(version.relations.len() as u64)
.with_prefix("Resolving")
.with_style(progress_style());
async fn download_file(
http: Arc<Client>,
repos: Arc<Vec<Repository>>,
mut target_dir: PathBuf,
all: bool,
rel: Relation,
) -> anyhow::Result<DownloadInfo> {
let link;
let link = match &rel.file {
Some(File::Link { link, .. }) => link,
Some(File::Maven {
artifact,
repository,
..
}) => {
if let Some(repo) = repos.iter().find(|r| &r.id == repository) {
match repo.repo_type {
for rel in version.relations {
pb.inc(1);
// Only mods
if rel.relation_type != RelationType::Mod {
pb.println(
format!("Skipping non-mod relation `{}`", &rel.id)
.info()
.to_string(),
);
continue;
}
let rel_id = rel.id;
let file = rel
.file
.with_context(|| format!("Relation `{}` has no file!", rel_id))?;
if !matches!(file.installer(), Installer::Dir(d) if all || d == Path::new("mods")) {
pb.println(
format!("Skipping excluded file `{:?}`", &file)
.info()
.to_string(),
);
continue;
}
let rels = match file {
File::Link {
link, installer, ..
} => (installer, link),
File::Maven {
installer,
artifact,
repository,
} => {
let repo = repos.get(&repository).with_context(|| {
format!("File references non-existant repository `{}`", repository)
})?;
let url = match repo.repo_type {
RepositoryType::Maven => {
link = Link::Http(mvn_artifact_to_url(&artifact, &repo)?);
&link
let url = mvn_artifact_to_url(&artifact, &repo)?;
pb.println(format!(
"{} {}",
"Resolved maven artifact with url".green(),
url.as_str().cyan().bold()
));
url
},
RepositoryType::Curseforge => {
let mut splits = artifact.split(':').skip(1);
let p_id = splits
.next()
.context("Couldn't parse curseforge artifact!")?;
@ -142,60 +122,114 @@ async fn download_file(
p_id, f_id
);
link = Link::Http(
Url::parse(http.get(url).send().await?.text().await?.trim())
.context("failed to parse curseforge URL")?,
);
&link
let url = Url::parse(http.get(url).send().await?.text().await?.trim())
.context("failed to parse curseforge URL")?;
pb.println(format!(
"{} {}",
"Resolved curseforge artifact with url".green(),
url.as_str().cyan().bold()
));
url
},
}
};
(installer, Link::Http(url))
},
};
links.push(rels);
}
pb.finish();
let mut files = vec![];
for (installer, link) in links {
let mut to = target.clone();
if all {
let dir = if let Installer::Dir(dir) = installer {
dir
} else {
bail!("Rel {:?} references non-existant repository!")
}
},
None => bail!("Rel {:?} has no file!", rel),
};
// we checked that we have a dir installer earlier
unreachable!()
};
if let (true, Installer::Dir(d)) = (all, rel.file.as_ref().unwrap().installer()) {
target_dir.push(d);
to.push(dir);
}
match link {
Link::File(p) => {
to.push(p.file_name().context("Local file has no file name!")?);
tokio::fs::copy(&p, to).await?;
println!(
"{} {}",
"Copied local file".green(),
p.to_string_lossy().cyan().bold()
);
},
Link::Http(url) => {
let file = url
.path_segments()
.context("File uses base URL without path!")?
.last()
.context("File uses empty URL!")?;
let file = percent_decode(file.as_bytes()).decode_utf8_lossy();
to.push(Borrow::<str>::borrow(&file));
files.push(FileToDownload { url, target: to });
},
}
}
tokio::fs::create_dir_all(&target_dir).await?;
match link {
Link::File(f) => {
let to = target_dir.join(f.file_name().context("File to copy is not a file!")?);
tokio::fs::copy(
f,
target_dir.join(f.file_name().context("File to copy is not a file!")?),
)
.await
.context("Failed to install file with file link.")?;
Ok(DownloadInfo::Local {
from: f.to_string_lossy().to_string(),
to: to.to_string_lossy().to_string(),
})
},
Link::Http(l) => {
let res = http.get(l.clone()).send().await?;
let p = target_dir.join(
Path::new(l.path())
.file_name()
.context("HTTP File has no file name!")?,
);
let mut file = TokioFile::create(&p).await?;
let status = res.status();
let mut stream = res.bytes_stream();
while let Some(b) = stream.next().await {
file.write_all_buf(&mut b?).await?;
}
Ok(DownloadInfo::Http {
status,
from: l.to_string(),
to: p.to_string_lossy().to_string(),
})
},
struct Cb {
pb: ProgressBar,
}
#[async_trait]
impl Callback for Cb {
type EndRes = Result<(), DownloadError>;
type StopInfo = DownloadError;
async fn on_download_complete(
&mut self,
res: Result<DownloadInfo, DownloadError>,
) -> CallbackStatus<Self::StopInfo> {
self.pb.inc(1);
match res {
Ok(i) => {
self.pb.println(format!(
"{} {} => {}",
i.status,
i.from.as_str().cyan().bold(),
i.to.to_string_lossy().cyan().bold()
));
CallbackStatus::Continue
},
Err(e) => CallbackStatus::Stop(e),
}
}
async fn on_completed(self, stop_info: Option<Self::StopInfo>) -> Self::EndRes {
self.pb.finish();
match stop_info {
Some(e) => Err(e),
None => Ok(()),
}
}
}
let pb = ProgressBar::new(files.len() as u64)
.with_prefix("Downloading")
.with_style(progress_style());
Downloader {
files,
callback: Cb { pb },
parellel_count: config.downloads.max_threads,
client: http,
}
.download()
.await?;
Ok(())
}

2
mpt/src/config.rs
View file

@ -17,6 +17,6 @@ pub struct Locations {
#[derive(Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct Downloads {
pub max_threads: u16,
pub max_threads: usize,
pub http_timeout: u32,
}

107
mpt/src/downloader.rs Normal file
View file

@ -0,0 +1,107 @@
use async_trait::async_trait;
use futures::{stream, StreamExt};
use reqwest::{Client, StatusCode};
use std::{path::PathBuf, sync::Arc};
use thiserror::Error;
use tokio::{fs::File, io::AsyncWriteExt};
use url::Url;
pub struct Downloader<C: Callback> {
pub callback: C,
pub files: Vec<FileToDownload>,
pub parellel_count: usize,
pub client: Arc<Client>,
}
impl<C: Callback> Downloader<C> {
pub async fn download(self) -> C::EndRes {
let Self {
mut callback,
files,
parellel_count: parallel_count,
client,
} = self;
let it = files
.into_iter()
.map(|f| Self::download_one(Arc::clone(&client), f.url, f.target));
let mut stream = stream::iter(it).buffer_unordered(parallel_count);
let mut stop_info = None;
while let Some(res) = stream.next().await {
match callback.on_download_complete(res).await {
CallbackStatus::Stop(i) => {
stop_info = Some(i);
break;
},
CallbackStatus::Continue => {},
}
}
callback.on_completed(stop_info).await
}
async fn download_one(
client: Arc<Client>,
url: Url,
target: PathBuf,
) -> Result<DownloadInfo, DownloadError> {
if let Some(parent) = target.parent() {
tokio::fs::create_dir_all(parent).await?;
}
let mut file = File::create(&target).await?;
let res = client.get(url.clone()).send().await?;
let status = res.status();
let mut stream = res.bytes_stream();
if let Some(b) = stream.next().await {
file.write_all_buf(&mut b?).await?;
}
Ok(DownloadInfo {
from: url,
to: target,
status,
})
}
}
pub struct FileToDownload {
pub url: Url,
pub target: PathBuf,
}
pub struct DownloadInfo {
pub from: Url,
pub to: PathBuf,
pub status: StatusCode,
}
#[derive(Debug, Error)]
pub enum DownloadError {
#[error("HTTP Error: {0}")]
HttpError(#[from] reqwest::Error),
#[error("Filesystem error: {0}")]
FilesystemError(#[from] std::io::Error),
}
pub enum CallbackStatus<I> {
Stop(I),
Continue,
}
#[async_trait]
pub trait Callback {
type EndRes;
type StopInfo;
async fn on_download_complete(
&mut self,
res: Result<DownloadInfo, DownloadError>,
) -> CallbackStatus<Self::StopInfo>;
async fn on_completed(self, stop_info: Option<Self::StopInfo>) -> Self::EndRes;
}

1
mpt/src/main.rs
View file

@ -3,6 +3,7 @@ use structopt::StructOpt;
mod commands;
mod config;
mod downloader;
mod forge;
mod util;

24
mpt/src/util.rs
View file

@ -1,4 +1,6 @@
use addonscript::manifest::{Manifest, Repository};
use crossterm::style::{Attribute, Attributes, Color, ContentStyle, Stylize};
use indicatif::ProgressStyle;
use std::path::Path;
use thiserror::Error;
use url::Url;
@ -80,3 +82,25 @@ mod tests {
);
}
}
pub trait CliStyle: Stylize {
fn info(self) -> Self::Styled;
}
impl<T: Stylize> CliStyle for T
where
T::Styled: AsRef<ContentStyle> + AsMut<ContentStyle>,
{
fn info(self) -> T::Styled {
let mut s = self.stylize();
s.as_mut().foreground_color = Some(Color::Cyan);
s.as_mut().attributes = Attributes::from(Attribute::Italic);
s
}
}
pub fn progress_style() -> ProgressStyle {
ProgressStyle::default_bar()
.template("{prefix:.bold} [{wide_bar:.green}] {pos}/{len}")
.progress_chars("█▇▆▅▄▃▂▁ ")
}