fix: make repositories optional

curseforge's API is so good, this endpoints only works half of the
time...

.drone.yml

@@ -5,11 +5,15 @@ steps:
   - name: test-linux
     image: rust
     commands:
+      - apt update
+      - apt -y install libluajit-5.1-dev
       - cargo test -v
 
   - name: clippy-linux
     image: rust
     commands:
+      - apt update
+      - apt -y install libluajit-5.1-dev
       - rustup component add clippy
       - cargo clippy
 ---
@@ -20,13 +24,17 @@ steps:
   - name: release-linux
     image: rust
     commands:
+      - apt update
+      - apt -y install libluajit-5.1-dev
       - cargo build --release -v
       - strip target/release/lmpt
 
   - name: release-windows
     image: lordmzte/rust-win
     commands:
-        - cargo build --release --target x86_64-pc-windows-gnu -v
+      - apt update
+      - apt -y install libluajit-5.1-dev
+      - cargo build --release --target x86_64-pc-windows-gnu -v
 
   - name: publish
     image: plugins/gitea-release

CHANGELOG.md

@@ -1,2 +1,15 @@
-# 0.1.0
-- initial release!
+# 0.1.4
+
+- fix optional `repositories` value being required
+
+# 0.1.3
+
+- add support for defines
+- update dependencies
+- switch from structopt to clap with macro
+
+# 0.1.2
+
+- optimize curseforge relation meta querying
+- add debug logging
+- import and init now set the modpack's ID to a kebab-case version of the name

addonscript/Cargo.toml

@@ -6,6 +6,6 @@ edition = "2018"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-serde = { version = "1.0.129", features = ["derive"] }
-thiserror = "1.0.26"
+serde = { version = "1.0.136", features = ["derive"] }
+thiserror = "1.0.30"
 url = { version = "2.2.2", features = ["serde"] }

addonscript/src/manifest/mod.rs

@@ -12,6 +12,7 @@ pub struct Manifest {
     #[serde(rename = "type")]
     pub manifest_type: ManifestType,
     pub versions: Vec<Version>,
+    #[serde(default)]
     pub repositories: Vec<Repository>,
     pub meta: Meta,
 }

mpt/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mpt"
-version = "0.1.0"
+version = "0.1.4"
 edition = "2018"
 
 [[bin]]
@@ -11,20 +11,24 @@ path = "src/main.rs"
 
 [dependencies]
 addonscript = { path = "../addonscript" }
-anyhow = "1.0.43"
-async-trait = "0.1.51"
-crossterm = "0.21.0"
+async-trait = "0.1.52"
+clap = { version = "3.1.5", features = ["derive"] }
+crossterm = "0.23.0"
 futures = "0.3.16"
+heck = "0.4.0"
 indicatif = "0.16.2"
-json5 = "0.3.0"
+json5 = "0.4.1"
+log = "0.4.14"
+miette = { version = "4.2.1", features = ["fancy"] }
+mlua = { version = "0.7.4", features = ["luajit", "serialize"] }
 percent-encoding = "2.1.0"
-reqwest = { version = "0.11.4", features = ["stream"] }
-serde = { version = "1.0.129", features = ["derive"] }
-serde_json = "1.0.67"
-structopt = "0.3.22"
-tera = "1.12.1"
-thiserror = "1.0.28"
-tokio = { version = "1.10.1", features = ["rt-multi-thread", "macros", "fs"] }
+reqwest = { version = "0.11.9", features = ["stream"] }
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = "1.0.79"
+simplelog = "0.11.2"
+tera = "1.15.0"
+thiserror = "1.0.30"
+tokio = { version = "1.17.0", features = ["rt-multi-thread", "macros", "fs"] }
 toml = "0.5.8"
 twitch = { path = "../twitch" }
 url = "2.2.2"

mpt/src/commands/buildtwitch.rs

@@ -21,9 +21,10 @@ use addonscript::manifest::{
     Repository,
     RepositoryType,
 };
-use anyhow::{bail, Context};
 use async_trait::async_trait;
 use indicatif::ProgressBar;
+use log::{debug, info};
+use miette::{bail, miette, IntoDiagnostic, WrapErr};
 use reqwest::Client;
 use std::{collections::HashMap, io::Write, path::Path, sync::Arc};
 use tokio::io::AsyncReadExt;
@@ -31,14 +32,14 @@ use twitch::manifest::Manifest as TwManifest;
 use walkdir::WalkDir;
 use zip::{CompressionMethod, ZipWriter};
 
-pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> anyhow::Result<()> {
+pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> miette::Result<()> {
     util::make_tmp_dir(&config).await?;
     let twitch_dir = config.locations.temp_dir.join("twitch");
 
     let mut version = manifest
         .versions
         .pop()
-        .context("Manifest has no versions!")?;
+        .ok_or_else(|| miette!("Manifest has no versions!"))?;
     let repos = util::repo_map(manifest.repositories);
 
     let mut modloader = None;
@@ -62,9 +63,9 @@ pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> anyho
 
             let version = rel
                 .versions
-                .context("Modloader is missing `versions` field!")?;
-            let version =
-                forge::parse_version(&version).context("Couldn't parse forge version!")?;
+                .ok_or_else(|| miette!("Modloader is missing `versions` field!"))?;
+            let version = forge::parse_version(&version)
+                .ok_or_else(|| miette!("Couldn't parse forge version!"))?;
             modloader = Some(format!("forge-{}", version));
 
             continue;
@@ -110,8 +111,10 @@ pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> anyho
 
                 match installer {
                     Installer::Dir(dir) => {
+                        info!("copying dir installer file {}", path.to_string_lossy());
                         tokio::fs::copy(config.locations.src.join(path), twitch_dir.join(dir))
-                            .await?;
+                            .await
+                            .into_diagnostic()?;
                     },
 
                     Installer::Override => {
@@ -120,7 +123,9 @@ pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> anyho
                             bail!("File with override installer is not directory!");
                         }
 
-                        util::copy_dir(path, twitch_dir.join("overrides")).await?;
+                        util::copy_dir(path, twitch_dir.join("overrides"))
+                            .await
+                            .into_diagnostic()?;
                     },
                 }
             },
@@ -190,15 +195,22 @@ pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> anyho
             .collect::<Vec<_>>(),
         manifest.meta.name.clone(),
         version.version.clone(),
-        version.mcversion.pop().context("mcversion is empty!")?,
+        version
+            .mcversion
+            .pop()
+            .ok_or_else(|| miette!("mcversion is empty!"))?,
         modloader,
     );
 
-    let json = serde_json::to_vec(&tw_manifest).context("Failed to serialize twitch manifest")?;
+    let json = serde_json::to_vec(&tw_manifest)
+        .into_diagnostic()
+        .wrap_err("Failed to serialize twitch manifest")?;
 
-    tokio::fs::write(twitch_dir.join("manifest.json"), json).await?;
+    tokio::fs::write(twitch_dir.join("manifest.json"), json)
+        .await
+        .into_diagnostic()?;
 
-    tokio::fs::create_dir_all("build").await?;
+    tokio::fs::create_dir_all("build").await.into_diagnostic()?;
 
     println!("{}", "Zipping pack.".info());
 
@@ -208,7 +220,8 @@ pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> anyho
             "build/{}-{}-twitch.zip",
             manifest.meta.name, version.version
         )))
-        .context("Failed to open zip file!")?,
+        .into_diagnostic()
+        .wrap_err("Failed to open zip file!")?,
     );
 
     let options =
@@ -216,26 +229,30 @@ pub async fn run((config, mut manifest): (Config, Manifest), all: bool) -> anyho
 
     let mut buf = vec![];
     for entry in WalkDir::new(&twitch_dir) {
-        let entry = entry?;
+        let entry = entry.into_diagnostic()?;
         let path = entry.path();
-        let to = path.strip_prefix(&twitch_dir)?;
+        let to = path.strip_prefix(&twitch_dir).into_diagnostic()?;
 
         if path.is_file() {
-            zip.start_file(to.to_string_lossy(), options)?;
+            zip.start_file(to.to_string_lossy(), options)
+                .into_diagnostic()?;
 
             tokio::fs::File::open(path)
-                .await?
+                .await
+                .into_diagnostic()?
                 .read_to_end(&mut buf)
-                .await?;
+                .await
+                .into_diagnostic()?;
 
-            zip.write_all(&buf)?;
+            zip.write_all(&buf).into_diagnostic()?;
 
             buf.clear();
         } else if !to.as_os_str().is_empty() {
-            zip.add_directory(to.to_string_lossy(), options)?;
+            zip.add_directory(to.to_string_lossy(), options)
+                .into_diagnostic()?;
         }
     }
-    zip.finish()?;
+    zip.finish().into_diagnostic()?;
 
     Ok(())
 }
@@ -245,7 +262,8 @@ fn sort_file(
     repos: &HashMap<String, Repository>,
     link_rels: &mut Vec<(Installer, Link)>,
     cf_rels: &mut Vec<(u32, u32)>,
-) -> anyhow::Result<()> {
+) -> miette::Result<()> {
+    debug!("Sorting file {:?}", &file);
     match file {
         File::Link {
             installer, link, ..
@@ -258,11 +276,12 @@ fn sort_file(
         } => {
             let repo = repos
                 .get(&repository)
-                .with_context(|| format!("File references unknown repository {}", &repository))?;
+                .ok_or_else(|| miette!("File references unknown repository {}", &repository))?;
             match repo.repo_type {
                 RepositoryType::Maven => {
-                    let url = util::mvn_artifact_to_url(&artifact, &repo)
-                        .context("Failed to convert maven artifact to url")?;
+                    let url = util::mvn_artifact_to_url(&artifact, repo)
+                        .into_diagnostic()
+                        .wrap_err("Failed to convert maven artifact to url")?;
 
                     link_rels.push((installer, Link::Http(url)));
                 },
@@ -270,8 +289,11 @@ fn sort_file(
                     let (p_id, f_id) = util::parse_curseforge_artifact(&artifact)?;
                     cf_rels.push((
                         p_id.parse()
-                            .context("Couldn't parse curseforge project ID!")?,
-                        f_id.parse().context("Couldn't parse curseforge file ID!")?,
+                            .into_diagnostic()
+                            .wrap_err("Couldn't parse curseforge project ID!")?,
+                        f_id.parse()
+                            .into_diagnostic()
+                            .wrap_err("Couldn't parse curseforge file ID!")?,
                     ));
                 },
             }

mpt/src/commands/clean.rs

@@ -1,9 +1,10 @@
 use crate::{config::Config, util::CliStyle};
 
-pub async fn run(config: Config) -> anyhow::Result<()> {
+pub async fn run(config: Config) -> miette::Result<()> {
     // These operations are so incredibly performance-critical that we absolutely
     // must execute them in parallel!
-    let (res1, res2) = tokio::join!(
+    // we ignore the results on purpose, so nothing fails when the dirs dont exist.
+    let (_, _) = tokio::join!(
         async {
             println!(
                 "{}",
@@ -17,8 +18,5 @@ pub async fn run(config: Config) -> anyhow::Result<()> {
         }
     );
 
-    res1?;
-    res2?;
-
     Ok(())
 }

mpt/src/commands/createmodlist.rs

@@ -8,10 +8,8 @@ use addonscript::manifest::{
     Repository,
     RepositoryType,
 };
-use anyhow::Context;
-use crossterm::style::Stylize;
-use futures::{stream, StreamExt};
-use indicatif::ProgressBar;
+use log::{debug, info};
+use miette::{miette, IntoDiagnostic, WrapErr};
 use reqwest::Client;
 use serde::Serialize;
 use std::{collections::HashMap, path::PathBuf, sync::Arc};
@@ -21,87 +19,133 @@ use twitch::api::AddonInfoResponse;
 const TEMPLATE: &str = include_str!("../../assets/modlist.html.tera");
 
 pub async fn run(
-    (config, mut manifest): (Config, Manifest),
+    (_config, mut manifest): (Config, Manifest),
     outfile: PathBuf,
-) -> anyhow::Result<()> {
+) -> miette::Result<()> {
     let version = manifest
         .versions
         .pop()
-        .context("Manifest has no versions!")?;
+        .ok_or_else(|| miette!("Manifest has no versions!"))?;
     let repos = util::repo_map(manifest.repositories);
 
+    info!("instatiating tera engine");
     let mut tera = Tera::default();
-    tera.add_raw_template("modlist", TEMPLATE)?;
+    tera.add_raw_template("modlist", TEMPLATE)
+        .into_diagnostic()?;
 
     let http = Arc::new(Client::new());
 
-    let len = version.relations.len();
-
-    let mut futures = stream::iter(version.relations.into_iter())
-        .map(|rel| get_meta(Arc::clone(&http), rel, &repos))
-        .buffer_unordered(config.downloads.max_threads);
-
+    println!("{}", "Resolving metas.".info());
     let mut metas = vec![];
-    let pb = ProgressBar::new(len as u64)
-        .with_style(util::progress_style())
-        .with_prefix("Resolving metadata");
-    while let Some(meta) = futures.next().await {
-        pb.inc(1);
-        let meta = meta?;
-        pb.println(format!(
-            "{} {}",
-            "Got meta for".green(),
-            AsRef::<str>::as_ref(&meta.name).cyan().bold()
-        ));
-        metas.push(meta);
+    let mut cf_rels = vec![];
+    for rel in version.relations {
+        let mi = get_meta(rel, &repos)?;
+
+        match mi {
+            MetaInfo::Meta(meta) => metas.push(meta),
+            MetaInfo::CfId(id) => cf_rels.push(id),
+        }
+    }
+
+    if !cf_rels.is_empty() {
+        println!("{}", "Querying CF metas.".info());
+
+        info!(
+            "Requesting addon info for {} curseforge mods",
+            cf_rels.len()
+        );
+
+        let res = http
+            .post("https://addons-ecs.forgesvc.net/api/v2/addon")
+            .body(
+                serde_json::to_string(&cf_rels)
+                    .into_diagnostic()
+                    .wrap_err("Failed to serialize curseforge relation IDs")?,
+            )
+            .header("Content-Type", "application/json")
+            .send()
+            .await
+            .into_diagnostic()
+            .wrap_err("Failed sending CF relation request")?
+            .bytes()
+            .await
+            .into_diagnostic()
+            .wrap_err("Failed getting CF relation response body")?;
+
+        let cf_metas = serde_json::from_slice::<Vec<AddonInfoResponse>>(&res)
+            .into_diagnostic()
+            .wrap_err("Failed deserializing CF relation response")?;
+
+        info!("Converting CF metas to AS metas");
+        let cf_metas = cf_metas.into_iter().map(|m| Meta {
+            name: m.name,
+            contributors: m
+                .authors
+                .into_iter()
+                .map(|a| Contributor {
+                    name: a.name,
+                    roles: vec!["author".into()],
+                })
+                .collect(),
+            description: Some(m.summary),
+            icon_url: m
+                .attachments
+                .into_iter()
+                .find(|a| a.is_default)
+                .map(|a| a.url.to_string()),
+            website_url: Some(m.website_url),
+        });
+
+        metas.extend(cf_metas);
     }
-    pb.finish();
 
     metas.sort_by_key(|m| m.name.to_ascii_lowercase());
 
     println!("{}", "Rendering modlist.".info());
 
-    let rendered = tera.render(
-        "modlist",
-        &tera::Context::from_serialize(&ModListContent {
-            metas,
-            pack_meta: manifest.meta,
-        })?,
-    )?;
+    let rendered = tera
+        .render(
+            "modlist",
+            &tera::Context::from_serialize(&ModListContent {
+                metas,
+                pack_meta: manifest.meta,
+            })
+            .into_diagnostic()?,
+        )
+        .into_diagnostic()?;
 
     println!("{}", "Writing file.".info());
     if let Some(parent) = outfile.parent() {
-        tokio::fs::create_dir_all(parent).await?;
+        tokio::fs::create_dir_all(parent).await.into_diagnostic()?;
     }
-    tokio::fs::write(outfile, rendered).await?;
+    tokio::fs::write(outfile, rendered)
+        .await
+        .into_diagnostic()?;
 
     Ok(())
 }
 
-async fn get_meta(
-    http: Arc<Client>,
-    rel: Relation,
-    repos: &HashMap<String, Repository>,
-) -> anyhow::Result<Meta> {
+fn get_meta(rel: Relation, repos: &HashMap<String, Repository>) -> miette::Result<MetaInfo> {
+    debug!("getting meta for {:?}", &rel);
     if let Some(meta) = rel.meta {
-        return Ok(meta);
+        return Ok(MetaInfo::Meta(meta));
     }
 
     if rel.file.is_none() {
-        return Ok(Meta {
+        return Ok(MetaInfo::Meta(Meta {
             name: rel.id.clone(),
             contributors: vec![],
             description: rel.versions.map(|v| format!("version {}", v)),
             icon_url: None,
             website_url: None,
-        });
+        }));
     }
 
     let file = rel.file.unwrap();
 
     match file {
         File::Link { link, id, .. } => {
-            Ok(Meta {
+            Ok(MetaInfo::Meta(Meta {
                 name: if let Some(id) = id {
                     id
                 } else {
@@ -113,7 +157,7 @@ async fn get_meta(
                 description: Some(format!("{:?}", &link)),
                 icon_url: None,
                 website_url: None,
-            })
+            }))
         },
         File::Maven {
             repository,
@@ -122,53 +166,24 @@ async fn get_meta(
         } => {
             let repo = repos
                 .get(&repository)
-                .context("File references unknown repository!")?;
+                .ok_or_else(|| miette!("File references unknown repository!"))?;
 
             match repo.repo_type {
-                RepositoryType::Maven => Ok(Meta {
+                RepositoryType::Maven => Ok(MetaInfo::Meta(Meta {
                     name: artifact,
                     contributors: vec![],
                     description: None,
                     icon_url: None,
                     website_url: None,
-                }),
+                })),
                 RepositoryType::Curseforge => {
                     let (p_id, _) = util::parse_curseforge_artifact(&artifact)?;
-                    let resp = http
-                        .get(format!(
-                            "https://addons-ecs.forgesvc.net/api/v2/addon/{}",
-                            p_id
-                        ))
-                        .send()
-                        .await?
-                        .bytes()
-                        .await?;
+                    let p_id = p_id
+                        .parse::<u32>()
+                        .into_diagnostic()
+                        .wrap_err("Failed to parse curseforge ID")?;
 
-                    let resp = serde_json::from_slice::<AddonInfoResponse>(&resp)
-                        .context("Failed to deserialize addon info!")?;
-
-                    let icon_url = resp.attachments.into_iter().find_map(|a| {
-                        if a.is_default {
-                            Some(a.url.into())
-                        } else {
-                            None
-                        }
-                    });
-
-                    Ok(Meta {
-                        name: resp.name,
-                        contributors: resp
-                            .authors
-                            .into_iter()
-                            .map(|a| Contributor {
-                                name: a.name,
-                                roles: vec![],
-                            })
-                            .collect(),
-                        description: Some(resp.summary),
-                        icon_url,
-                        website_url: Some(resp.website_url),
-                    })
+                    Ok(MetaInfo::CfId(p_id))
                 },
             }
         },
@@ -180,3 +195,8 @@ struct ModListContent {
     metas: Vec<Meta>,
     pack_meta: Meta,
 }
+
+enum MetaInfo {
+    Meta(Meta),
+    CfId(u32),
+}

mpt/src/commands/downloadmods.rs

@@ -19,11 +19,11 @@ use addonscript::manifest::{
     Repository,
     RepositoryType,
 };
-use anyhow::Context;
 use async_trait::async_trait;
 use crossterm::style::Stylize;
 use futures::{stream, StreamExt};
 use indicatif::ProgressBar;
+use miette::{miette, IntoDiagnostic, WrapErr};
 use percent_encoding::percent_decode;
 use reqwest::Client;
 use std::{
@@ -38,14 +38,16 @@ pub async fn run(
     (config, manifest): (Config, Manifest),
     target: PathBuf,
     all: bool,
-) -> anyhow::Result<()> {
+) -> miette::Result<()> {
     let http = Arc::new(Client::new());
     let Manifest {
         mut versions,
         repositories,
         ..
     } = manifest;
-    let version = versions.pop().context("Manifest has no Versions!")?;
+    let version = versions
+        .pop()
+        .ok_or_else(|| miette!("Manifest has no Versions!"))?;
 
     let repos = util::repo_map(repositories);
 
@@ -68,7 +70,7 @@ pub async fn run(
         let rel_id = rel.id;
         let file = rel
             .file
-            .with_context(|| format!("Relation `{}` has no file!", rel_id))?;
+            .ok_or_else(|| miette!("Relation `{}` has no file!", rel_id))?;
 
         if !matches!(file.installer(), Installer::Dir(d) if all || d == Path::new("mods")) {
             pb.println(
@@ -111,8 +113,11 @@ pub async fn run(
 
         match link {
             Link::File(p) => {
-                to.push(p.file_name().context("Local file has no file name!")?);
-                tokio::fs::copy(&p, to).await?;
+                to.push(
+                    p.file_name()
+                        .ok_or_else(|| miette!("Local file has no file name!"))?,
+                );
+                tokio::fs::copy(&p, to).await.into_diagnostic()?;
                 println!(
                     "{} {}",
                     "Copied local file".green(),
@@ -122,9 +127,9 @@ pub async fn run(
             Link::Http(url) => {
                 let file = url
                     .path_segments()
-                    .context("File uses base URL without path!")?
+                    .ok_or_else(|| miette!("File uses base URL without path!"))?
                     .last()
-                    .context("File uses empty URL!")?;
+                    .ok_or_else(|| miette!("File uses empty URL!"))?;
                 let file = percent_decode(file.as_bytes()).decode_utf8_lossy();
                 to.push(Borrow::<str>::borrow(&file));
                 if !to.exists() {
@@ -188,7 +193,7 @@ async fn process_file(
     repos: &HashMap<String, Repository>,
     pb: &ProgressBar,
     http: Arc<Client>,
-) -> anyhow::Result<(Installer, Link)> {
+) -> miette::Result<(Installer, Link)> {
     Ok(match file {
         File::Link {
             link, installer, ..
@@ -198,13 +203,13 @@ async fn process_file(
             artifact,
             repository,
         } => {
-            let repo = repos.get(&repository).with_context(|| {
-                format!("File references non-existant repository `{}`", repository)
+            let repo = repos.get(&repository).ok_or_else(|| {
+                miette!("File references non-existant repository `{}`", repository)
             })?;
 
             let url = match repo.repo_type {
                 RepositoryType::Maven => {
-                    let url = mvn_artifact_to_url(&artifact, &repo)?;
+                    let url = mvn_artifact_to_url(&artifact, repo)?;
                     pb.println(format!(
                         "{} {}",
                         "Resolved maven artifact with url".green(),
@@ -220,8 +225,34 @@ async fn process_file(
                         p_id, f_id
                     );
 
-                    let url = Url::parse(http.get(url).send().await?.text().await?.trim())
-                        .context("failed to parse curseforge URL")?;
+                    let mut failed = false;
+                    let url = loop {
+                        let maybe_url = Url::parse(
+                            http.get(&url)
+                                .send()
+                                .await
+                                .into_diagnostic()?
+                                .text()
+                                .await
+                                .into_diagnostic()?
+                                .trim(),
+                        );
+
+                        if let (false, Err(_)) = (failed, &maybe_url) {
+                            pb.println(
+                                "Failed to resolve artifact, as curseforge is too incompetent to \
+                                 host a stable API. Retrying..."
+                                    .red()
+                                    .to_string(),
+                            );
+                            failed = true;
+                            continue;
+                        }
+
+                        break maybe_url
+                            .into_diagnostic()
+                            .wrap_err("Failed to parse curseforge URL even after 2 tries lol")?;
+                    };
 
                     pb.println(format!(
                         "{} {}",

mpt/src/commands/import.rs

@@ -0,0 +1,136 @@
+use addonscript::{
+    manifest::{
+        installer::Installer,
+        link::Link,
+        Contributor,
+        File,
+        Manifest,
+        ManifestType,
+        Meta,
+        Relation,
+        RelationType,
+        Repository,
+        RepositoryType,
+        Version,
+    },
+    util::default_file_opts,
+};
+use crossterm::style::Stylize;
+use heck::ToKebabCase;
+use log::info;
+use miette::{bail, IntoDiagnostic, WrapErr};
+use twitch::manifest::Manifest as TwManifest;
+use url::Url;
+
+use std::path::PathBuf;
+
+use crate::config::Config;
+
+pub async fn run(config: Config, infile: PathBuf) -> miette::Result<()> {
+    if config.locations.src.join("modpack.json").exists() ||
+        config.locations.src.join("modpack.json5").exists()
+    {
+        bail!("Manifest already exists!");
+    }
+
+    let mut data = serde_json::from_slice::<TwManifest>(
+        &tokio::fs::read(infile)
+            .await
+            .into_diagnostic()
+            .wrap_err("Failed to read twitch manifest")?,
+    )
+    .into_diagnostic()
+    .wrap_err("Failed to parse twitch manifest")?;
+
+    info!("converting twitch mods to AS relations");
+    let mut relations = data
+        .files
+        .into_iter()
+        .map(|f| Relation {
+            id: f.project_id.to_string(),
+            file: Some(File::Maven {
+                installer: Installer::Dir("mods".into()),
+                artifact: format!("curse:{}:{}", f.project_id, f.file_id),
+                repository: "curseforge".into(),
+            }),
+            versions: None,
+            meta: None,
+            relation_type: RelationType::Mod,
+            options: default_file_opts(),
+        })
+        .collect::<Vec<_>>();
+
+    if let Some(ml) = data.minecraft.mod_loaders.pop() {
+        info!("fount modloader {:?}", &ml);
+        let mut splits = ml.id.split('-');
+        if !matches!(splits.next(), Some("forge")) {
+            bail!("Twitch manifest contains invalid or unknown modloader!");
+        }
+
+        let forgever = splits.next();
+        if forgever.is_none() {
+            bail!("Modloader in twitch manifest missing version!");
+        }
+        let forgever = forgever.unwrap();
+
+        relations.push(Relation {
+            id: "forge".into(),
+            file: None,
+            relation_type: RelationType::Modloader,
+            options: default_file_opts(),
+            versions: Some(format!(
+                "[{mcver}-{forgever}]",
+                mcver = &data.minecraft.version,
+                forgever = forgever,
+            )),
+            meta: None,
+        });
+    }
+
+    let manif = Manifest {
+        id: data.name.to_kebab_case(),
+        manifest_type: ManifestType::Modpack,
+        versions: vec![Version {
+            mcversion: vec![data.minecraft.version],
+            version: data.version,
+            files: vec![File::Link {
+                id: Some("overrides".into()),
+                link: Link::File("overrides".into()),
+                installer: Installer::Override,
+                options: Some(default_file_opts()),
+            }],
+            relations,
+        }],
+        repositories: vec![Repository {
+            id: "curseforge".into(),
+            repo_type: RepositoryType::Curseforge,
+            url: Url::parse("https://cursemaven.com").unwrap(), // unwrap is ok on fixed value
+        }],
+        meta: Meta {
+            name: data.name,
+            contributors: vec![Contributor {
+                roles: vec!["owner".into()],
+                name: data.author,
+            }],
+            description: None,
+            icon_url: None,
+            website_url: None,
+        },
+    };
+
+    tokio::fs::create_dir_all(&config.locations.src)
+        .await
+        .into_diagnostic()?;
+    tokio::fs::write(
+        config.locations.src.join("modpack.json5"),
+        serde_json::to_vec_pretty(&manif)
+            .into_diagnostic()
+            .wrap_err("Failed to generate json data")?,
+    )
+    .await
+    .into_diagnostic()?;
+
+    println!("{}", "Imported manifest!".green());
+
+    Ok(())
+}

mpt/src/commands/init.rs

@@ -16,7 +16,6 @@ use addonscript::{
     },
     util::default_file_opts,
 };
-use anyhow::Context;
 use crossterm::{
     execute,
     style::{
@@ -31,6 +30,8 @@ use crossterm::{
     },
     ExecutableCommand,
 };
+use heck::ToKebabCase;
+use miette::{IntoDiagnostic, WrapErr};
 use reqwest::Client;
 use std::path::Path;
 use url::Url;
@@ -47,7 +48,7 @@ pub async fn run(
     modpack_name: String,
     author_name: String,
     mcversion: String,
-) -> anyhow::Result<()> {
+) -> miette::Result<()> {
     let mut stdout = std::io::stdout();
 
     execute!(
@@ -65,44 +66,57 @@ pub async fn run(
         Print(&mcversion),
         ResetColor,
         Print('\n'),
-    )?;
+    )
+    .into_diagnostic()?;
 
     let config_path = Path::new("modpacktoolsconfig.toml");
     if !config_path.exists() {
-        tokio::fs::write(config_path, DEFAULT_CONFIG).await?;
+        tokio::fs::write(config_path, DEFAULT_CONFIG)
+            .await
+            .into_diagnostic()?;
 
-        stdout.execute(PrintStyledContent(
-            "Created config!\n"
-                .with(Color::Green)
-                .attribute(Attribute::Bold),
-        ))?;
+        stdout
+            .execute(PrintStyledContent(
+                "Created config!\n"
+                    .with(Color::Green)
+                    .attribute(Attribute::Bold),
+            ))
+            .into_diagnostic()?;
     } else {
-        stdout.execute(PrintStyledContent(
-            "Config already exists, skipping...\n"
-                .with(Color::Red)
-                .attribute(Attribute::Italic),
-        ))?;
+        stdout
+            .execute(PrintStyledContent(
+                "Config already exists, skipping...\n"
+                    .with(Color::Red)
+                    .attribute(Attribute::Italic),
+            ))
+            .into_diagnostic()?;
     }
 
-    let config = tokio::fs::read(config_path).await?;
+    let config = tokio::fs::read(config_path).await.into_diagnostic()?;
     let Config {
         locations: Locations { src, .. },
         ..
-    } = toml::from_slice::<Config>(&config).context("failed to deserialize config")?;
+    } = toml::from_slice::<Config>(&config)
+        .into_diagnostic()
+        .wrap_err("failed to deserialize config")?;
 
     let path = Path::new(&src);
 
     if path.join("modpack.json").exists() || path.join("modpack.json5").exists() {
-        stdout.execute(PrintStyledContent(
-            "Manifest already exists, skipping...\n"
-                .with(Color::Red)
-                .attribute(Attribute::Italic),
-        ))?;
+        stdout
+            .execute(PrintStyledContent(
+                "Manifest already exists, skipping...\n"
+                    .with(Color::Red)
+                    .attribute(Attribute::Italic),
+            ))
+            .into_diagnostic()?;
     } else {
         let mut relations = vec![];
-        stdout.execute(PrintStyledContent(
-            "Trying to find newest forge version...\n".with(Color::Magenta),
-        ))?;
+        stdout
+            .execute(PrintStyledContent(
+                "Trying to find newest forge version...\n".with(Color::Magenta),
+            ))
+            .into_diagnostic()?;
 
         if let Some(ver) = forge::newest_forge_version(&Client::new(), &mcversion).await? {
             execute!(
@@ -114,7 +128,8 @@ pub async fn run(
                 Print(&ver),
                 ResetColor,
                 Print('\n'),
-            )?;
+            )
+            .into_diagnostic()?;
 
             relations.push(Relation {
                 id: "forge".into(),
@@ -140,15 +155,17 @@ pub async fn run(
                 SetForegroundColor(Color::Red),
                 Print(" skipping forge...\n"),
                 ResetColor,
-            )?;
+            )
+            .into_diagnostic()?;
         }
 
         // also create overrides
-        tokio::fs::create_dir_all(path.join("overrides")).await?;
+        tokio::fs::create_dir_all(path.join("overrides"))
+            .await
+            .into_diagnostic()?;
 
         let data = serde_json::to_vec_pretty(&Manifest {
-            // TODO rename this to snake_case
-            id: modpack_name.clone(),
+            id: modpack_name.to_kebab_case(),
             manifest_type: ManifestType::Modpack,
             versions: vec![Version {
                 version: "1.0".into(),
@@ -176,31 +193,43 @@ pub async fn run(
                 icon_url: None,
                 website_url: None,
             },
-        })?;
+        })
+        .into_diagnostic()
+        .wrap_err("Failed to generate json data")?;
 
-        tokio::fs::write(path.join("modpack.json5"), data).await?;
+        tokio::fs::write(path.join("modpack.json5"), data)
+            .await
+            .into_diagnostic()?;
 
-        stdout.execute(PrintStyledContent(
-            "Created manifest!\n"
-                .with(Color::Green)
-                .attribute(Attribute::Bold),
-        ))?;
+        stdout
+            .execute(PrintStyledContent(
+                "Created manifest!\n"
+                    .with(Color::Green)
+                    .attribute(Attribute::Bold),
+            ))
+            .into_diagnostic()?;
     }
 
     if Path::new(".gitignore").exists() {
-        stdout.execute(PrintStyledContent(
-            ".gitignore exists, skipping...\n"
-                .with(Color::Red)
-                .attribute(Attribute::Italic),
-        ))?;
+        stdout
+            .execute(PrintStyledContent(
+                ".gitignore exists, skipping...\n"
+                    .with(Color::Red)
+                    .attribute(Attribute::Italic),
+            ))
+            .into_diagnostic()?;
     } else {
-        tokio::fs::write(".gitignore", DEFAULT_GITIGNORE).await?;
+        tokio::fs::write(".gitignore", DEFAULT_GITIGNORE)
+            .await
+            .into_diagnostic()?;
 
-        stdout.execute(PrintStyledContent(
-            "Created .gitignore!\n"
-                .with(Color::Green)
-                .attribute(Attribute::Bold),
-        ))?;
+        stdout
+            .execute(PrintStyledContent(
+                "Created .gitignore!\n"
+                    .with(Color::Green)
+                    .attribute(Attribute::Bold),
+            ))
+            .into_diagnostic()?;
     }
 
     Ok(())

mpt/src/commands/mod.rs

@@ -2,4 +2,5 @@ pub mod buildtwitch;
 pub mod clean;
 pub mod createmodlist;
 pub mod downloadmods;
+pub mod import;
 pub mod init;

mpt/src/downloader.rs

@@ -1,6 +1,8 @@
 use async_trait::async_trait;
 use crossterm::style::Stylize;
 use futures::{stream, StreamExt};
+use log::info;
+use miette::Diagnostic;
 use reqwest::{Client, StatusCode};
 use std::{path::PathBuf, sync::Arc};
 use thiserror::Error;
@@ -58,6 +60,8 @@ impl<C: Callback> Downloader<C> {
         url: Url,
         target: PathBuf,
     ) -> Result<DownloadInfo, DownloadError> {
+        info!("Downloading {}", &url);
+
         if let Some(parent) = target.parent() {
             tokio::fs::create_dir_all(parent).await?;
         }
@@ -116,7 +120,7 @@ impl DownloadInfo {
 }
 
 /// An error that can occur while a file is being downloaded
-#[derive(Debug, Error)]
+#[derive(Debug, Diagnostic, Error)]
 pub enum DownloadError {
     #[error("HTTP Error: {0}")]
     HttpError(#[from] reqwest::Error),

mpt/src/forge.rs

@@ -1,5 +1,6 @@
 use std::collections::HashMap;
 
+use miette::IntoDiagnostic;
 use reqwest::Client;
 use serde::Deserialize;
 
@@ -13,15 +14,19 @@ struct ForgeVersionResponse {
 pub async fn newest_forge_version(
     http: &Client,
     mcversion: &str,
-) -> anyhow::Result<Option<String>> {
+) -> miette::Result<Option<String>> {
     let resp = http
         .get("https://files.minecraftforge.net/net/minecraftforge/forge/promotions_slim.json")
         .send()
-        .await?
+        .await
+        .into_diagnostic()?
         .bytes()
-        .await?;
+        .await
+        .into_diagnostic()?;
 
-    let mut resp = json5::from_str::<ForgeVersionResponse>(std::str::from_utf8(&resp)?)?;
+    let mut resp =
+        json5::from_str::<ForgeVersionResponse>(std::str::from_utf8(&resp).into_diagnostic()?)
+            .into_diagnostic()?;
 
     Ok(resp.promos.remove(&format!("{}-latest", mcversion)))
 }

mpt/src/main.rs

@@ -1,5 +1,8 @@
+use clap::{Parser, Subcommand};
+use log::{info, LevelFilter};
+use miette::{IntoDiagnostic, WrapErr};
+use simplelog::{ColorChoice, TermLogger, TerminalMode};
 use std::path::PathBuf;
-use structopt::StructOpt;
 
 mod commands;
 mod config;
@@ -7,82 +10,127 @@ mod downloader;
 mod forge;
 mod util;
 
-#[derive(StructOpt)]
+#[derive(Parser)]
 struct Opt {
-    #[structopt(subcommand)]
+    /// enable verbose logging
+    #[clap(short, long, parse(from_occurrences))]
+    verbose: u8,
+
+    /// add a parameter to be used by the build script
+    /// (only useful with lua manifests)
+    #[clap(short = 'D')]
+    defines: Vec<String>,
+
+    #[clap(subcommand)]
     cmd: Command,
 }
 
-#[derive(StructOpt)]
+#[derive(Subcommand)]
 enum Command {
-    #[structopt(about = "Initializes a new modpack")]
+    /// Initializes a new modpack
     Init {
-        #[structopt(help = "The name of the modpack")]
+        /// The name of the modpack
         modpack_name: String,
-        #[structopt(help = "Name of the modpack author")]
+        /// Name of the modpack author
         name: String,
-        #[structopt(help = "The minecraft version of the modpack")]
+
+        /// The minecraft version of the modpack
         mcversion: String,
     },
 
-    #[structopt(name = "downloadmods", about = "Downloads mods of the pack")]
+    /// Downloads mods of the pack
+    #[clap(name = "downloadmods")]
     DownloadMods {
-        #[structopt(help = "Directory to download mods to")]
+        /// Directory to download mods to
         dir: PathBuf,
-        #[structopt(short, long, help = "Download all relations and not just mods")]
+
+        /// Download all relations and not just mods
+        #[clap(short, long)]
         all: bool,
     },
 
-    #[structopt(name = "buildtwitch", about = "Builds a twitch export of the pack")]
+    /// Builds a twitch export of the pack
+    #[clap(name = "buildtwitch")]
     BuildTwitch {
-        #[structopt(
-            short,
-            long,
-            help = "Downloads all relations instead of just required ones"
-        )]
+        /// Downloads all relations instead of just required ones
+        #[clap(short, long)]
         all: bool,
     },
 
-    #[structopt(about = "Deletes artifacts and temporary files")]
+    /// Deletes artifacts and temporary files
     Clean,
 
-    #[structopt(
-        name = "createmodlist",
-        about = "Creates a HTML list of the pack's mods."
-    )]
+    /// Creates a HTML list of the pack's mods
+    #[clap(name = "createmodlist")]
     CreateModList {
-        #[structopt(
-            default_value = "build/modlist.html",
-            help = "File to write the mod list to"
-        )]
+        /// File to write the mod list to
+        #[clap(default_value = "build/modlist.html")]
         outfile: PathBuf,
     },
+
+    /// Imports a twitch manifest file and converts it to an addonscript
+    /// modpack.json
+    Import {
+        /// Twitch manifest to convert
+        infile: PathBuf,
+    },
 }
 
 #[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    let Opt { cmd } = Opt::from_args();
+async fn main() -> miette::Result<()> {
+    let Opt {
+        cmd,
+        defines,
+        verbose,
+    } = Opt::parse();
+
+    let log_level = match verbose {
+        0 => LevelFilter::Off,
+        1 => LevelFilter::Info,
+        2 => LevelFilter::Debug,
+        _ => LevelFilter::Trace,
+    };
+
+    TermLogger::init(
+        log_level,
+        simplelog::ConfigBuilder::new()
+            .set_time_to_local(true)
+            .set_target_level(LevelFilter::Error)
+            .build(),
+        TerminalMode::Stderr,
+        ColorChoice::Auto,
+    )
+    .into_diagnostic()
+    .wrap_err("Failed to init logger")?;
+
+    /// runs a given command, if the first arg is config or manifest, the
+    /// manifest or config file is passed to the comand respectively.
+    macro_rules! run_cmd {
+        (config: $cmd:ident $($args:expr),* $(,)?) => {
+            run_cmd!($cmd util::parse_config().await?, $($args),*)
+        };
+
+        (manifest: $cmd:ident $($args:expr),* $(,)?) => {
+            run_cmd!($cmd util::parse_config_and_manifest(defines, stringify!($cmd)).await?, $($args),*)
+        };
+
+        ($cmd:ident $($args:expr),* $(,)?) => {{
+            info!("Running command {}", stringify!($cmd));
+            commands::$cmd::run($($args),*).await?;
+        }}
+    }
 
     match cmd {
         Command::Init {
             modpack_name,
             name,
             mcversion,
-        } => commands::init::run(modpack_name, name, mcversion).await?,
-
-        Command::DownloadMods { dir, all } => {
-            commands::downloadmods::run(util::parse_config_and_manifest().await?, dir, all).await?
-        },
-
-        Command::BuildTwitch { all } => {
-            commands::buildtwitch::run(util::parse_config_and_manifest().await?, all).await?
-        },
-
-        Command::Clean => commands::clean::run(util::parse_config().await?).await?,
-
-        Command::CreateModList { outfile } => {
-            commands::createmodlist::run(util::parse_config_and_manifest().await?, outfile).await?
-        },
+        } => run_cmd!(init modpack_name, name, mcversion),
+        Command::DownloadMods { dir, all } => run_cmd!(manifest: downloadmods dir, all),
+        Command::BuildTwitch { all } => run_cmd!(manifest: buildtwitch all),
+        Command::Clean => run_cmd!(config: clean),
+        Command::CreateModList { outfile } => run_cmd!(manifest: createmodlist outfile),
+        Command::Import { infile } => run_cmd!(config: import infile),
     }
 
     Ok(())

mpt/src/util.rs

@@ -1,7 +1,9 @@
 use addonscript::manifest::{link::Link, Manifest, Repository};
-use anyhow::Context;
 use crossterm::style::{Attribute, Attributes, Color, ContentStyle, Stylize};
 use indicatif::ProgressStyle;
+use log::info;
+use miette::{Diagnostic, IntoDiagnostic, WrapErr};
+use mlua::{Lua, LuaSerdeExt};
 use percent_encoding::percent_decode;
 use std::{
     collections::HashMap,
@@ -16,32 +18,80 @@ use walkdir::WalkDir;
 use crate::config::Config;
 
 /// reads and parses the config from the current working directory
-pub async fn parse_config() -> anyhow::Result<Config> {
-    let conf = tokio::fs::read("modpacktoolsconfig.toml").await?;
-    Ok(toml::from_slice(&conf)?)
+pub async fn parse_config() -> miette::Result<Config> {
+    info!("reading config");
+    let conf = tokio::fs::read("modpacktoolsconfig.toml")
+        .await
+        .into_diagnostic()
+        .wrap_err("Failed to read config")?;
+
+    toml::from_slice(&conf)
+        .into_diagnostic()
+        .wrap_err("Failed to parse config")
 }
 
 /// parses the config from the current working directory, reads the location of
 /// the manifest file from it, parses the manifest and returns both the conig
 /// and the manifest.
-pub async fn parse_config_and_manifest() -> anyhow::Result<(Config, Manifest)> {
+pub async fn parse_config_and_manifest(
+    defines: Vec<String>,
+    command: &str,
+) -> miette::Result<(Config, Manifest)> {
     let config = parse_config().await?;
     let src = Path::new(&config.locations.src);
 
+    let mut is_lua = false;
     let path = if src.join("modpack.json5").exists() {
         src.join("modpack.json5")
-    } else {
+    } else if src.join("modpack.json").exists() {
         src.join("modpack.json")
+    } else {
+        is_lua = true;
+        src.join("modpack.lua")
     };
 
-    let data = tokio::fs::read(path).await?;
-    let data = std::str::from_utf8(&data)?;
-    let manifest = json5::from_str::<Manifest>(data)?;
+    info!("reading manifest");
+
+    let data = tokio::fs::read(path)
+        .await
+        .into_diagnostic()
+        .wrap_err("Failed to read manifest")?;
+
+    let manifest = if is_lua {
+        let lua = Lua::new();
+
+        let mpt = lua.create_table().into_diagnostic()?;
+        mpt.set("defines", defines).into_diagnostic()?;
+        mpt.set("command", command).into_diagnostic()?;
+
+        lua.globals().set("mpt", mpt).into_diagnostic()?;
+
+        lua.load(&data)
+            .exec()
+            .into_diagnostic()
+            .wrap_err("Failed to execute lua manifest")?;
+        let lua_manifest = lua
+            .globals()
+            .get("manifest")
+            .into_diagnostic()
+            .wrap_err("Failed to get manifest value")?;
+        lua.from_value(lua_manifest).into_diagnostic().wrap_err(
+            "Failed to deserialize lua manifest. Did you set the global `manifest` to the correct \
+             data?",
+        )?
+    } else {
+        let data = std::str::from_utf8(&data)
+            .into_diagnostic()
+            .wrap_err("Manifest is invalid UTF-8")?;
+        json5::from_str::<Manifest>(data)
+            .into_diagnostic()
+            .wrap_err("Failed to parse manifest")?
+    };
 
     Ok((config, manifest))
 }
 
-#[derive(Debug, Error, PartialEq, Eq)]
+#[derive(Debug, Error, Diagnostic, PartialEq, Eq)]
 pub enum MavenParseError {
     #[error("Maven Artifact specifier has invalid format!")]
     InvalidFormat,
@@ -104,10 +154,12 @@ pub fn progress_style() -> ProgressStyle {
 }
 
 /// creates the modpacktools temporary directory (set in the config)
-pub async fn make_tmp_dir(config: &Config) -> anyhow::Result<()> {
+pub async fn make_tmp_dir(config: &Config) -> miette::Result<()> {
+    info!("creating temporary directory");
     tokio::fs::create_dir_all(&config.locations.temp_dir)
         .await
-        .context("Failed to create temporary directory")?;
+        .into_diagnostic()
+        .wrap_err("Failed to create temporary directory")?;
     Ok(())
 }
 
@@ -132,7 +184,7 @@ pub fn parse_curseforge_artifact(artifact: &str) -> Result<(&str, &str), MavenPa
     Ok((p_id, f_id))
 }
 
-#[derive(Debug, Error, PartialEq, Eq)]
+#[derive(Debug, Error, Diagnostic, PartialEq, Eq)]
 pub enum UrlFileNameError {
     #[error("URL is base URL without file name!")]
     BaseUrl,
@@ -144,8 +196,7 @@ pub enum UrlFileNameError {
 pub fn url_file_name(url: &Url) -> Result<String, UrlFileNameError> {
     let file_name = url
         .path_segments()
-        .map(Iterator::last)
-        .flatten()
+        .and_then(Iterator::last)
         .ok_or(UrlFileNameError::BaseUrl)?;
 
     if file_name.is_empty() {
@@ -156,7 +207,7 @@ pub fn url_file_name(url: &Url) -> Result<String, UrlFileNameError> {
     Ok(String::from_utf8(bytes)?)
 }
 
-#[derive(Debug, Error)]
+#[derive(Debug, Error, Diagnostic)]
 pub enum LinkFileNameError {
     #[error("Empty path has no file name!")]
     EmptyPath,
@@ -177,6 +228,12 @@ pub fn link_file_name(link: &Link) -> Result<String, LinkFileNameError> {
 
 /// Copies a directory inclding all files
 pub async fn copy_dir(from: PathBuf, to: PathBuf) -> io::Result<()> {
+    info!(
+        "Copying directory {} to {}",
+        from.to_string_lossy(),
+        to.to_string_lossy()
+    );
+
     for file in WalkDir::new(&from) {
         let file = file?;
 

twitch/Cargo.toml

@@ -7,6 +7,6 @@ edition = "2018"
 
 [dependencies]
 addonscript = { path = "../addonscript" }
-serde = { version = "1.0.130", features = ["derive"] }
-thiserror = "1.0.28"
+serde = { version = "1.0.136", features = ["derive"] }
+thiserror = "1.0.30"
 url = { version = "2.2.2", features = ["serde"] }

twitch/src/manifest.rs

@@ -67,8 +67,8 @@ pub enum ManifestType {
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct Minecraft {
-    version: String,
-    mod_loaders: Vec<ModLoader>,
+    pub version: String,
+    pub mod_loaders: Vec<ModLoader>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]