Anvilcraft/HexCasting
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Somewhat improve resource loading

Browse source
This commit is contained in:
object-Object 2023-08-08 22:45:09 -04:00
parent ed425f6280
commit 2e2f318be3
17 changed files with 229 additions and 177 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
doc/TODO.md Normal file
View file

@ -0,0 +1,6 @@
- [x] Better resource loading
- [ ] Sandbox for Jinja
- [ ] First-class addon support
- [ ] Language picker
- [ ] Version picker
- [ ] Re-add edified wood recipe to [Common/src/main/resources/assets/hexcasting/patchouli_books/thehexbook/en_us/entries/items/edified.json](items/edified) when it actually exists

57
doc/properties.toml
View file

@ -1,16 +1,24 @@
# general properties/settings
modid = "hexcasting"
book_name = "thehexbook"
book = "hexcasting:thehexbook"
url = "https://gamma-delta.github.io/HexMod"
is_0_black = false
# NOTE: _Raw means "don't apply variable interpolation to this value"
pattern_regex = {_Raw='make\(\s*"(?P<name>[a-zA-Z0-9_\/]+)",\s*(?:new )?(?:ActionRegistryEntry|OperationAction)\(\s*HexPattern\.fromAngles\(\s*"(?P<signature>[aqweds]+)",\s*HexDir.(?P<startdir>\w+)\)'}
resource_dirs = [
# hot path (these are the most common directories, so put them first)
"{_common.src}/main/resources",
# cold path (order doesn't matter below this point)
"{_common.src}/generated/resources",
"{_fabric.src}/main/resources",
"{_fabric.src}/generated/resources",
"{_forge.src}/main/resources",
"{_forge.src}/generated/resources",
]
template = "main.html.jinja"
template_dirs = []
template_packages = []
# NOTE: "!Raw" means "don't apply variable interpolation to this value"
_pattern_regex = {"!Raw"='make\(\s*"(?P<name>[a-zA-Z0-9_\/]+)",\s*(?:new )?(?:ActionRegistryEntry|OperationAction)\(\s*HexPattern\.fromAngles\(\s*"(?P<signature>[aqweds]+)",\s*HexDir.(?P<startdir>\w+)\)'}
spoilered_advancements = [
"hexcasting:opened_eyes",
@ -20,6 +28,10 @@ spoilered_advancements = [
]
entry_id_blacklist = []
template = "main.html.jinja"
template_dirs = []
template_packages = []
[template_args]
title = "Hex Book"
@ -50,36 +62,27 @@ sneak = "Left Shift"
jump = "Space"
[[pattern_stubs]]
path = "{^_common.package}/common/lib/hex/HexActions.java"
regex = "{^_pattern_regex}"
[[pattern_stubs]]
path = "{^_fabric.package}/FabricHexInitializer.kt"
regex = "{^_pattern_regex}"
# platforms
[common]
[_common]
src = "../Common/src"
package = "{src}/main/java/at/petrak/hexcasting"
resources = "{src}/main/resources"
generated = "{src}/generated/resources"
[[common.pattern_stubs]]
path = "{^package}/common/lib/hex/HexActions.java"
regex = "{^^pattern_regex}"
[fabric]
[_fabric]
src = "../Fabric/src"
package = "{src}/main/java/at/petrak/hexcasting/fabric"
resources = "{src}/main/resources"
generated = "{src}/generated/resources"
recipes = "{generated}/data/{^modid}/recipes"
tags = "{generated}/data/{^modid}/tags"
[[fabric.pattern_stubs]]
path = "{^package}/FabricHexInitializer.kt"
regex = "{^^pattern_regex}"
[forge]
[_forge]
src = "../Forge/src"
package = "{src}/main/java/at/petrak/hexcasting/forge"
resources = "{src}/main/resources"
generated = "{src}/generated/resources"
recipes = "{generated}/data/{^modid}/recipes"
tags = "{generated}/data/{^modid}/tags"

2
doc/src/hexdoc/_templates/pages/patchouli/image.html.jinja
View file

@ -3,7 +3,7 @@
{% block inner_body %}
<p class="img-wrapper">
{% for image in page.images %}
<img src="{{ props.asset_url(image) }}"></img>
<img src="{{ props.get_asset_url(image) }}"></img>
{% endfor %}
</p>
{{ super() }}

2
doc/src/hexdoc/hexcasting/hex_book.py
View file

@ -59,7 +59,7 @@ class HexBookType(
startdir=Direction[groups["startdir"]],
signature=groups["signature"],
# is_per_world=bool(is_per_world), # FIXME: idfk how to do this now
id=ResourceLocation(props.modid, groups["name"]),
id=props.mod_loc(groups["name"]),
)

9
doc/src/hexdoc/hexdoc.py
View file

@ -1,5 +1,7 @@
import io
import logging
import os
import sys
from argparse import ArgumentParser
from dataclasses import dataclass
from pathlib import Path
@ -64,10 +66,15 @@ def main(args: Args | None = None) -> None:
# treat all paths as relative to the location of the properties file by cd-ing there
with cd(args.properties_file.parent):
# set stdout to utf-8 so printing to pipe or redirect doesn't break on Windows
# (common windows L)
assert isinstance(sys.stdout, io.TextIOWrapper)
sys.stdout.reconfigure(encoding="utf-8")
# set up logging
logging.basicConfig(
style="{",
format="[{levelname}][{name}] {message}",
format="\033[1m[{relativeCreated:.02f} | {levelname} | {name}]\033[0m {message}",
)
logger = logging.getLogger(__name__)
if args.verbose:

2
doc/src/hexdoc/minecraft/i18n.py
View file

@ -112,7 +112,7 @@ class I18n:
# TODO: load ALL of the i18n files, return dict[str, _Lookup] | None
# or maybe dict[(str, str), LocalizedStr]
# we could also use that to ensure all i18n files have the same set of keys
path = props.assets_dir / "lang" / props.i18n.filename
path = props.find_resource("assets", "lang", props.mod_loc(props.i18n.filename))
raw_lookup = load_and_flatten_json_dict(path) | props.i18n.extra
# validate and insert

21
doc/src/hexdoc/minecraft/recipe/abstract_recipes.py
View file

@ -20,7 +20,8 @@ class Recipe(TypeTaggedUnion[AnyPropsContext], group="hexdoc.Recipe", type=None)
info: ValidationInfo,
):
"""Loads the recipe from json if the actual value is a resource location str."""
if not info.context or isinstance(values, (dict, Recipe)):
context = cast(AnyPropsContext, info.context)
if not context or isinstance(values, (dict, Recipe)):
return values
# if necessary, convert the id to a ResourceLocation
@ -31,18 +32,6 @@ class Recipe(TypeTaggedUnion[AnyPropsContext], group="hexdoc.Recipe", type=None)
id = values
# load the recipe
context = cast(AnyPropsContext, info.context)
# TODO: this is ugly and not super great for eg. hexbound
forge_path = context["props"].forge.recipes / f"{id.path}.json"
fabric_path = context["props"].fabric.recipes / f"{id.path}.json"
# this is to ensure the recipe at least exists on all platforms
# because we've had issues with that before (eg. Hexal's Mote Nexus)
if not forge_path.exists():
raise ValueError(f"Recipe {id} missing from path {forge_path}")
logging.getLogger(__name__).debug(
f"Load {cls}\n id: {id}\n path: {fabric_path}"
)
return load_json_dict(fabric_path) | {"id": id}
path = context["props"].find_resource("data", "recipes", id)
logging.getLogger(__name__).debug(f"Load {cls}\n id: {id}\n path: {path}")
return load_json_dict(path) | {"id": id}

7
doc/src/hexdoc/patchouli/book.py
View file

@ -75,7 +75,8 @@ class Book(Generic[AnyContext, AnyBookContext], HexDocModel[AnyBookContext]):
@classmethod
def prepare(cls, props: Properties) -> tuple[dict[str, Any], BookContext]:
# read the raw dict from the json file
data = load_json_dict(props.book_path)
path = props.find_resource("data", "patchouli_books", props.book / "book")
data = load_json_dict(path)
# set up the deserialization context object
assert isinstance_or_raise(data["i18n"], bool)
@ -124,8 +125,8 @@ class Book(Generic[AnyContext, AnyBookContext], HexDocModel[AnyBookContext]):
self._categories: dict[ResourceLocation, Category] = Category.load_all(context)
# load entries
for path in context["props"].entries_dir.rglob("*.json"):
entry = Entry.load(path, context)
for id, path in context["props"].find_book_assets("entries"):
entry = Entry.load(id, path, context)
# i used the entry to insert the entry (pretty sure thanos said that)
self._categories[entry.category_id].entries.append(entry)

37
doc/src/hexdoc/patchouli/book_models.py
View file

@ -1,10 +1,11 @@
from abc import ABC, abstractmethod
import logging
from abc import ABC
from pathlib import Path
from typing import Any, Generic, TypeVar, cast, dataclass_transform
from typing import Generic, Self, TypeVar, dataclass_transform
from pydantic import ValidationInfo, model_validator
from hexdoc.utils import AnyContext, HexDocFileModel, Properties, ResourceLocation
from hexdoc.utils import AnyContext, ResourceLocation
from hexdoc.utils.deserialize import load_json_dict
from hexdoc.utils.model import HexDocModel
from .text.formatting import FormatContext
@ -19,26 +20,18 @@ AnyBookContext = TypeVar("AnyBookContext", bound=BookContext)
@dataclass_transform()
class BookFileModel(
Generic[AnyContext, AnyBookContext],
HexDocFileModel[AnyBookContext],
HexDocModel[AnyBookContext],
ABC,
):
id: ResourceLocation
@classmethod
@abstractmethod
def _id_base_dir(cls, props: Properties) -> Path:
...
def load(cls, id: ResourceLocation, path: Path, context: AnyBookContext) -> Self:
logging.getLogger(__name__).debug(f"Load {cls}\n path: {path}")
@model_validator(mode="before")
def _pre_root(cls, values: dict[str, Any], info: ValidationInfo) -> dict[str, Any]:
if not info.context:
return values
context = cast(AnyBookContext, info.context)
return values | {
"id": ResourceLocation.from_file(
modid=context["props"].modid,
base_dir=cls._id_base_dir(context["props"]),
path=values.pop("__path"),
)
}
try:
data = load_json_dict(path) | {"id": id}
return cls.model_validate(data, context=context)
except Exception as e:
e.add_note(f"File: {path}")
raise

14
doc/src/hexdoc/patchouli/category.py
View file

@ -1,10 +1,9 @@
from pathlib import Path
from typing import Self
from pydantic import Field
from hexdoc.minecraft import LocalizedStr
from hexdoc.utils import ItemStack, Properties, ResourceLocation
from hexdoc.utils import ItemStack, ResourceLocation
from hexdoc.utils.types import Sortable, sorted_dict
from .book_models import BookContext, BookFileModel
@ -37,9 +36,9 @@ class Category(BookFileModel[BookContext, BookContext], Sortable):
categories: dict[ResourceLocation, Self] = {}
# load
for path in context["props"].categories_dir.rglob("*.json"):
category = cls.load(path, context)
categories[category.id] = category
for id, path in context["props"].find_book_assets("categories"):
category = cls.load(id, path, context)
categories[id] = category
# late-init _parent_cmp_key
# track iterations to avoid an infinite loop if for some reason there's a cycle
@ -68,11 +67,6 @@ class Category(BookFileModel[BookContext, BookContext], Sortable):
# return sorted by sortnum, which requires parent to be initialized
return sorted_dict(categories)
@classmethod
def _id_base_dir(cls, props: Properties) -> Path:
# implement BookModelFile
return props.categories_dir
@property
def is_spoiler(self) -> bool:
return all(entry.is_spoiler for entry in self.entries)

7
doc/src/hexdoc/patchouli/entry.py
View file

@ -1,10 +1,9 @@
from pathlib import Path
from typing import cast
from pydantic import Field, ValidationInfo, model_validator
from hexdoc.minecraft import LocalizedStr
from hexdoc.utils import Color, ItemStack, Properties, ResourceLocation
from hexdoc.utils import Color, ItemStack, ResourceLocation
from hexdoc.utils.types import Sortable
from .book_models import BookContext, BookFileModel
@ -36,10 +35,6 @@ class Entry(BookFileModel[BookContext, BookContext], Sortable):
extra_recipe_mappings: dict[ItemStack, int] | None = None
entry_color: Color | None = None # this is undocumented lmao
@classmethod
def _id_base_dir(cls, props: Properties) -> Path:
return props.entries_dir
@property
def _cmp_key(self) -> tuple[bool, int, LocalizedStr]:
# implement Sortable

3
doc/src/hexdoc/utils/__init__.py
View file

@ -1,6 +1,5 @@
__all__ = [
"HexDocModel",
"HexDocFileModel",
"InternallyTaggedUnion",
"Color",
"AnyContext",
@ -18,7 +17,7 @@ __all__ = [
"TypeTaggedUnion",
]
from .model import DEFAULT_CONFIG, AnyContext, HexDocFileModel, HexDocModel
from .model import DEFAULT_CONFIG, AnyContext, HexDocModel
from .properties import AnyPropsContext, Properties, PropsContext
from .resource import Entity, ItemStack, ResLoc, ResourceLocation
from .tagged_union import (

2
doc/src/hexdoc/utils/deserialize.py
View file

@ -1,3 +1,4 @@
import logging
import re
from pathlib import Path
from typing import Any, TypeGuard, TypeVar, get_origin
@ -42,6 +43,7 @@ JSONValue = JSONDict | list["JSONValue"] | str | int | float | bool | None
def load_json_dict(path: Path) -> JSONDict:
logging.getLogger(__name__).debug(f"Load json from {path}")
data = pyjson5.decode(path.read_text("utf-8"))
assert isinstance_or_raise(data, dict)
return data

31
doc/src/hexdoc/utils/model.py
View file

@ -1,13 +1,9 @@
import logging
from pathlib import Path
from typing import TYPE_CHECKING, Any, Generic, Self, TypeVar, dataclass_transform
from typing import TYPE_CHECKING, Any, Generic, TypeVar, dataclass_transform
from pydantic import BaseModel, ConfigDict
from pydantic import BaseModel, ConfigDict, model_validator
from pydantic.config import ConfigDict
from typing_extensions import TypedDict
from .deserialize import load_json_dict
if TYPE_CHECKING:
from pydantic.root_model import Model
@ -50,13 +46,16 @@ class HexDocModel(Generic[AnyContext], BaseModel):
@dataclass_transform()
class HexDocFileModel(HexDocModel[AnyContext]):
@classmethod
def load(cls, path: Path, context: AnyContext) -> Self:
logging.getLogger(__name__).debug(f"Load {cls}\n path: {path}")
data = load_json_dict(path) | {"__path": path}
try:
return cls.model_validate(data, context=context)
except Exception as e:
e.add_note(f"File: {path}")
raise
class HexDocStripHiddenModel(HexDocModel[AnyContext]):
"""Base model which removes all keys starting with _ before validation."""
@model_validator(mode="before")
def _pre_root_strip_hidden(cls, values: Any) -> Any:
if not isinstance(values, dict):
return values
return {
key: value
for key, value in values.items()
if not (isinstance(key, str) and key.startswith("_"))
}

169
doc/src/hexdoc/utils/properties.py
View file

@ -1,16 +1,19 @@
from __future__ import annotations
import re
from collections.abc import Iterator
from pathlib import Path
from typing import Annotated, Any, Self, TypeVar
from typing import Annotated, Any, Literal, Self, TypeVar
from pydantic import AfterValidator, Field, HttpUrl
from typing_extensions import TypedDict
from .model import HexDocModel
from .model import HexDocStripHiddenModel
from .resource import ResourceLocation
from .toml_placeholders import load_toml_with_placeholders
ResourceType = Literal["assets", "data"]
NoTrailingSlashHttpUrl = Annotated[
str,
HttpUrl,
@ -18,14 +21,13 @@ NoTrailingSlashHttpUrl = Annotated[
]
class PatternStubProps(HexDocModel[Any], extra="ignore"):
class PatternStubProps(HexDocStripHiddenModel[Any]):
path: Path
regex: re.Pattern[str]
class XplatProps(HexDocModel[Any], extra="ignore"):
class XplatProps(HexDocStripHiddenModel[Any]):
src: Path
package: Path
pattern_stubs: list[PatternStubProps] | None = None
resources: Path
@ -35,28 +37,31 @@ class PlatformProps(XplatProps):
tags: Path
class I18nProps(HexDocModel[Any], extra="ignore"):
class I18nProps(HexDocStripHiddenModel[Any]):
default_lang: str
filename: str
extra: dict[str, str] = Field(default_factory=dict)
keys: dict[str, str] = Field(default_factory=dict)
class Properties(HexDocModel[Any], extra="ignore"):
class Properties(HexDocStripHiddenModel[Any]):
modid: str
book_name: str
book: ResourceLocation
url: NoTrailingSlashHttpUrl
is_0_black: bool
"""If true, the style `$(0)` changes the text color to black; otherwise it resets
the text color to the default."""
template: str
template_dirs: list[Path]
template_packages: list[tuple[str, Path]]
resource_dirs: list[Path]
spoilered_advancements: set[ResourceLocation]
entry_id_blacklist: set[ResourceLocation]
template: str
template_dirs: list[Path]
template_packages: list[tuple[str, Path]]
template_args: dict[str, Any]
base_asset_urls: dict[str, NoTrailingSlashHttpUrl]
@ -64,73 +69,111 @@ class Properties(HexDocModel[Any], extra="ignore"):
i18n: I18nProps
common: XplatProps
fabric: PlatformProps # TODO: some way to make these optional for addons
forge: PlatformProps
pattern_stubs: list[PatternStubProps]
@classmethod
def load(cls, path: Path) -> Self:
return cls.model_validate(load_toml_with_placeholders(path))
@property
def resources_dir(self):
return self.common.resources
def mod_loc(self, path: str) -> ResourceLocation:
"""Returns a ResourceLocation with self.modid as the namespace."""
return ResourceLocation(self.modid, path)
@property
def lang(self):
return self.i18n.default_lang
def get_asset_url(self, id: ResourceLocation) -> str:
base_url = self.base_asset_urls[id.namespace]
return f"{base_url}/{id.file_path_stub('assets').as_posix()}"
@property
def book_path(self) -> Path:
"""eg. `resources/data/hexcasting/patchouli_books/thehexbook/book.json`"""
return (
self.resources_dir
/ "data"
/ self.modid
/ "patchouli_books"
/ self.book_name
/ "book.json"
def find_book_assets(self, folder: Literal["categories", "entries", "templates"]):
return self.find_resources(
type="assets",
folder="patchouli_books",
base_id=self.book / self.i18n.default_lang / folder,
)
@property
def assets_dir(self) -> Path:
"""eg. `resources/assets/hexcasting`"""
return self.resources_dir / "assets" / self.modid
def find_resource(
self,
type: ResourceType,
folder: str,
id: ResourceLocation,
) -> Path:
"""Find the first file with this resource location in `resource_dirs`.
@property
def book_assets_dir(self) -> Path:
"""eg. `resources/assets/hexcasting/patchouli_books/thehexbook`"""
return self.assets_dir / "patchouli_books" / self.book_name
If no file extension is provided, `.json` is assumed.
@property
def categories_dir(self) -> Path:
return self.book_assets_dir / self.lang / "categories"
Raises FileNotFoundError if the file does not exist.
"""
@property
def entries_dir(self) -> Path:
return self.book_assets_dir / self.lang / "entries"
# check in each directory, return the first that exists
path_stub = id.file_path_stub(type, folder)
for resource_dir in self.resource_dirs:
path = resource_dir / path_stub
if path.is_file():
return path
@property
def platforms(self) -> list[XplatProps]:
platforms = [self.common]
if self.fabric:
platforms.append(self.fabric)
if self.forge:
platforms.append(self.forge)
return platforms
raise FileNotFoundError(f"Path {path_stub} not found in any resource dir")
@property
def pattern_stubs(self):
return [
stub
for platform in self.platforms
if platform.pattern_stubs
for stub in platform.pattern_stubs
]
def find_resources(
self,
type: ResourceType,
folder: str,
base_id: ResourceLocation,
glob: str = "**/*",
) -> Iterator[tuple[ResourceLocation, Path]]:
"""Search for a glob under a given resource location in all of `resource_dirs`.
def asset_url(self, asset: ResourceLocation, path: str = "assets") -> str:
base_url = self.base_asset_urls[asset.namespace]
return f"{base_url}/{path}/{asset.full_path}"
The path of the returned resource location is relative to the path of base_id.
If no file extension is provided for glob, `.json` is assumed.
Raises FileNotFoundError if no files were found in any resource dir.
For example:
```py
props.book = ResLoc("hexcasting", "thehexbook")
lang = "en_us"
props.find_resources(
type="assets",
folder="patchouli_books",
base_id=props.book / lang / "entries",
glob="**/*",
)
# [(hexcasting:basics/couldnt_cast, .../resources/assets/hexcasting/patchouli_books/thehexbook/en_us/entries/basics/couldnt_cast.json)]
```
"""
# eg. assets/hexcasting/patchouli_books/thehexbook/en_us/entries
base_path_stub = base_id.file_path_stub(type, folder, assume_json=False)
# glob for json files if not provided
if not Path(glob).suffix:
glob += ".json"
# find all files matching the resloc
found_any = False
for resource_dir in self.resource_dirs:
# eg. .../resources/assets/hexcasting/patchouli_books/thehexbook/en_us/entries
base_path = resource_dir / base_path_stub
# eg. .../resources/assets/hexcasting/patchouli_books/thehexbook/en_us/entries/**/*.json
for path in base_path.glob(glob):
# only yield actual files
if not path.is_file():
continue
found_any = True
# determine the resource location of this file
path_stub = path.relative_to(base_path).with_suffix("")
id = ResourceLocation(base_id.namespace, path_stub.as_posix())
yield id, path
# if we never yielded any files, raise an error
if not found_any:
raise FileNotFoundError(
f"No files found under {base_path_stub} in any resource dir"
)
class PropsContext(TypedDict):

35
doc/src/hexdoc/utils/resource.py
View file

@ -7,7 +7,7 @@
import re
from fnmatch import fnmatch
from pathlib import Path
from typing import Any, ClassVar, Self
from typing import Any, ClassVar, Literal, Self
from pydantic import field_validator, model_serializer, model_validator
from pydantic.dataclasses import dataclass
@ -62,17 +62,13 @@ class BaseResourceLocation:
return value.lower()
@field_validator("path")
def _lower_path(cls, value: str):
return value.lower()
def _validate_path(cls, value: str):
return value.lower().rstrip("/")
@model_serializer
def _ser_model(self) -> str:
return str(self)
@property
def full_path(self) -> str:
return f"{self.namespace}/{self.path}"
def __repr__(self) -> str:
return f"{self.namespace}:{self.path}"
@ -93,6 +89,31 @@ class ResourceLocation(BaseResourceLocation, regex=_make_regex()):
def match(self, pattern: Self) -> bool:
return fnmatch(str(self), str(pattern))
def file_path_stub(
self,
type: Literal["assets", "data"],
folder: str = "",
assume_json: bool = True,
) -> Path:
"""Returns the path to find this resource within a resource directory.
If `assume_json` is True and no file extension is provided, `.json` is assumed.
For example:
```py
ResLoc("hexcasting", "thehexbook/book").file_path_stub("data", "patchouli_books")
# data/hexcasting/patchouli_books/thehexbook/book.json
```
"""
# if folder is an empty string, Path won't add an extra slash
path = Path(type) / self.namespace / folder / self.path
if assume_json and not path.suffix:
return path.with_suffix(".json")
return path
def __truediv__(self, other: str) -> Self:
return ResourceLocation(self.namespace, f"{self.path}/{other}")
# pure unadulterated laziness
ResLoc = ResourceLocation

2
doc/src/hexdoc/utils/toml_placeholders.py
View file

@ -86,7 +86,7 @@ def _handle_child(
expanded.add((id(stack[-1]), key))
update(key, value)
case {"_Raw": raw} if len(value) == 1:
case {"!Raw": raw} if len(value) == 1:
# interpolaten't
expanded.add((id(stack[-1]), key))
update(key, raw)