diff --git a/.github/actions/install-artifact-wheel/action.yml b/.github/actions/install-artifact-wheel/action.yml new file mode 100644 index 00000000..1a154a5a --- /dev/null +++ b/.github/actions/install-artifact-wheel/action.yml @@ -0,0 +1,32 @@ +name: Install wheel from artifact +description: Install wheel from artifact + +inputs: + name: + description: Artifact name. + required: true + python-version: + description: Version range or exact version of Python to use. + required: true + +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + cache: pip + + - name: Download artifact + uses: actions/download-artifact@v3 + with: + name: ${{ inputs.name }} + path: _dist + + - name: Install wheel + shell: bash + run: | + wheels=( _dist/*-py3-none-any.whl ) + pip install "${wheels[0]}" + rm -r _dist diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml index 4c30f7ef..112e5b23 100644 --- a/.github/workflows/build_docs.yml +++ b/.github/workflows/build_docs.yml @@ -17,13 +17,14 @@ on: required: false env: + PYPI_PACKAGE: hexdoc HEXDOC: hexdoc doc/properties.toml --ci permissions: contents: read concurrency: - group: "hexdoc" + group: "docgen" cancel-in-progress: false jobs: @@ -37,28 +38,34 @@ jobs: outputs: matrix: ${{ steps.list-langs.outputs.matrix }} + release: ${{ steps.check-release.outputs.release }} steps: - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 + - uses: actions/setup-python@v4 with: python-version: "3.11" + cache: pip - - name: Install docgen + - name: Install docgen from source run: pip install . hatch - - name: List languages + - name: List book languages id: list-langs run: echo "matrix=$($HEXDOC --list-langs)" >> "$GITHUB_OUTPUT" + - name: Check if this is a release + id: check-release + run: | + release=${{ github.event_name == 'workflow_dispatch' && inputs.publish == 'PyPI' || startsWith(github.ref, 'refs/tags') || startsWith(github.event.head_commit.message, '[Release]') }} + echo "release=$release" >> "$GITHUB_OUTPUT" + - name: Export web book - run: $HEXDOC --export-only + run: $HEXDOC --export-only --release $release - name: Bump version if: github.event_name == 'workflow_dispatch' && inputs.segment - run: hatch version ${{ inputs.segment }} + run: hatch version "${{ inputs.segment }}" - name: Commit changes uses: stefanzweifel/git-auto-commit-action@v4 @@ -66,19 +73,13 @@ jobs: commit_message: Build web book from ${{ github.ref }} - name: Build docgen - run: hatch build + run: hatch build _site/dist - - name: Upload hexdoc artifact + - name: Upload docgen artifact uses: actions/upload-artifact@v3 with: - name: hexdoc-build - path: dist/ - - - name: Copy build to Pages - run: | - mkdir -p _site/dist - cp dist/*.whl _site/dist/latest.whl - cp dist/*.tar.gz _site/dist/latest.tar.gz + name: docgen-build + path: _site/dist/ - name: Upload temporary Pages artifact uses: actions/upload-artifact@v3 @@ -86,118 +87,22 @@ jobs: name: github-pages-tmp path: _site/ - generate: - runs-on: ubuntu-latest - needs: build - continue-on-error: true - - strategy: - fail-fast: false - matrix: - lang: ${{ fromJson(needs.build.outputs.matrix) }} - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Download hexdoc artifact - uses: actions/download-artifact@v3 - with: - name: hexdoc-build - - - name: Install docgen - run: pip install *.whl - - - name: Build web book - run: $HEXDOC --lang ${{ matrix.lang }} -o _site - - - name: Upload temporary Pages artifact - uses: actions/upload-artifact@v3 - with: - name: github-pages-tmp - path: _site/ - - bundle-pages: - runs-on: ubuntu-latest - needs: generate - - steps: - - uses: actions/checkout@v3 - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: "3.11" - - - name: Download hexdoc artifact - uses: actions/download-artifact@v3 - with: - name: hexdoc-build - - - name: Install docgen - run: pip install *.whl - - - name: Download temporary Pages artifact - uses: actions/download-artifact@v3 - with: - name: github-pages-tmp - path: _site/ - - - name: Check default lang - run: $HEXDOC --check-default-lang _site - - - name: Fix permissions - run: | - chmod -c -R +rX "_site/" | while read line; do - echo "::warning title=Invalid file permissions automatically fixed::$line" - done - - - name: Upload Pages artifact - uses: actions/upload-pages-artifact@v2 - - deploy-pages: - runs-on: ubuntu-latest - needs: bundle-pages - - environment: - name: github-pages - url: ${{ steps.deployment.outputs.page_url }} - - permissions: - id-token: write - pages: write - - steps: - - name: Deploy to Pages - id: deployment - uses: actions/deploy-pages@v2 - with: - timeout: 300000 # 5 minutes - publish-pypi: runs-on: ubuntu-latest needs: build - if: |- - github.event_name == 'workflow_dispatch' && inputs.publish == 'PyPI' || - startsWith(github.ref, 'refs/tags') || - startsWith(github.event.head_commit.message, '[Release]') + if: needs.build.outputs.release == true environment: name: pypi - url: https://pypi.org/p/hexdoc - + url: https://pypi.org/p/${{ env.PYPI_PACKAGE }} permissions: id-token: write steps: - - name: Download hexdoc artifact + - name: Download docgen artifact uses: actions/download-artifact@v3 with: - name: hexdoc-build + name: docgen-build path: dist - name: Publish to PyPI @@ -210,19 +115,105 @@ jobs: environment: name: testpypi - url: https://test.pypi.org/p/hexdoc - + url: https://test.pypi.org/p/${{ env.PYPI_PACKAGE }} permissions: id-token: write steps: - - name: Download hexdoc artifact + - name: Download docgen artifact uses: actions/download-artifact@v3 with: - name: hexdoc-build + name: docgen-build path: dist - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: https://test.pypi.org/legacy/ + + generate: + runs-on: ubuntu-latest + needs: build + continue-on-error: true + + strategy: + fail-fast: false + matrix: + lang: ${{ fromJson(needs.build.outputs.matrix) }} + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-artifact-wheel + with: + name: docgen-build + python-version: "3.11" + + - name: Build web book + run: $HEXDOC --lang "${{ matrix.lang }}" -o _site --release "${{ needs.build.outputs.release }}" + + - name: Upload temporary Pages artifact + uses: actions/upload-artifact@v3 + with: + name: github-pages-tmp + path: _site/ + + bundle-pages: + runs-on: ubuntu-latest + needs: [build, generate, publish-pypi] + + # allow publish-pypi to be skipped, but run after it + if: always() && !cancelled() && !failure() && needs.generate.result == 'success' + + steps: + - uses: actions/checkout@v3 + - uses: ./.github/actions/install-artifact-wheel + with: + name: docgen-build + python-version: "3.11" + + - name: Checkout current Pages + uses: actions/checkout@v3 + continue-on-error: true + with: + ref: gh-pages + path: _site/ + + - name: Download temporary Pages artifact + uses: actions/download-artifact@v3 + with: + name: github-pages-tmp + path: _new_site/ + + - name: Add new docs to site + run: hexdoc_merge --source _new_site --dest _site --release "${{ needs.build.outputs.release == true }}" + + - name: Fix permissions + run: | + chmod -c -R +rX "_site/" | while read line; do + echo "::warning title=Invalid file permissions automatically fixed::$line" + done + + - name: Upload Pages artifact + uses: actions/upload-artifact@v3 + with: + name: github-pages + path: _site/ + + deploy-pages: + runs-on: ubuntu-latest + needs: bundle-pages + + permissions: + contents: write + + steps: + - name: Download Pages artifact + uses: actions/download-artifact@v3 + with: + name: github-pages + path: _site/ + + - name: Deploy to Pages + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: _site diff --git a/doc/properties.toml b/doc/properties.toml index fd8d75b3..94f00487 100644 --- a/doc/properties.toml +++ b/doc/properties.toml @@ -44,7 +44,6 @@ mod_name = "Hex Casting" author = "petrak@, Alwinfy" description = "The Hex Book, all in one place." icon_href = "logo.png" -is_bleeding_edge = true show_landing_text = true diff --git a/doc/src/hexdoc/__version__.py b/doc/src/hexdoc/__version__.py index 110b13f1..2150e998 100644 --- a/doc/src/hexdoc/__version__.py +++ b/doc/src/hexdoc/__version__.py @@ -1 +1,10 @@ -__version__ = "1.0.dev1" +# This file is auto-generated by hatch-gradle-version. +# Only the value of PY_VERSION is editable. +# All changes to other values will be ignored and overwritten. + +PY_VERSION = "1.0.dev0" + +# Everything below this line is completely auto-generated. + +GRADLE_VERSION = "0.11.1-7" +FULL_VERSION = "0.11.1.1.0rc7.dev0" diff --git a/doc/src/hexdoc/_templates/main.html.jinja b/doc/src/hexdoc/_templates/main.html.jinja index 1912bca9..a32a3bae 100644 --- a/doc/src/hexdoc/_templates/main.html.jinja +++ b/doc/src/hexdoc/_templates/main.html.jinja @@ -2,7 +2,7 @@ {{ title }} - + @@ -11,8 +11,8 @@ - - + + diff --git a/doc/src/hexdoc/hexdoc.py b/doc/src/hexdoc/hexdoc.py index e2b446ff..6a3c4723 100644 --- a/doc/src/hexdoc/hexdoc.py +++ b/doc/src/hexdoc/hexdoc.py @@ -14,15 +14,17 @@ from jinja2.sandbox import SandboxedEnvironment from pydantic import field_validator, model_validator from hexdoc.hexcasting.hex_book import load_hex_book -from hexdoc.minecraft.i18n import I18n -from hexdoc.patchouli.book import Book -from hexdoc.utils import Properties +from hexdoc.minecraft import I18n +from hexdoc.patchouli import Book +from hexdoc.utils import HexdocModel, ModResourceLoader, Properties from hexdoc.utils.cd import cd -from hexdoc.utils.model import HexdocModel -from hexdoc.utils.resource_loader import ModResourceLoader +from hexdoc.utils.path import write_to_path +from .__version__ import GRADLE_VERSION from .jinja_extensions import IncludeRawExtension, hexdoc_block, hexdoc_wrap +MARKER_NAME = ".hexdoc-meta-sitemap-marker.json" + def strip_empty_lines(text: str) -> str: return "\n".join(s for s in text.splitlines() if s.strip()) @@ -38,11 +40,11 @@ class Args(HexdocModel): ci: bool allow_missing: bool lang: str | None + release: bool output_dir: Path | None export_only: bool list_langs: bool - check_default_lang: Path | None @classmethod def parse_args(cls, args: Sequence[str] | None = None) -> Self: @@ -54,10 +56,10 @@ class Args(HexdocModel): parser.add_argument("--ci", action="store_true") parser.add_argument("--allow-missing", action="store_true") parser.add_argument("--lang", type=str, default=None) + parser.add_argument("--release", default=False) group = parser.add_mutually_exclusive_group(required=True) - group.add_argument("--output_dir", "-o", type=Path) - group.add_argument("--check-default-lang", type=Path) + group.add_argument("--output-dir", "-o", type=Path) group.add_argument("--export-only", action="store_true") group.add_argument("--list-langs", action="store_true") @@ -66,7 +68,6 @@ class Args(HexdocModel): @field_validator( "properties_file", "output_dir", - "check_default_lang", mode="after", ) def _resolve_path(cls, value: Path | None): @@ -83,13 +84,7 @@ class Args(HexdocModel): self.verbose = True # exactly one of these must be truthy (should be enforced by group above) - assert ( - bool(self.output_dir) - + self.export_only - + self.list_langs - + bool(self.check_default_lang) - == 1 - ) + assert bool(self.output_dir) + self.export_only + self.list_langs == 1 return self @@ -104,6 +99,12 @@ class Args(HexdocModel): return logging.DEBUG +class SitemapMarker(HexdocModel): + version: str + lang: str + path: str + + def main(args: Args | None = None) -> None: # allow passing Args for test cases, but parse by default if args is None: @@ -122,19 +123,10 @@ def main(args: Args | None = None) -> None: format="\033[1m[{relativeCreated:.02f} | {levelname} | {name}]\033[0m {message}", level=args.log_level, ) + logger = logging.getLogger(__name__) props = Properties.load(args.properties_file) - if args.check_default_lang: - dir_path = args.check_default_lang - for path in [ - dir_path / "index.html", - dir_path / props.default_lang / "index.html", - ]: - if not path.is_file(): - raise FileNotFoundError(path) - return - # just list the languages and exit if args.list_langs: with ModResourceLoader.load_all(props, export=False) as loader: @@ -210,30 +202,54 @@ def main(args: Args | None = None) -> None: template = env.get_template(props.template.main) - # set up the output directory - subprocess.run(["git", "clean", "-fdX", args.output_dir]) - args.output_dir.mkdir(parents=True, exist_ok=True) - static_dir = props.template.static_dir - if static_dir and static_dir.is_dir(): - shutil.copytree(static_dir, args.output_dir, dirs_exist_ok=True) + subprocess.run(["git", "clean", "-fdX", args.output_dir]) - # render each language separately - for lang, book in books.items(): - docs = strip_empty_lines( - template.render( - **props.template.args, - book=book, - props=props, + versions = ["latest"] + if args.release: + # root should be the latest released version + versions += ["", GRADLE_VERSION] + + # render each version and language separately + for version in versions: + for lang, book in books.items(): + # /index.html + # /lang/index.html + # /v/version/index.html + # /v/version/lang/index.html + parts = ["v", version] if version else [] + if lang != props.default_lang: + parts.append(lang) + + output_dir = args.output_dir / Path(*parts) + url = "/".join([props.url] + parts) + + logger.info(f"Rendering {output_dir}") + docs = strip_empty_lines( + template.render( + **props.template.args, + book=book, + props=props, + url=url, + is_bleeding_edge=version != "latest", + ) ) - ) - lang_output_dir = args.output_dir / lang - lang_output_dir.mkdir(parents=True, exist_ok=True) - (lang_output_dir / "index.html").write_text(docs, "utf-8") + write_to_path(output_dir / "index.html", docs) + if static_dir: + shutil.copytree(static_dir, output_dir, dirs_exist_ok=True) - if lang == props.default_lang: - (args.output_dir / "index.html").write_text(docs, "utf-8") + # marker file for updating the sitemap later + # we use this because matrix doesn't have outputs + # this feels scuffed but it does work + if version: + (output_dir / MARKER_NAME).write_text( + SitemapMarker( + version=version, + lang=lang, + path="/" + "/".join(parts), + ).model_dump_json() + ) if __name__ == "__main__": diff --git a/doc/src/hexdoc/hexdoc_merge.py b/doc/src/hexdoc/hexdoc_merge.py new file mode 100644 index 00000000..2d9f4ff9 --- /dev/null +++ b/doc/src/hexdoc/hexdoc_merge.py @@ -0,0 +1,71 @@ +import json +import shutil +from argparse import ArgumentParser +from collections import defaultdict +from pathlib import Path +from typing import Self, Sequence + +from hexdoc.hexdoc import MARKER_NAME, SitemapMarker +from hexdoc.utils import HexdocModel +from hexdoc.utils.path import write_to_path + + +def strip_empty_lines(text: str) -> str: + return "\n".join(s for s in text.splitlines() if s.strip()) + + +# CLI arguments +class Args(HexdocModel): + """example: main.py properties.toml -o out.html""" + + source: Path + dest: Path + release: bool + + @classmethod + def parse_args(cls, args: Sequence[str] | None = None) -> Self: + parser = ArgumentParser(allow_abbrev=False) + + parser.add_argument("--source", type=Path, required=True) + parser.add_argument("--dest", type=Path, required=True) + parser.add_argument("--release", default=False) + + return cls.model_validate(vars(parser.parse_args(args))) + + +def main(): + args = Args.parse_args() + + if not (args.source / "latest" / "index.html").is_file(): + raise FileNotFoundError(args.source / "index.html") + + args.dest.mkdir(parents=True, exist_ok=True) + + if args.release: + # remove current latest-released book in the destination + for path in args.dest.iterdir(): + if path.name not in ["v", "meta"]: + shutil.rmtree(path) + + new_sitemap = defaultdict[str, dict[str, str]]() + for marker_path in args.source.rglob(MARKER_NAME): + # add new(?) version to the sitemap + marker = SitemapMarker.model_validate_json(marker_path.read_text("utf-8")) + new_sitemap[marker.version][marker.lang] = marker.path + + # delete the corresponding directory in the destination + shutil.rmtree(args.dest / marker_path.relative_to(args.source)) + + sitemap_path = args.dest / "meta" / "sitemap.json" + + if sitemap_path.is_file(): + sitemap = json.loads(sitemap_path.read_text("utf-8")) | new_sitemap + else: + sitemap = new_sitemap + + shutil.copytree(args.source, args.dest, dirs_exist_ok=True) + write_to_path(sitemap_path, json.dumps(sitemap)) + + +if __name__ == "__main__": + main() diff --git a/doc/src/hexdoc/minecraft/i18n.py b/doc/src/hexdoc/minecraft/i18n.py index 88972ffc..0e1cd228 100644 --- a/doc/src/hexdoc/minecraft/i18n.py +++ b/doc/src/hexdoc/minecraft/i18n.py @@ -16,8 +16,8 @@ from hexdoc.utils.deserialize import ( decode_and_flatten_json_dict, isinstance_or_raise, ) +from hexdoc.utils.path import replace_suffixes from hexdoc.utils.resource_loader import LoaderContext -from hexdoc.utils.types import replace_suffixes @total_ordering diff --git a/doc/src/hexdoc/utils/path.py b/doc/src/hexdoc/utils/path.py new file mode 100644 index 00000000..e17b8a56 --- /dev/null +++ b/doc/src/hexdoc/utils/path.py @@ -0,0 +1,40 @@ +from pathlib import Path + + +def strip_suffixes(path: Path) -> Path: + """Removes all suffixes from a path. This is helpful because `path.with_suffix("")` + only removes the last suffix. + + For example: + ```py + path = Path("lang/en_us.flatten.json5") + strip_suffixes(path) # lang/en_us + path.with_suffix("") # lang/en_us.flatten + ``` + """ + while path.suffix: + path = path.with_suffix("") + return path + + +def replace_suffixes(path: Path, suffix: str) -> Path: + """Replaces all suffixes of a path. This is helpful because `path.with_suffix()` + only replaces the last suffix. + + For example: + ```py + path = Path("lang/en_us.flatten.json5") + replace_suffixes(path, ".json") # lang/en_us.json + path.with_suffix(".json") # lang/en_us.flatten.json + ``` + """ + return strip_suffixes(path).with_suffix(suffix) + + +def write_to_path(path: Path, data: str | bytes, encoding: str = "utf-8"): + path.parent.mkdir(parents=True, exist_ok=True) + match data: + case str(): + path.write_text(data, encoding) + case bytes(): + path.write_bytes(data) diff --git a/doc/src/hexdoc/utils/resource_loader.py b/doc/src/hexdoc/utils/resource_loader.py index 7931751f..55736f0e 100644 --- a/doc/src/hexdoc/utils/resource_loader.py +++ b/doc/src/hexdoc/utils/resource_loader.py @@ -9,9 +9,10 @@ from typing import Callable, Literal, Self, TypeVar, overload from pydantic.dataclasses import dataclass +from hexdoc.__version__ import GRADLE_VERSION from hexdoc.utils.deserialize import decode_json_dict from hexdoc.utils.model import DEFAULT_CONFIG, HexdocModel, ValidationContext -from hexdoc.utils.types import strip_suffixes +from hexdoc.utils.path import strip_suffixes, write_to_path from .properties import Properties from .resource import PathResourceDir, ResourceLocation, ResourceType @@ -77,7 +78,7 @@ class ModResourceLoader: loader.export( path=HexdocMetadata.path(props.modid), data=HexdocMetadata( - book_url=props.url, + book_url=f"{props.url}/v/{GRADLE_VERSION}", ).model_dump_json(), ) @@ -364,8 +365,7 @@ class ModResourceLoader: case (str(out_data), Path() as out_path): pass - out_path.parent.mkdir(parents=True, exist_ok=True) - out_path.write_text(out_data, "utf-8") + write_to_path(out_path, out_data) class LoaderContext(ValidationContext): diff --git a/doc/src/hexdoc/utils/types.py b/doc/src/hexdoc/utils/types.py index 118c27ac..64e3dc60 100644 --- a/doc/src/hexdoc/utils/types.py +++ b/doc/src/hexdoc/utils/types.py @@ -1,7 +1,6 @@ import string from abc import ABC, abstractmethod from enum import Enum, unique -from pathlib import Path from typing import Any, Mapping, Protocol, TypeVar from pydantic import field_validator, model_validator @@ -101,33 +100,3 @@ class TryGetEnum(Enum): return cls(value) except ValueError: return None - - -def strip_suffixes(path: Path) -> Path: - """Removes all suffixes from a path. This is helpful because `path.with_suffix("")` - only removes the last suffix. - - For example: - ```py - path = Path("lang/en_us.flatten.json5") - strip_suffixes(path) # lang/en_us - path.with_suffix("") # lang/en_us.flatten - ``` - """ - while path.suffix: - path = path.with_suffix("") - return path - - -def replace_suffixes(path: Path, suffix: str) -> Path: - """Replaces all suffixes of a path. This is helpful because `path.with_suffix()` - only replaces the last suffix. - - For example: - ```py - path = Path("lang/en_us.flatten.json5") - replace_suffixes(path, ".json") # lang/en_us.json - path.with_suffix(".json") # lang/en_us.flatten.json - ``` - """ - return strip_suffixes(path).with_suffix(suffix) diff --git a/doc/test/test_snapshots.py b/doc/test/test_snapshots.py index 37c9e74a..dd29ff45 100644 --- a/doc/test/test_snapshots.py +++ b/doc/test/test_snapshots.py @@ -11,17 +11,21 @@ PROPS = "doc/properties.toml" def test_file(tmp_path: Path, snapshot: SnapshotAssertion): # generate output docs html file and assert it hasn't changed vs. the snapshot - out_path = tmp_path / "out.html" - main(Args.parse_args([PROPS, "-o", out_path.as_posix()])) + main( + Args.parse_args( + [PROPS, "--lang", "en_us", "--release", "-o", tmp_path.as_posix()] + ) + ) + out_path = tmp_path / "index.html" assert out_path.read_text("utf-8") == snapshot def test_cmd(tmp_path: Path, snapshot: SnapshotAssertion): # as above, but running the command we actually want to be using - out_path = tmp_path / "out.html" subprocess.run( - ["hexdoc", PROPS, "-o", out_path.as_posix()], + ["hexdoc", PROPS, "--lang", "en_us", "--release", "-o", tmp_path.as_posix()], stdout=sys.stdout, stderr=sys.stderr, ) + out_path = tmp_path / "index.html" assert out_path.read_text("utf-8") == snapshot diff --git a/pyproject.toml b/pyproject.toml index 1332a7c1..f87bacfe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["hatchling", "hatch-gradle-version>=0.3.0"] +requires = ["hatchling", "hatch-gradle-version>=0.4.0"] build-backend = "hatchling.build" # project metadata @@ -34,6 +34,7 @@ dev = [ [project.scripts] hexdoc = "hexdoc.hexdoc:main" +hexdoc_merge = "hexdoc.hexdoc_merge:main" # Gradle version/deps