worldmap-viewer

git clone https://git.godosa.eu/worldmap-viewer

master

raw · 4669 bytes

"""Ship a built world to a public map server, and tidy it there.

  python deploy.py bundle WORLD_DIR DEST [--res N] [--dry-run]   # copy what a --public server reads
  python deploy.py dedupe WORLD_DIR [--res N]                    # on the server, after a copy

bundle copies, with rsync (DEST may be host:/path): config/, places/regions.json and out/r<res>/ — but not the
refined areas or tiles of earlier builds, and never pins, exports, lore or build inputs (a public server serves
none of them). rsync -a keeps the files' modification times, which the build fingerprint needs (else every
refined area and tile would be rebuilt there); it writes plainly (no preallocation), so btrfs compresses. Nothing
is deleted at DEST: the server prunes what it no longer uses, and its own saved tiles stay.

A copy does not keep the blocks that eras' serve caches shared (dedupe.py): run `dedupe` on the server after it."""
from __future__ import annotations

import argparse
import subprocess
import sys
from pathlib import Path

import worldgen_path  # noqa: F401  (mapgen on sys.path)

import dedupe
import refine
import servecache
import tiles


def fingerprints(root: Path, res: int) -> list[str]:
    """The current build fingerprints of a world's base build and built eras."""
    from mapgen import config as C
    seed = int(C.load(root)[0]["build"]["seed"])
    return [tiles.fingerprint(d, seed) for _, d in refine.world_dirs(root, res, log=lambda m: None)]


def stale_rules(root: Path, res: int) -> list[str]:
    """rsync exclude rules for refined areas and tile folders of builds other than the current ones."""
    fps = fingerprints(root, res)
    out = Path(root) / "out" / f"r{res}"
    tags = tuple(f"v{tiles.VERSION}.{tiles.DRAW}-{fp}" for fp in fps)
    keep_regions = {refine.results_dir(out / "regions", fp).name for fp in fps} | {refine.AREAS}
    rules = []
    rdir = out / "regions"
    for d in sorted(rdir.iterdir()) if rdir.is_dir() else ():
        if d.is_dir() and d.name not in keep_regions:
            rules.append(f"- /out/r{res}/regions/{d.name}/")
    for tdir in sorted(out.glob("**/tiles")):
        if tdir.is_dir() and "regions" not in tdir.relative_to(out).parts:
            for d in sorted(tdir.iterdir()):
                if d.is_dir() and not d.name.startswith(tags):
                    rules.append(f"- /{d.relative_to(root).as_posix()}/")
    return rules


def filter_rules(root: Path, res: int) -> list[str]:
    return [*stale_rules(root, res),
            "- **/__pycache__/", "- *.part", "- .*.npz",
            "+ /config/***",
            "+ /places/", "+ /places/regions.json",
            "+ /out/", f"+ /out/r{res}/***",
            "- *"]


def bundle(root: Path, dest: str, res: int, dry_run: bool = False, rsync: str = "rsync") -> int:
    rules = filter_rules(root, res)
    cmd = [rsync, "-a", "--info=stats1", *(["--dry-run", "--itemize-changes"] if dry_run else []),
           *(arg for r in rules for arg in ("--filter", r)), f"{Path(root).resolve()}/", dest]
    return subprocess.run(cmd).returncode


def dedupe_world(root: Path, res: int) -> int:
    """Share identical blocks between the eras' serve caches again (lost in a copy). Bytes shared."""
    out = Path(root) / "out" / f"r{res}"
    shared = 0
    for d in sorted(out.glob(f"**/{servecache.PREFIX}*")):
        if d.is_dir() and (d / servecache.DONE).exists():
            shared += dedupe.dedupe_dirs(d, [p for p in servecache.peers(d) if p < d])
    return shared


def main(argv=None) -> int:
    ap = argparse.ArgumentParser(prog="deploy.py", description=__doc__.split("\n\n")[0])
    sub = ap.add_subparsers(dest="cmd", required=True)
    b = sub.add_parser("bundle", help="copy what a --public server reads to DEST (rsync; host:/path works)")
    b.add_argument("world")
    b.add_argument("dest")
    b.add_argument("--res", type=int)
    b.add_argument("--dry-run", action="store_true", help="list what would be copied")
    d = sub.add_parser("dedupe", help="on the server after a copy: eras' serve caches share their blocks again")
    d.add_argument("world")
    d.add_argument("--res", type=int)
    a = ap.parse_args(argv)
    root = Path(a.world).resolve()
    from mapgen import config as C
    res = a.res if a.res is not None else int(C.load(root)[0]["build"]["res_final"])
    if not (root / "out" / f"r{res}" / "cells.npz").exists():
        print(f"error: no r{res} build in {root}", file=sys.stderr)
        return 2
    if a.cmd == "bundle":
        return bundle(root, a.dest, res, a.dry_run)
    print(f"shared {dedupe_world(root, res) / 1e9:.2f} GB")
    return 0


if __name__ == "__main__":
    sys.exit(main())