worldmap-viewer: initial public history

1 files changed, 105 insertions, 0 deletions

diff --git a/deploy.py b/deploy.py
new file mode 100644
index 0000000..0fb5c8e
--- /dev/null
+++ b/deploy.py
@@ -0,0 +1,105 @@
+"""Ship a built world to a public map server, and tidy it there.
+
+  python deploy.py bundle WORLD_DIR DEST [--res N] [--dry-run]   # copy what a --public server reads
+  python deploy.py dedupe WORLD_DIR [--res N]                    # on the server, after a copy
+
+bundle copies, with rsync (DEST may be host:/path): config/, places/regions.json and out/r<res>/ — but not the
+refined areas or tiles of earlier builds, and never pins, exports, lore or build inputs (a public server serves
+none of them). rsync -a keeps the files' modification times, which the build fingerprint needs (else every
+refined area and tile would be rebuilt there); it writes plainly (no preallocation), so btrfs compresses. Nothing
+is deleted at DEST: the server prunes what it no longer uses, and its own saved tiles stay.
+
+A copy does not keep the blocks that eras' serve caches shared (dedupe.py): run `dedupe` on the server after it."""
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from pathlib import Path
+
+import worldgen_path  # noqa: F401  (mapgen on sys.path)
+
+import dedupe
+import refine
+import servecache
+import tiles
+
+
+def fingerprints(root: Path, res: int) -> list[str]:
+    """The current build fingerprints of a world's base build and built eras."""
+    from mapgen import config as C
+    seed = int(C.load(root)[0]["build"]["seed"])
+    return [tiles.fingerprint(d, seed) for _, d in refine.world_dirs(root, res, log=lambda m: None)]
+
+
+def stale_rules(root: Path, res: int) -> list[str]:
+    """rsync exclude rules for refined areas and tile folders of builds other than the current ones."""
+    fps = fingerprints(root, res)
+    out = Path(root) / "out" / f"r{res}"
+    tags = tuple(f"v{tiles.VERSION}.{tiles.DRAW}-{fp}" for fp in fps)
+    keep_regions = {refine.results_dir(out / "regions", fp).name for fp in fps} | {refine.AREAS}
+    rules = []
+    rdir = out / "regions"
+    for d in sorted(rdir.iterdir()) if rdir.is_dir() else ():
+        if d.is_dir() and d.name not in keep_regions:
+            rules.append(f"- /out/r{res}/regions/{d.name}/")
+    for tdir in sorted(out.glob("**/tiles")):
+        if tdir.is_dir() and "regions" not in tdir.relative_to(out).parts:
+            for d in sorted(tdir.iterdir()):
+                if d.is_dir() and not d.name.startswith(tags):
+                    rules.append(f"- /{d.relative_to(root).as_posix()}/")
+    return rules
+
+
+def filter_rules(root: Path, res: int) -> list[str]:
+    return [*stale_rules(root, res),
+            "- **/__pycache__/", "- *.part", "- .*.npz",
+            "+ /config/***",
+            "+ /places/", "+ /places/regions.json",
+            "+ /out/", f"+ /out/r{res}/***",
+            "- *"]
+
+
+def bundle(root: Path, dest: str, res: int, dry_run: bool = False, rsync: str = "rsync") -> int:
+    rules = filter_rules(root, res)
+    cmd = [rsync, "-a", "--info=stats1", *(["--dry-run", "--itemize-changes"] if dry_run else []),
+           *(arg for r in rules for arg in ("--filter", r)), f"{Path(root).resolve()}/", dest]
+    return subprocess.run(cmd).returncode
+
+
+def dedupe_world(root: Path, res: int) -> int:
+    """Share identical blocks between the eras' serve caches again (lost in a copy). Bytes shared."""
+    out = Path(root) / "out" / f"r{res}"
+    shared = 0
+    for d in sorted(out.glob(f"**/{servecache.PREFIX}*")):
+        if d.is_dir() and (d / servecache.DONE).exists():
+            shared += dedupe.dedupe_dirs(d, [p for p in servecache.peers(d) if p < d])
+    return shared
+
+
+def main(argv=None) -> int:
+    ap = argparse.ArgumentParser(prog="deploy.py", description=__doc__.split("\n\n")[0])
+    sub = ap.add_subparsers(dest="cmd", required=True)
+    b = sub.add_parser("bundle", help="copy what a --public server reads to DEST (rsync; host:/path works)")
+    b.add_argument("world")
+    b.add_argument("dest")
+    b.add_argument("--res", type=int)
+    b.add_argument("--dry-run", action="store_true", help="list what would be copied")
+    d = sub.add_parser("dedupe", help="on the server after a copy: eras' serve caches share their blocks again")
+    d.add_argument("world")
+    d.add_argument("--res", type=int)
+    a = ap.parse_args(argv)
+    root = Path(a.world).resolve()
+    from mapgen import config as C
+    res = a.res if a.res is not None else int(C.load(root)[0]["build"]["res_final"])
+    if not (root / "out" / f"r{res}" / "cells.npz").exists():
+        print(f"error: no r{res} build in {root}", file=sys.stderr)
+        return 2
+    if a.cmd == "bundle":
+        return bundle(root, a.dest, res, a.dry_run)
+    print(f"shared {dedupe_world(root, res) / 1e9:.2f} GB")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())