worldmap-viewer: initial public history

1 files changed, 120 insertions, 0 deletions

diff --git a/tilecap.py b/tilecap.py
new file mode 100644
index 0000000..e730d87
--- /dev/null
+++ b/tilecap.py
@@ -0,0 +1,120 @@
+"""A size cap for the saved tiles (a public server on a shared disk): past the cap, the tiles used least recently
+are deleted down to (1 − SLACK) × cap, so a sweep is rare. Tiles of the kept layers at zooms ≤ keep_z (the
+prerendered map shipped with the server) are never deleted and do not count; other layers at those zooms do (they
+render on demand, and fifteen layers' worth would outgrow any disk). "Used" = the file's mtime: a tile read from disk is touched
+(the data disk may be mounted noatime). Only files shaped <layer>/<z>/<x>/<y>.<ext> are ever touched or deleted."""
+from __future__ import annotations
+
+import os
+import threading
+from pathlib import Path
+
+SLACK = 0.1
+TILE_EXTS = (".jpg", ".png", ".bin")
+
+
+KEEP_LAYERS = ("relief", "mesh")                     # what a public deploy prerenders
+
+
+def tile_key(path) -> tuple[str, int] | None:
+    """(layer, zoom) of a saved tile's path, or None when it is not one (manifests, .part files, anything else)."""
+    p = Path(path)
+    if p.suffix not in TILE_EXTS or not p.stem.isdigit():
+        return None
+    parts = p.parts
+    if len(parts) < 5 or not (parts[-2].isdigit() and parts[-3].isdigit()):
+        return None
+    return parts[-4], int(parts[-3])
+
+
+class TileCap:
+    def __init__(self, cap_bytes: int, keep_z: int = -1, keep_layers=KEEP_LAYERS):
+        self.cap, self.keep_z, self.keep_layers = int(cap_bytes), int(keep_z), tuple(keep_layers)
+        self.roots: list[Path] = []
+        self.total = None                                       # counted bytes; None until the first sweep
+        self.lock = threading.Lock()
+        self.sweeping = False
+
+    def add_root(self, root) -> None:
+        root = Path(root)
+        with self.lock:
+            if root not in self.roots:
+                self.roots.append(root)
+
+    def counts(self, path) -> bool:
+        k = tile_key(path)
+        return k is not None and not (k[0] in self.keep_layers and k[1] <= self.keep_z)
+
+    def used(self, path) -> None:
+        """A tile was read from disk: it is now the most recently used."""
+        if self.counts(path):
+            try:
+                os.utime(path)
+            except OSError:
+                pass
+
+    def saved(self, path, nbytes: int) -> None:
+        """A tile was written: count it, and sweep in the background once past the cap."""
+        if not self.counts(path):
+            return
+        with self.lock:
+            if self.total is None:
+                return                                          # the first sweep counts it
+            self.total += nbytes
+            over = self.total > self.cap
+        if over:
+            self.start()
+
+    def start(self) -> None:
+        with self.lock:
+            if self.sweeping:
+                return
+            self.sweeping = True
+        threading.Thread(target=self.sweep, daemon=True).start()
+
+    def _files(self):
+        for root in list(self.roots):
+            for d, _, names in os.walk(root):
+                for n in names:
+                    f = os.path.join(d, n)
+                    if self.counts(f):
+                        try:
+                            st = os.stat(f)
+                        except OSError:
+                            continue
+                        yield st.st_mtime_ns, st.st_size, f
+
+    def sweep(self) -> int:
+        """Count the tiles; past the cap, delete the least recently used down to (1 − SLACK) × cap.
+        Returns the bytes deleted."""
+        with self.lock:
+            self.sweeping = True
+        freed = 0
+        try:
+            files = list(self._files())
+            total = sum(s for _, s, _ in files)
+            if total > self.cap:
+                goal = self.cap * (1 - SLACK)
+                files.sort()
+                for _, size, f in files:
+                    if total <= goal:
+                        break
+                    try:
+                        os.unlink(f)
+                    except FileNotFoundError:
+                        pass
+                    except OSError:
+                        continue
+                    total -= size
+                    freed += size
+                    for parent in (Path(f).parent, Path(f).parent.parent):   # empty <x>/ and <z>/ folders go too
+                        try:
+                            parent.rmdir()
+                        except OSError:
+                            break
+            with self.lock:
+                self.total = total
+        finally:
+            with self.lock:
+                self.sweeping = False
+        return freed