worldmap-viewer

git clone https://git.godosa.eu/worldmap-viewer

master

raw · 4552 bytes

"""A size cap for the saved tiles (a public server on a shared disk): past the cap, the tiles used least recently
are deleted down to (1 − SLACK) × cap, so a sweep is rare. Tiles of the kept layers at zooms ≤ keep_z (the
prerendered map shipped with the server) are never deleted and do not count; other layers at those zooms do (they
render on demand, and fifteen layers' worth would outgrow any disk). "Used" = the file's mtime: a tile read from disk is touched
(the data disk may be mounted noatime). Only files shaped <layer>/<z>/<x>/<y>.<ext> are ever touched or deleted."""
from __future__ import annotations

import os
import threading
from pathlib import Path

SLACK = 0.1
TILE_EXTS = (".jpg", ".png", ".bin")


KEEP_LAYERS = ("relief", "mesh")                     # what a public deploy prerenders


def tile_key(path) -> tuple[str, int] | None:
    """(layer, zoom) of a saved tile's path, or None when it is not one (manifests, .part files, anything else)."""
    p = Path(path)
    if p.suffix not in TILE_EXTS or not p.stem.isdigit():
        return None
    parts = p.parts
    if len(parts) < 5 or not (parts[-2].isdigit() and parts[-3].isdigit()):
        return None
    return parts[-4], int(parts[-3])


class TileCap:
    def __init__(self, cap_bytes: int, keep_z: int = -1, keep_layers=KEEP_LAYERS):
        self.cap, self.keep_z, self.keep_layers = int(cap_bytes), int(keep_z), tuple(keep_layers)
        self.roots: list[Path] = []
        self.total = None                                       # counted bytes; None until the first sweep
        self.lock = threading.Lock()
        self.sweeping = False

    def add_root(self, root) -> None:
        root = Path(root)
        with self.lock:
            if root not in self.roots:
                self.roots.append(root)

    def counts(self, path) -> bool:
        k = tile_key(path)
        return k is not None and not (k[0] in self.keep_layers and k[1] <= self.keep_z)

    def used(self, path) -> None:
        """A tile was read from disk: it is now the most recently used."""
        if self.counts(path):
            try:
                os.utime(path)
            except OSError:
                pass

    def saved(self, path, nbytes: int) -> None:
        """A tile was written: count it, and sweep in the background once past the cap."""
        if not self.counts(path):
            return
        with self.lock:
            if self.total is None:
                return                                          # the first sweep counts it
            self.total += nbytes
            over = self.total > self.cap
        if over:
            self.start()

    def start(self) -> None:
        with self.lock:
            if self.sweeping:
                return
            self.sweeping = True
        threading.Thread(target=self.sweep, daemon=True).start()

    def _files(self):
        for root in list(self.roots):
            for d, _, names in os.walk(root):
                for n in names:
                    f = os.path.join(d, n)
                    if self.counts(f):
                        try:
                            st = os.stat(f)
                        except OSError:
                            continue
                        yield st.st_mtime_ns, st.st_size, f

    def sweep(self) -> int:
        """Count the tiles; past the cap, delete the least recently used down to (1 − SLACK) × cap.
        Returns the bytes deleted."""
        with self.lock:
            self.sweeping = True
        freed = 0
        try:
            files = list(self._files())
            total = sum(s for _, s, _ in files)
            if total > self.cap:
                goal = self.cap * (1 - SLACK)
                files.sort()
                for _, size, f in files:
                    if total <= goal:
                        break
                    try:
                        os.unlink(f)
                    except FileNotFoundError:
                        pass
                    except OSError:
                        continue
                    total -= size
                    freed += size
                    for parent in (Path(f).parent, Path(f).parent.parent):   # empty <x>/ and <z>/ folders go too
                        try:
                            parent.rmdir()
                        except OSError:
                            break
            with self.lock:
                self.total = total
        finally:
            with self.lock:
                self.sweeping = False
        return freed