"""XYZ tile server + slippy map + generation (lidar-maps image). Serves the tile pyramid described in `tiles.py` using the OpenStreetMap/Google Maps scheme, so it can be used directly as an imagery background in JOSM, iD, QGIS, uMap, MapLibre or OsmAnd: GET /tiles/{layer}/{z}/{x}/{y}.png 256 px, RGBA PNG (canonical) GET /tiles/{layer}/{z}/{x}/{y}@2x.avif 512 px, AVIF (internal interface; .webp/.png also accepted) GET /tiles/{layer}.json TileJSON 3.0.0 GET /tiles/wmts.xml WMTS 1.0.0 capabilities (QGIS…) GET /tiles/josm.imagery.xml JOSM imagery file GET /api/map/meta layers, zooms, extent, version GET /api/map/tile?lat&lng tile under the point (Tile tab) GET /api/map/ign?col&row IGN catalogue record of a tile GET|POST|DELETE /api/map/defaults server-wide default display POST /api/map/warm pre-render an extent GET|POST /api/map/background background maintenance state/scan GET /api/export/frame, /api/export/pdf printable PDF sheet GET / and /assets/* viewing interface Tiles are rendered on demand, then cached on disk (`output/index_xyz/`). In two-machine mode, `LIDAR_MAPS_URL` points to an upstream server: a tile that is missing locally is fetched from it (a few tens of KB) instead of fetching whole source tiles. Tile generation (taken over from the former webapp): GET /api/status → state of the current (or last) run, queue, tiles GET /api/tiles → tile inventory (index_tiles.json) — acts as the upstream tile server (LIDAR_SOURCE_URL) POST /api/preview → 1 km cells of a WGS84 bbox that need processing POST /api/generate → downloads (IGN Géoplateforme), then processes POST /api/stop → stops the current run POST /api/queue/clear → empties the queue GET /api/cell → L93 tile under a point (manual selection) On the full image (worker, processing machine), generation is local: `python -m lidar_pipeline` as a subprocess, one job at a time, concurrent requests go to a persistent queue. On the lightweight image (Raspberry Pi), `LIDAR_GENERATION_URL` delegates everything to the processing machine — the produced tiles are fetched on demand through `LIDAR_SOURCE_URL` (inventory /api/tiles + static mounts visualisations/, index_thumbs/, index_subtiles/). `LIDAR_API_TOKEN` (worker) / `LIDAR_REMOTE_TOKEN` (lightweight) optionally protect the calls; `LIDAR_REGEN_CIDR` restricts generation to the local network (Generation tab hidden elsewhere). On a small machine (Raspberry Pi), two settings invert the load: - `LIDAR_TILE_CACHE_ONLY=1` — browsing no longer renders stored levels: a missing tile is served transparent (`X-Tile-Pending: 1`, never cached by the browser) and appears once the maintenance has processed it (levels that are not stored are still rendered on the fly); - `LIDAR_TILE_BACKGROUND=1` — a background task watches the source tiles (new or regenerated) and queues their pyramid for update, one render at a time, at low priority (`os.nice`) and paced (`LIDAR_TILE_BACKGROUND_PAUSE`). First start = rebuild of the whole pyramid up to `LIDAR_TILE_BACKGROUND_MAX_Z` (tiles that are already fresh only cost a `stat`). `POST /api/map/background` forces a scan, `POST /api/map/warm` remains the exhaustive manual pre-render. """ import json import logging import math import os import re import signal import subprocess import sys import threading import time import urllib.parse import urllib.request import uuid import xml.sax.saxutils as _xml from collections import deque from contextlib import asynccontextmanager from functools import lru_cache from pathlib import Path from typing import Optional from fastapi import Depends, FastAPI, Header, HTTPException, Request from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, Response from fastapi.staticfiles import StaticFiles as _StaticFiles from pydantic import BaseModel, Field from starlette.concurrency import run_in_threadpool from . import tiles as tiles_mod from .mapui import ASSETS_DIRNAME, render_html, ui_version, write_map_assets OUTPUT_DIR = Path(os.environ.get("LIDAR_OUTPUT_DIR", "/data/output")) INPUT_DIR = Path(os.environ.get("LIDAR_INPUT_DIR", "/data/input")) PORT = int(os.environ.get("LIDAR_PORT", "8975")) # Upstream tile server (processing machine) — two-machine mode. MAPS_URL = (os.environ.get("LIDAR_MAPS_URL") or "").rstrip("/") ATTRIBUTION = os.environ.get( "LIDAR_ATTRIBUTION", 'LiDAR HD © IGN ' '— Licence Ouverte 2.0') ATTRIBUTION_TEXT = re.sub(r"<[^>]+>", "", ATTRIBUTION).replace("©", "©").replace("—", "—") # Concurrent renders: low cap by default — the map is often served from a # small machine (Raspberry Pi), and a render decodes an AVIF source tile of # several tens of MB. Excess requests wait on the semaphore without using # CPU. `LIDAR_TILE_WORKERS` raises the cap on a well-equipped machine. TILE_WORKERS = max(1, int(os.environ.get("LIDAR_TILE_WORKERS", "2") or 2)) _render_sem = threading.Semaphore(TILE_WORKERS) # Remote processing machine (lightweight map such as a Raspberry Pi): when # set, generation (IGN download + processing) is delegated to it. GENERATION_URL = (os.environ.get("LIDAR_GENERATION_URL") or "").rstrip("/") # Shared token: the lightweight map presents it to the processing machine # (LIDAR_REMOTE_TOKEN here, LIDAR_API_TOKEN over there). Optional. REMOTE_TOKEN = os.environ.get("LIDAR_REMOTE_TOKEN") or None API_TOKEN = os.environ.get("LIDAR_API_TOKEN") or None # Network allowed to start tile (re)generation (/api/generate, and also # /api/preview, /api/stop, /api/queue/clear): comma-separated list of CIDRs, # empty = no restriction. Default: loopback + RFC 1918 private ranges — # covers the LAN, the Docker host (172.x gateways) and localhost, while # rejecting clients coming from the Internet. The client IP is the # connection IP (preserved by Docker DNAT); if it is inside the allowed # network, X-Forwarded-For is honoured (local reverse proxy — a remote # client cannot forge it since its direct connection is already outside). REGEN_CIDR = (os.environ.get( "LIDAR_REGEN_CIDR", "127.0.0.0/8,::1,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16") or "").strip() # Generation resolution (policy: 0.2 m only — see AGENTS.md). GENERATE_RESOLUTIONS = (0.2,) GROUND_CLASS_METHODS = ("auto", "ign", "smrf", "csf") MAX_CELLS = 400 # safeguard: ~400 km² max per request (drawn areas) MAX_CELLS_ALL = 10000 # safeguard: global "complete everything" pass (tiles already present) LOG_FILE = OUTPUT_DIR / ".generation.log" JOB_FILE = OUTPUT_DIR / ".generation.job.json" QUEUE_FILE = OUTPUT_DIR / ".generation.queue.json" # --------------------------------------------------------------------------- # Background pyramid maintenance (small machines) # --------------------------------------------------------------------------- # # `LIDAR_TILE_BACKGROUND=1` + `LIDAR_TILE_CACHE_ONLY=1` invert the load: # browsing no longer renders stored levels (cache reads only), a poller # rescans the source tiles at regular intervals (and as soon as the # inventory changes) and every new/regenerated source tile queues its tile # pyramid; low-priority renderers (os.nice) drain the queue at one local # render per LIDAR_TILE_BACKGROUND_PAUSE seconds (upstream downloads are not # paced). First pass = unknown source tile: the whole pyramid is rebuilt # (tiles that are already fresh only cost a stat). def _env_flag(name): """Environment flag (1/true/yes/on/oui). Re-read on every call: testable without reloading the module.""" return (os.environ.get(name, "").strip().lower() in ("1", "true", "yes", "on", "oui")) def _bg_scale(): return 1 if os.environ.get("LIDAR_TILE_BACKGROUND_SCALE", "2").strip() == "1" else 2 def _bg_conf(): """Maintenance settings (re-read on every call, see the module doc).""" return { "interval": max(10.0, float(os.environ.get( "LIDAR_TILE_BACKGROUND_INTERVAL", "120") or 120)), # Default: up to native (18@2x = 0.2 m/px) — complete pyramid. "max_z": max(tiles_mod.TILE_MIN_Z, int(os.environ.get( "LIDAR_TILE_BACKGROUND_MAX_Z", "") or (tiles_mod.TILE_MAX_NATIVE_Z - ( 0 if _bg_scale() == 1 else 1)))), "scale": _bg_scale(), "fmt": (os.environ.get("LIDAR_TILE_BACKGROUND_FMT", "avif") or "avif").strip(), "pause": max(0.0, float(os.environ.get( "LIDAR_TILE_BACKGROUND_PAUSE", "1.0") or 1.0)), "queue_max": max(1024, int(os.environ.get( "LIDAR_TILE_BACKGROUND_QUEUE_MAX", "65536") or 65536)), } _bg = {"threads": [], "stop": threading.Event(), "queue": deque(), "queued": set(), "guard": threading.Lock(), "snapshot": None, "prefetch": deque(), "incomplete": {}, "running": False, "stats": {"rendues": 0, "a_jour": 0, "vides": 0, "scans": 0, "dernier_scan": None, "ajouts_dernier_scan": 0}} def _bg_enqueue(items, front=False, rejected=None): """Add tiles (layer, z, x, y) to the queue: deduplicated, bounded. rejected: set filled with the tiles refused for lack of room. front=True: at the head of the queue, in the given order (freshly rendered source tiles); a tile already queued is moved up. The backlog is not truncated to make room for them (its source tiles would never be resumed): the queue then briefly overflows by the added head. """ added = 0 with _bg["guard"]: if front: fresh = [it for it in items] keep = set(fresh) rest = [it for it in _bg["queue"] if it not in keep] _bg["queue"].clear() _bg["queue"].extend(fresh[:_bg_conf()["queue_max"]]) _bg["queue"].extend(rest) if rejected is not None: rejected.update(fresh[_bg_conf()["queue_max"]:]) added = sum(1 for it in fresh if it not in _bg["queued"]) _bg["queued"] = set(_bg["queue"]) return added for item in items: if item in _bg["queued"]: continue if len(_bg["queue"]) >= _bg_conf()["queue_max"]: if rejected is not None: rejected.add(item) continue _bg["queue"].append(item) _bg["queued"].add(item) added += 1 return added def _bg_pop(): with _bg["guard"]: if _bg["queue"]: item = _bg["queue"].popleft() _bg["queued"].discard(item) return item return None def background_scan(output_dir=None): """Detect new or regenerated source tiles and queue their tiles. Returns the number of tiles added. The comparison uses the most recent mtime of each source tile's sources: a regenerated source tile only invalidates its own tiles (same rule as the cache). """ output_dir = Path(output_dir or OUTPUT_DIR) conf = _bg_conf() first_scan = _bg["snapshot"] is None snapshot = {} changed = {} resume = {} # unchanged source tile, pyramid not fully queued yet → resume for layer, per_cell in tiles_mod.source_index(output_dir, force=True).items(): for (col, row), tiers in per_cell.items(): newest = 0.0 for group in tiers: for src in group: m = src.mtime() if m and m > newest: newest = m snapshot[(layer, col, row)] = newest if (_bg["snapshot"] is not None and _bg["snapshot"].get((layer, col, row)) == newest): if (layer, col, row) in _bg["incomplete"]: resume[(layer, col, row)] = _bg["incomplete"][(layer, col, row)] changed.setdefault(layer, []).append((col, row)) continue changed.setdefault(layer, []).append((col, row)) _bg["snapshot"] = snapshot # Map fed by an upstream tile server (Pi): the sources of new or modified # source tiles are fetched and KEPT locally, without waiting for a visit — # levels rendered on the fly stay available when the rendering container # is down. # Every source tile is re-checked (plain existence tests): a source # missed while upstream was down is picked up again at the next scan. if tiles_mod.REMOTE_SOURCE_URL: index = tiles_mod.source_index(output_dir) pending = {str(s_.path) for s_ in _bg["prefetch"]} for per_cell in index.values(): for tiers in per_cell.values(): for src in tiles_mod.sources_to_keep(tiers): key = str(src.path) if src.url is not None and key not in pending and not src.path.is_file(): _bg["prefetch"].append(src) pending.add(key) # Bounded queue (queue_max): over a large territory, the first scan used # to fill it and the rest of the pyramid was never generated. Each source # tile therefore keeps its position: tiles refused for lack of room are # resumed at the next scans (at the tail, behind the backlog), from the # first refused one. Source tiles that just changed go to the head (the # pyramid follows the current run, except on the first scan). order = lambda t: (t[1], t[0], t[2], t[3]) # noqa: E731 — low levels first items, retry, per_cell = [], [], {} for layer, cells in changed.items(): for col, row in cells: cell = (layer, col, row) bbox = tiles_mod.cell_bounds_wgs84(col, row) mine = [] for z in range(tiles_mod.TILE_MIN_Z, conf["max_z"] + 1): if not (tiles_mod.zoom_supported(z, conf["scale"]) and tiles_mod.zoom_cached(z, conf["scale"])): continue for x, y in tiles_mod.tiles_in_bounds(bbox, z): mine.append((layer, z, x, y)) mine.sort(key=order) start = resume.get(cell, 0) per_cell[cell] = (start, mine) (retry if cell in resume else items).extend(mine[start:]) items.sort(key=order) retry.sort(key=order) rejected = set() added = _bg_enqueue(items, front=first_scan is False, rejected=rejected) added += _bg_enqueue(retry, rejected=rejected) incomplete = {} for cell, (start, mine) in per_cell.items(): for i in range(start, len(mine)): if mine[i] in rejected: incomplete[cell] = i break _bg["incomplete"] = incomplete _bg["stats"]["scans"] += 1 _bg["stats"]["dernier_scan"] = time.time() _bg["stats"]["ajouts_dernier_scan"] = added cells_changed = sum(len(v) for v in changed.values()) if items: logger.info(f"Tile maintenance: {cells_changed} source tile(s) " f"changed, {added} tile(s) queued") return added def _bg_process_one(): """Process one tile from the queue, otherwise one source to fetch. Truthy if some work was done: "telechargee" for a tile downloaded from upstream (chained without a pause: the worker can take it), True for a local render or a source fetch (followed by the pause). False when there was nothing to do or the source fetch failed. Tiles first: light and visible immediately, they must not wait behind thousands of sources (only useful to the high zooms rendered on the fly). Tiles that are already up to date are skipped without a pause. """ conf = _bg_conf() while True: item = _bg_pop() if item is None: break layer, z, x, y = item _data, state = tiles_mod.cached_tile(OUTPUT_DIR, layer, z, x, y, conf["scale"], conf["fmt"]) if state == "pending": break _bg["stats"]["a_jour" if state == "fresh" else "vides"] += 1 if item is None: try: src = _bg["prefetch"].popleft() except IndexError: return False if src.ensure(): _bg["stats"]["sources_locales"] = _bg["stats"].get("sources_locales", 0) + 1 return True return False # upstream down: retried at the next scan if MAPS_URL: # Upstream tile server: the pyramid is DOWNLOADED in the background # (a few tens of KB, no source tile decoding); the map fills up # without any visit and is then served without upstream. suffix = "@2x" if conf["scale"] == 2 else "" data = _fetch_upstream(f"tiles/{layer}/{z}/{x}/{y}{suffix}.{conf['fmt']}") if data: tiles_mod._write_atomic( tiles_mod.tile_cache_path(OUTPUT_DIR, layer, z, x, y, conf["scale"], conf["fmt"]), data) _bg["stats"]["telechargees"] = _bg["stats"].get("telechargees", 0) + 1 return "telechargee" # No upstream, or upstream down: local render from the kept sources (autonomy). tiles_mod.get_tile(OUTPUT_DIR, layer, z, x, y, conf["scale"], conf["fmt"]) _bg["stats"]["rendues"] += 1 return True def _bg_worker(): try: os.nice(10) # minimal footprint: rendering yields to other workloads except (AttributeError, OSError): # pragma: no cover — nice unavailable pass while not _bg["stop"].is_set(): try: rendered = _bg_process_one() except Exception as e: # noqa: BLE001 — one tile does not stop the queue logger.warning(f"Tile maintenance: render failed ({e})") rendered = False if rendered == "telechargee": continue # download: no local computation, no pause if rendered: _bg["stop"].wait(_bg_conf()["pause"]) # pacing: one render, one pause else: _bg["stop"].wait(0.5) def _inventory_mark(): """Cheap signature of the tile inventory (local file or upstream payload), used to trigger a scan as soon as a source tile arrives.""" try: return (OUTPUT_DIR / "index_tiles.json").stat().st_mtime except OSError: pass if not tiles_mod.REMOTE_SOURCE_URL: return None try: payload = tiles_mod._remote_payload() return payload.get("stamp") if payload else None except Exception: # noqa: BLE001 return None BG_WATCH_S = 10.0 # inventory watch interval during runs def _bg_poller(): last_scan, last_mark = 0.0, _inventory_mark() while not _bg["stop"].is_set(): mark = _inventory_mark() due = time.time() - last_scan >= _bg_conf()["interval"] if due or mark != last_mark: try: background_scan() except Exception as e: # noqa: BLE001 — next scan on the next round logger.warning(f"Tile maintenance: scan failed ({e})") last_scan, last_mark = time.time(), mark _bg["stop"].wait(min(BG_WATCH_S, _bg_conf()["interval"])) def start_background(): """Start the maintenance: 1 poller + TILE_WORKERS paced renderers.""" if _bg["running"]: return _bg["stop"].clear() _bg["running"] = True threads = [threading.Thread(target=_bg_poller, daemon=True, name="lidar-bg-scan")] threads += [threading.Thread(target=_bg_worker, daemon=True, name=f"lidar-bg-render-{i}") for i in range(TILE_WORKERS)] for t in threads: t.start() _bg["threads"] = threads conf = _bg_conf() logger.info(f"Tile maintenance enabled: z ≤ {conf['max_z']} " f"@{conf['scale']}x {conf['fmt']}, scan every " f"{conf['interval']:.0f} s, {conf['pause']:.1f} s pause " f"between local renders (downloads not paced)") def stop_background(): _bg["stop"].set() _bg["running"] = False @lru_cache(maxsize=8) def _transparent_tile(scale=1, fmt="png"): return tiles_mod.transparent_tile(scale, fmt) @asynccontextmanager async def _lifespan(_app): if _env_flag("LIDAR_TILE_BACKGROUND"): start_background() yield stop_background() _LAYER_RE = re.compile(r"^[a-z0-9_]{1,40}$") _NAME_RE = re.compile(r"^(\d+)(@2x)?\.(png|webp|avif)$") _MAX_UPSTREAM_BYTES = 8 * 1024 * 1024 logger = logging.getLogger("lidar") if not logger.handlers: _h = logging.StreamHandler(sys.stdout) _h.setFormatter(logging.Formatter("%(message)s")) logger.addHandler(_h) logger.setLevel(logging.INFO) app = FastAPI(title="LiDAR map — XYZ tiles", lifespan=_lifespan) # Interface assets: baked into the image at build time, regenerated here as a # fallback (dev, package installed without baking). _assets_dir = Path(__file__).resolve().parent / ASSETS_DIRNAME if not (_assets_dir / "app.js").is_file(): try: write_map_assets(_assets_dir) except OSError: import tempfile _assets_dir = Path(tempfile.gettempdir()) / "lidar_map_assets" write_map_assets(_assets_dir) _pipeline_import_cache = {} def _pipeline_importable(): """True if the processing pipeline can be imported (full image).""" if "ok" not in _pipeline_import_cache: try: import lidar_pipeline.pipeline # noqa: F401 _pipeline_import_cache["ok"] = True except Exception: _pipeline_import_cache["ok"] = False return _pipeline_import_cache["ok"] logger.info(f"LiDAR map (XYZ tiles) — output {OUTPUT_DIR}" + (f", upstream {MAPS_URL}" if MAPS_URL else ", local rendering") + f", {TILE_WORKERS} concurrent render(s)" + f", {tiles_mod.FETCH_WORKERS} concurrent download(s)" + f", source cache {tiles_mod.SOURCE_CACHE_BYTES // (1024 * 1024)} MB") if GENERATION_URL: logger.info(f"Tile generation backend: {GENERATION_URL} " f"(LIDAR_GENERATION_URL)") elif _pipeline_importable(): logger.info("Tile generation backend: local (full pipeline)") else: logger.info("Tile generation backend: none (lightweight image without " "LIDAR_GENERATION_URL) — Generation tab hidden") # --- Source tiles served statically (inventory /api/tiles + fetching) ------- # A lightweight machine (LIDAR_SOURCE_URL pointing here) lists these tiles via # /api/tiles, then downloads them from these mounts: thumbnails, quadrants and # full tiles. Immutable browser cache when the URL carries ?v= (mtime set by # build_index), short revalidation otherwise. class _VersionedStaticFiles(_StaticFiles): async def get_response(self, path, scope): resp = await super().get_response(path, scope) qs = urllib.parse.parse_qs( scope.get("query_string", b"").decode("ascii", "replace")) versioned = qs.get("v", [""])[0].isdigit() resp.headers["Cache-Control"] = ( "public, max-age=31536000, immutable" if versioned else "public, max-age=300, must-revalidate") return resp for _name in ("visualisations", "index_thumbs", "index_subtiles", "index_overview"): _dir = OUTPUT_DIR / _name _dir.mkdir(parents=True, exist_ok=True) app.mount(f"/{_name}", _VersionedStaticFiles(directory=str(_dir)), name=_name) # --------------------------------------------------------------------------- # Utilities # --------------------------------------------------------------------------- @app.middleware("http") async def _cors(request: Request, call_next): """Open CORS on tiles: required by iD, uMap and MapLibre.""" if request.method == "OPTIONS" and request.url.path.startswith("/tiles/"): return Response(status_code=204, headers=_cors_headers()) response = await call_next(request) if request.url.path.startswith("/tiles/"): response.headers.update(_cors_headers()) return response def _cors_headers(): return {"Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "GET, OPTIONS", "Access-Control-Allow-Headers": "*"} def _base_url(request: Request): """Public URL of the service (reverse proxy honoured).""" proto = request.headers.get("x-forwarded-proto") or request.url.scheme host = request.headers.get("x-forwarded-host") or request.headers.get("host") if not host: host = f"{request.url.hostname}:{request.url.port or PORT}" return f"{proto}://{host}" def _layer_or_404(layer): if not _LAYER_RE.match(layer or ""): raise HTTPException(404, "unknown layer") if layer not in tiles_mod.available_layers(OUTPUT_DIR): raise HTTPException(404, f"layer unavailable: {layer}") return layer def _layer_infos(): """Available layers + labels/descriptions from the shared registry.""" from .index import KEYWORD_TO_STEP, VIZ_LABELS, VIZ_LEGENDS out = [] for key in tiles_mod.available_layers(OUTPUT_DIR): legend = VIZ_LEGENDS.get(key, {}) out.append({"key": key, "label": VIZ_LABELS.get(key, key), "description": legend.get("description", ""), "reading": list(legend.get("reading", ())), # Pipeline --only step name (generation selector) "step": KEYWORD_TO_STEP.get(key, key)}) return out # --- upstream circuit breaker ---------------------------------------------- _UPSTREAM = {"fails": 0, "until": 0.0} _UPSTREAM_OFFLINE_AFTER = 3 _UPSTREAM_OFFLINE_SECONDS = 120.0 def _upstream_offline(): return time.time() < _UPSTREAM["until"] def _upstream_mark(ok): if ok: _UPSTREAM.update(fails=0, until=0.0) return _UPSTREAM["fails"] += 1 # Re-armed after expiry: an upstream that is still down is suspended again. if _UPSTREAM["fails"] >= _UPSTREAM_OFFLINE_AFTER and not _upstream_offline(): _UPSTREAM["until"] = time.time() + _UPSTREAM_OFFLINE_SECONDS logger.info("Upstream tile server unreachable — attempts suspended for 2 min") def _fetch_upstream(rel_path): """Fetch a tile from LIDAR_MAPS_URL (None if unavailable).""" if not MAPS_URL or _upstream_offline(): return None url = f"{MAPS_URL}/{rel_path.lstrip('/')}" try: req = urllib.request.Request(url, headers={"User-Agent": "lidar-maps-cache"}) with urllib.request.urlopen(req, timeout=20) as r: data = r.read(_MAX_UPSTREAM_BYTES + 1) if len(data) > _MAX_UPSTREAM_BYTES: raise IOError("upstream tile too large") _upstream_mark(True) return data except Exception as e: # noqa: BLE001 — upstream down: local/empty render _upstream_mark(False) logger.debug(f"Upstream tile unavailable ({url}): {e}") return None # --- rendering: deduplication of concurrent requests ----------------------- _locks = {} _locks_guard = threading.Lock() def _acquire_lock(key): """Reference-counted per-tile lock: two simultaneous requests render once. The counter avoids two pitfalls: a dictionary that grows forever (millions of tiles) and a lock removed while another thread still waits on it — which would allow two concurrent renders of the same tile. """ with _locks_guard: entry = _locks.get(key) if entry is None: entry = _locks[key] = [threading.Lock(), 0] entry[1] += 1 return entry def _release_lock(key, entry): with _locks_guard: entry[1] -= 1 if entry[1] <= 0: _locks.pop(key, None) def _tile_bytes(layer, z, x, y, scale, fmt): """Encoded tile: cache → local render → upstream. None if the area is empty.""" key = (layer, z, x, y, scale, fmt) entry = _acquire_lock(key) try: with entry[0]: try: with _render_sem: data = tiles_mod.get_tile(OUTPUT_DIR, layer, z, x, y, scale, fmt) except Exception as e: # noqa: BLE001 — one tile does not break the map logger.warning(f"Tile render failed ({layer} {z}/{x}/{y}): {e}") data = None # Inventory from upstream with no source tile under this tile: # upstream knows no more, no point asking it. if data is None and MAPS_URL and not ( tiles_mod.REMOTE_SOURCE_URL and not tiles_mod._contributing(OUTPUT_DIR, layer, z, x, y, scale)): suffix = "@2x" if scale == 2 else "" data = _fetch_upstream(f"tiles/{layer}/{z}/{x}/{y}{suffix}.{fmt}") if data: tiles_mod._write_atomic( tiles_mod.tile_cache_path(OUTPUT_DIR, layer, z, x, y, scale, fmt), data) finally: _release_lock(key, entry) return data _MEDIA = {"png": "image/png", "webp": "image/webp", "avif": "image/avif"} # --------------------------------------------------------------------------- # Tiles # --------------------------------------------------------------------------- @app.get("/tiles/{layer}/{z}/{x}/{name}") async def tile(layer: str, z: int, x: int, name: str, v: Optional[str] = None): """XYZ tile (OSM scheme). `name` = `{y}.{png|webp|avif}` or `{y}@2x.{png|webp|avif}`.""" m = _NAME_RE.match(name) if not m: raise HTTPException(404, "invalid tile name") y = int(m.group(1)) scale = 2 if m.group(2) else 1 fmt = m.group(3) _layer_or_404(layer) if not tiles_mod.zoom_supported(z, scale): raise HTTPException(404, f"zoom out of range ({tiles_mod.TILE_MIN_Z}" f"–{tiles_mod.TILE_MAX_NATIVE_Z})") n = 2 ** z if not (0 <= x < n and 0 <= y < n): raise HTTPException(404, "tile outside the grid") if _env_flag("LIDAR_TILE_CACHE_ONLY") and tiles_mod.zoom_cached(z, scale): # Cache-only mode (stored levels): browsing never triggers a local # RENDER. A missing tile (e.g. zoom > LIDAR_TILE_BACKGROUND_MAX_Z, # never covered by the maintenance) is fetched from the upstream # server LIDAR_MAPS_URL — a plain download of a few tens of KB, # cached, with no decoding or computation on this machine. data, state = await run_in_threadpool( tiles_mod.cached_tile, OUTPUT_DIR, layer, z, x, y, scale, fmt) pending = state == "pending" empty = state == "empty" if pending and MAPS_URL: suffix = "@2x" if scale == 2 else "" key = (layer, z, x, y, scale, fmt) entry = _acquire_lock(key) try: with entry[0]: data = _fetch_upstream(f"tiles/{layer}/{z}/{x}/{y}{suffix}.{fmt}") if data: tiles_mod._write_atomic( tiles_mod.tile_cache_path(OUTPUT_DIR, layer, z, x, y, scale, fmt), data) pending = False empty = False finally: _release_lock(key, entry) if pending: # Stale tile (source tile regenerated): the old one stays displayed # until the new one arrives, rather than a transparent hole. try: data = tiles_mod.tile_cache_path(OUTPUT_DIR, layer, z, x, y, scale, fmt).read_bytes() except OSError: data = None else: data = await run_in_threadpool(_tile_bytes, layer, z, x, y, scale, fmt) pending = False empty = data is None stale = pending and bool(data) if empty or (pending and not stale): data = _transparent_tile(scale, fmt) headers = dict(_cors_headers()) if pending: # Tile still to come (rendered by the maintenance): never cached by # the browser, the next navigation requests it again and finds it. headers["Cache-Control"] = "no-store" headers["X-Tile-Pending"] = "1" else: headers["Cache-Control"] = ("public, max-age=31536000, immutable" if v else "public, max-age=300, must-revalidate") headers["X-Tile-Empty"] = "1" if (empty or (pending and not stale)) else "0" return Response(content=data, media_type=_MEDIA[fmt], headers=headers) # --------------------------------------------------------------------------- # Discovery: TileJSON, WMTS, JOSM # --------------------------------------------------------------------------- @app.get("/tiles/josm.imagery.xml") def josm_imagery(request: Request): """JOSM imagery file: adds every layer at once.""" base = _base_url(request) bounds = tiles_mod.grid_bounds_wgs84(OUTPUT_DIR) entries = [] for info in _layer_infos(): b = "" if bounds: b = (f' \n') entries.append( " \n" f" LiDAR HD — {_xml.escape(info['label'])}\n" f" lidar-hd-{info['key'].replace('_', '-')}\n" " tms\n" f" \n" f" {tiles_mod.TILE_MIN_Z}\n" f" {tiles_mod.TILE_MAX_NATIVE_Z}\n" f"{b}" f" {_xml.escape(info['description'])}\n" f" {_xml.escape(ATTRIBUTION_TEXT)}\n" " https://geoservices.ign.fr/lidarhd\n" " other\n" " ") xml = ('\n' '\n' + "\n".join(entries) + "\n\n") return Response(content=xml, media_type="application/xml", headers=_cors_headers()) @app.get("/tiles/wmts.xml") def wmts_capabilities(request: Request): """WMTS 1.0.0 capabilities (QGIS, ArcGIS) on the GoogleMapsCompatible grid.""" base = _base_url(request) bounds = tiles_mod.grid_bounds_wgs84(OUTPUT_DIR) or [-180.0, -85.0, 180.0, 85.0] layers = [] for info in _layer_infos(): layers.append(f""" {_xml.escape(info['label'])} {_xml.escape(info['description'])} {bounds[0]:.6f} {bounds[1]:.6f} {bounds[2]:.6f} {bounds[3]:.6f} {info['key']} image/png GoogleMapsCompatible """) matrices = [] for z in range(tiles_mod.TILE_MIN_Z, tiles_mod.TILE_MAX_NATIVE_Z + 1): matrices.append(f""" {z} {559082264.0287178 / (2 ** z):.10f} -20037508.3427892 20037508.3427892 256 256 {2 ** z} {2 ** z} """) xml = f""" LiDAR HD map {_xml.escape(ATTRIBUTION_TEXT)} OGC WMTS 1.0.0 Licence Ouverte 2.0 {chr(10).join(layers)} GoogleMapsCompatible urn:ogc:def:crs:EPSG:6.18:3:3857 urn:ogc:def:wkss:OGC:1.0:GoogleMapsCompatible {chr(10).join(matrices)} """ return Response(content=xml, media_type="application/xml", headers=_cors_headers()) @app.get("/tiles/{layer}.json") def tilejson(layer: str, request: Request): """TileJSON 3.0.0 (MapLibre, QGIS, Felt…).""" _layer_or_404(layer) from .index import VIZ_LABELS, VIZ_LEGENDS base = _base_url(request) bounds = tiles_mod.grid_bounds_wgs84(OUTPUT_DIR) or [-180.0, -85.0, 180.0, 85.0] center = [(bounds[0] + bounds[2]) / 2, (bounds[1] + bounds[3]) / 2, min(tiles_mod.TILE_MAX_NATIVE_Z, 15)] return JSONResponse({ "tilejson": "3.0.0", "name": f"LiDAR HD — {VIZ_LABELS.get(layer, layer)}", "description": VIZ_LEGENDS.get(layer, {}).get("description", ""), "attribution": ATTRIBUTION, "scheme": "xyz", "tiles": [f"{base}/tiles/{layer}/{{z}}/{{x}}/{{y}}.png"], "minzoom": tiles_mod.TILE_MIN_Z, "maxzoom": tiles_mod.TILE_MAX_NATIVE_Z, "bounds": bounds, "center": center, }, headers=_cors_headers()) # --------------------------------------------------------------------------- # Default display configuration # --------------------------------------------------------------------------- # The display set from the map (main layer, relief / precision / compare # mode, base map) can be frozen as the configuration served to EVERY new # browser: the useful setting is not lost in a localStorage and is shared # without a link. The relief intensity is never part of it. DEFAULTS_FILE = OUTPUT_DIR / ".map-defaults.json" _defaults_lock = threading.Lock() def _load_defaults(): """Saved default configuration, or None if there is none.""" try: data = json.loads(DEFAULTS_FILE.read_text(encoding="utf-8")) except (OSError, ValueError): return None return data if isinstance(data, dict) else None def _clamp01(value, fallback=1.0): try: return min(1.0, max(0.0, float(value))) except (TypeError, ValueError): return fallback def _sanitize_defaults(req): """Keep only what is known and bounded: real layer, valid mode, base opacity 0–1.""" from .index import PRECISION_VIZ, VIEW_MODES known = set(tiles_mod.available_layers(OUTPUT_DIR)) - {PRECISION_VIZ} base_in = req.base if isinstance(req.base, dict) else {} base = {"on": bool(base_in.get("on", True)), "opacity": _clamp01(base_in.get("opacity"), 0.85), # None = no explicit choice (the base map follows the theme on the # map side); only a boolean value received becomes a frozen choice. "dark": None if base_in.get("dark") is None else bool(base_in.get("dark"))} # v: 3 distinguishes this format (dark possibly None = not chosen) from # the old one, which always wrote dark:true (see map_meta). return {"main": req.main if req.main in known else None, "mode": req.mode if req.mode in VIEW_MODES else "relief", "base": base, "v": 3, "saved_at": time.time()} class DefaultsRequest(BaseModel): main: Optional[str] = Field(None, description="main display layer") mode: str = Field("relief", description="relief | precision | compare") precision_opacity: Optional[float] = Field(None, description="ignored (former \"both\" mode)") base: dict = Field(default_factory=dict, description="base map: on, opacity, dark") @app.get("/api/map/defaults") def get_defaults(): """Default configuration served to new browsers.""" return {"defaults": _load_defaults()} @app.post("/api/map/defaults") def set_defaults(req: DefaultsRequest): """Freeze the current state as the default configuration (all browsers).""" data = _sanitize_defaults(req) with _defaults_lock: tmp = DEFAULTS_FILE.with_suffix(".tmp") DEFAULTS_FILE.parent.mkdir(parents=True, exist_ok=True) tmp.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") os.replace(tmp, DEFAULTS_FILE) logger.info(f"Default display configuration saved: " f"{data['main'] or 'registry main layer'}, mode {data['mode']}") return {"enregistré": True, "defaults": data} @app.delete("/api/map/defaults") def clear_defaults(): """Revert to the registry settings (index.py).""" with _defaults_lock: existed = DEFAULTS_FILE.exists() DEFAULTS_FILE.unlink(missing_ok=True) return {"supprimé": existed} # --------------------------------------------------------------------------- # Map API # --------------------------------------------------------------------------- @app.get("/api/map/meta") def map_meta(): """Layers, zooms, extent and version — the whole initial interface state.""" from .index import (DEFAULT_VIEW_MODE, PRECISION_VIZ, VIEW_MODES, default_main_layer) infos = _layer_infos() keys = [i["key"] for i in infos] saved = _load_defaults() or {} # Configuration frozen from the map, otherwise registry settings (index.py). # Frozen layer gone, or file from the former layer stack (order/on keys): # registry main layer. main = saved.get("main") if main not in keys or main == PRECISION_VIZ: main = default_main_layer(keys) mode = saved.get("mode") if saved.get("mode") in VIEW_MODES else DEFAULT_VIEW_MODE # Pre-v3 file (former webapp): base.dark:true always, never a real # choice — served as "not chosen" (None, the base map follows the theme). base_default = saved.get("base") if isinstance(base_default, dict) and saved.get("v") != 3: base_default = dict(base_default, dark=None) return { "layers": infos, "default_main": main, "precision_layer": PRECISION_VIZ if PRECISION_VIZ in keys else None, "default_mode": mode, "default_base": base_default, "defaults_saved": bool(saved), # The interface consumes 512 px tiles (half as many requests as with # 256: decisive over HTTP/1.1); OSM clients keep the canonical 256. "tile_url": "tiles/{layer}/{z}/{x}/{y}@2x.avif", # Even standard levels only (reduced storage): at odd zooms the # interface downscales the tiles of the level above. "even_levels": tiles_mod.TILE_EVEN_LEVELS, "tile_size": tiles_mod.TILE_SIZE * 2, "zoom_offset": -1, "min_zoom": tiles_mod.TILE_MIN_Z, # Standard numbering, like the Leaflet tile level (the @2x URL is one # step below via zoom_offset): 19 = native 0.2 m/px, served as 18@2x. # Announcing 18 capped the display at 0.4 m/px. "max_native_zoom": tiles_mod.TILE_MAX_NATIVE_Z, "bounds": tiles_mod.grid_bounds_wgs84(OUTPUT_DIR), "stamp": tiles_mod.tiles_stamp(OUTPUT_DIR), "attribution": ATTRIBUTION, "attribution_text": ATTRIBUTION_TEXT, "ui_version": ui_version(), } # IGN record (STAC catalogue) cached per tile: a single network call; an # unpublished tile is re-checked at most once a day. IGN_META_RETRY_S = 86400 def _ign_tile_meta(col, row): """IGN metadata of a tile (acquisition, download link).""" from .fetch_ign import tile_ign_metadata meta_dir = OUTPUT_DIR / "ign_meta" path = meta_dir / f"{col:04d}_{row:04d}.json" try: data = json.loads(path.read_text(encoding="utf-8")) if data.get("found") or time.time() - data.get("checked", 0) < IGN_META_RETRY_S: return data except (OSError, ValueError): pass try: data = tile_ign_metadata(col, row) except Exception as e: logger.warning(f"IGN catalogue unreachable for {col}_{row}: {e}") return {"found": False, "error": "IGN catalogue unreachable"} data["checked"] = time.time() try: meta_dir.mkdir(parents=True, exist_ok=True) tmp = path.with_suffix(".tmp") tmp.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") os.replace(tmp, path) except OSError: pass return data def _calage_summary(basename, suffix): """Summary of the vertical strip alignment (DTM sidecar), if present.""" path = OUTPUT_DIR / "DTM" / f"{basename}_dtm{suffix}_stripalign.json" try: data = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): return None offsets = data.get("offsets") or {} lines = data.get("lines") or {} return { "beams": sorted(set(offsets) | set(lines), key=lambda k: int(k) if str(k).isdigit() else 0), "offsets_cm": {k: round(v * 100, 1) for k, v in offsets.items()}, "lines_rms_cm": {k: round(v.get("rms_m", 0) * 100, 1) for k, v in lines.items()}, "model": data.get("line_model"), } @app.get("/api/map/ign") def map_tile_ign(col: int, row: int): """IGN record of a tile (acquisition, sensors, download link). Separate from /api/map/tile: selecting a tile displays immediately, even when the IGN catalogue is slow or unreachable. """ return _ign_tile_meta(col, row) @app.get("/api/map/tile") def map_tile_info(lat: float, lng: float): """LHD tile under a WGS84 point: extent, generation, strip alignment. Always answers (whether the tile is generated or not) and without any network call; the IGN record is loaded separately (/api/map/ign). """ from .index import _collect_tile_metadata, _res_suffix_str, scan_tiles from .quality import read_quality x, y = tiles_mod.wgs84_to_l93(lng, lat) col, row = int(x // 1000), int(y // 1000) + 1 x0, y0, x1, y1 = col * 1000, (row - 1) * 1000, (col + 1) * 1000, row * 1000 lons, lats = tiles_mod._transformer("EPSG:2154", "EPSG:4326").transform( [x0, x1, x1, x0], [y1, y1, y0, y0]) base = { "col": col, "row": row, "name": f"LHD_FXX_{col:04d}_{row:04d}_PTS_LAMB93_IGN69", "l93_bbox": [x0, y0, x1, y1], # NW, NE, SE, SW corners (lat, lng): selection outline on the map "corners": [[round(la, 7), round(lo, 7)] for lo, la in zip(lons, lats)], } best = None for t in scan_tiles(OUTPUT_DIR / "visualisations"): if (t["col"], t["row"]) != (col, row): continue if best is None or t["resolution"] < best["resolution"]: best = t # Quality (density, dates): under the real name of the rendered tile when # there is one (best["basename"] may differ from the nominal name computed # from col/row); nominal name otherwise, for lack of anything better. quality_name = best["basename"] if best else base["name"] q = read_quality(OUTPUT_DIR, quality_name) base["quality"] = ({"ground_density": q.get("ground_density"), "acq_start": q.get("acq_start"), "acq_end": q.get("acq_end")} if q else None) if best is None: return {"found": False, **base} meta = _collect_tile_metadata(best, OUTPUT_DIR / "DTM") suffix = _res_suffix_str(best["resolution"]) return { "found": True, **base, "name": best["basename"], "resolution": best["resolution"], "method": meta.get("method"), "generated": meta.get("generated"), "layers": sorted(best["viz"]), "suffix": suffix, "calage": _calage_summary(best["basename"], suffix), } class WarmRequest(BaseModel): layers: list = Field(default_factory=list, description="layers to pre-render (default: all)") z_min: int = Field(tiles_mod.TILE_MIN_Z, description="minimum zoom") z_max: int = Field(16, description="maximum zoom (19 = native resolution)") bounds: Optional[list] = Field(None, description="[west, south, east, north] WGS84; " "default: extent of the tiles") scale: int = Field(1, description="1 = 256 px tiles, 2 = 512 px (@2x)") fmt: str = Field("png", description="png, webp or avif") _warm = {"running": False, "done": None, "result": None} @app.post("/api/map/warm") def map_warm(req: WarmRequest): """Pre-render tiles in the background (speeds up the first visit).""" if _warm["running"]: raise HTTPException(409, "a warm-up is already running") layers = [l for l in req.layers if _LAYER_RE.match(str(l))] or \ tiles_mod.available_layers(OUTPUT_DIR) if req.fmt not in _MEDIA: raise HTTPException(400, f"invalid format: {req.fmt}") _warm.update(running=True, done=None, result=None) def _run(): try: _warm["result"] = tiles_mod.warm( OUTPUT_DIR, layers, req.z_min, req.z_max, req.bounds, max(1, min(2, req.scale)), req.fmt) except Exception as e: # noqa: BLE001 — reported through the state _warm["result"] = {"erreur": str(e)} finally: _warm["running"] = False _warm["done"] = time.time() threading.Thread(target=_run, daemon=True).start() return {"demarré": True, "couches": layers} @app.get("/api/map/warm") def map_warm_status(): return dict(_warm) @app.get("/api/map/background") def background_status(): """Background maintenance state: queue, counters, configuration.""" with _bg["guard"]: queued = len(_bg["queue"]) return {"active": _bg["running"], "en_file": queued, "configuration": _bg_conf(), "stats": dict(_bg["stats"])} @app.post("/api/map/background") def background_trigger(): """Force an immediate scan of the source tiles (rendered afterwards in the background).""" if not _env_flag("LIDAR_TILE_BACKGROUND"): raise HTTPException(409, "background maintenance disabled " "(LIDAR_TILE_BACKGROUND)") return {"tuiles_en_file": background_scan()} @app.get("/healthz") def healthz(): return {"ok": True, "layers": len(tiles_mod.available_layers(OUTPUT_DIR))} # --------------------------------------------------------------------------- # Tile generation (ported from the former webapp) # --------------------------------------------------------------------------- # Two mutually exclusive modes: # - LIDAR_GENERATION_URL set (lightweight map): /api/preview, /api/generate, # /api/status, /api/stop, /api/queue/clear and /api/cell are forwarded to # the processing machine (itself served by this module, full image); # - otherwise, if the pipeline can be imported (full image): local generation # — `python -m lidar_pipeline` as a subprocess, one job at a time, # concurrent requests go to the queue (persisted, survives restarts). # On the lightweight image without LIDAR_GENERATION_URL, /api/status answers # available=false: the interface hides the Generation tab. def _generation_available(): return bool(GENERATION_URL) or _pipeline_importable() def _require_token(x_lidar_token: str = Header(None)): """Protect mutating routes when LIDAR_API_TOKEN is set (no-op otherwise).""" if API_TOKEN: import hmac presented = x_lidar_token or "" if not hmac.compare_digest(presented, API_TOKEN): raise HTTPException(401, "missing or invalid API token") def _ip_in_regen_cidr(ip): """True if the IP belongs to one of the allowed CIDRs (LIDAR_REGEN_CIDR).""" if not REGEN_CIDR: return True # restriction disabled if not ip: return False import ipaddress try: addr = ipaddress.ip_address(ip) except ValueError: return False for part in REGEN_CIDR.split(","): part = part.strip() if not part: continue try: if addr in ipaddress.ip_network(part, strict=False): return True except ValueError: continue return False def _client_ip(request): """Originating client IP for the generation restriction. The direct peer must be inside the allowed network; only then does X-Forwarded-For (local reverse proxy) designate the real client behind it. An external client connecting directly is judged on its connection IP, whatever X-Forwarded-For it announces. """ if request is None: return None peer = getattr(request, "client", None) peer_ip = getattr(peer, "host", None) if peer else None if not _ip_in_regen_cidr(peer_ip): return peer_ip # outside the network (or unknown): judged on this IP xff = request.headers.get("x-forwarded-for") if hasattr(request, "headers") else None if xff: first = xff.split(",")[0].strip() if first: return first return peer_ip def _require_lan_for_generation(request: Request): """Restrict generation routes to local-network clients (LIDAR_REGEN_CIDR).""" ip = _client_ip(request) if not _ip_in_regen_cidr(ip): raise HTTPException( 403, f"tile generation is restricted to the local network" f"{f' ({REGEN_CIDR})' if REGEN_CIDR else ''} — " f"client: {ip or 'unknown IP'}") def _proxy_generation(method, path, payload=None, timeout=30): """Forward a generation call to the processing machine. Returns the remote JSON response. Remote HTTP errors (e.g. 409 nothing to stop) are relayed as is; an unreachable machine becomes an explicit 503 for the interface. """ import urllib.error import urllib.request req = urllib.request.Request( GENERATION_URL + path, data=json.dumps(payload).encode("utf-8") if payload is not None else None, method=method) req.add_header("Content-Type", "application/json") if REMOTE_TOKEN: req.add_header("X-Lidar-Token", REMOTE_TOKEN) try: with urllib.request.urlopen(req, timeout=timeout) as resp: return json.loads(resp.read().decode("utf-8")) except urllib.error.HTTPError as e: try: detail = json.loads(e.read().decode("utf-8")).get("detail", str(e)) except (ValueError, OSError): detail = str(e) raise HTTPException(e.code, detail) except (urllib.error.URLError, OSError) as e: raise HTTPException(503, f"processing machine unreachable " f"({GENERATION_URL}): {e}") def _viz_step_names(): """Names of the accepted visualizations (pipeline VIZ_STEPS steps). On the lightweight image (without processing dependencies), falls back to the index registry — same order as VIZ_STEPS. """ try: from .pipeline import VIZ_STEPS return [name for name, _ in VIZ_STEPS] except ImportError: from .index import KEYWORD_TO_STEP return list(KEYWORD_TO_STEP.values()) def _panel_viz_steps(): """Layers actually displayed by the map, as --only step names.""" from .index import panel_steps return panel_steps() or _viz_step_names() def _viz_step_labels(): """Steps → labels (readable progress log).""" from .index import VIZ_LABELS, step_to_keyword return {step: VIZ_LABELS.get(step_to_keyword(step), step) for step in _viz_step_names()} class PreviewRequest(BaseModel): bbox: Optional[list] = Field(None, description="[west, south, east, north] in WGS84 " "(required unless all_missing=true)") regenerate: bool = Field(False, description="Include tiles that are already generated") viz: Optional[list] = Field(None, description="Requested visualizations, pipeline step " "names (e.g. aspect, wavelet); default: the " "layers displayed in the panel. " "An existing tile counts as done only " "if it has all of them") all_missing: bool = Field(False, description="Ignore the bbox: every LHD tile " "present in input/ that lacks at least " "one of the requested visualizations (no " "download)") class GenerateRequest(BaseModel): tiles: list = Field(..., description="list of [col, row] (L93 km integers)") regenerate: bool = Field(False, description="Regenerate tiles that are already generated " "(visualizations redone, classification kept)") # Classification (IGN, ground only) and edge stitching (100 m band taken # from the neighbouring tiles) are enforced: no setting on the map side. # Legacy fields a client may still send are ignored. viz: list = Field(None, description="Visualizations to generate, pipeline step " "names (e.g. aspect, wavelet, slope); " "default: the layers displayed in the panel") all_missing: bool = Field(False, description="Ignore the tile list: process every " "LHD tile present in input/ that " "lacks at least one of the requested " "visualizations (no download)") def bbox_to_cells(w, s, e, n): """1 km L93 cells (col, row) intersecting a WGS84 bbox. A cell (col, row) covers X ∈ [col, col+1] km, Y ∈ [row-1, row] km. Conversion via pyproj (present in every image), falling back to the index's affine approximation. """ from .index import _approx_wgs84_to_l93 try: from pyproj import Transformer transformer = Transformer.from_crs("EPSG:4326", "EPSG:2154", always_xy=True) xs, ys = transformer.transform([w, e, w, e], [s, s, n, n]) except ImportError: pts = [_approx_wgs84_to_l93(lon, lat) for lon, lat in ((w, s), (e, s), (w, n), (e, n))] xs = [p[0] for p in pts] ys = [p[1] for p in pts] min_x, max_x = min(xs) + 0.5, max(xs) - 0.5 # shrink by 1 m (0.5 m per side): edges excluded min_y, max_y = min(ys) + 0.5, max(ys) - 0.5 if max_x <= min_x or max_y <= min_y: return [] cols = range(int(math.floor(min_x / 1000)), int(math.floor(max_x / 1000)) + 1) rows = range(int(math.floor(min_y / 1000)) + 1, int(math.floor(max_y / 1000)) + 2) return [(c, r) for r in rows for c in cols] def point_to_cell(lon, lat): """1 km L93 cell (col, row) containing a WGS84 point.""" from .index import _approx_wgs84_to_l93 try: from pyproj import Transformer transformer = Transformer.from_crs("EPSG:4326", "EPSG:2154", always_xy=True) x, y = transformer.transform(lon, lat) except ImportError: x, y = _approx_wgs84_to_l93(lon, lat) return int(math.floor(x / 1000)), int(math.floor(y / 1000)) + 1 def processed_cells(output_dir): """Set of cells (col, row) that already have visualizations.""" from .index import scan_tiles tiles = scan_tiles(Path(output_dir) / "visualisations") return {(t["col"], t["row"]) for t in tiles} def laz_cells(input_dir): """Coordinates (col, row) of the LHD tiles present in input/.""" from .index import parse_basename_coords cells = set() for f in list(Path(input_dir).glob("*.laz")) + list(Path(input_dir).glob("*.las")): name = f.name for ext in ('.copc.laz', '.copc.las', '.laz', '.las'): if name.lower().endswith(ext): name = name[:-len(ext)] break coords = parse_basename_coords(name) if coords: cells.add(coords) return sorted(cells) def complete_cells(output_dir, viz_steps): """Cells that have ALL the requested visualizations at 0.2 m.""" from .index import cells_with_all_viz, step_to_keyword keys = [step_to_keyword(v) for v in viz_steps] return cells_with_all_viz(Path(output_dir) / "visualisations", keys, GENERATE_RESOLUTIONS) def missing_cells_with_corners(cells, output_dir, include_done=False, viz=None): """Filter the cells to process and compute their WGS84 corners. Returns [{col, row, corners: [[lat, lon] × 4, SW, SE, NE, NW]}]. """ from .index import attach_gps_bounds if viz: done = complete_cells(output_dir, viz) else: done = processed_cells(output_dir) todo = [{"col": c, "row": r} for (c, r) in cells if include_done or (c, r) not in done] if todo: attach_gps_bounds(todo) return todo def _attach_frame_bounds(tiles): """Add their WGS84 corners to unfinished tiles (outline on the map). The interface draws a coloured outline around the tiles of the current run (running, pending, failed); the corners come from the L93 grid (col, row) → exact bounds via PROJ. Finished tiles have no outline: corners are omitted to lighten /api/status polling. """ from .index import attach_gps_bounds, parse_basename_coords todo = [] for tile in tiles: if tile.get("state") == "done": continue coords = parse_basename_coords(tile.get("name") or "") if coords: todo.append({"col": coords[0], "row": coords[1]}) attach_gps_bounds(todo) by_cell = {(t["col"], t["row"]): t for t in todo} for tile in tiles: if tile.get("state") == "done": continue coords = parse_basename_coords(tile.get("name") or "") entry = by_cell.get(coords) if coords else None if entry: tile["corners"] = entry.get("corners") or [] def _tail_log(n_lines): """Last lines of the generation log (.generation.log).""" try: text = LOG_FILE.read_text(encoding="utf-8", errors="replace") except OSError: return [] lines = [l for l in text.splitlines() if l.strip()] return lines[-n_lines:] # --- Job state + queue (persisted on disk) ---------------------------------- _job: dict = {"proc": None, "started": None, "returncode": None, "cmd": None, "finished": None, "qid": None, "run_id": None} # RLock (reentrant): /api/status builds its response UNDER the lock and calls # _queue_summary(), which takes it again — a plain Lock would deadlock there. _job_lock = threading.RLock() _queue = [] _queue_seq = 0 def _save_job_state(): """Write the job state to disk — survives a server restart.""" state = {k: _job.get(k) for k in ("started", "returncode", "cmd", "finished", "qid", "run_id")} try: JOB_FILE.write_text(json.dumps(state), encoding="utf-8") except OSError: pass def _load_job_state(): """Reload the last job state when the server starts.""" try: state = json.loads(JOB_FILE.read_text(encoding="utf-8")) except (OSError, ValueError): return with _job_lock: _job["started"] = state.get("started") _job["returncode"] = state.get("returncode") _job["cmd"] = state.get("cmd") _job["finished"] = state.get("finished") _job["qid"] = state.get("qid") _job["run_id"] = state.get("run_id") _load_job_state() def _save_queue(): """Write the queue to disk (atomic, best-effort).""" try: tmp = QUEUE_FILE.with_suffix(".tmp") tmp.write_text(json.dumps(_queue), encoding="utf-8") os.replace(tmp, QUEUE_FILE) except OSError: pass def _load_queue(): """Reload the queue at startup (and its id counter).""" global _queue_seq try: data = json.loads(QUEUE_FILE.read_text(encoding="utf-8")) if isinstance(data, list): _queue[:] = [it for it in data if isinstance(it, dict) and isinstance(it.get("req"), dict)] ids = [int(it.get("id") or 0) for it in _queue] _queue_seq = max(ids) if ids else 0 except (OSError, ValueError): pass _load_queue() def _queue_summary(): """Queue overview for /api/status (requests + timestamps).""" with _job_lock: return [{"id": it.get("id"), "tuiles": len(it["req"].get("tiles") or []), "en_file_depuis": it.get("queued_at")} for it in _queue] EDGE_BUFFER_METERS = 100.0 # edge stitching: band width (m) def _build_command(tiles, regenerate=False, viz=None): """Generation command: IGN download + file processing. IGN classification (ground only) and edge stitching (EDGE_BUFFER_METERS band filled with the ground points of the 8 neighbouring tiles) are always applied. With regenerate=True, the visualizations of tiles that are already present are redone (--force), classification kept. """ from .fetch_ign import tile_filename # -u: unbuffered output — the .generation.log log must be read in real # time by /api/status (progress shown in the UI). cmd = [sys.executable, "-u", "-m", "lidar_pipeline", str(INPUT_DIR), "-o", str(OUTPUT_DIR), "-r", ",".join(str(r) for r in GENERATE_RESOLUTIONS), "--only", *(viz or _panel_viz_steps()), "--ground-classification", "ign", "--ign-classes", "sol", "--edge-buffer", f"{EDGE_BUFFER_METERS:g}"] if regenerate: cmd += ["--force"] if os.environ.get("LIDAR_GPU", "") == "1": cmd += ["-g", "all"] workers = os.environ.get("LIDAR_WORKERS", "auto").strip() if workers.lower() not in ("auto", ""): cmd += ["-w", workers] # Inventory regenerated after every finished tile (now the pipeline # default anyway, kept explicit here): the map polls it (/api/tiles) and # new tiles appear live. cmd += ["--incremental-index"] cmd += ["--fetch-tiles"] cmd += [f"{c},{r}" for (c, r) in tiles] cmd += ["--file"] cmd += [tile_filename(c, r) for (c, r) in tiles] return cmd def _resolve_request(req): """Validate a generation request and resolve its tile list. Returns (tiles [(col, row)], viz [step names]). Re-run when each queued request starts: the tiles present in input/ may have changed between queuing and the start of the run. """ names = _viz_step_names() viz = [v for v in (req.viz or []) if v] or _panel_viz_steps() invalid = [v for v in viz if v not in names] if invalid: raise HTTPException( 400, f"invalid visualization: {', '.join(invalid)} " f"(expected: {', '.join(names)})") if req.all_missing: done = set() if req.regenerate else complete_cells(OUTPUT_DIR, viz) tiles = [c for c in laz_cells(INPUT_DIR) if c not in done or req.regenerate] if not tiles: raise HTTPException(400, "no tile to complete in input/") if len(tiles) > MAX_CELLS_ALL: raise HTTPException( 400, f"too many tiles to complete ({len(tiles)}) — max " f"{MAX_CELLS_ALL} per run; start the global pass again " f"once this one is over") else: tiles = [] for pair in req.tiles: if not (isinstance(pair, list) and len(pair) == 2): raise HTTPException(400, f"invalid tile: {pair!r} (expected [col, row])") tiles.append((int(pair[0]), int(pair[1]))) if not tiles: raise HTTPException(400, "no tile provided") if len(tiles) > MAX_CELLS: raise HTTPException(400, f"too many tiles ({len(tiles)}) — max {MAX_CELLS}") # North-to-south generation (decreasing row, then increasing col): the # workers take tiles in submission order and the map fills from top to # bottom during the run. tiles.sort(key=lambda cr: (-cr[1], cr[0])) return tiles, viz def _launch_job(tiles, viz, req, qid=None): """Start a pipeline run. Call under _job_lock, with the server idle.""" cmd = _build_command(tiles, regenerate=req.regenerate, viz=viz) LOG_FILE.parent.mkdir(parents=True, exist_ok=True) log_fh = open(LOG_FILE, "w", encoding="utf-8") # New run: event log reset and run id passed to the pipeline # (LIDAR_RUN_ID): late events from a cancelled run (surviving workers) # are filtered out when reading. from .progress import reset_events reset_events(OUTPUT_DIR) run_id = uuid.uuid4().hex[:12] _job.update({"proc": None, "started": time.time(), "returncode": None, "cmd": cmd, "finished": None, "qid": qid, "run_id": run_id}) _save_job_state() # start_new_session: the pipeline and its workers/PDAL form their own # process group — /api/stop can kill it as a whole without touching the # server, and the pipeline's SIGTERM handler (killpg) stays confined. p = subprocess.Popen(cmd, stdout=log_fh, stderr=subprocess.STDOUT, cwd="/app" if Path("/app").exists() else None, env=dict(os.environ, LIDAR_RUN_ID=run_id), start_new_session=True) # proc set BEFORE the watcher thread: a run that dies instantly would # trigger _watch → _start_next_queued with _job["proc"] still None # (server "idle") and a second run would start in parallel. _job["proc"] = p def _watch(): rc = p.wait() log_fh.close() with _job_lock: _job["returncode"] = rc _job["finished"] = time.time() _save_job_state() # The run is over: the next queued request can start. _start_next_queued() threading.Thread(target=_watch, daemon=True).start() return cmd def _start_next_queued(): """Start the first queued request as soon as the server is idle.""" while True: with _job_lock: proc = _job["proc"] if proc is not None and proc.poll() is None: return False if not _queue: return False item = _queue.pop(0) _save_queue() try: req = GenerateRequest(**item["req"]) tiles, viz = _resolve_request(req) except HTTPException as e: logger.warning("Queued request skipped: %s", e.detail) continue with _job_lock: proc = _job["proc"] if proc is not None and proc.poll() is None: _queue.insert(0, item) # another run started: put back at the head _save_queue() return False _launch_job(tiles, viz, req, qid=item.get("id")) return True @app.get("/api/status") def generation_status(request: Request = None): """Generation state: current run, queue, tiles one by one. available: this server can generate (local pipeline or LIDAR_GENERATION_URL). regen_allowed: the browser's IP may start runs (LIDAR_REGEN_CIDR) — the interface hides the Generation tab otherwise. Lightweight map: the state comes from the processing machine; offline: running=null (no false "finished"). """ regen_allowed = _ip_in_regen_cidr(_client_ip(request)) available = _generation_available() if GENERATION_URL: try: data = _proxy_generation("GET", "/api/status", timeout=10) except HTTPException as e: if e.status_code != 503: raise return {"running": None, "distant": True, "worker_offline": True, "available": available, "regen_allowed": regen_allowed} data["distant"] = True data["available"] = available data["regen_allowed"] = regen_allowed return data from .progress import progress_snapshot tiles = progress_snapshot(OUTPUT_DIR, viz_labels=_viz_step_labels(), run_id=_job.get("run_id")) _attach_frame_bounds(tiles) with _job_lock: pending = bool(_queue) if pending: # Non-empty queue and idle server (e.g. container restarted while # requests were queued): the next request starts. _start_next_queued() with _job_lock: proc = _job["proc"] running = proc is not None and proc.poll() is None return { "running": running, "started": _job["started"], "returncode": _job["returncode"], "cmd": _job["cmd"], "qid": _job.get("qid"), "queue": _queue_summary(), "log": _tail_log(40), "available": available, "regen_allowed": regen_allowed, "tiles": tiles, } @app.get("/api/tiles") def tiles_inventory(stamp: Optional[float] = None): """Tile inventory (index_tiles.json) for lightweight machines. Written by the pipeline after every finished tile (unless --no-index): the thumbnails, sub-tiles and source tiles it references are served by the static mounts of the same name. ?stamp=X → light response if nothing changed. """ path = OUTPUT_DIR / "index_tiles.json" try: mtime = path.stat().st_mtime if stamp is not None and abs(mtime - stamp) < 1e-4: return {"stamp": mtime, "tiles": None} data = json.loads(path.read_text(encoding="utf-8")) except (OSError, ValueError): return {"stamp": None, "tiles": None} data["stamp"] = mtime return data # --- PDF export (printable field sheet) --- # Always computed locally (never delegated to LIDAR_GENERATION_URL): the # lightweight map alone must be able to export. One export at a time: the # sheet image takes up to ~40 MB (Pi, mem_limit 1g). _export_lock = threading.Lock() @app.get("/api/export/frame") def export_frame(lat: float, lon: float, paper: str = "A4", orient: str = "paysage", scale: int = 2000): """Printable frame (WGS84 corners) centred on (lat, lon) for these settings.""" from . import export_pdf try: return export_pdf.frame(lat, lon, paper, orient, scale) except ValueError as e: raise HTTPException(400, str(e)) @app.get("/api/export/pdf") def export_pdf_file(lat: float, lon: float, paper: str = "A4", orient: str = "paysage", scale: int = 2000, title: Optional[str] = None): """PDF sheet of the oriented relief centred on (lat, lon).""" from . import export_pdf if not _export_lock.acquire(blocking=False): raise HTTPException(429, "a PDF export is already running, try again in a few seconds") try: data, name = export_pdf.build_pdf(OUTPUT_DIR, lat, lon, paper, orient, scale, title=title) except ValueError as e: raise HTTPException(400, str(e)) except export_pdf.NoDataError as e: raise HTTPException(404, str(e)) except Exception: # noqa: BLE001 — clean message on the map side logger.exception("PDF export failed") raise HTTPException(500, "PDF export failed (see the server logs)") finally: _export_lock.release() return Response(data, media_type="application/pdf", headers={"Content-Disposition": f'attachment; filename="{name}"', "Cache-Control": "no-store"}) @app.post("/api/preview", dependencies=[Depends(_require_token), Depends(_require_lan_for_generation)]) def preview(req: PreviewRequest): """1 km cells intersecting a WGS84 bbox (or all_missing pass). Includes missing tiles AND incomplete existing tiles (lacking one of the requested visualizations). Returns {count, capped, cells:[{col, row, corners}]}. """ if GENERATION_URL: # Lightweight map: the remote machine knows the full input/ and output/ return _proxy_generation("POST", "/api/preview", json.loads(req.json())) names = _viz_step_names() viz = [v for v in (req.viz or []) if v] or _panel_viz_steps() invalid = [v for v in viz if v not in names] if invalid: raise HTTPException( 400, f"invalid visualization: {', '.join(invalid)} " f"(expected: {', '.join(names)})") if req.all_missing: # Global pass: LHD tiles already present in input/ (no download) cells = laz_cells(INPUT_DIR) capped = len(cells) > MAX_CELLS_ALL todo = missing_cells_with_corners(cells[:MAX_CELLS_ALL], OUTPUT_DIR, include_done=req.regenerate, viz=viz) else: if not req.bbox or len(req.bbox) != 4: raise HTTPException(400, "expected bbox: [west, south, east, north]") w, s, e, n = (float(v) for v in req.bbox) cells = bbox_to_cells(w, s, e, n) capped = len(cells) > MAX_CELLS todo = missing_cells_with_corners(cells[:MAX_CELLS], OUTPUT_DIR, include_done=req.regenerate, viz=viz) return {"count": len(todo), "capped": capped, "cells": todo} @app.get("/api/cell") def cell_at_point(lat: float, lng: float): """1 km L93 cell containing a WGS84 point + corners for the map.""" if GENERATION_URL: return _proxy_generation("GET", f"/api/cell?lat={lat}&lng={lng}") from .index import attach_gps_bounds col, row = point_to_cell(lng, lat) tile = {"col": col, "row": row} attach_gps_bounds([tile]) return {"col": col, "row": row, "corners": tile.get("corners") or []} @app.post("/api/generate", dependencies=[Depends(_require_token), Depends(_require_lan_for_generation)]) def generate(req: GenerateRequest): if GENERATION_URL: # Lightweight map: download + processing on the remote machine, which # validates the parameters, queues and returns its own state. data = _proxy_generation("POST", "/api/generate", json.loads(req.json())) data["distant"] = True return data tiles, viz = _resolve_request(req) with _job_lock: proc = _job["proc"] running = proc is not None and proc.poll() is None if running: # A run is in progress: the request is queued and will start when # it ends — the running work is never interrupted. global _queue_seq _queue_seq += 1 item = {"id": _queue_seq, "req": json.loads(req.json()), "queued_at": time.time()} _queue.append(item) _save_queue() return {"demarré": False, "en_file": len(_queue), "qid": item["id"], "tuiles": len(tiles)} cmd = _launch_job(tiles, viz, req) return {"demarré": True, "tuiles": len(tiles), "commande": " ".join(cmd)} @app.post("/api/queue/clear", dependencies=[Depends(_require_token), Depends(_require_lan_for_generation)]) def queue_clear(): """Remove the pending requests (the current run is not touched).""" if GENERATION_URL: return _proxy_generation("POST", "/api/queue/clear") with _job_lock: n = len(_queue) _queue.clear() _save_queue() return {"retirées": n} @app.post("/api/stop", dependencies=[Depends(_require_token), Depends(_require_lan_for_generation)]) def stop_generation(): """Stop the current generation (SIGTERM to the pipeline, SIGKILL as a fallback).""" if GENERATION_URL: return _proxy_generation("POST", "/api/stop") with _job_lock: proc = _job["proc"] if proc is None or proc.poll() is not None: raise HTTPException(409, "no generation in progress") try: proc.terminate() # SIGTERM → clean pipeline shutdown except OSError: pass def _escalate(): try: proc.wait(timeout=15) except subprocess.TimeoutExpired: try: os.killpg(proc.pid, signal.SIGKILL) except (OSError, ProcessLookupError): try: proc.kill() except OSError: pass threading.Thread(target=_escalate, daemon=True).start() logger.info("Generation stop requested (SIGTERM to the pipeline)") return {"arrêt": "demandé"} # --------------------------------------------------------------------------- # Interface # --------------------------------------------------------------------------- @app.get("/", response_class=HTMLResponse) def root(): return HTMLResponse(render_html(), headers={ "Cache-Control": "no-cache, no-store, must-revalidate"}) @app.get("/assets/{file_path:path}") def assets(file_path: str, v: str = None): """Interface assets (revalidated: they follow the image, not the cache).""" base = _assets_dir.resolve() p = (_assets_dir / file_path).resolve() if base not in p.parents or not p.is_file(): raise HTTPException(404, f"asset not found: {file_path}") return FileResponse(str(p), headers={"Cache-Control": "no-cache, must-revalidate"}) if __name__ == "__main__": import uvicorn _cert = os.environ.get("LIDAR_SSL_CERTFILE") _key = os.environ.get("LIDAR_SSL_KEYFILE") _ssl = {"ssl_certfile": _cert, "ssl_keyfile": _key} if _cert and _key else {} uvicorn.run(app, host="0.0.0.0", port=PORT, **_ssl)