Add web map with zone generation API, side job queue, and restore historical DTM hole rendering
- webapp.py: FastAPI serving the continuous map (port 8973) with /api/preview, /api/generate and /api/status; tiles are downloaded from IGN and processed in a logged subprocess, tracked live in a side "File de génération" panel that survives page reloads - fetch_ign.py: download missing 1 km LiDAR HD tiles from the IGN geoplateforme before processing - index.py: tile thumbnails and 500 m subtiles are now invalidated by mtime so regenerating a tile refreshes its cached images; progress logging per tile - dtm.py: back to the historical gap handling (small gaps filled by fillnodata only, larger holes left as nodata rendered black); lowest-return floor only via --bare-earth, IGN class selection via --ign-classes - cli.py: positional input now optional (--rebuild-index works alone) - docker-compose.yml: serve (GPU, port 8973) and process services; launch via docker compose only (documented in AGENTS.md/AGENTS.md) - tests: 131 passing, incl. regressions for thumbnail staleness, --rebuild-index without input, and nodata rendering
This commit is contained in:
Antoine Jacquin
254
lidar_pipeline/webapp.py
Normal file
254
lidar_pipeline/webapp.py
Normal file
@ -0,0 +1,254 @@
|
||||
"""Serveur web de la carte LiDAR : sert l'index et expose l'API de génération.
|
||||
|
||||
Lancé via `./run.sh --serve [PORT]` (input/ monté en écriture pour permettre
|
||||
le téléchargement IGN). Endpoints :
|
||||
|
||||
GET / → carte interactive (output/index.html)
|
||||
GET /api/status → état de la génération en cours (ou dernière terminée)
|
||||
POST /api/preview → cellules 1 km intersectant une bbox WGS84 (option
|
||||
regenerate=true pour inclure celles déjà générées)
|
||||
POST /api/generate → télécharge (géoplateforme IGN) puis traite des cellules
|
||||
(option regenerate=true ajoute --force --force-classification ;
|
||||
option ground_class choisit la méthode de classification du sol)
|
||||
|
||||
Fichiers statiques : /assets (interface), /index_thumbs, /index_subtiles,
|
||||
/visualisations, /DTM.
|
||||
Un seul job à la fois : la génération lance `python -m lidar_pipeline` en
|
||||
sous-processus avec --fetch-tiles + --file, journalisé dans .generation.log.
|
||||
"""
|
||||
|
||||
import math
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
INPUT_DIR = Path(os.environ.get("LIDAR_INPUT_DIR", "/data/input"))
|
||||
OUTPUT_DIR = Path(os.environ.get("LIDAR_OUTPUT_DIR", "/data/output"))
|
||||
PORT = int(os.environ.get("LIDAR_PORT", "8973"))
|
||||
LOG_FILE = OUTPUT_DIR / ".generation.log"
|
||||
MAX_CELLS = 400 # garde-fou : ~400 km² max par demande
|
||||
|
||||
app = FastAPI(title="Carte LiDAR — génération de zones")
|
||||
|
||||
# assets/ (CSS/JS de l'interface) est créé dès le démarrage pour que le monteur
|
||||
# statique soit actif même avant la première génération de l'index.
|
||||
_assets_dir = OUTPUT_DIR / "assets"
|
||||
_assets_dir.mkdir(parents=True, exist_ok=True)
|
||||
for name in ("index_thumbs", "index_subtiles", "visualisations", "DTM"):
|
||||
_dir = OUTPUT_DIR / name
|
||||
if _dir.exists():
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
app.mount(f"/{name}", StaticFiles(directory=str(_dir)), name=name)
|
||||
|
||||
|
||||
@app.get("/assets/{file_path:path}")
|
||||
def assets(file_path: str):
|
||||
"""Sert les fichiers de l'interface sans cache (régénérés à chaque rebuild)."""
|
||||
base = _assets_dir.resolve()
|
||||
p = (_assets_dir / file_path).resolve()
|
||||
if base not in p.parents or not p.is_file():
|
||||
raise HTTPException(404, f"asset introuvable : {file_path}")
|
||||
return FileResponse(str(p), headers={"Cache-Control": "no-cache, must-revalidate"})
|
||||
|
||||
|
||||
# Méthodes de classification du sol acceptées (mêmes valeurs que --ground-classification).
|
||||
GROUND_CLASS_METHODS = ("auto", "ign", "smrf", "csf")
|
||||
|
||||
|
||||
class PreviewRequest(BaseModel):
|
||||
bbox: list = Field(..., description="[ouest, sud, est, nord] en WGS84")
|
||||
regenerate: bool = Field(False, description="Inclure les tuiles déjà générées")
|
||||
|
||||
|
||||
class GenerateRequest(BaseModel):
|
||||
tiles: list = Field(..., description="liste [col, row] (entiers km L93)")
|
||||
regenerate: bool = Field(False, description="Régénérer les tuiles déjà générées")
|
||||
ground_class: str = Field("ign",
|
||||
description="Méthode de classification du sol : "
|
||||
"auto, ign, smrf, csf")
|
||||
ign_classes: str = Field("sol",
|
||||
description="Classes LAS pour le MNT IGN : liste noms ou "
|
||||
"codes séparés par virgules — sol(2), "
|
||||
"unclassified(1), eau(9), virtuel(66), "
|
||||
"pont(17), sursol(64). Mode pur, "
|
||||
"aucune retouche. (défaut: sol)")
|
||||
bare_earth: bool = Field(False,
|
||||
description="Sol nu : DTM au retour le plus bas de "
|
||||
"chaque cellule (requalifie le point le plus "
|
||||
"bas en terrain)")
|
||||
|
||||
|
||||
# --- État du job de génération -------------------------------------------
|
||||
_job = {"proc": None, "started": None, "returncode": None, "cmd": None}
|
||||
_job_lock = threading.Lock()
|
||||
|
||||
|
||||
def bbox_to_cells(w, s, e, n):
|
||||
"""Cellules L93 de 1 km (col, row) intersectant une bbox WGS84.
|
||||
|
||||
Une cellule (col, row) couvre X ∈ [col, col+1] km, Y ∈ [row-1, row] km.
|
||||
"""
|
||||
from rasterio.warp import transform as warp_transform
|
||||
lons, lats = warp_transform("EPSG:4326", "EPSG:2154", [w, e, w, e], [s, s, n, n])
|
||||
# warp_transform renvoie (xs, ys) dans la CRS cible
|
||||
min_x, max_x = min(lons) + 0.5, max(lons) - 0.5 # rétrécit d'1 m : bords exclus
|
||||
min_y, max_y = min(lats) + 0.5, max(lats) - 0.5
|
||||
if max_x <= min_x or max_y <= min_y:
|
||||
return []
|
||||
cols = range(int(math.floor(min_x / 1000)), int(math.floor(max_x / 1000)) + 1)
|
||||
rows = range(int(math.floor(min_y / 1000)) + 1, int(math.floor(max_y / 1000)) + 2)
|
||||
return [(c, r) for r in rows for c in cols]
|
||||
|
||||
|
||||
def processed_cells(output_dir):
|
||||
"""Ensemble des cellules (col, row) ayant déjà des visualisations."""
|
||||
from .index import scan_tiles
|
||||
tiles = scan_tiles(Path(output_dir) / "visualisations")
|
||||
return {(t["col"], t["row"]) for t in tiles}
|
||||
|
||||
|
||||
def missing_cells_with_corners(cells, output_dir, include_done=False):
|
||||
"""Filtre les cellules déjà traitées et calcule leurs coins WGS84.
|
||||
|
||||
Retourne [{col, row, corners: [[lat, lon] × 4 SW,SE,NE,NW}].
|
||||
"""
|
||||
from .index import attach_gps_bounds
|
||||
done = processed_cells(output_dir)
|
||||
todo = [{"col": c, "row": r} for (c, r) in cells
|
||||
if include_done or (c, r) not in done]
|
||||
if todo:
|
||||
attach_gps_bounds(todo)
|
||||
return todo
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def root():
|
||||
index = OUTPUT_DIR / "index.html"
|
||||
if not index.exists():
|
||||
return JSONResponse({"erreur": "index.html introuvable — lancez d'abord le pipeline"},
|
||||
status_code=404)
|
||||
# index.html est régénéré à chaque passe du pipeline : on interdit le cache
|
||||
# navigateur pour ne pas servir une version périmée (ex. menu de génération).
|
||||
return FileResponse(
|
||||
str(index), media_type="text/html",
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-store, must-revalidate",
|
||||
"Pragma": "no-cache",
|
||||
"Expires": "0",
|
||||
})
|
||||
|
||||
|
||||
@app.get("/api/status")
|
||||
def status():
|
||||
with _job_lock:
|
||||
proc = _job["proc"]
|
||||
running = proc is not None and proc.poll() is None
|
||||
return {
|
||||
"running": running,
|
||||
"started": _job["started"],
|
||||
"returncode": _job["returncode"],
|
||||
"cmd": _job["cmd"],
|
||||
"log": _tail_log(40),
|
||||
}
|
||||
|
||||
|
||||
def _tail_log(n_lines):
|
||||
try:
|
||||
lines = LOG_FILE.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
return lines[-n_lines:]
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
|
||||
@app.post("/api/preview")
|
||||
def preview(req: PreviewRequest):
|
||||
if len(req.bbox) != 4:
|
||||
raise HTTPException(400, "bbox attendue : [ouest, sud, est, nord]")
|
||||
w, s, e, n = (float(v) for v in req.bbox)
|
||||
cells = bbox_to_cells(w, s, e, n)
|
||||
capped = len(cells) > MAX_CELLS
|
||||
todo = missing_cells_with_corners(cells[:MAX_CELLS], OUTPUT_DIR,
|
||||
include_done=req.regenerate)
|
||||
return {"count": len(todo), "capped": capped, "cells": todo}
|
||||
|
||||
|
||||
def _build_command(tiles, regenerate=False, ground_class="ign", bare_earth=False, ign_classes="sol"):
|
||||
"""Commande de génération : téléchargement IGN + traitement des fichiers.
|
||||
|
||||
La classification du sol est choisie via `ground_class` (défaut : "ign",
|
||||
pré-classification IGN ; le pipeline bascule sur SMRF si un fichier ne la
|
||||
contient pas). Avec ign_classes, on choisit les classes LAS extraites
|
||||
pour le MNT (mode pur, ex. "sol,unclassified" pour combler les trous
|
||||
sans retouche). Avec regenerate=True, force la reclassification et la
|
||||
régénération des visualisations des tuiles déjà présentes. Avec
|
||||
bare_earth=True, le DTM est ramené au retour le plus bas de chaque
|
||||
cellule (sol nu).
|
||||
"""
|
||||
from .fetch_ign import tile_filename
|
||||
# -u : sortie non bufferisée — le journal .generation.log doit être
|
||||
# lu en temps réel par /api/status (progression affichée dans l'UI).
|
||||
cmd = [sys.executable, "-u", "-m", "lidar_pipeline", str(INPUT_DIR),
|
||||
"-o", str(OUTPUT_DIR), "-r", "0.5,0.2", "--only", "aspect",
|
||||
"--ground-classification", ground_class,
|
||||
"--ign-classes", ign_classes]
|
||||
if bare_earth:
|
||||
cmd += ["--bare-earth"]
|
||||
if regenerate:
|
||||
cmd += ["--force", "--force-classification"]
|
||||
if os.environ.get("LIDAR_GPU", "") == "1":
|
||||
cmd += ["-g", "all", "-w", os.environ.get("LIDAR_WORKERS", "2")]
|
||||
cmd += ["--fetch-tiles"]
|
||||
cmd += [f"{c},{r}" for (c, r) in tiles]
|
||||
cmd += ["--file"]
|
||||
cmd += [tile_filename(c, r) for (c, r) in tiles]
|
||||
return cmd
|
||||
|
||||
|
||||
@app.post("/api/generate")
|
||||
def generate(req: GenerateRequest):
|
||||
tiles = []
|
||||
for pair in req.tiles:
|
||||
if not (isinstance(pair, list) and len(pair) == 2):
|
||||
raise HTTPException(400, f"tuile invalide : {pair!r} (attendu [col, row])")
|
||||
tiles.append((int(pair[0]), int(pair[1])))
|
||||
if not tiles:
|
||||
raise HTTPException(400, "aucune tuile fournie")
|
||||
if len(tiles) > MAX_CELLS:
|
||||
raise HTTPException(400, f"trop de tuiles ({len(tiles)}) — max {MAX_CELLS}")
|
||||
if req.ground_class not in GROUND_CLASS_METHODS:
|
||||
raise HTTPException(
|
||||
400, f"méthode de classification invalide : {req.ground_class!r} "
|
||||
f"(attendu : {', '.join(GROUND_CLASS_METHODS)})")
|
||||
|
||||
with _job_lock:
|
||||
proc = _job["proc"]
|
||||
if proc is not None and proc.poll() is None:
|
||||
raise HTTPException(409, "une génération est déjà en cours")
|
||||
cmd = _build_command(tiles, regenerate=req.regenerate, ground_class=req.ground_class, bare_earth=req.bare_earth, ign_classes=req.ign_classes)
|
||||
LOG_FILE.parent.mkdir(parents=True, exist_ok=True)
|
||||
log_fh = open(LOG_FILE, "w", encoding="utf-8")
|
||||
_job.update({"proc": None, "started": time.time(), "returncode": None, "cmd": cmd})
|
||||
p = subprocess.Popen(cmd, stdout=log_fh, stderr=subprocess.STDOUT,
|
||||
cwd="/app" if Path("/app").exists() else None)
|
||||
|
||||
def _watch():
|
||||
rc = p.wait()
|
||||
log_fh.close()
|
||||
with _job_lock:
|
||||
_job["returncode"] = rc
|
||||
|
||||
threading.Thread(target=_watch, daemon=True).start()
|
||||
_job["proc"] = p
|
||||
return {"demarré": True, "tuiles": len(tiles), "commande": " ".join(cmd)}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=PORT)
|
||||
Reference in New Issue
Block a user