Add web map with zone generation API, side job queue, and restore historical DTM hole rendering
- webapp.py: FastAPI serving the continuous map (port 8973) with /api/preview, /api/generate and /api/status; tiles are downloaded from IGN and processed in a logged subprocess, tracked live in a side "File de génération" panel that survives page reloads - fetch_ign.py: download missing 1 km LiDAR HD tiles from the IGN geoplateforme before processing - index.py: tile thumbnails and 500 m subtiles are now invalidated by mtime so regenerating a tile refreshes its cached images; progress logging per tile - dtm.py: back to the historical gap handling (small gaps filled by fillnodata only, larger holes left as nodata rendered black); lowest-return floor only via --bare-earth, IGN class selection via --ign-classes - cli.py: positional input now optional (--rebuild-index works alone) - docker-compose.yml: serve (GPU, port 8973) and process services; launch via docker compose only (documented in AGENTS.md/AGENTS.md) - tests: 131 passing, incl. regressions for thumbnail staleness, --rebuild-index without input, and nodata rendering
This commit is contained in:
Antoine Jacquin
162
lidar_pipeline/fetch_ign.py
Normal file
162
lidar_pipeline/fetch_ign.py
Normal file
@ -0,0 +1,162 @@
|
||||
"""Téléchargement des dalles LiDAR HD de l'IGN pour les tuiles non générées.
|
||||
|
||||
Catalogue STAC (à jour) : https://browser.stac.teledetection.fr/collections/lidarhd
|
||||
API : https://api.stac.teledetection.fr/collections/lidarhd/items
|
||||
Fichiers (géoplateforme): https://data.geopf.fr/telechargement/download/...
|
||||
|
||||
Chaque dalle couvre 1 km × 1 km en Lambert 93 et est nommée par son coin
|
||||
nord-ouest : LHD_FXX_{col}_{row}_PTS_LAMB93_IGN69.copc.laz
|
||||
(col = X ouest en km, row = Y nord en km, cf. propriété STAC
|
||||
"lidarhd:coordonnees_NW" au format "0816-6847").
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger("lidar")
|
||||
|
||||
_STAC_ITEMS_URL = "https://api.stac.teledetection.fr/collections/lidarhd/items"
|
||||
_HEADERS = {"User-Agent": "Mozilla/5.0 (lidar-archeo-pipeline)"}
|
||||
|
||||
|
||||
def parse_tile_specs(args):
|
||||
"""Convertit des spécifications "col,row" ou "col:row" en liste de tuples.
|
||||
|
||||
Args:
|
||||
args: liste de chaînes (ex: ["1055,6882", "1056:6883"]).
|
||||
|
||||
Returns:
|
||||
Liste de tuples (col, row).
|
||||
|
||||
Raises:
|
||||
ValueError: si une spécification est mal formée.
|
||||
"""
|
||||
specs = []
|
||||
for raw in args:
|
||||
text = raw.strip().replace(":", ",").replace(";", ",")
|
||||
parts = [p.strip() for p in text.split(",") if p.strip()]
|
||||
if len(parts) != 2:
|
||||
raise ValueError(f"Spécification de tuile invalide: {raw!r} (attendu: col,row)")
|
||||
try:
|
||||
col, row = int(parts[0]), int(parts[1])
|
||||
except ValueError:
|
||||
raise ValueError(f"Spécification de tuile invalide: {raw!r} (col et row doivent être des entiers)")
|
||||
specs.append((col, row))
|
||||
return specs
|
||||
|
||||
|
||||
def tile_filename(col, row):
|
||||
"""Nom de fichier LAZ standard d'une dalle (col, row)."""
|
||||
return f"LHD_FXX_{col:04d}_{row:04d}_PTS_LAMB93_IGN69.copc.laz"
|
||||
|
||||
|
||||
def _bbox_wgs84(col, row):
|
||||
"""Bbox WGS84 de la dalle (col,row) pour la requête STAC (peut être élargie)."""
|
||||
try:
|
||||
from rasterio.warp import transform as warp_transform
|
||||
xs = [col * 1000, (col + 1) * 1000, col * 1000, (col + 1) * 1000]
|
||||
ys = [(row - 1) * 1000] * 2 + [row * 1000] * 2
|
||||
lons, lats = warp_transform('EPSG:2154', 'EPSG:4326', xs, ys)
|
||||
except Exception:
|
||||
from .index import _approx_l93_to_wgs84
|
||||
pts = [_approx_l93_to_wgs84(x, y)
|
||||
for x in (col * 1000, (col + 1) * 1000)
|
||||
for y in ((row - 1) * 1000, row * 1000)]
|
||||
lons = [p[0] for p in pts]
|
||||
lats = [p[1] for p in pts]
|
||||
pad = 0.005 # ~500 m de marge pour éviter les erreurs d'arrondi aux bords
|
||||
return (min(lons) - pad, min(lats) - pad, max(lons) + pad, max(lats) + pad)
|
||||
|
||||
|
||||
def match_feature(features, col, row):
|
||||
"""Retourne l'item STAC correspondant à la dalle (col,row), sinon None."""
|
||||
want = f"{col:04d}-{row:04d}"
|
||||
for feature in features:
|
||||
props = feature.get("properties", {})
|
||||
if props.get("lidarhd:coordonnees_NW") == want:
|
||||
return feature
|
||||
return None
|
||||
|
||||
|
||||
def find_tile_url(col, row, timeout=20):
|
||||
"""Cherche l'URL de téléchargement de la dalle (col,row) dans le catalogue STAC.
|
||||
|
||||
Returns:
|
||||
URL (str) ou None si la dalle n'est pas (encore) publiée par l'IGN.
|
||||
"""
|
||||
w, s, e, n = _bbox_wgs84(col, row)
|
||||
query = urllib.parse.urlencode({"bbox": f"{w:.6f},{s:.6f},{e:.6f},{n:.6f}", "limit": 50})
|
||||
req = urllib.request.Request(f"{_STAC_ITEMS_URL}?{query}", headers=_HEADERS)
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||||
data = json.loads(response.read().decode("utf-8"))
|
||||
feature = match_feature(data.get("features", []), col, row)
|
||||
if not feature:
|
||||
return None
|
||||
return feature.get("assets", {}).get("data", {}).get("href")
|
||||
|
||||
|
||||
def download_file(url, dest_path, timeout=120, chunk=1024 * 1024):
|
||||
"""Télécharge url vers dest_path en streaming. Retourne la taille en octets."""
|
||||
req = urllib.request.Request(url, headers=_HEADERS)
|
||||
t0 = time.time()
|
||||
with urllib.request.urlopen(req, timeout=timeout) as response, open(dest_path, "wb") as out:
|
||||
done = 0
|
||||
while True:
|
||||
block = response.read(chunk)
|
||||
if not block:
|
||||
break
|
||||
out.write(block)
|
||||
done += len(block)
|
||||
elapsed = time.time() - t0
|
||||
logger.info(f" {done / 1e6:.0f} Mo en {elapsed:.0f}s"
|
||||
f" ({done / 1e6 / max(elapsed, 0.1):.1f} Mo/s)")
|
||||
return done
|
||||
|
||||
|
||||
def fetch_tiles(input_dir, specs, output_dir=None):
|
||||
"""Télécharge les dalles IGN spécifiées, sauf celles déjà présentes/générées.
|
||||
|
||||
Args:
|
||||
input_dir: dossier des fichiers LAZ (écriture autorisée requise).
|
||||
specs: liste de tuples (col, row).
|
||||
output_dir: dossier de sortie (optionnel) — permet d'ignorer les
|
||||
tuiles dont les visualisations existent déjà.
|
||||
|
||||
Returns:
|
||||
Liste des chemins téléchargés.
|
||||
"""
|
||||
input_dir = Path(input_dir)
|
||||
downloaded = []
|
||||
for col, row in specs:
|
||||
name = tile_filename(col, row)
|
||||
dest = input_dir / name
|
||||
if dest.exists():
|
||||
logger.info(f" {name} : déjà présent dans input/ — aucun téléchargement")
|
||||
continue
|
||||
if output_dir is not None:
|
||||
vis_dir = Path(output_dir) / "visualisations"
|
||||
if list(vis_dir.glob(f"LHD_FXX_{col:04d}_{row:04d}_PTS*")):
|
||||
logger.info(f" {name} : visualisations déjà générées — ignorée")
|
||||
continue
|
||||
logger.info(f" {name} : recherche dans le catalogue IGN...")
|
||||
try:
|
||||
url = find_tile_url(col, row)
|
||||
except Exception as e:
|
||||
logger.warning(f" ✗ {name} : erreur catalogue ({e})")
|
||||
continue
|
||||
if not url:
|
||||
logger.warning(f" ✗ {name} : introuvable dans le catalogue IGN (zone non publiée ?)")
|
||||
continue
|
||||
logger.info(f" {name} : téléchargement depuis la géoplateforme...")
|
||||
try:
|
||||
download_file(url, dest)
|
||||
logger.info(f" ✓ {name} téléchargée")
|
||||
downloaded.append(dest)
|
||||
except Exception as e:
|
||||
dest.unlink(missing_ok=True)
|
||||
logger.warning(f" ✗ {name} : échec du téléchargement ({e})")
|
||||
return downloaded
|
||||
Reference in New Issue
Block a user