Add web map with zone generation API, side job queue, and restore historical DTM hole rendering

- webapp.py: FastAPI serving the continuous map (port 8973) with
  /api/preview, /api/generate and /api/status; tiles are downloaded
  from IGN and processed in a logged subprocess, tracked live in a
  side "File de génération" panel that survives page reloads
- fetch_ign.py: download missing 1 km LiDAR HD tiles from the IGN
  geoplateforme before processing
- index.py: tile thumbnails and 500 m subtiles are now invalidated by
  mtime so regenerating a tile refreshes its cached images; progress
  logging per tile
- dtm.py: back to the historical gap handling (small gaps filled by
  fillnodata only, larger holes left as nodata rendered black);
  lowest-return floor only via --bare-earth, IGN class selection via
  --ign-classes
- cli.py: positional input now optional (--rebuild-index works alone)
- docker-compose.yml: serve (GPU, port 8973) and process services;
  launch via docker compose only (documented in AGENTS.md/AGENTS.md)
- tests: 131 passing, incl. regressions for thumbnail staleness,
  --rebuild-index without input, and nodata rendering
This commit is contained in:
Antoine Jacquin
2026-08-31 18:07:14 +02:00
parent 35bd827790
commit 8ca65155db
19 changed files with 3676 additions and 771 deletions

View File

@ -1,7 +1,10 @@
"""DTM generation from classified LiDAR point clouds.
Handles ground classification via PDAL (SMRF or CSF) and DTM rasterisation
using scipy binned_statistic_2d. Zones without LiDAR data remain as NaN.
Handles ground classification via PDAL (IGN supplier pre-classification,
SMRF or CSF) and DTM rasterisation
using scipy binned_statistic_2d. Gaps without LiDAR data (common in
complex/rocky terrain) are filled with a terrain-aware interpolation so the
DTM stays continuous.
"""
import json
@ -16,8 +19,67 @@ from scipy.stats import binned_statistic_2d
logger = logging.getLogger("lidar")
# Classes LAS exploitables de la pré-classification LiDAR HD (noms → codes)
IGN_CLASS_NAMES = {
"sol": 2,
"unclassified": 1,
"non-classe": 1,
"eau": 9,
"virtuel": 66,
"pont": 17,
"sursol": 64,
}
def _create_ground_pipeline(input_laz, output_las, method):
def parse_ign_classes(spec):
"""Convertit une liste de classes IGN (noms ou codes) en codes LAS triés.
Args:
spec: Chaîne séparée par virgules, ex. "sol,unclassified" ou "2,1".
Returns:
Liste triée de codes LAS uniques.
Raises:
ValueError: Si un élément n'est ni un nom connu ni un code LAS 0-255,
ou si la liste est vide.
"""
codes = set()
for token in str(spec).split(","):
token = token.strip().lower()
if not token:
continue
if token in IGN_CLASS_NAMES:
codes.add(IGN_CLASS_NAMES[token])
else:
try:
code = int(token)
except ValueError:
raise ValueError(
f"Classe IGN inconnue: '{token}' "
f"(noms: {', '.join(sorted(IGN_CLASS_NAMES))} ou code LAS 0-255)")
if not 0 <= code <= 255:
raise ValueError(f"Code LAS hors bornes (0-255): {code}")
codes.add(code)
if not codes:
raise ValueError("Aucune classe IGN fournie")
return sorted(codes)
def ign_method_label(codes):
"""Étiquette de méthode encodant les classes IGN (ex. 'ign_1_2').
'ign' seul = sol uniquement (code 2), rétrocompatible avec les fichiers de
classification existants. Toute autre combinaison est encodée dans le nom
pour invalider le cache et déclencher la reclassification.
"""
codes = sorted(codes)
if codes == [2]:
return "ign"
return "ign_" + "_".join(str(c) for c in codes)
def _create_ground_pipeline(input_laz, output_las, method, ign_codes=None):
"""Create a PDAL pipeline JSON for ground classification.
All methods include a ReturnNumber/NumberOfReturns >= 1 filter to handle
@ -33,7 +95,10 @@ def _create_ground_pipeline(input_laz, output_las, method):
Args:
input_laz: Path to input LAZ/LAS file.
output_las: Path to output classified LAS file.
method: Ground classification method ('smrf' or 'csf').
method: Ground classification method ('ign', 'smrf' or 'csf').
ign_codes: LAS class codes to extract with the 'ign' method
(default: [2] = sol). Multiple ranges on Classification are
logically ORed by filters.range (documented PDAL semantics).
Returns:
JSON string of the PDAL pipeline.
@ -44,6 +109,38 @@ def _create_ground_pipeline(input_laz, output_las, method):
"limits": "ReturnNumber[1:],NumberOfReturns[1:]"
}
# Classification filter (ground points only)
ground_filter = {
"type": "filters.range",
"limits": "Classification[2:2]"
}
# LiDAR HD IGN : le fichier est pré-classifié par le fournisseur.
# On réutilise la classification telle quelle (mode pur) : les classes
# extraites sont paramétrables — par défaut le sol seul (2), mais on peut
# ajouter p.ex. unclassified (1) pour combler les trous sans retouche.
# Les plages multiples sur Classification sont combinées en OU logique
# par filters.range (sémantique PDAL documentée).
if method == 'ign':
codes = sorted(ign_codes) if ign_codes else [2]
ign_filter = {
"type": "filters.range",
"limits": ",".join(f"Classification[{c}:{c}]" for c in codes)
}
pipeline = {
"pipeline": [
str(input_laz),
return_filter,
ign_filter,
{
"type": "writers.las",
"filename": str(output_las),
"extra_dims": "all"
}
]
}
return json.dumps(pipeline)
# Reset Classification to 0 before preprocessing
reset_classification = {
"type": "filters.assign",
@ -68,12 +165,6 @@ def _create_ground_pipeline(input_laz, output_las, method):
"multiplier": 3.0
}
# Classification filter (ground points only)
ground_filter = {
"type": "filters.range",
"limits": "Classification[2:2]"
}
# Method-specific ground classification filter
if method == 'smrf':
ground_step = {
@ -85,9 +176,12 @@ def _create_ground_pipeline(input_laz, output_las, method):
"scalar": 1.25
}
elif method == 'csf':
# resolution 1.0 m : un cloth à 0.5 m (= 4 M particules pour 1 km²)
# rend la classification ~4× plus lente sans gain visible sur le MNT
# (la résolution finale du MNT vient de la rasterisation, pas du cloth).
ground_step = {
"type": "filters.csf",
"resolution": 0.5,
"resolution": 1.0,
"rigidness": 3,
"smooth": True,
"threshold": 0.5
@ -119,6 +213,11 @@ def create_smrf_pipeline(input_laz, output_las):
return _create_ground_pipeline(input_laz, output_las, 'smrf')
def create_ign_pipeline(input_laz, output_las):
"""Create a PDAL pipeline JSON using the IGN supplier pre-classification."""
return _create_ground_pipeline(input_laz, output_las, 'ign')
def create_csf_pipeline(input_laz, output_las):
"""Create a PDAL pipeline JSON for CSF ground classification."""
return _create_ground_pipeline(input_laz, output_las, 'csf')
@ -258,7 +357,7 @@ def detect_ground_method(laz_file):
laz_file: Path to input LAZ/LAS file.
Returns:
String: 'smrf' or 'csf'
String: 'ign', 'smrf' or 'csf'
"""
import laspy
@ -280,6 +379,22 @@ def detect_ground_method(laz_file):
logger.warning(f" Nuage vide (0 points) — méthode par défaut: SMRF")
return 'smrf'
# LiDAR HD IGN : les données livrées sont pré-classifiées par le fournisseur
# (classe 2 = sol). C'est la base la plus rapide (~10 s) et de référence.
# Le MNT est ensuite complété par le retour le plus bas par cellule +
# interpolation (voir create_dtm_fast), ce qui « rattrape » les trous de la
# pré-classification (forêt dense / relief). On la préfère donc dès qu'une
# part raisonnable des points est classée sol, plutôt que de refiltrer.
try:
cls = np.asarray(las.classification, dtype=np.int32)
ground_ratio = float(np.mean(cls == 2))
except Exception:
ground_ratio = 0.0
if ground_ratio >= 0.2:
logger.info(f" → Méthode: IGN (pré-classification fournisseur — "
f"{ground_ratio * 100:.1f}% de points classe 2)")
return 'ign'
z = np.array(las.z)
# Height variance (always available)
@ -317,14 +432,17 @@ def detect_ground_method(laz_file):
return method
def classify_ground(laz_file, temp_dir, method='auto', force=False):
def classify_ground(laz_file, temp_dir, method='auto', force=False, ign_classes="sol"):
"""Classify ground points using PDAL ground classification filter.
Args:
laz_file: Path to input LAZ/LAS file.
temp_dir: Directory for temporary files (pipeline.json, ground.las).
method: Ground classification method ('auto', 'smrf', or 'csf').
method: Ground classification method ('auto', 'ign', 'smrf' or 'csf').
force: If True, reclassify even if output file already exists.
ign_classes: Classes LAS extraites par la méthode IGN (noms ou codes
séparés par virgules, ex. "sol,unclassified"). Ignoré pour les
autres méthodes.
Returns:
Path to classified ground LAS file, or None on failure.
@ -338,11 +456,17 @@ def classify_ground(laz_file, temp_dir, method='auto', force=False):
else:
logger.info(f" Classification sol: {method.upper()} (forcé)")
# Les classes IGN sont encodées dans le nom de fichier (ex. ign_1_2)
# pour qu'un changement de classes invalide le cache et déclenche la
# reclassification.
ign_codes = parse_ign_classes(ign_classes) if method == 'ign' else None
method_label = ign_method_label(ign_codes) if ign_codes else method
# Use shared basename extraction function
from .pipeline import _file_basename
laz_base = _file_basename(laz_file)
output_las = temp_dir / f"{laz_base}_ground_{method}.las"
output_las = temp_dir / f"{laz_base}_ground_{method_label}.las"
if output_las.exists() and not force:
logger.info(f" Classification {method.upper()} déjà effectuée — fichier existant réutilisé")
@ -352,8 +476,8 @@ def classify_ground(laz_file, temp_dir, method='auto', force=False):
logger.info(f" Reclassification forcée — suppression de {output_las.name}")
output_las.unlink()
pipeline_json = _create_ground_pipeline(laz_file, output_las, method)
pipeline_file = temp_dir / f"pipeline_{method}.json"
pipeline_json = _create_ground_pipeline(laz_file, output_las, method, ign_codes=ign_codes)
pipeline_file = temp_dir / f"pipeline_{method_label}.json"
with open(pipeline_file, 'w') as f:
f.write(pipeline_json)
@ -367,9 +491,9 @@ def classify_ground(laz_file, temp_dir, method='auto', force=False):
if output_las.exists() and output_las.stat().st_size < 100:
logger.error(f" ✗ Fichier ground vide (taille < 100 octets)")
output_las.unlink(missing_ok=True)
# Fallback: if CSF produced no ground points, retry with SMRF
if method == 'csf':
return _fallback_to_smrf(laz_file, temp_dir, laz_base, force)
# Fallback: si la méthode ne produit aucun point sol, réessayer avec SMRF
if method in ('csf', 'ign'):
return _fallback_to_smrf(laz_file, temp_dir, laz_base, force, source=method_label)
return None
logger.info(f" ✓ Classification sol {method.upper()} terminée")
return output_las
@ -377,9 +501,9 @@ def classify_ground(laz_file, temp_dir, method='auto', force=False):
error_msg = e.stderr.decode() if e.stderr else str(e)
logger.warning(f" ✗ Erreur classification PDAL ({method.upper()}): {error_msg}")
# Fallback: if CSF failed, retry with SMRF
if method == 'csf':
return _fallback_to_smrf(laz_file, temp_dir, laz_base, force)
# Fallback: si CSF ou la pré-classification échouent, réessayer avec SMRF
if method in ('csf', 'ign'):
return _fallback_to_smrf(laz_file, temp_dir, laz_base, force, source=method_label)
# Try repairing file with laspy if PDAL fails on EVLR/VLR
if 'VLR' in error_msg or 'Invalid' in error_msg:
@ -405,28 +529,30 @@ def classify_ground(laz_file, temp_dir, method='auto', force=False):
return None
def _fallback_to_smrf(laz_file, temp_dir, laz_base, force=False):
"""Retry ground classification with SMRF when CSF fails.
def _fallback_to_smrf(laz_file, temp_dir, laz_base, force=False, source='csf'):
"""Retry ground classification with SMRF when CSF/IGN fails.
CSF (Cloth Simulation Filter) can fail on certain terrain types where
SMRF (Simple Morphological Filter) succeeds. This fallback ensures
processing continues even when auto-detection selects CSF incorrectly.
SMRF (Simple Morphological Filter) succeeds, and a file without usable
pre-classification produces an empty ground extract. This fallback ensures
processing continues even when the selected method fails.
Args:
laz_file: Path to input LAZ/LAS file.
temp_dir: Directory for temporary files.
laz_base: Base name for the file.
force: If True, reclassify even if output exists.
source: Method that failed ('csf' or 'ign').
Returns:
Path to classified ground LAS file, or None on failure.
"""
logger.info(f" → Basculement CSF → SMRF (fallback)")
logger.info(f" → Basculement {source.upper()} → SMRF (fallback)")
# Clean up failed CSF output if it exists
csf_output = temp_dir / f"{laz_base}_ground_csf.las"
if csf_output.exists():
csf_output.unlink(missing_ok=True)
# Clean up failed output if it exists
failed_output = temp_dir / f"{laz_base}_ground_{source}.las"
if failed_output.exists():
failed_output.unlink(missing_ok=True)
output_las = temp_dir / f"{laz_base}_ground_smrf.las"
@ -481,7 +607,118 @@ def _repair_laz_with_laspy(input_laz, output_las):
return False
def create_dtm_fast(las_file, basename, dtm_dir, resolution, force=False, output_suffix=""):
def _interpolate_holes(dtm, downsample=8):
"""Fill remaining NaN holes with a terrain-aware surface interpolation.
In complex / rocky terrain the ground under-classification leaves interior
holes far too large for a 1 m gap fill, which otherwise become flat
nearest-neighbor patches in the downstream layers. This helper triangulates
the valid cells on a downsampled grid (linear, nearest as a fallback for
cells outside the data hull) and bilinearly upsamples the result, keeping
the operation fast even for large rasters.
Args:
dtm: 2-D float array (may contain NaN holes).
downsample: Coarsening factor for the interpolation grid.
Returns:
Tuple (filled_array, filled_count).
"""
holes = np.isnan(dtm)
if not holes.any():
return dtm, 0
valid = ~holes
if not valid.any():
return dtm, 0
height, width = dtm.shape
step = max(1, downsample)
coarse = dtm[::step, ::step].astype(np.float64)
c_valid = ~np.isnan(coarse)
c_holes = np.isnan(coarse)
if not c_holes.any() or not c_valid.any():
return dtm, 0
from scipy.interpolate import griddata
from scipy.ndimage import map_coordinates
cy, cx = np.where(c_valid)
c_coords = np.column_stack([cx, cy]).astype(np.float64)
c_vals = coarse[c_valid]
hy, hx = np.where(c_holes)
h_coords = np.column_stack([hx, hy]).astype(np.float64)
interp = griddata(c_coords, c_vals, h_coords, method='linear')
bad = np.isnan(interp)
if bad.any():
interp[bad] = griddata(c_coords, c_vals, h_coords[bad], method='nearest')
coarse_filled = coarse.copy()
coarse_filled[c_holes] = interp
# Coarse cell i represents fine column/row i*step, so fine index c maps to
# coarse coordinate c/step (no half-cell offset).
rows = np.arange(height) / step
cols = np.arange(width) / step
grid_y, grid_x = np.meshgrid(rows, cols, indexing='ij')
upsampled = map_coordinates(coarse_filled, [grid_y, grid_x], order=1)
filled = dtm.copy()
filled[holes] = upsampled[holes]
return filled, int(holes.sum())
def _min_return_grid(laz_file, width, height, bounds, chunk_size=2_000_000):
"""Rasterize the per-cell minimum z (lowest return) of the full point cloud.
In complex/forested terrain the ground is under-classified, leaving DTM
holes. Filling them with the *lowest measured return* of the cell (Wack &
Wimmer 2002) recovers a real ground surface (forest floor, rock, clearing)
instead of a pure interpolation. The read is streamed in chunks so memory
stays bounded to the output grid regardless of the point count.
Args:
laz_file: Path to the full (unclassified) LAZ/LAS file.
width, height: Output grid dimensions (pixels).
bounds: (min_x, min_y, max_x, max_y) the grid covers.
chunk_size: Points per streaming chunk.
Returns:
(height, width) float32 array of per-cell min z (NaN where no point).
"""
import laspy
min_x, min_y, max_x, max_y = bounds
grid = np.full((height, width), np.nan, dtype=np.float32)
rng = [[min_x, max_x], [min_y, max_y]]
def process(points):
if len(points) == 0:
return
x = np.asarray(points.x, dtype=np.float64)
y = np.asarray(points.y, dtype=np.float64)
z = np.asarray(points.z, dtype=np.float64)
st = binned_statistic_2d(x, y, z, statistic='min',
bins=[width, height], range=rng)
# Match the DTM convention: .T then flip Y (north at top).
cell_min = st.statistic.T[::-1, :].astype(np.float32)
# fmin ignores NaN so cells without a point in this chunk stay NaN.
np.fmin(grid, cell_min, out=grid)
try:
with laspy.open(str(laz_file)) as las:
for chunk in las.chunk_iterator(chunk_size):
process(chunk)
except Exception as e:
logger.warning(f" Lecture streaming impossible ({e}) — lecture complète")
las = _read_with_pdal(laz_file)
if las is None:
return grid
process(las)
return grid
def create_dtm_fast(las_file, basename, dtm_dir, resolution, force=False,
output_suffix="", source_laz=None, bare_earth=False,
pure=False):
"""Create DTM using fast binning method with gap filling.
Args:
@ -491,6 +728,15 @@ def create_dtm_fast(las_file, basename, dtm_dir, resolution, force=False, output
resolution: Grid resolution in meters per pixel.
force: If True, regenerate even if DTM already exists.
output_suffix: Suffix for output filename (e.g. '_r0p2' for additional resolutions).
source_laz: Optionnel : chemin du LAZ complet (non classé). Utilisé
uniquement avec bare_earth (plancher au retour le plus bas).
bare_earth: If True, pull the DTM down to the lowest measured return of
each cell (bare-earth floor). This requalifies the lowest point of
every column as terrain, recovering the ground under dense
vegetation / steep relief that the ground classifier rejected.
pure: Sans effet (conservé pour compatibilité). Fonctionnement
historique rétabli : petits trous comblés par fillnodata, grands
trous laissés en nodata (rendus en noir dans les rendus).
Returns:
Path to output DTM GeoTIFF, or None on failure.
@ -542,7 +788,19 @@ def create_dtm_fast(las_file, basename, dtm_dir, resolution, force=False, output
dtm = stat.statistic.T
dtm = dtm[::-1, :] # Flip Y so north is at top
# Fill small gaps (< 1m from existing data) while keeping large gaps as NaN
# Comblement « historique » (fonctionnement d'origine, rétabli) :
# seuls les petits trous proches des données sont remplis ; les grands
# trous restent en nodata et apparaissent en noir dans les rendus.
# Le plancher au retour le plus bas n'est appliqué qu'à la demande
# explicite (--bare-earth).
if bare_earth and source_laz is not None:
min_grid = _min_return_grid(source_laz, width, height,
(min_x, min_y, max_x, max_y))
lower = ~np.isnan(min_grid) & (min_grid < dtm)
dtm = np.where(lower, min_grid, dtm)
logger.info(f" Sol nu : {int(lower.sum()):,} cellules raménées au retour le plus bas")
# Fill small gaps (< 1 m from data) precisely — comme avant
nan_count = np.count_nonzero(np.isnan(dtm))
if nan_count > 0:
total = dtm.size
@ -558,8 +816,6 @@ def create_dtm_fast(las_file, basename, dtm_dir, resolution, force=False, output
if filled_count > 0:
dtm = np.where(small_gap_mask, dtm_filled, dtm)
logger.info(f" {filled_count:,} petits trous comblés (< {max_gap_pixels}px)")
remaining = np.count_nonzero(np.isnan(dtm))
logger.info(f" {remaining:,} pixels restent sans données (grands écarts)")
# Save as GeoTIFF
output_tif = dtm_dir / f"{basename}_dtm{output_suffix}.tif"