code_public/lidar_rendu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add web map with zone generation API, side job queue, and restore historical DTM hole rendering

Browse Source 
- webapp.py: FastAPI serving the continuous map (port 8973) with
  /api/preview, /api/generate and /api/status; tiles are downloaded
  from IGN and processed in a logged subprocess, tracked live in a
  side "File de génération" panel that survives page reloads
- fetch_ign.py: download missing 1 km LiDAR HD tiles from the IGN
  geoplateforme before processing
- index.py: tile thumbnails and 500 m subtiles are now invalidated by
  mtime so regenerating a tile refreshes its cached images; progress
  logging per tile
- dtm.py: back to the historical gap handling (small gaps filled by
  fillnodata only, larger holes left as nodata rendered black);
  lowest-return floor only via --bare-earth, IGN class selection via
  --ign-classes
- cli.py: positional input now optional (--rebuild-index works alone)
- docker-compose.yml: serve (GPU, port 8973) and process services;
  launch via docker compose only (documented in AGENTS.md/AGENTS.md)
- tests: 131 passing, incl. regressions for thumbnail staleness,
  --rebuild-index without input, and nodata rendering
This commit is contained in:
Antoine Jacquin
2026-08-31 18:07:14 +02:00
parent 35bd827790
commit 8ca65155db
19 changed files with 3676 additions and 771 deletions
Download Patch File Download Diff File Expand all files Collapse all files

128
lidar_pipeline/pipeline.py
View File

@ -109,7 +109,7 @@ VIZ_STEPS = [
class LidarArchaeoPipeline:
"""Orchestrates the LiDAR archaeological analysis pipeline."""
def __init__(self, input_dir, output_dir, resolution=0.5, workers=1, force=False, ground_method='auto', force_classify=False, keep_tif=False, quality=98, only_viz=None, skip_viz=None, output_format='avif', gpu_ids=None, no_index=False):
def __init__(self, input_dir, output_dir, resolution=0.5, workers=1, force=False, ground_method='auto', ign_classes="sol", force_classify=False, keep_tif=False, bare_earth=False, quality=98, only_viz=None, skip_viz=None, output_format='avif', gpu_ids=None, no_index=False):
self.input_dir = Path(input_dir)
self.output_dir = Path(output_dir)
# Accept single float or comma-separated string for multi-resolution
@ -123,7 +123,9 @@ class LidarArchaeoPipeline:
self.workers = workers
self.force = force
self.ground_method = ground_method
self.ign_classes = ign_classes
self.force_classify = force_classify
self.bare_earth = bare_earth
self.keep_tif = keep_tif
self.quality = quality
self.only_viz = only_viz
@ -212,7 +214,7 @@ class LidarArchaeoPipeline:
else:
return file_vis_dir / f"{basename}_{name}.{ext}"
def generate_all_visualizations(self, dtm_file, basename, resolution=None, vis_dir=None):
def generate_all_visualizations(self, dtm_file, basename, resolution=None, vis_dir=None, force_images=None):
"""Generate all archaeological visualizations for one DTM file.
Optimisation: SharedDEM is only computed if at least one visualization
@ -229,9 +231,10 @@ class LidarArchaeoPipeline:
total = len(self.viz_steps)
# Phase 1: determine which visualizations need generation
force_viz = self.force if force_images is None else force_images
needs_generation = {} # name -> True/False
for name, func in self.viz_steps:
if self.force:
if force_viz:
needs_generation[name] = True
else:
expected_webp = self._expected_output_path(name, basename, file_vis_dir, self.output_format)
@ -265,8 +268,8 @@ class LidarArchaeoPipeline:
vis_results[name] = self._expected_output_path(name, basename, file_vis_dir, self.output_format)
continue
# When --force, delete existing TIF to ensure clean regeneration
if self.force:
# When regenerating, delete existing TIF to ensure clean regeneration
if force_viz:
for tif in file_vis_dir.glob(f"{basename}_{name}.tif"):
tif.unlink(missing_ok=True)
if name == 'pos_open':
@ -325,6 +328,48 @@ class LidarArchaeoPipeline:
res_str = f"{resolution}".replace('.', 'p')
return f"_r{res_str}"
def _dtm_method_path(self, basename, res_suffix):
"""Sidecar path storing which ground method produced a DTM."""
return self.dtm_dir / f"{basename}_dtm{res_suffix}_method.txt"
def _dtm_method_name(self, basename, res_suffix):
"""Read the recorded ground method for a DTM, or None if unknown."""
p = self._dtm_method_path(basename, res_suffix)
if p.exists():
try:
return p.read_text(encoding="utf-8").strip() or None
except Exception:
return None
return None
def _effective_ground_method(self):
"""Méthode effective pour le suivi de cache, classes IGN incluses.
La méthode 'ign' est étiquetée avec les classes choisies (ex. 'ign_1_2')
pour qu'un changement de --ign-classes déclenche la reclassification.
"""
if self.ground_method == 'ign':
from .dtm import parse_ign_classes, ign_method_label
return ign_method_label(parse_ign_classes(self.ign_classes))
return self.ground_method
def _dtm_method_matches(self, basename, res_suffix):
"""True if the recorded ground method matches the requested one.
A DTM without a recorded method is treated as matching so the existing
cache is preserved; the method is adopted on its next reclassification.
"""
recorded = self._dtm_method_name(basename, res_suffix)
return recorded is None or recorded == self._effective_ground_method()
def _write_dtm_method(self, basename, res_suffix):
"""Record the ground classification method used to build a DTM."""
try:
self._dtm_method_path(basename, res_suffix).write_text(
self._effective_ground_method(), encoding="utf-8")
except Exception:
pass
def process_file(self, laz_file):
"""Process a single LAZ file through the full pipeline.
@ -349,26 +394,37 @@ class LidarArchaeoPipeline:
# Step 1: Ground classification (shared across all resolutions)
las_file = None
t_classif = 0
dtm_rebuilt = False
# The ground method is shared across resolutions. It is recorded per DTM
# in a sidecar so that changing --ground-classification invalidates the
# cache; otherwise the cached DTM would be reused and the new method
# never applied (nothing would change visually).
primary_suffix = self._res_suffix(self.resolutions[0])
method_matches = self._dtm_method_matches(basename, primary_suffix)
for i, res in enumerate(self.resolutions):
res_suffix = self._res_suffix(res)
dtm_path = self.dtm_dir / f"{basename}_dtm{res_suffix}.tif"
if dtm_path.exists():
import rasterio
try:
with rasterio.open(dtm_path) as src:
existing_res = abs(src.transform.a)
if abs(existing_res - res) > 0.01:
logger.info(f" DTM{res_suffix} existant à {existing_res}m/px — résolution demandée {res}m/px → régénération")
dtm_path.unlink()
else:
if i == 0:
logger.info(f"[1/5] Classification du sol — sautée (DTM existant)")
logger.info(f"[2/5] Génération DTM {res}m/px — sautée (DTM existant)")
if dtm_path.exists() and not self.force_classify:
if method_matches:
import rasterio
try:
with rasterio.open(dtm_path) as src:
existing_res = abs(src.transform.a)
if abs(existing_res - res) > 0.01:
logger.info(f" DTM{res_suffix} existant à {existing_res}m/px — résolution demandée {res}m/px → régénération")
dtm_path.unlink()
else:
logger.info(f" DTM {res}m/px déjà existant — ignoré")
continue
except Exception:
logger.warning(f"Impossible de lire le DTM existant — régénération")
if i == 0:
logger.info(f"[1/5] Classification du sol — sautée (DTM existant)")
logger.info(f"[2/5] Génération DTM {res}m/px — sautée (DTM existant)")
else:
logger.info(f" DTM {res}m/px déjà existant — ignoré")
continue
except Exception:
logger.warning(f"Impossible de lire le DTM existant — régénération")
dtm_path.unlink()
else:
logger.info(f" DTM{res_suffix} produit par {self._dtm_method_name(basename, primary_suffix) or '?'} ≠ {self._effective_ground_method()} → reclassification")
dtm_path.unlink()
# Need to classify/generate DTM for this resolution
@ -376,7 +432,7 @@ class LidarArchaeoPipeline:
# First time: do ground classification
logger.info("[1/5] Classification du sol...")
t1 = time.time()
las_file = classify_ground(laz_file, self.temp_dir, method=self.ground_method, force=self.force_classify)
las_file = classify_ground(laz_file, self.temp_dir, method=self.ground_method, force=self.force_classify, ign_classes=self.ign_classes)
t_classif = time.time() - t1
if not las_file:
logger.error(f" ✗ Échec classification ({t_classif:.1f}s)")
@ -386,7 +442,15 @@ class LidarArchaeoPipeline:
# Generate DTM at this resolution
logger.info(f"{'[2/5]' if i == 0 else ' '} Génération DTM {res}m/px...")
t2 = time.time()
dtm_file = create_dtm_fast(las_file, basename, self.dtm_dir, res, force=self.force, output_suffix=res_suffix)
# Classification IGN → mode pur : DTM = rasterisation brute des
# classes choisies, sans plancher ni comblement (sauf --bare-earth)
pure_ign = "_ground_ign" in Path(las_file).name
dtm_file = create_dtm_fast(las_file, basename, self.dtm_dir, res,
force=self.force or self.force_classify,
output_suffix=res_suffix,
source_laz=laz_file,
bare_earth=self.bare_earth,
pure=pure_ign)
t_dtm = time.time() - t2
if not dtm_file:
logger.error(f" ✗ Échec DTM {res}m/px ({t_dtm:.1f}s)")
@ -394,6 +458,8 @@ class LidarArchaeoPipeline:
return False # Primary resolution failure is fatal
continue # Additional resolution failure is non-fatal
logger.info(f" ✓ DTM {res}m/px terminé ({t_dtm:.1f}s)")
dtm_rebuilt = True
self._write_dtm_method(basename, res_suffix)
# Process each resolution: visualizations + PDF
all_vis_results = {}
@ -420,16 +486,18 @@ class LidarArchaeoPipeline:
vis_dir.mkdir(exist_ok=True)
self.generate_all_visualizations(dtm_path, basename, actual_res, vis_dir=vis_dir)
self.generate_all_visualizations(
dtm_path, basename, actual_res, vis_dir=vis_dir,
force_images=self.force or self.force_classify or dtm_rebuilt)
t_total = time.time() - t_start
logger.info(f"✓ {basename} terminé en {t_total:.1f}s")
_file_filter.basename = None
return True
def process_all(self):
"""Process all LAZ files in input directory."""
files = self.find_laz_files()
def process_all(self, files=None):
"""Process all LAZ files in input directory (or an explicit list)."""
files = files if files is not None else self.find_laz_files()
if not files:
logger.error("Aucun fichier LAZ/LAS trouvé !")
@ -466,7 +534,7 @@ class LidarArchaeoPipeline:
active_ids = self.gpu_ids if self.gpu_ids else available_gpu_ids()
resolutions_str = ','.join(str(r) for r in self.resolutions)
future_to_file = {
executor.submit(_process_file_standalone, str(laz_file), str(self.input_dir), str(self.output_dir), resolutions_str, self.force, self.ground_method, self.force_classify, self.keep_tif, self.quality, self.only_viz, self.skip_viz, self.output_format, active_ids[file_idx % len(active_ids)] if active_ids else None): laz_file
executor.submit(_process_file_standalone, str(laz_file), str(self.input_dir), str(self.output_dir), resolutions_str, self.force, self.ground_method, self.ign_classes, self.force_classify, self.keep_tif, self.bare_earth, self.quality, self.only_viz, self.skip_viz, self.output_format, active_ids[file_idx % len(active_ids)] if active_ids else None): laz_file
for file_idx, laz_file in enumerate(files)
}
done = 0
@ -545,7 +613,7 @@ class LidarArchaeoPipeline:
logger.warning(f" Note: Impossible de supprimer les fichiers temporaires: {e}")
def _process_file_standalone(laz_file_str, input_dir, output_dir, resolution, force=False, ground_method='auto', force_classify=False, keep_tif=False, quality=98, only_viz=None, skip_viz=None, output_format='avif', gpu_id=None):
def _process_file_standalone(laz_file_str, input_dir, output_dir, resolution, force=False, ground_method='auto', ign_classes="sol", force_classify=False, keep_tif=False, bare_earth=False, quality=98, only_viz=None, skip_viz=None, output_format='avif', gpu_id=None):
"""Standalone function for multiprocessing — creates its own pipeline instance.
Each worker gets its own temp directory to avoid file conflicts.
@ -572,7 +640,7 @@ def _process_file_standalone(laz_file_str, input_dir, output_dir, resolution, fo
worker_logger.addHandler(handler)
worker_logger.addFilter(_file_filter)
pipeline = LidarArchaeoPipeline(input_dir, output_dir, resolution=resolution, workers=1, force=force, ground_method=ground_method, force_classify=force_classify, keep_tif=keep_tif, quality=quality, only_viz=only_viz, skip_viz=skip_viz, output_format=output_format)
pipeline = LidarArchaeoPipeline(input_dir, output_dir, resolution=resolution, workers=1, force=force, ground_method=ground_method, ign_classes=ign_classes, force_classify=force_classify, keep_tif=keep_tif, bare_earth=bare_earth, quality=quality, only_viz=only_viz, skip_viz=skip_viz, output_format=output_format)
basename = _file_basename(laz_file_str)
pipeline.temp_dir = pipeline.output_dir / "temp" / basename
pipeline.temp_dir.mkdir(exist_ok=True)