code_public/lidar_rendu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Split webapp for Raspberry Pi deployment, remote generation API and sync

Browse Source 
La webapp (carte + vignettes) et la génération de tuiles se déploient sur
deux machines : image légère Dockerfile.webapp (FastAPI + Pillow AVIF
natif + pyproj) sur Raspberry Pi, pipeline complet sur la machine de
traitement. LIDAR_GENERATION_URL délègue /api/generate, /api/preview et
/api/status ; /api/sync ramène les tuiles par rsync puis régénère
vignettes et index localement. Token partagé optionnel
(LIDAR_API_TOKEN/LIDAR_REMOTE_TOKEN). Retire du dépôt les journaux
internes (.swival, audit-findings) et les données (data/, notebooks/).
Doc : docs/DEPLOY_WEBAPP.md.
This commit is contained in:
Antoine Jacquin
2026-09-02 19:44:39 +02:00
parent ed3e90ea89
commit 422f58d772
24 changed files with 2390 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files

63
lidar_pipeline/pipeline.py
View File

@ -54,6 +54,7 @@ class FilePrefixFilter(logging.Filter):
# Module-level filter instance so process_file can set it
_file_filter = FilePrefixFilter()
from .progress import report_event
from .dtm import classify_ground, create_dtm_fast
from .visualizations import (
SharedDEM,
@ -62,7 +63,7 @@ from .visualizations import (
generate_mslrm, generate_sailore,
generate_roughness, generate_wavelet,
generate_solar,
generate_svf, generate_aniso_open,
generate_svf,
generate_flow_accumulation,
generate_anomaly_mask,
)
@ -83,7 +84,6 @@ VIZ_STEPS = [
('pos_open', lambda d, b, v, r, shared=None: generate_openness(d, b, v, r, positive=True, shared=shared)),
('neg_open', lambda d, b, v, r, shared=None: generate_openness(d, b, v, r, positive=False, shared=shared)),
('svf', generate_svf),
('aniso_open', generate_aniso_open),
('roughness', generate_roughness),
('wavelet', generate_wavelet),
('flow_acc', generate_flow_accumulation),
@ -109,7 +109,7 @@ VIZ_STEPS = [
class LidarArchaeoPipeline:
"""Orchestrates the LiDAR archaeological analysis pipeline."""
def __init__(self, input_dir, output_dir, resolution=0.5, workers=1, force=False, ground_method='auto', ign_classes="sol", force_classify=False, keep_tif=False, bare_earth=False, quality=98, only_viz=None, skip_viz=None, output_format='avif', gpu_ids=None, no_index=False):
def __init__(self, input_dir, output_dir, resolution=0.5, workers=1, force=False, ground_method='auto', ign_classes="sol", force_classify=False, keep_tif=False, bare_earth=False, quality=98, only_viz=None, skip_viz=None, output_format='avif', gpu_ids=None, no_index=False, incremental_index=False):
self.input_dir = Path(input_dir)
self.output_dir = Path(output_dir)
# Accept single float or comma-separated string for multi-resolution
@ -133,6 +133,8 @@ class LidarArchaeoPipeline:
self.output_format = output_format
self.gpu_ids = gpu_ids
self.no_index = no_index
self.incremental_index = incremental_index
self._last_index_rebuild = 0.0
self.temp_dir = self.output_dir / "temp"
if not self.input_dir.exists():
@ -265,6 +267,7 @@ class LidarArchaeoPipeline:
for idx, (name, func) in enumerate(self.viz_steps, 1):
if not needs_generation[name]:
logger.info(f" [{idx}/{total}] {name}: déjà existant, ignoré")
self._report(basename, "viz", "skip", name, res=resolution)
vis_results[name] = self._expected_output_path(name, basename, file_vis_dir, self.output_format)
continue
@ -283,6 +286,7 @@ class LidarArchaeoPipeline:
tif.unlink(missing_ok=True)
logger.info(f" [{idx}/{total}] {name}...")
self._report(basename, "viz", "start", name, res=resolution)
t0 = time.time()
try:
# IGN overlays don't use SharedDEM (they download external data)
@ -295,11 +299,14 @@ class LidarArchaeoPipeline:
elapsed = time.time() - t0
if result:
logger.info(f" [{idx}/{total}] ✓ {name} ({elapsed:.1f}s)")
self._report(basename, "viz", "ok", name, res=resolution)
else:
logger.warning(f" [{idx}/{total}] ✗ {name} — no output ({elapsed:.1f}s)")
self._report(basename, "viz", "fail", name, res=resolution)
except Exception as e:
vis_results[name] = None
logger.error(f" [{idx}/{total}] ✗ {name}: {e}", exc_info=True)
self._report(basename, "viz", "fail", name, res=resolution)
# Free GPU memory between visualizations to prevent OOM
gpu_cleanup()
@ -370,6 +377,10 @@ class LidarArchaeoPipeline:
except Exception:
pass
def _report(self, basename, phase, state, detail=None, res=None):
"""Émet un événement de progression (file de génération, best-effort)."""
report_event(self.output_dir, basename, phase, state, detail=detail, res=res)
def process_file(self, laz_file):
"""Process a single LAZ file through the full pipeline.
@ -389,6 +400,7 @@ class LidarArchaeoPipeline:
# Validate file integrity before any processing
from .dtm import validate_laz
if not validate_laz(laz_file):
self._report(basename, "tile", "fail", "fichier LAZ invalide")
return False
# Step 1: Ground classification (shared across all resolutions)
@ -417,8 +429,10 @@ class LidarArchaeoPipeline:
if i == 0:
logger.info(f"[1/5] Classification du sol — sautée (DTM existant)")
logger.info(f"[2/5] Génération DTM {res}m/px — sautée (DTM existant)")
self._report(basename, "classif", "skip", "DTM existant")
else:
logger.info(f" DTM {res}m/px déjà existant — ignoré")
self._report(basename, "dtm", "skip", res=res)
continue
except Exception:
logger.warning(f"Impossible de lire le DTM existant — régénération")
@ -431,16 +445,21 @@ class LidarArchaeoPipeline:
if las_file is None:
# First time: do ground classification
logger.info("[1/5] Classification du sol...")
self._report(basename, "classif", "start")
t1 = time.time()
las_file = classify_ground(laz_file, self.temp_dir, method=self.ground_method, force=self.force_classify, ign_classes=self.ign_classes)
t_classif = time.time() - t1
if not las_file:
logger.error(f" ✗ Échec classification ({t_classif:.1f}s)")
self._report(basename, "classif", "fail")
self._report(basename, "tile", "fail", "échec classification")
return False
logger.info(f" ✓ Classification terminée ({t_classif:.1f}s)")
self._report(basename, "classif", "ok")
# Generate DTM at this resolution
logger.info(f"{'[2/5]' if i == 0 else ' '} Génération DTM {res}m/px...")
self._report(basename, "dtm", "start", res=res)
t2 = time.time()
# Classification IGN → mode pur : DTM = rasterisation brute des
# classes choisies, sans plancher ni comblement (sauf --bare-earth)
@ -454,10 +473,13 @@ class LidarArchaeoPipeline:
t_dtm = time.time() - t2
if not dtm_file:
logger.error(f" ✗ Échec DTM {res}m/px ({t_dtm:.1f}s)")
self._report(basename, "dtm", "fail", res=res)
if i == 0:
self._report(basename, "tile", "fail", "échec DTM")
return False # Primary resolution failure is fatal
continue # Additional resolution failure is non-fatal
logger.info(f" ✓ DTM {res}m/px terminé ({t_dtm:.1f}s)")
self._report(basename, "dtm", "ok", res=res)
dtm_rebuilt = True
self._write_dtm_method(basename, res_suffix)
@ -492,9 +514,36 @@ class LidarArchaeoPipeline:
t_total = time.time() - t_start
logger.info(f"✓ {basename} terminé en {t_total:.1f}s")
self._report(basename, "tile", "ok", f"{t_total:.0f}s")
_file_filter.basename = None
return True
def _rebuild_index_incremental(self):
"""Régénère la carte juste après une tuile terminée (mode incrémental).
Réécrit index.html + index_tiles.json pour que la webapp affiche la
tuile sans attendre la fin du run. Anti-rebond : 3 s minimum entre
deux passes — les tuiles terminées pendant l'intervalle sont couvertes
par la passe suivante ou par la passe finale. Les logs de build_index
sont masqués pour ne pas noyer le journal du run.
"""
if self.no_index:
return
now = time.time()
if now - self._last_index_rebuild < 3.0:
return
self._last_index_rebuild = now
try:
from .index import build_index
saved_level = logger.getEffectiveLevel()
logger.setLevel(logging.WARNING)
try:
build_index(self.output_dir, self.output_format)
finally:
logger.setLevel(saved_level)
except Exception as e:
logger.debug(f"Rebuild incrémental de l'index ignoré : {e}")
def process_all(self, files=None):
"""Process all LAZ files in input directory (or an explicit list)."""
files = files if files is not None else self.find_laz_files()
@ -547,9 +596,13 @@ class LidarArchaeoPipeline:
results[laz_file.name] = success
status = "✓" if success else "✗"
logger.info(f" [{done}/{len(files)}] {status} {laz_file.name}")
if success and self.incremental_index:
self._rebuild_index_incremental()
except Exception as e:
logger.error(f" [{done}/{len(files)}] ✗ {laz_file.name}: {e}")
logger.debug(f" Traceback:", exc_info=True)
report_event(self.output_dir, _file_basename(laz_file),
"tile", "fail", detail=str(e))
results[laz_file.name] = False
except KeyboardInterrupt:
logger.info("Interruption — annulation des travaux en cours...")
@ -564,12 +617,16 @@ class LidarArchaeoPipeline:
logger.info(f"--- Fichier {idx}/{total} ---")
try:
results[laz_file.name] = self.process_file(laz_file)
if results[laz_file.name] and self.incremental_index:
self._rebuild_index_incremental()
except KeyboardInterrupt:
logger.info("Interruption — arrêt immédiat.")
return
except Exception as e:
logger.error(f"✗ Erreur traitement {laz_file.name}: {e}")
logger.debug("Traceback:", exc_info=True)
report_event(self.output_dir, _file_basename(laz_file),
"tile", "fail", detail=str(e))
results[laz_file.name] = False
# Summary