Skip SharedDEM computation when all visualizations already exist

Two optimizations to avoid ~2min wasted per file on re-runs:

1. pipeline.py: Check which visualizations need regeneration before
   computing SharedDEM. If all WebP outputs exist, skip SharedDEM
   entirely. If only IGN overlays need updating, also skip SharedDEM.

2. visualizations.py: Make SharedDEM attributes lazy (filled, gradient,
   lrm_15) so only the data actually needed is computed. For example,
   if only hillshade is regenerated, LRM at 15m is never calculated.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
2026-05-14 20:40:51 +02:00
parent cf3e680b02
commit c6c804749e
2 changed files with 113 additions and 46 deletions

View File

@ -162,11 +162,24 @@ class LidarArchaeoPipeline:
return False
return True
@staticmethod
def _expected_webp_path(name, basename, file_vis_dir):
"""Return the expected WebP filename for a visualization step."""
if name == 'pos_open':
return file_vis_dir / f"{basename}_positive_openness.webp"
elif name == 'neg_open':
return file_vis_dir / f"{basename}_negative_openness.webp"
elif name == 'hillshade':
return file_vis_dir / f"{basename}_hillshade_multi.webp"
else:
return file_vis_dir / f"{basename}_{name}.webp"
def generate_all_visualizations(self, dtm_file, basename, resolution=None):
"""Generate all archaeological visualizations for one DTM file.
Args:
resolution: Actual resolution from DTM geotransform. If None, uses self.resolution.
Optimisation: SharedDEM is only computed if at least one visualization
needs to be generated. When all WebP outputs exist, SharedDEM is
skipped entirely (saves ~2min per file on re-runs).
"""
if resolution is None:
resolution = self.resolution
@ -175,22 +188,49 @@ class LidarArchaeoPipeline:
# Create per-file subdirectory
file_vis_dir = self.vis_dir / basename
file_vis_dir.mkdir(exist_ok=True)
# Pre-compute shared DEM data (gradient, NaN mask, LRM) once for all visualizations
logger.info(" Pré-calcul données partagées (gradient, LRM)...")
t_shared = time.time()
shared = SharedDEM(dtm_file, resolution)
logger.info(f" ✓ Données partagées prêtes ({time.time()-t_shared:.1f}s)")
vis_results = {}
total = len(VIZ_STEPS)
# Phase 1: determine which visualizations need generation
needs_generation = {} # name -> True/False
for name, func in VIZ_STEPS:
if self.force:
needs_generation[name] = True
else:
expected_webp = self._expected_webp_path(name, basename, file_vis_dir)
needs_generation[name] = not expected_webp.exists()
to_generate = [n for n, needed in needs_generation.items() if needed]
ign_only = all(name in ('ortho', 'topo') for name in to_generate)
needs_shared = any(name not in ('ortho', 'topo') for name in to_generate)
if not to_generate:
logger.info(" Toutes les visualisations déjà existantes — ignorées")
# Still need to return results dict for PDF check
vis_results = {}
for name, func in VIZ_STEPS:
vis_results[name] = self._expected_webp_path(name, basename, file_vis_dir)
return vis_results
# Phase 2: compute SharedDEM only if needed
shared = None
if needs_shared:
logger.info(" Pré-calcul données partagées (gradient, LRM)...")
t_shared = time.time()
shared = SharedDEM(dtm_file, resolution)
logger.info(f" ✓ Données partagées prêtes ({time.time()-t_shared:.1f}s)")
# Phase 3: generate visualizations
vis_results = {}
for idx, (name, func) in enumerate(VIZ_STEPS, 1):
if not needs_generation[name]:
logger.info(f" [{idx}/{total}] {name}: déjà existant, ignoré")
vis_results[name] = self._expected_webp_path(name, basename, file_vis_dir)
continue
# When --force, delete existing TIF to ensure clean regeneration
if self.force:
for tif in file_vis_dir.glob(f"{basename}_{name}.tif"):
tif.unlink(missing_ok=True)
# Special cases for differently-named TIFs
if name == 'pos_open':
for tif in file_vis_dir.glob(f"{basename}_positive_openness.tif"):
tif.unlink(missing_ok=True)
@ -201,26 +241,6 @@ class LidarArchaeoPipeline:
for tif in file_vis_dir.glob(f"{basename}_hillshade_multi.tif"):
tif.unlink(missing_ok=True)
# Check if output WebP already exists (skip unless --force)
if not self.force:
# Determine expected WebP filename from the viz name
# Special cases for openness and IGN overlays
if name == 'pos_open':
expected_webp = file_vis_dir / f"{basename}_positive_openness.webp"
elif name == 'neg_open':
expected_webp = file_vis_dir / f"{basename}_negative_openness.webp"
elif name == 'hillshade':
expected_webp = file_vis_dir / f"{basename}_hillshade_multi.webp"
elif name in ('ortho', 'topo'):
expected_webp = file_vis_dir / f"{basename}_{name}.webp"
else:
expected_webp = file_vis_dir / f"{basename}_{name}.webp"
if expected_webp.exists():
logger.info(f" [{idx}/{total}] {name}: déjà existant, ignoré")
vis_results[name] = expected_webp # Track as existing file
continue
logger.info(f" [{idx}/{total}] {name}...")
t0 = time.time()
try: