code_public/lidar_rendu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improve visualizations: adaptive scales, revert z-score to std normalization

Browse Source 
- MSRM/TPI/roughness/anomalies: revert z-score (x-mean)/std to std normalization x/std
  to preserve contrast and visibility of linear features (paths, ditches, trenches)
- MSRM: adaptive scales based on resolution, archaeological weight combination
- TPI: extend from 2 to 4 scales (3m/15m/50m/200m) with weighted combination
- Hillshade: 8 directions instead of 4, altitude 35° instead of 30°
- LRM: adaptive sigma based on resolution
- Openness: doubled radius (100m instead of 50m)
- Roughness: multi-scale (3m fine + 15m broad) instead of single 5x5 window
- Anomalies: uses MSRM multi-scale relief instead of single LRM 15m
- Wavelet: 8 adaptive scales, std normalization, archaeological weights
- Remove svf (Sky-View Factor) and local_dominance visualizations
- Add AVIF format support (default), quality 98
- Add multi-resolution support (-r 0.5,0.2)
- Improve Ctrl+C handling for immediate process termination
- Update rendering.py descriptions for all modified visualizations

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
2026-05-14 23:12:08 +02:00
parent ac56ba8084
commit d334892880
8 changed files with 344 additions and 180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
lidar_pipeline/pipeline.py
View File

@ -58,10 +58,10 @@ from .dtm import classify_ground, create_dtm_fast
from .visualizations import (
SharedDEM,
generate_hillshade, generate_slope, generate_aspect, generate_curvature,
generate_lrm, generate_svf, generate_openness,
generate_lrm, generate_openness,
generate_mslrm, generate_tpi, generate_sailore,
generate_roughness, generate_anomalies, generate_wavelet,
generate_flow, generate_local_dominance,
generate_flow,
)
from .gpu import gpu_cleanup
from .ign import generate_ign_overlay
@ -76,7 +76,6 @@ VIZ_STEPS = [
('slope', generate_slope),
('aspect', generate_aspect),
('curvature', generate_curvature),
('svf', generate_svf),
('lrm', generate_lrm),
('pos_open', lambda d, b, v, r, shared=None: generate_openness(d, b, v, r, positive=True, shared=shared)),
('neg_open', lambda d, b, v, r, shared=None: generate_openness(d, b, v, r, positive=False, shared=shared)),
@ -87,7 +86,6 @@ VIZ_STEPS = [
('anomalies', generate_anomalies),
('wavelet', generate_wavelet),
('flow', generate_flow),
('local_dominance', generate_local_dominance),
('ortho', lambda d, b, v, r: generate_ign_overlay(
d, b, v, r,
layer='ORTHOIMAGERY.ORTHOPHOTOS',
@ -108,7 +106,7 @@ VIZ_STEPS = [
class LidarArchaeoPipeline:
"""Orchestrates the LiDAR archaeological analysis pipeline."""
def __init__(self, input_dir, output_dir, resolution=0.5, workers=1, force=False, ground_method='auto', force_classify=False, keep_tif=False, quality=85, only_viz=None, skip_viz=None):
def __init__(self, input_dir, output_dir, resolution=0.5, workers=1, force=False, ground_method='auto', force_classify=False, keep_tif=False, quality=98, only_viz=None, skip_viz=None, output_format='avif'):
self.input_dir = Path(input_dir)
self.output_dir = Path(output_dir)
# Accept single float or comma-separated string for multi-resolution
@ -127,6 +125,7 @@ class LidarArchaeoPipeline:
self.quality = quality
self.only_viz = only_viz
self.skip_viz = skip_viz
self.output_format = output_format
self.temp_dir = self.output_dir / "temp"
if not self.input_dir.exists():
@ -137,9 +136,8 @@ class LidarArchaeoPipeline:
self.dtm_dir = self.output_dir / "DTM"
self.vis_dir = self.output_dir / "visualisations"
self.pdf_dir = self.output_dir / "rapports"
for d in [self.dtm_dir, self.vis_dir, self.pdf_dir]:
for d in [self.dtm_dir, self.vis_dir]:
d.mkdir(exist_ok=True)
# Filter visualizations based on --only / --skip
@ -169,7 +167,7 @@ class LidarArchaeoPipeline:
logger.info(f" Classification sol : {self.ground_method}")
logger.info(f" Force classif.: {'OUI' if self.force_classify else 'non'}")
logger.info(f" Keep TIFF : {'OUI' if self.keep_tif else 'non'}")
logger.info(f" Qualité WebP: {self.quality if self.quality < 100 else 'lossless'}")
logger.info(f" Qualité {self.output_format.upper()}: {self.quality if self.quality < 100 else 'lossless'}")
if only_viz:
logger.info(f" Visualisations: uniquement {', '.join(only_viz)}")
elif skip_viz:
@ -197,18 +195,19 @@ class LidarArchaeoPipeline:
return True
@staticmethod
def _expected_webp_path(name, basename, file_vis_dir):
"""Return the expected WebP filename for a visualization step."""
def _expected_output_path(name, basename, file_vis_dir, output_format='avif'):
"""Return the expected output filename for a visualization step."""
ext = 'avif' if output_format == 'avif' else 'webp'
if name == 'pos_open':
return file_vis_dir / f"{basename}_positive_openness.webp"
return file_vis_dir / f"{basename}_positive_openness.{ext}"
elif name == 'neg_open':
return file_vis_dir / f"{basename}_negative_openness.webp"
return file_vis_dir / f"{basename}_negative_openness.{ext}"
elif name == 'hillshade':
return file_vis_dir / f"{basename}_hillshade_multi.webp"
return file_vis_dir / f"{basename}_hillshade_multi.{ext}"
else:
return file_vis_dir / f"{basename}_{name}.webp"
return file_vis_dir / f"{basename}_{name}.{ext}"
def generate_all_visualizations(self, dtm_file, basename, resolution=None):
def generate_all_visualizations(self, dtm_file, basename, resolution=None, vis_dir=None):
"""Generate all archaeological visualizations for one DTM file.
Optimisation: SharedDEM is only computed if at least one visualization
@ -219,8 +218,8 @@ class LidarArchaeoPipeline:
resolution = self.resolution
logger.info(" Génération visualisations:")
# Create per-file subdirectory
file_vis_dir = self.vis_dir / basename
# Use provided vis_dir (for multi-resolution subdirectories) or default
file_vis_dir = vis_dir if vis_dir else (self.vis_dir / basename)
file_vis_dir.mkdir(exist_ok=True)
total = len(self.viz_steps)
@ -230,7 +229,7 @@ class LidarArchaeoPipeline:
if self.force:
needs_generation[name] = True
else:
expected_webp = self._expected_webp_path(name, basename, file_vis_dir)
expected_webp = self._expected_output_path(name, basename, file_vis_dir, self.output_format)
needs_generation[name] = not expected_webp.exists()
to_generate = [n for n, needed in needs_generation.items() if needed]
@ -242,7 +241,7 @@ class LidarArchaeoPipeline:
# Still need to return results dict for PDF check
vis_results = {}
for name, func in self.viz_steps:
vis_results[name] = self._expected_webp_path(name, basename, file_vis_dir)
vis_results[name] = self._expected_output_path(name, basename, file_vis_dir, self.output_format)
return vis_results
# Phase 2: compute SharedDEM only if needed
@ -258,7 +257,7 @@ class LidarArchaeoPipeline:
for idx, (name, func) in enumerate(self.viz_steps, 1):
if not needs_generation[name]:
logger.info(f" [{idx}/{total}] {name}: déjà existant, ignoré")
vis_results[name] = self._expected_webp_path(name, basename, file_vis_dir)
vis_results[name] = self._expected_output_path(name, basename, file_vis_dir, self.output_format)
continue
# When --force, delete existing TIF to ensure clean regeneration
@ -296,8 +295,9 @@ class LidarArchaeoPipeline:
# Free GPU memory between visualizations to prevent OOM
gpu_cleanup()
# Convert to WebP (only newly generated TIFs, not skipped ones)
logger.info(" Conversion images WebP:")
# Convert to output format (only newly generated TIFs, not skipped ones)
fmt_label = self.output_format.upper()
logger.info(f" Conversion images {fmt_label}:")
source_info = {
'method': self.ground_method,
'date': datetime.now().strftime('%Y-%m-%d'),
@ -305,9 +305,9 @@ class LidarArchaeoPipeline:
}
for name, tif_file in vis_results.items():
if tif_file and isinstance(tif_file, Path) and tif_file.suffix == '.tif' and tif_file.exists():
webp_file = tif_to_png(tif_file, file_vis_dir, resolution, keep_tif=self.keep_tif, source_info=source_info, quality=self.quality)
if webp_file:
logger.info(f" ✓ {webp_file.name}")
img_file = tif_to_png(tif_file, file_vis_dir, resolution, keep_tif=self.keep_tif, source_info=source_info, quality=self.quality, output_format=self.output_format)
if img_file:
logger.info(f" ✓ {img_file.name}")
# Clean up remaining TIF files unless --keep-tif
if not self.keep_tif:
@ -411,17 +411,15 @@ class LidarArchaeoPipeline:
if len(self.resolutions) > 1:
logger.info(f" --- Résolution {res}m/px ---")
# For additional resolutions, use suffixed subdirectory and basename
# For additional resolutions, use suffixed subdirectory
if res_suffix:
vis_dir = self.vis_dir / f"{basename}{res_suffix}"
pdf_basename = f"{basename}{res_suffix}"
else:
vis_dir = self.vis_dir / basename
pdf_basename = basename
vis_dir.mkdir(exist_ok=True)
self.generate_all_visualizations(dtm_path, basename, actual_res)
self.generate_all_visualizations(dtm_path, basename, actual_res, vis_dir=vis_dir)
t_total = time.time() - t_start
logger.info(f"✓ {basename} terminé en {t_total:.1f}s")
@ -452,29 +450,42 @@ class LidarArchaeoPipeline:
logger.info(f"Traitement parallèle avec {self.workers} workers...")
logger.info(f"Fichiers: {len(files)}")
with ProcessPoolExecutor(max_workers=self.workers) as executor:
# Pass resolutions as comma-separated string for multiprocessing serialization
resolutions_str = ','.join(str(r) for r in self.resolutions)
future_to_file = {
executor.submit(_process_file_standalone, str(laz_file), str(self.input_dir), str(self.output_dir), self.resolution, self.force, self.ground_method, self.force_classify, self.keep_tif, self.quality, self.only_viz, self.skip_viz): laz_file
executor.submit(_process_file_standalone, str(laz_file), str(self.input_dir), str(self.output_dir), resolutions_str, self.force, self.ground_method, self.force_classify, self.keep_tif, self.quality, self.only_viz, self.skip_viz, self.output_format): laz_file
for laz_file in files
}
done = 0
for future in as_completed(future_to_file):
laz_file = future_to_file[future]
done += 1
try:
success = future.result()
results[laz_file.name] = success
status = "✓" if success else "✗"
logger.info(f" [{done}/{len(files)}] {status} {laz_file.name}")
except Exception as e:
logger.error(f" [{done}/{len(files)}] ✗ {laz_file.name}: {e}")
logger.debug(f" Traceback:", exc_info=True)
results[laz_file.name] = False
try:
for future in as_completed(future_to_file):
laz_file = future_to_file[future]
done += 1
try:
success = future.result()
results[laz_file.name] = success
status = "✓" if success else "✗"
logger.info(f" [{done}/{len(files)}] {status} {laz_file.name}")
except Exception as e:
logger.error(f" [{done}/{len(files)}] ✗ {laz_file.name}: {e}")
logger.debug(f" Traceback:", exc_info=True)
results[laz_file.name] = False
except KeyboardInterrupt:
logger.info("Interruption — annulation des travaux en cours...")
for f in future_to_file:
f.cancel()
executor.shutdown(wait=False, cancel_futures=True)
logger.info("Travaux annulés.")
return
else:
total = len(files)
for idx, laz_file in enumerate(files, 1):
logger.info(f"--- Fichier {idx}/{total} ---")
try:
results[laz_file.name] = self.process_file(laz_file)
except KeyboardInterrupt:
logger.info("Interruption — arrêt immédiat.")
return
except Exception as e:
logger.error(f"✗ Erreur traitement {laz_file.name}: {e}")
logger.debug("Traceback:", exc_info=True)
@ -500,7 +511,6 @@ class LidarArchaeoPipeline:
logger.info(f"\nRésultats dans: {self.output_dir}")
logger.info(f" • DTM : {self.dtm_dir}")
logger.info(f" • Visualisations: {self.vis_dir}")
logger.info(f" • Rapports PDF : {self.pdf_dir}")
# Clean up temporary files
logger.info("Nettoyage des fichiers temporaires...")
@ -516,7 +526,7 @@ class LidarArchaeoPipeline:
logger.warning(f" Note: Impossible de supprimer les fichiers temporaires: {e}")
def _process_file_standalone(laz_file_str, input_dir, output_dir, resolution, force=False, ground_method='auto', force_classify=False, keep_tif=False, quality=85, only_viz=None, skip_viz=None):
def _process_file_standalone(laz_file_str, input_dir, output_dir, resolution, force=False, ground_method='auto', force_classify=False, keep_tif=False, quality=98, only_viz=None, skip_viz=None, output_format='avif'):
"""Standalone function for multiprocessing — creates its own pipeline instance.
Each worker gets its own temp directory to avoid file conflicts.
@ -537,7 +547,7 @@ def _process_file_standalone(laz_file_str, input_dir, output_dir, resolution, fo
worker_logger.addHandler(handler)
worker_logger.addFilter(_file_filter)
pipeline = LidarArchaeoPipeline(input_dir, output_dir, resolution=resolution, workers=1, force=force, ground_method=ground_method, force_classify=force_classify, keep_tif=keep_tif, quality=quality, only_viz=only_viz, skip_viz=skip_viz)
pipeline = LidarArchaeoPipeline(input_dir, output_dir, resolution=resolution, workers=1, force=force, ground_method=ground_method, force_classify=force_classify, keep_tif=keep_tif, quality=quality, only_viz=only_viz, skip_viz=skip_viz, output_format=output_format)
basename = _file_basename(laz_file_str)
pipeline.temp_dir = pipeline.output_dir / "temp" / basename
pipeline.temp_dir.mkdir(exist_ok=True)