From eac482874dd3b4e1bc9fa7e5e2f2df98c38d7003 Mon Sep 17 00:00:00 2001 From: Jacquin Antoine Date: Thu, 14 May 2026 00:08:25 +0200 Subject: [PATCH] Fix bugs and improve pipeline flexibility MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix gpu_cleanup import missing in visualizations.py (NameError in workers) - Fix t_pdf referenced before assignment when PDF is skipped - Skip classification+DTM when DTM exists regardless of --force - --force now only regenerates WebP/PDF, not classification/DTM - --force-classification forces reclassification when needed - Add laspy repair fallback for corrupt LAZ files (EVLR errors) - Keep DTM TIF by default for reuse (--no-keep-tif to delete) - Increase space between image and bottom cartouche (0.12→0.19) Co-Authored-By: Claude Opus 4.6 --- CLAUDE.md | 5 ++-- lidar_pipeline/cli.py | 6 ++-- lidar_pipeline/dtm.py | 49 +++++++++++++++++++++++++++++++- lidar_pipeline/pipeline.py | 15 +++++----- lidar_pipeline/rendering.py | 4 +-- lidar_pipeline/visualizations.py | 2 +- run.sh | 18 ++++++------ 7 files changed, 74 insertions(+), 25 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 2a58a9f..0a19c45 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -17,7 +17,8 @@ All commands run inside Docker. Use `./run.sh` as the primary interface. ./run.sh --test # Run unit tests ./run.sh -g --file LHD_FXX_1000_6882_PTS_LAMB93_IGN69.copc # Single file ./run.sh --ground-classification pmf # Force PMF ground classification -./run.sh -g --keep-tif # Keep intermediate TIFF files +./run.sh -g --keep-tif # Keep intermediate TIFF files (default: kept) +./run.sh -g --no-keep-tif # Delete intermediate TIFF files ./run.sh # Print help (no args) ``` @@ -82,6 +83,6 @@ Uses `ProcessPoolExecutor` with `'spawn'` start method (required for CUDA). Each - **Language**: UI messages and comments in French. Code identifiers in English. - **Logging**: Use `logger = logging.getLogger("lidar")`. Prefix per-file logs via `_file_filter.basename`. - **GPU pattern**: `arr_gpu = to_gpu(arr)` → compute → `result = to_cpu(arr_gpu)` → `gpu_cleanup()` between visualizations. -- **Output format**: Visualizations saved as WebP. TIFF intermediates deleted after conversion unless `--keep-tif`. No COGs or viewer — only WebP + PDF report remain. +- **Output format**: Visualizations saved as WebP. DTM TIFF kept by default for reuse (use `--no-keep-tif` to delete). `--force` regenerates WebPs without re-classifying if DTM exists. No COGs or viewer — only WebP + PDF report remain. - **Compression**: TIF intermediates use `deflate` compression (faster than LZW for float32 data). - **Tests**: Run only inside Docker via `./run.sh --test`. Synthetic DEM fixture in `tests/conftest.py`. \ No newline at end of file diff --git a/lidar_pipeline/cli.py b/lidar_pipeline/cli.py index 9b1578a..66800dd 100644 --- a/lidar_pipeline/cli.py +++ b/lidar_pipeline/cli.py @@ -118,9 +118,9 @@ Exemples: help="Reclassifier le sol même si le fichier .las existe déjà" ) parser.add_argument( - "--keep-tif", + "--no-keep-tif", action="store_true", - help="Conserver les fichiers TIFF intermédiaires (sinon supprimés après conversion WebP)" + help="Supprimer les fichiers TIFF intermédiaires après conversion WebP (par défaut: conservés)" ) parser.add_argument( "--ground-classification", @@ -176,7 +176,7 @@ Exemples: force=args.force, ground_method=args.ground_classification, force_classify=args.force_classification, - keep_tif=args.keep_tif, + keep_tif=not args.no_keep_tif, ) # If --file is specified, process only matching files diff --git a/lidar_pipeline/dtm.py b/lidar_pipeline/dtm.py index 3038b0d..533f349 100644 --- a/lidar_pipeline/dtm.py +++ b/lidar_pipeline/dtm.py @@ -246,10 +246,57 @@ def classify_ground(laz_file, temp_dir, method='auto', force=False): logger.info(f" ✓ Classification sol {method.upper()} terminée") return output_las except subprocess.CalledProcessError as e: - logger.error(f" ✗ Erreur classification PDAL ({method.upper()}): {e.stderr.decode()}") + error_msg = e.stderr.decode() if e.stderr else str(e) + logger.warning(f" ✗ Erreur classification PDAL ({method.upper()}): {error_msg}") + + # Try repairing file with laspy if PDAL fails on EVLR/VLR + if 'VLR' in error_msg or 'Invalid' in error_msg: + logger.info(f" → Tentative de réparation du fichier avec laspy...") + repaired_las = temp_dir / f"{laz_base}_repaired.las" + if _repair_laz_with_laspy(laz_file, repaired_las): + # Retry PDAL pipeline with repaired file + pipeline_json = _create_ground_pipeline(repaired_las, output_las, method) + with open(pipeline_file, 'w') as f: + f.write(pipeline_json) + try: + subprocess.run( + ["pdal", "pipeline", str(pipeline_file)], + capture_output=True, check=True + ) + logger.info(f" ✓ Classification sol {method.upper()} terminée (fichier réparé)") + return output_las + except subprocess.CalledProcessError as e2: + error_msg2 = e2.stderr.decode() if e2.stderr else str(e2) + logger.error(f" ✗ Échec classification même après réparation: {error_msg2}") + else: + logger.error(f" ✗ Impossible de réparer le fichier") return None +def _repair_laz_with_laspy(input_laz, output_las): + """Try to repair a corrupt LAZ file by re-reading with laspy and saving as LAS. + + Works around PDAL errors like 'Invalid Extended VLR size' by stripping + problematic VLR/EVLR metadata during re-save. + + Args: + input_laz: Path to corrupt LAZ/LAS file. + output_las: Path for repaired LAS output. + + Returns: + True if repair succeeded, False otherwise. + """ + import laspy + try: + las = laspy.read(str(input_laz)) + las.write(str(output_las)) + logger.info(f" ✓ Fichier réparé via laspy ({len(las.points):,} points)") + return True + except Exception as e: + logger.warning(f" ✗ Réparation laspy échouée: {e}") + return False + + def create_dtm_fast(las_file, basename, dtm_dir, resolution, force=False): """Create DTM using fast binning method with gap filling. diff --git a/lidar_pipeline/pipeline.py b/lidar_pipeline/pipeline.py index 9373a1f..82d462f 100644 --- a/lidar_pipeline/pipeline.py +++ b/lidar_pipeline/pipeline.py @@ -275,9 +275,11 @@ class LidarArchaeoPipeline: logger.info(f"FICHIER : {basename}") logger.info("=" * 60) - # Skip ground classification + DTM if DTM already exists (unless --force) + # Skip ground classification + DTM if DTM already exists + # --force only affects visualizations/PDF, not classification/DTM + # Use --force-classification to force reclassification dtm_path = self.dtm_dir / f"{basename}_dtm.tif" - if dtm_path.exists() and not self.force: + if dtm_path.exists(): logger.info("[1/5] Classification du sol — sautée (DTM existant)") logger.info("[2/5] Génération DTM — sautée (DTM existant)") dtm_file = dtm_path @@ -297,7 +299,7 @@ class LidarArchaeoPipeline: # Step 2: Generate DTM logger.info("[2/5] Génération DTM...") t2 = time.time() - dtm_file = create_dtm_fast(las_file, basename, self.dtm_dir, self.resolution, force=self.force) + dtm_file = create_dtm_fast(las_file, basename, self.dtm_dir, self.resolution) t_dtm = time.time() - t2 if not dtm_file: logger.error(f" ✗ Échec DTM ({t_dtm:.1f}s)") @@ -309,6 +311,7 @@ class LidarArchaeoPipeline: self.generate_all_visualizations(dtm_file, basename) # Step 4: PDF report + t_pdf = 0 file_vis_dir = self.vis_dir / basename pdf_file = self.pdf_dir / f"{basename}_rapport.pdf" if pdf_file.exists() and not self.force: @@ -320,10 +323,8 @@ class LidarArchaeoPipeline: t_pdf = time.time() - t4 logger.info(f" ✓ Rapport PDF terminé ({t_pdf:.1f}s)") - # Step 5: Clean up DTM TIF unless --keep-tif - if not self.keep_tif: - for dtm_file in sorted(self.dtm_dir.glob(f"{basename}_dtm.tif")): - dtm_file.unlink(missing_ok=True) + # Step 5: Keep DTM TIF for reuse (regenerating WebPs skips classification) + # Use --no-keep-tif to delete DTM files after processing t_total = time.time() - t_start logger.info(f"✓ {basename} terminé en {t_total:.1f}s") diff --git a/lidar_pipeline/rendering.py b/lidar_pipeline/rendering.py index d4bed61..0c6077d 100644 --- a/lidar_pipeline/rendering.py +++ b/lidar_pipeline/rendering.py @@ -387,9 +387,9 @@ def tif_to_png(tif_file, vis_dir, resolution, keep_tif=False, source_info=None): # Fixed data area position — identical for ALL visualization types # This ensures overlay/superposition works across all WebP images data_left = 0.08 - data_bottom = 0.12 + data_bottom = 0.19 data_width_frac = 0.74 - data_height_frac = 0.78 + data_height_frac = 0.71 ax = fig.add_axes([data_left, data_bottom, data_width_frac, data_height_frac]) if is_rgba or is_rgb: diff --git a/lidar_pipeline/visualizations.py b/lidar_pipeline/visualizations.py index bda618e..a4121a0 100644 --- a/lidar_pipeline/visualizations.py +++ b/lidar_pipeline/visualizations.py @@ -18,7 +18,7 @@ import rasterio from scipy.ndimage import generic_filter from scipy.stats import binned_statistic_2d -from .gpu import HAS_GPU, to_gpu, to_cpu, xp_gaussian_filter, xp_uniform_filter, xp_minimum_filter +from .gpu import HAS_GPU, to_gpu, to_cpu, xp_gaussian_filter, xp_uniform_filter, xp_minimum_filter, gpu_cleanup logger = logging.getLogger("lidar") diff --git a/run.sh b/run.sh index 1609c2d..dbd166f 100755 --- a/run.sh +++ b/run.sh @@ -9,7 +9,7 @@ # -v Mode verbeux (timestamps + niveaux) # --debug Mode debug (détails internes fichier:ligne) # -f / --force Régénérer tous les fichiers même si existants -# --keep-tif Conserver les fichiers TIFF intermédiaires +# --no-keep-tif Supprimer les TIFF intermédiaires (par défaut: conservés) # --force-classification # Reclassifier le sol même si le fichier .las existe déjà # --ground-classification {auto,smrf,pmf,csf} @@ -40,7 +40,7 @@ if [ $# -eq 0 ]; then echo " -f / --force Régénérer tous les fichiers même si les WebP existent" echo " --force-classification" echo " Reclassifier le sol même si le fichier .las existe" - echo " --keep-tif Conserver les fichiers TIFF intermédiaires" + echo " --no-keep-tif Supprimer les TIFF intermédiaires (défaut: conservés)" echo " --ground-classification {auto,smrf,pmf,csf}" echo " Méthode de classification du sol (défaut: auto)" echo " --file NOM... Traiter un ou plusieurs fichiers LAZ (nom complet sans .laz/.las)" @@ -52,7 +52,7 @@ if [ $# -eq 0 ]; then echo " $0 -g -w 4 # GPU + 4 workers" echo " $0 -g -v # GPU + verbeux" echo " $0 -g -r 0.2 # Haute résolution" - echo " $0 -g --force # Tout régénérer (WebP + classification)" + echo " $0 -g --force # Régénérer les WebP" echo " $0 -g --force-classification # Reclassifier le sol seulement" echo " $0 -g --ground-classification pmf # Forcer PMF" echo " $0 -g --file LHD_...IGN69.copc # Un fichier" @@ -67,7 +67,7 @@ FORCE_FLAG="" FILE_ARGS="" GROUND_METHOD="" FORCE_CLASSIFY_FLAG="" -KEEP_TIF_FLAG="" +NO_KEEP_TIF_FLAG="" # Parse arguments manually (more robust than getopts for mixed short/long options) while [ $# -gt 0 ]; do @@ -80,7 +80,7 @@ while [ $# -gt 0 ]; do --debug) VERBOSE_FLAG="--debug"; shift ;; --force) FORCE_FLAG="--force"; shift ;; --force-classification) FORCE_CLASSIFY_FLAG="--force-classification"; shift ;; - --keep-tif) KEEP_TIF_FLAG="--keep-tif"; shift ;; + --no-keep-tif) NO_KEEP_TIF_FLAG="--no-keep-tif"; shift ;; --ground-classification) GROUND_METHOD="$2"; shift 2 ;; --ground-classification=*) GROUND_METHOD="${1#--ground-classification=}"; shift ;; --file) shift; while [ $# -gt 0 ] && [[ ! "$1" =~ ^- ]]; do FILE_ARGS="$FILE_ARGS $1"; shift; done ;; @@ -99,7 +99,7 @@ while [ $# -gt 0 ]; do echo " -f / --force Régénérer tous les fichiers même si les WebP existent" echo " --force-classification" echo " Reclassifier le sol même si le fichier .las existe" - echo " --keep-tif Conserver les fichiers TIFF intermédiaires" + echo " --no-keep-tif Supprimer les TIFF intermédiaires (défaut: conservés)" echo " --ground-classification {auto,smrf,pmf,csf}" echo " Méthode de classification du sol (défaut: auto)" echo " --file NOM... Traiter un ou plusieurs fichiers LAZ (nom complet sans .laz/.las)" @@ -111,7 +111,7 @@ while [ $# -gt 0 ]; do echo " $0 -g -w 4 # GPU + 4 workers" echo " $0 -g -v # GPU + verbeux" echo " $0 -g -r 0.2 # Haute résolution" - echo " $0 -g --force # Tout régénérer (WebP + classification)" + echo " $0 -g --force # Régénérer les WebP" echo " $0 -g --force-classification # Reclassifier le sol seulement" echo " $0 -g --ground-classification pmf # Forcer PMF" echo " $0 -g --file LHD_...IGN69.copc # Un fichier" @@ -159,14 +159,14 @@ echo " GPU : $([ -n "$GPU_FLAG" ] && echo 'OUI' || echo 'non')" echo " Verbeux : $([ -n "$VERBOSE_FLAG" ] && echo 'OUI' || echo 'non')" echo " Force : $([ -n "$FORCE_FLAG" ] && echo 'OUI' || echo 'non')" echo " Force classif.: $([ -n "$FORCE_CLASSIFY_FLAG" ] && echo 'OUI' || echo 'non')" -echo " Keep TIFF : $([ -n "$KEEP_TIF_FLAG" ] && echo 'OUI' || echo 'non')" +echo " Keep TIFF : $([ -n "$NO_KEEP_TIF_FLAG" ] && echo 'non' || echo 'OUI')" echo " Classification sol : $([ -n "$GROUND_METHOD" ] && echo "$GROUND_METHOD" || echo 'auto')" if [ -n "$FILE_ARGS" ]; then echo " Fichiers :${FILE_ARGS}" fi echo "============================================" -CMD_ARGS="-o /data/output -r $RESOLUTION -w $WORKERS $VERBOSE_FLAG $FORCE_FLAG $FORCE_CLASSIFY_FLAG $KEEP_TIF_FLAG" +CMD_ARGS="-o /data/output -r $RESOLUTION -w $WORKERS $VERBOSE_FLAG $FORCE_FLAG $FORCE_CLASSIFY_FLAG $NO_KEEP_TIF_FLAG" if [ -n "$GROUND_METHOD" ]; then CMD_ARGS="$CMD_ARGS --ground-classification $GROUND_METHOD" fi