Fix bugs and improve pipeline flexibility
- Fix gpu_cleanup import missing in visualizations.py (NameError in workers) - Fix t_pdf referenced before assignment when PDF is skipped - Skip classification+DTM when DTM exists regardless of --force - --force now only regenerates WebP/PDF, not classification/DTM - --force-classification forces reclassification when needed - Add laspy repair fallback for corrupt LAZ files (EVLR errors) - Keep DTM TIF by default for reuse (--no-keep-tif to delete) - Increase space between image and bottom cartouche (0.12→0.19) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
@ -17,7 +17,8 @@ All commands run inside Docker. Use `./run.sh` as the primary interface.
|
||||
./run.sh --test # Run unit tests
|
||||
./run.sh -g --file LHD_FXX_1000_6882_PTS_LAMB93_IGN69.copc # Single file
|
||||
./run.sh --ground-classification pmf # Force PMF ground classification
|
||||
./run.sh -g --keep-tif # Keep intermediate TIFF files
|
||||
./run.sh -g --keep-tif # Keep intermediate TIFF files (default: kept)
|
||||
./run.sh -g --no-keep-tif # Delete intermediate TIFF files
|
||||
./run.sh # Print help (no args)
|
||||
```
|
||||
|
||||
@ -82,6 +83,6 @@ Uses `ProcessPoolExecutor` with `'spawn'` start method (required for CUDA). Each
|
||||
- **Language**: UI messages and comments in French. Code identifiers in English.
|
||||
- **Logging**: Use `logger = logging.getLogger("lidar")`. Prefix per-file logs via `_file_filter.basename`.
|
||||
- **GPU pattern**: `arr_gpu = to_gpu(arr)` → compute → `result = to_cpu(arr_gpu)` → `gpu_cleanup()` between visualizations.
|
||||
- **Output format**: Visualizations saved as WebP. TIFF intermediates deleted after conversion unless `--keep-tif`. No COGs or viewer — only WebP + PDF report remain.
|
||||
- **Output format**: Visualizations saved as WebP. DTM TIFF kept by default for reuse (use `--no-keep-tif` to delete). `--force` regenerates WebPs without re-classifying if DTM exists. No COGs or viewer — only WebP + PDF report remain.
|
||||
- **Compression**: TIF intermediates use `deflate` compression (faster than LZW for float32 data).
|
||||
- **Tests**: Run only inside Docker via `./run.sh --test`. Synthetic DEM fixture in `tests/conftest.py`.
|
||||
@ -118,9 +118,9 @@ Exemples:
|
||||
help="Reclassifier le sol même si le fichier .las existe déjà"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keep-tif",
|
||||
"--no-keep-tif",
|
||||
action="store_true",
|
||||
help="Conserver les fichiers TIFF intermédiaires (sinon supprimés après conversion WebP)"
|
||||
help="Supprimer les fichiers TIFF intermédiaires après conversion WebP (par défaut: conservés)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ground-classification",
|
||||
@ -176,7 +176,7 @@ Exemples:
|
||||
force=args.force,
|
||||
ground_method=args.ground_classification,
|
||||
force_classify=args.force_classification,
|
||||
keep_tif=args.keep_tif,
|
||||
keep_tif=not args.no_keep_tif,
|
||||
)
|
||||
|
||||
# If --file is specified, process only matching files
|
||||
|
||||
@ -246,10 +246,57 @@ def classify_ground(laz_file, temp_dir, method='auto', force=False):
|
||||
logger.info(f" ✓ Classification sol {method.upper()} terminée")
|
||||
return output_las
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.error(f" ✗ Erreur classification PDAL ({method.upper()}): {e.stderr.decode()}")
|
||||
error_msg = e.stderr.decode() if e.stderr else str(e)
|
||||
logger.warning(f" ✗ Erreur classification PDAL ({method.upper()}): {error_msg}")
|
||||
|
||||
# Try repairing file with laspy if PDAL fails on EVLR/VLR
|
||||
if 'VLR' in error_msg or 'Invalid' in error_msg:
|
||||
logger.info(f" → Tentative de réparation du fichier avec laspy...")
|
||||
repaired_las = temp_dir / f"{laz_base}_repaired.las"
|
||||
if _repair_laz_with_laspy(laz_file, repaired_las):
|
||||
# Retry PDAL pipeline with repaired file
|
||||
pipeline_json = _create_ground_pipeline(repaired_las, output_las, method)
|
||||
with open(pipeline_file, 'w') as f:
|
||||
f.write(pipeline_json)
|
||||
try:
|
||||
subprocess.run(
|
||||
["pdal", "pipeline", str(pipeline_file)],
|
||||
capture_output=True, check=True
|
||||
)
|
||||
logger.info(f" ✓ Classification sol {method.upper()} terminée (fichier réparé)")
|
||||
return output_las
|
||||
except subprocess.CalledProcessError as e2:
|
||||
error_msg2 = e2.stderr.decode() if e2.stderr else str(e2)
|
||||
logger.error(f" ✗ Échec classification même après réparation: {error_msg2}")
|
||||
else:
|
||||
logger.error(f" ✗ Impossible de réparer le fichier")
|
||||
return None
|
||||
|
||||
|
||||
def _repair_laz_with_laspy(input_laz, output_las):
|
||||
"""Try to repair a corrupt LAZ file by re-reading with laspy and saving as LAS.
|
||||
|
||||
Works around PDAL errors like 'Invalid Extended VLR size' by stripping
|
||||
problematic VLR/EVLR metadata during re-save.
|
||||
|
||||
Args:
|
||||
input_laz: Path to corrupt LAZ/LAS file.
|
||||
output_las: Path for repaired LAS output.
|
||||
|
||||
Returns:
|
||||
True if repair succeeded, False otherwise.
|
||||
"""
|
||||
import laspy
|
||||
try:
|
||||
las = laspy.read(str(input_laz))
|
||||
las.write(str(output_las))
|
||||
logger.info(f" ✓ Fichier réparé via laspy ({len(las.points):,} points)")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f" ✗ Réparation laspy échouée: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def create_dtm_fast(las_file, basename, dtm_dir, resolution, force=False):
|
||||
"""Create DTM using fast binning method with gap filling.
|
||||
|
||||
|
||||
@ -275,9 +275,11 @@ class LidarArchaeoPipeline:
|
||||
logger.info(f"FICHIER : {basename}")
|
||||
logger.info("=" * 60)
|
||||
|
||||
# Skip ground classification + DTM if DTM already exists (unless --force)
|
||||
# Skip ground classification + DTM if DTM already exists
|
||||
# --force only affects visualizations/PDF, not classification/DTM
|
||||
# Use --force-classification to force reclassification
|
||||
dtm_path = self.dtm_dir / f"{basename}_dtm.tif"
|
||||
if dtm_path.exists() and not self.force:
|
||||
if dtm_path.exists():
|
||||
logger.info("[1/5] Classification du sol — sautée (DTM existant)")
|
||||
logger.info("[2/5] Génération DTM — sautée (DTM existant)")
|
||||
dtm_file = dtm_path
|
||||
@ -297,7 +299,7 @@ class LidarArchaeoPipeline:
|
||||
# Step 2: Generate DTM
|
||||
logger.info("[2/5] Génération DTM...")
|
||||
t2 = time.time()
|
||||
dtm_file = create_dtm_fast(las_file, basename, self.dtm_dir, self.resolution, force=self.force)
|
||||
dtm_file = create_dtm_fast(las_file, basename, self.dtm_dir, self.resolution)
|
||||
t_dtm = time.time() - t2
|
||||
if not dtm_file:
|
||||
logger.error(f" ✗ Échec DTM ({t_dtm:.1f}s)")
|
||||
@ -309,6 +311,7 @@ class LidarArchaeoPipeline:
|
||||
self.generate_all_visualizations(dtm_file, basename)
|
||||
|
||||
# Step 4: PDF report
|
||||
t_pdf = 0
|
||||
file_vis_dir = self.vis_dir / basename
|
||||
pdf_file = self.pdf_dir / f"{basename}_rapport.pdf"
|
||||
if pdf_file.exists() and not self.force:
|
||||
@ -320,10 +323,8 @@ class LidarArchaeoPipeline:
|
||||
t_pdf = time.time() - t4
|
||||
logger.info(f" ✓ Rapport PDF terminé ({t_pdf:.1f}s)")
|
||||
|
||||
# Step 5: Clean up DTM TIF unless --keep-tif
|
||||
if not self.keep_tif:
|
||||
for dtm_file in sorted(self.dtm_dir.glob(f"{basename}_dtm.tif")):
|
||||
dtm_file.unlink(missing_ok=True)
|
||||
# Step 5: Keep DTM TIF for reuse (regenerating WebPs skips classification)
|
||||
# Use --no-keep-tif to delete DTM files after processing
|
||||
|
||||
t_total = time.time() - t_start
|
||||
logger.info(f"✓ {basename} terminé en {t_total:.1f}s")
|
||||
|
||||
@ -387,9 +387,9 @@ def tif_to_png(tif_file, vis_dir, resolution, keep_tif=False, source_info=None):
|
||||
# Fixed data area position — identical for ALL visualization types
|
||||
# This ensures overlay/superposition works across all WebP images
|
||||
data_left = 0.08
|
||||
data_bottom = 0.12
|
||||
data_bottom = 0.19
|
||||
data_width_frac = 0.74
|
||||
data_height_frac = 0.78
|
||||
data_height_frac = 0.71
|
||||
|
||||
ax = fig.add_axes([data_left, data_bottom, data_width_frac, data_height_frac])
|
||||
if is_rgba or is_rgb:
|
||||
|
||||
@ -18,7 +18,7 @@ import rasterio
|
||||
from scipy.ndimage import generic_filter
|
||||
from scipy.stats import binned_statistic_2d
|
||||
|
||||
from .gpu import HAS_GPU, to_gpu, to_cpu, xp_gaussian_filter, xp_uniform_filter, xp_minimum_filter
|
||||
from .gpu import HAS_GPU, to_gpu, to_cpu, xp_gaussian_filter, xp_uniform_filter, xp_minimum_filter, gpu_cleanup
|
||||
|
||||
logger = logging.getLogger("lidar")
|
||||
|
||||
|
||||
18
run.sh
18
run.sh
@ -9,7 +9,7 @@
|
||||
# -v Mode verbeux (timestamps + niveaux)
|
||||
# --debug Mode debug (détails internes fichier:ligne)
|
||||
# -f / --force Régénérer tous les fichiers même si existants
|
||||
# --keep-tif Conserver les fichiers TIFF intermédiaires
|
||||
# --no-keep-tif Supprimer les TIFF intermédiaires (par défaut: conservés)
|
||||
# --force-classification
|
||||
# Reclassifier le sol même si le fichier .las existe déjà
|
||||
# --ground-classification {auto,smrf,pmf,csf}
|
||||
@ -40,7 +40,7 @@ if [ $# -eq 0 ]; then
|
||||
echo " -f / --force Régénérer tous les fichiers même si les WebP existent"
|
||||
echo " --force-classification"
|
||||
echo " Reclassifier le sol même si le fichier .las existe"
|
||||
echo " --keep-tif Conserver les fichiers TIFF intermédiaires"
|
||||
echo " --no-keep-tif Supprimer les TIFF intermédiaires (défaut: conservés)"
|
||||
echo " --ground-classification {auto,smrf,pmf,csf}"
|
||||
echo " Méthode de classification du sol (défaut: auto)"
|
||||
echo " --file NOM... Traiter un ou plusieurs fichiers LAZ (nom complet sans .laz/.las)"
|
||||
@ -52,7 +52,7 @@ if [ $# -eq 0 ]; then
|
||||
echo " $0 -g -w 4 # GPU + 4 workers"
|
||||
echo " $0 -g -v # GPU + verbeux"
|
||||
echo " $0 -g -r 0.2 # Haute résolution"
|
||||
echo " $0 -g --force # Tout régénérer (WebP + classification)"
|
||||
echo " $0 -g --force # Régénérer les WebP"
|
||||
echo " $0 -g --force-classification # Reclassifier le sol seulement"
|
||||
echo " $0 -g --ground-classification pmf # Forcer PMF"
|
||||
echo " $0 -g --file LHD_...IGN69.copc # Un fichier"
|
||||
@ -67,7 +67,7 @@ FORCE_FLAG=""
|
||||
FILE_ARGS=""
|
||||
GROUND_METHOD=""
|
||||
FORCE_CLASSIFY_FLAG=""
|
||||
KEEP_TIF_FLAG=""
|
||||
NO_KEEP_TIF_FLAG=""
|
||||
|
||||
# Parse arguments manually (more robust than getopts for mixed short/long options)
|
||||
while [ $# -gt 0 ]; do
|
||||
@ -80,7 +80,7 @@ while [ $# -gt 0 ]; do
|
||||
--debug) VERBOSE_FLAG="--debug"; shift ;;
|
||||
--force) FORCE_FLAG="--force"; shift ;;
|
||||
--force-classification) FORCE_CLASSIFY_FLAG="--force-classification"; shift ;;
|
||||
--keep-tif) KEEP_TIF_FLAG="--keep-tif"; shift ;;
|
||||
--no-keep-tif) NO_KEEP_TIF_FLAG="--no-keep-tif"; shift ;;
|
||||
--ground-classification) GROUND_METHOD="$2"; shift 2 ;;
|
||||
--ground-classification=*) GROUND_METHOD="${1#--ground-classification=}"; shift ;;
|
||||
--file) shift; while [ $# -gt 0 ] && [[ ! "$1" =~ ^- ]]; do FILE_ARGS="$FILE_ARGS $1"; shift; done ;;
|
||||
@ -99,7 +99,7 @@ while [ $# -gt 0 ]; do
|
||||
echo " -f / --force Régénérer tous les fichiers même si les WebP existent"
|
||||
echo " --force-classification"
|
||||
echo " Reclassifier le sol même si le fichier .las existe"
|
||||
echo " --keep-tif Conserver les fichiers TIFF intermédiaires"
|
||||
echo " --no-keep-tif Supprimer les TIFF intermédiaires (défaut: conservés)"
|
||||
echo " --ground-classification {auto,smrf,pmf,csf}"
|
||||
echo " Méthode de classification du sol (défaut: auto)"
|
||||
echo " --file NOM... Traiter un ou plusieurs fichiers LAZ (nom complet sans .laz/.las)"
|
||||
@ -111,7 +111,7 @@ while [ $# -gt 0 ]; do
|
||||
echo " $0 -g -w 4 # GPU + 4 workers"
|
||||
echo " $0 -g -v # GPU + verbeux"
|
||||
echo " $0 -g -r 0.2 # Haute résolution"
|
||||
echo " $0 -g --force # Tout régénérer (WebP + classification)"
|
||||
echo " $0 -g --force # Régénérer les WebP"
|
||||
echo " $0 -g --force-classification # Reclassifier le sol seulement"
|
||||
echo " $0 -g --ground-classification pmf # Forcer PMF"
|
||||
echo " $0 -g --file LHD_...IGN69.copc # Un fichier"
|
||||
@ -159,14 +159,14 @@ echo " GPU : $([ -n "$GPU_FLAG" ] && echo 'OUI' || echo 'non')"
|
||||
echo " Verbeux : $([ -n "$VERBOSE_FLAG" ] && echo 'OUI' || echo 'non')"
|
||||
echo " Force : $([ -n "$FORCE_FLAG" ] && echo 'OUI' || echo 'non')"
|
||||
echo " Force classif.: $([ -n "$FORCE_CLASSIFY_FLAG" ] && echo 'OUI' || echo 'non')"
|
||||
echo " Keep TIFF : $([ -n "$KEEP_TIF_FLAG" ] && echo 'OUI' || echo 'non')"
|
||||
echo " Keep TIFF : $([ -n "$NO_KEEP_TIF_FLAG" ] && echo 'non' || echo 'OUI')"
|
||||
echo " Classification sol : $([ -n "$GROUND_METHOD" ] && echo "$GROUND_METHOD" || echo 'auto')"
|
||||
if [ -n "$FILE_ARGS" ]; then
|
||||
echo " Fichiers :${FILE_ARGS}"
|
||||
fi
|
||||
echo "============================================"
|
||||
|
||||
CMD_ARGS="-o /data/output -r $RESOLUTION -w $WORKERS $VERBOSE_FLAG $FORCE_FLAG $FORCE_CLASSIFY_FLAG $KEEP_TIF_FLAG"
|
||||
CMD_ARGS="-o /data/output -r $RESOLUTION -w $WORKERS $VERBOSE_FLAG $FORCE_FLAG $FORCE_CLASSIFY_FLAG $NO_KEEP_TIF_FLAG"
|
||||
if [ -n "$GROUND_METHOD" ]; then
|
||||
CMD_ARGS="$CMD_ARGS --ground-classification $GROUND_METHOD"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user