code_public/lidar_rendu
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Translate the whole project to English and fix outdated comments and help

Browse Source 
Comments, docstrings, logs, CLI help, map UI, legends, PDF sheet, scripts,
compose files and AGENTS.md are now English. Data keys stay unchanged
(relief_oriente, densite_sol, visualisations/, API JSON keys, link params).
Wrong comments and help defaults found along the way are corrected.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Antoine
2026-09-27 23:16:45 +02:00
parent cc1c22d2b8
commit fb892ea9f2
52 changed files with 4356 additions and 4323 deletions
Download Patch File Download Diff File Expand all files Collapse all files

173
lidar_pipeline/gpu.py
View File

@ -4,7 +4,8 @@ Auto-selects the best NVIDIA GPU (highest compute capability first).
Uses CuPy Device API (not CUDA_VISIBLE_DEVICES) so JIT compilation
works correctly for any architecture (sm_89, sm_120, etc.).
All workers share the selected GPU.
In the worker pool, each process is pinned to one GPU (or forced to CPU)
by gpu_worker_slots / set_active_gpu / force_cpu, bounded by free VRAM.
"""
import logging
@ -27,10 +28,10 @@ _gpu_reason = None
# GPU restriction from -g flag (host-level indices)
_restricted_gpu_ids: list[int] | None = None
# Vrai si CUDA_VISIBLE_DEVICES a été écrit par _init_gpu lui-même (choix du
# meilleur GPU). Cette valeur NE DOIT PAS être traitée comme une restriction
# externe : sinon available_gpu_ids() ne retourne plus que le GPU choisi et
# tous les workers reçoivent le même gpu_id (GPU 1 jamais utilisé).
# True if CUDA_VISIBLE_DEVICES was written by _init_gpu itself (best-GPU
# pick). That value MUST NOT be treated as an external restriction: otherwise
# available_gpu_ids() returns only the chosen GPU and every worker gets the
# same gpu_id (GPU 1 never used).
_env_set_by_init = False
# Discovered GPU candidates (populated by _pick_gpu)
@ -82,7 +83,7 @@ except Exception:
# ---------------------------------------------------------------------------
_cp = None
_cp_ndimage = None
_cupy_ndarray = None # type ref qui survit à disable_gpu() pour que to_cpu() récupère les tableaux orphelins
_cupy_ndarray = None # type ref that survives disable_gpu() so to_cpu() can still recover orphaned arrays
_gpu_initialized = False
@ -91,8 +92,8 @@ def _filter_candidates(gpus: list) -> list:
nvidia-smi lists ALL GPUs even when CUDA_VISIBLE_DEVICES is set
(driver 580.x behavior), so we must filter manually.
La variable écrite par _init_gpu (choix auto du meilleur GPU) est ignorée :
seules les restrictions externes (run.sh -g, compose) comptent.
The value written by _init_gpu (automatic best-GPU pick) is ignored:
only external restrictions (run.sh -g, compose) count.
"""
# Filter by CUDA_VISIBLE_DEVICES if set externally
cuda_visible = os.environ.get('CUDA_VISIBLE_DEVICES')
@ -112,12 +113,12 @@ def _filter_candidates(gpus: list) -> list:
def _runtime_candidates():
"""Détection GPU de repli via le runtime CuPy (sans nvidia-smi).
"""Fallback GPU detection through the CuPy runtime (no nvidia-smi).
nvidia-smi peut dépasser son timeout quand le système est chargé :
_pick_gpu() retourne alors [] et les workers passent à tort en CPU.
Les indices CuPy sont renumérotés selon CUDA_VISIBLE_DEVICES — on les
remappe en indices hôtes pour rester compatible avec _filter_candidates.
nvidia-smi can exceed its timeout on a loaded system: _pick_gpu() then
returns [] and workers wrongly fall back to CPU. CuPy indices are
renumbered according to CUDA_VISIBLE_DEVICES, so they are mapped back to
host indices to stay compatible with _filter_candidates.
"""
try:
import cupy as _cp_runtime
@ -160,10 +161,10 @@ def _init_gpu():
candidates = _filter_candidates(_gpu_candidates)
if not candidates:
# Repli runtime : nvidia-smi a échoué (timeout système chargé)
# Runtime fallback: nvidia-smi failed (timeout on a loaded system)
candidates = _filter_candidates(_runtime_candidates())
if not candidates:
logger.info("Pas de GPU utilisable — mode CPU uniquement")
logger.info("No usable GPU — CPU-only mode")
_cp = None
_cp_ndimage = None
HAS_GPU = False
@ -194,7 +195,7 @@ def _init_gpu():
_cupy_ndarray = _real_cupy.ndarray
return
except Exception as e:
logger.warning(f"GPU indisponible (CUDA_VISIBLE_DEVICES={cuda_visible}): {e}")
logger.warning(f"GPU unavailable (CUDA_VISIBLE_DEVICES={cuda_visible}): {e}")
_cp = None
_cp_ndimage = None
HAS_GPU = False
@ -220,12 +221,12 @@ def _init_gpu():
_working_gpu = (idx, name, mem_mi)
break
else:
logger.warning(f"GPU {idx} ({name}, sm_{cap_str}) faux positif CPU fallback")
logger.warning(f"GPU {idx} ({name}, sm_{cap_str}) non compatible: "
f"{result.stderr.strip().splitlines()[-1] if result.stderr else 'inconnue'}")
logger.warning(f"GPU {idx} ({name}, sm_{cap_str}): false positive, the test ran on another device")
logger.warning(f"GPU {idx} ({name}, sm_{cap_str}) not compatible: "
f"{result.stderr.strip().splitlines()[-1] if result.stderr else 'unknown'}")
if _working_gpu is None:
logger.info("Pas de GPU utilisable — mode CPU uniquement")
logger.info("No usable GPU — CPU-only mode")
_cp = None
_cp_ndimage = None
HAS_GPU = False
@ -281,12 +282,12 @@ def restrict_gpus(gpu_ids: list[int], set_env_var: bool = False):
def set_active_gpu(gpu_id):
"""Restrict to a single GPU by host-level index.
Appelé par les workers spawnés (_process_file_standalone) : CuPy n'y est
pas encore initialisé (le worker n'exécute pas log_gpu_status), il faut
donc déclencher _init_gpu() ici, sinon le worker retombe silencieusement
en CPU. On réduit aussi CUDA_VISIBLE_DEVICES à ce seul GPU avant l'init
pour que le device 0 du worker soit le bon : sans cela, tous les workers
avec plusieurs GPU visibles partagent le premier d'entre eux.
Called by spawned workers (_process_file_standalone): CuPy is not yet
initialized there (the worker does not run log_gpu_status), so
_init_gpu() must be triggered here, otherwise the worker silently falls
back to CPU. CUDA_VISIBLE_DEVICES is also narrowed to this single GPU
before init so that the worker's device 0 is the right one: without it,
all workers with several visible GPUs share the first of them.
"""
global _restricted_gpu_ids
_restricted_gpu_ids = [gpu_id]
@ -295,10 +296,10 @@ def set_active_gpu(gpu_id):
def force_cpu():
"""Interdit le GPU à ce processus (worker en excédent de VRAM).
"""Forbid the GPU for this process (worker beyond the VRAM budget).
Aucun candidat ne survit au filtre et CUDA_VISIBLE_DEVICES vide empêche
CuPy de créer un contexte (~300 Mo de VRAM par processus sinon).
No candidate survives the filter, and an empty CUDA_VISIBLE_DEVICES stops
CuPy from creating a context (~300 MB of VRAM per process otherwise).
"""
global _restricted_gpu_ids
_restricted_gpu_ids = []
@ -306,20 +307,20 @@ def force_cpu():
_init_gpu()
# Pic de VRAM d'un worker (Mo) : contexte CUDA (~300) + calage conjoint des
# lignes sur CuPy (~70-100 o par point sol en float64, ~15 M points par dalle
# 0,2 m + bande de 100 m) + transformée de distance de _fill_nans (indices
# int32 ×2 sur ~7000² px ≈ 400 Mo) ; le pool CuPy garde ses blocs entre deux
# étapes. Estimation depuis le code, à ajuster par LIDAR_GPU_WORKER_MIB.
# Peak VRAM of one worker (MiB): CUDA context (~300) + joint scan-line
# adjustment on CuPy (~70-100 bytes per ground point in float64, ~15 M points
# per 0.2 m tile + 100 m buffer) + _fill_nans distance transform (int32
# indices x2 over ~7000² px ≈ 400 MB); the CuPy pool keeps its blocks between
# steps. Estimated from the code, tune with LIDAR_GPU_WORKER_MIB.
GPU_WORKER_MIB = int(os.environ.get("LIDAR_GPU_WORKER_MIB", "2048") or 2048)
# VRAM laissée libre sur chaque GPU (affichage, autres processus).
# VRAM left free on each GPU (display, other processes).
GPU_RESERVE_MIB = int(os.environ.get("LIDAR_GPU_RESERVE_MIB", "512") or 512)
def gpu_free_mib():
"""VRAM libre par GPU (indice hôte → Mo) via nvidia-smi, sans contexte CUDA.
"""Free VRAM per GPU (host index → MiB) via nvidia-smi, without a CUDA context.
Dictionnaire vide si la mesure échoue : l'appelant ne borne alors rien.
Empty dict if the query fails: the caller then applies no bound.
"""
try:
import subprocess
@ -341,13 +342,12 @@ def gpu_free_mib():
def gpu_worker_slots(gpu_ids, n_workers, free_mib=None):
"""Place de chaque worker du pool : indice GPU, -1 (CPU forcé) ou None.
"""Slot of each pool worker: GPU index, -1 (forced CPU) or None.
Chaque GPU reçoit au plus (VRAM libre − réserve) / GPU_WORKER_MIB
workers (au moins un), entrelacés entre GPU ; l'excédent tourne en CPU
au lieu de saturer la VRAM (OOM). Sans GPU : None partout (choix laissé
au worker). VRAM inconnue : round-robin sans borne (comportement
historique).
Each GPU gets at most (free VRAM − reserve) / GPU_WORKER_MIB workers (at
least one), interleaved across GPUs; the surplus runs on CPU instead of
exhausting VRAM (OOM). No GPU: None everywhere (the worker decides).
Unknown VRAM: unbounded round-robin (legacy behaviour).
"""
if not gpu_ids:
return [None] * n_workers
@ -383,9 +383,9 @@ def log_gpu_status():
if isinstance(name, bytes):
name = name.decode()
mem_gb = props['totalGlobalMem'] // (1024**3)
gpu_info = f"GPU: {name} ({mem_gb} Go VRAM) — ID {_best_gpu_id}"
gpu_info = f"GPU: {name} ({mem_gb} GB VRAM) — ID {_best_gpu_id}"
except Exception:
gpu_info = f"GPU: {_gpu_name} ({_gpu_mem_gb} Go VRAM)"
gpu_info = f"GPU: {_gpu_name} ({_gpu_mem_gb} GB VRAM)"
logger.info(gpu_info)
# List other GPUs for info
@ -403,11 +403,11 @@ def log_gpu_status():
idx = int(parts[0])
if idx != _best_gpu_id:
cap = parts[2]
logger.info(f" GPU {idx}: {parts[1]} (sm_{cap}) — disponible")
logger.info(f" GPU {idx}: {parts[1]} (sm_{cap}) — available")
except Exception:
pass
else:
logger.info(f"Pas de GPU utilisable — mode CPU uniquement")
logger.info("No usable GPU — CPU-only mode")
# ---------------------------------------------------------------------------
@ -420,9 +420,9 @@ def to_gpu(arr):
try:
return _cp.asarray(arr.astype(np.float32))
except Exception as e:
# L'erreur d'origine (souvent OOM) ne doit pas être avalée :
# sans elle, un repli CPU est indissociable d'un simple bug.
logger.warning(f"Transfert GPU échoué ({e}) — repli CPU pour ce calcul")
# The original error (often OOM) must not be swallowed:
# without it, a CPU fallback is indistinguishable from a plain bug.
logger.warning(f"GPU transfer failed ({e}) — falling back to CPU")
disable_gpu()
return arr.astype(np.float32)
@ -430,7 +430,7 @@ def to_gpu(arr):
def to_cpu(arr):
"""Bring array back to CPU (numpy). No-op if already on CPU.
Fonctionne même après disable_gpu() grâce à la réf de type _cupy_ndarray.
Works even after disable_gpu() thanks to the _cupy_ndarray type reference.
"""
if _cupy_ndarray is not None and isinstance(arr, _cupy_ndarray):
try:
@ -454,7 +454,7 @@ def xp_gaussian_filter(arr, sigma):
try:
return _cp_ndimage.gaussian_filter(arr, sigma)
except Exception as e:
logger.warning(f"Filtre gaussien GPU échoué ({e}) — repli CPU")
logger.warning(f"GPU Gaussian filter failed ({e}) — falling back to CPU")
arr = to_cpu(arr)
return ndimage.gaussian_filter(arr, sigma)
@ -464,19 +464,20 @@ def xp_uniform_filter(arr, size):
try:
return _cp_ndimage.uniform_filter(arr, size)
except Exception as e:
logger.warning(f"Filtre uniform GPU échoué ({e}) — repli CPU")
logger.warning(f"GPU uniform filter failed ({e}) — falling back to CPU")
arr = to_cpu(arr)
return ndimage.uniform_filter(arr, size)
def xp_zoom(arr, factor, order=1):
"""Agrandissement aligné sur les centres de pixels (grid_mode) : chaque
pixel source couvre exactement factor×factor pixels, sans décalage."""
"""Upsampling with grid_mode (pixel-center alignment over the full image
extent): each source pixel covers exactly factor×factor output pixels,
with no half-pixel shift."""
if _cp is not None and isinstance(arr, _cp.ndarray):
try:
return _cp_ndimage.zoom(arr, factor, order=order, mode='nearest', grid_mode=True)
except Exception as e:
logger.warning(f"Zoom GPU échoué ({e}) — repli CPU")
logger.warning(f"GPU zoom failed ({e}) — falling back to CPU")
arr = to_cpu(arr)
return ndimage.zoom(arr, factor, order=order, mode='nearest', grid_mode=True)
@ -486,7 +487,7 @@ def xp_minimum_filter(arr, footprint=None, size=None):
try:
return _cp_ndimage.minimum_filter(arr, footprint=footprint, size=size)
except Exception as e:
logger.warning(f"Filtre minimum GPU échoué ({e}) — repli CPU")
logger.warning(f"GPU minimum filter failed ({e}) — falling back to CPU")
arr = to_cpu(arr)
return ndimage.minimum_filter(arr, footprint=footprint, size=size)
@ -496,26 +497,26 @@ def xp_maximum_filter(arr, footprint=None, size=None):
try:
return _cp_ndimage.maximum_filter(arr, footprint=footprint, size=size)
except Exception as e:
logger.warning(f"Filtre maximum GPU échoué ({e}) — repli CPU")
logger.warning(f"GPU maximum filter failed ({e}) — falling back to CPU")
arr = to_cpu(arr)
return ndimage.maximum_filter(arr, footprint=footprint, size=size)
# ---------------------------------------------------------------------------
# Rasterisation MNT — moyenne z par cellule (bincount 2D)
# DTM rasterization — mean z per cell (2D bincount)
# ---------------------------------------------------------------------------
def _bin_mean_core(lib, xs, ys, zs, width, height, x_range, y_range):
"""Moyenne z par cellule d'une grille régulière, backend-agnostique.
"""Mean z per cell of a regular grid, backend-agnostic.
`lib` = numpy ou cupy (mêmes primitives). Sémantique calquée sur
`lib` = numpy or cupy (same primitives). Semantics mirror
scipy.stats.binned_statistic_2d(statistic='mean', bins=[width, height],
range=[[xmin, xmax], [ymin, ymax]]) : points hors emprise ignorés, valeur
exactement sur le bord droit/haut rangée dans la dernière cellule.
Retourne (height, width) float64, NaN sur les cellules vides.
range=[[xmin, xmax], [ymin, ymax]]): points outside the extent are
ignored, a value exactly on the right/top edge goes into the last cell.
Returns (height, width) float64, NaN on empty cells.
float64 obligatoire pour x/y : à des coordonnées Lambert 93 (~1e6 m) la
résolution float32 est ~6 cm — grossière devant un pixel de 0,2 m.
float64 is required for x/y: at Lambert 93 coordinates (~1e6 m) float32
resolution is ~6 cm (Y ~7e6 m: ~50 cm), far too coarse for a 0.2 m pixel.
"""
(xmin, xmax), (ymin, ymax) = x_range, y_range
x = lib.asarray(xs, dtype=lib.float64)
@ -523,9 +524,9 @@ def _bin_mean_core(lib, xs, ys, zs, width, height, x_range, y_range):
z = lib.asarray(zs, dtype=lib.float64)
inside = (x >= xmin) & (x <= xmax) & (y >= ymin) & (y <= ymax)
x, y, z = x[inside], y[inside], z[inside]
# floor((v - min) / pas) ; les valeurs intérieures sont >= 0 donc la
# troncature astype == floor. Le clip range le bord droit (et l'arrondi
# float adjacent) dans la dernière cellule, comme scipy.
# floor((v - min) / step); interior values are >= 0 so astype truncation
# == floor. The clip puts the right edge (and the adjacent float rounding)
# into the last cell, like scipy.
ix = ((x - xmin) * (float(width) / (xmax - xmin))).astype(lib.int64)
iy = ((y - ymin) * (float(height) / (ymax - ymin))).astype(lib.int64)
ix = lib.clip(ix, 0, width - 1)
@ -539,13 +540,13 @@ def _bin_mean_core(lib, xs, ys, zs, width, height, x_range, y_range):
def bin_mean_2d(xs, ys, zs, width, height, x_range, y_range):
"""Rasterisation « moyenne par cellule » sur GPU (appelant : dtm.create_dtm_fast).
""""Mean per cell" rasterization on GPU (caller: dtm.create_dtm_fast).
Retourne un tableau numpy (height, width) float64 (NaN = cellule vide),
ou None si le GPU est indisponible ou échoue (OOM le plus souvent) —
l'appelant retombe alors sur scipy. Un échec ici n'appelle PAS
disable_gpu() : la rastérisation est ponctuelle, les visualisations qui
suivent doivent garder leur accélérateur.
Returns a numpy (height, width) float64 array (NaN = empty cell), or None
if the GPU is unavailable or fails (usually OOM) — the caller then falls
back to scipy. A failure here does NOT call disable_gpu(): rasterization
is a one-off step, and the visualizations that follow must keep their
accelerator.
"""
if not _gpu_available():
return None
@ -554,7 +555,7 @@ def bin_mean_2d(xs, ys, zs, width, height, x_range, y_range):
x_range, y_range)
return to_cpu(result)
except Exception as e:
logger.warning(f"Rasterisation GPU échouée ({e}) — repli scipy")
logger.warning(f"GPU rasterization failed ({e}) — falling back to scipy")
gpu_cleanup()
return None
@ -575,13 +576,13 @@ def gpu_cleanup():
def disable_gpu():
"""Disable GPU acceleration for the rest of this process.
Libère le memory pool GPU avant de nuller les références (anti-fuite VRAM).
Garde _cupy_ndarray vivant pour que to_cpu() récupère les tableaux orphelins.
Frees the GPU memory pool before dropping the references (avoids VRAM
leaks). Keeps _cupy_ndarray alive so to_cpu() can recover orphaned arrays.
"""
global HAS_GPU, _cp, _cp_ndimage
if not HAS_GPU:
return
logger.warning("GPU désactivé — passage en mode CPU pour la suite du processus")
logger.warning("GPU disabled — switching to CPU for the rest of this process")
gpu_cleanup()
HAS_GPU = False
_cp = None
@ -598,14 +599,14 @@ def safe_gpu_call(func, *args, **kwargs):
try:
return func(*args, **kwargs)
except Exception as e:
# GPU actif : on TOUTE erreur (OOM, types numpy/cupy mêlés après un
# échec de transfert en cours de run, ...) le GPU est désactivé et le
# calcul est retranché en CPU — une panne partielle ne doit pas
# faire échouer la visualisation entière. En mode CPU, on relance
# l'erreur d'origine (déjà en CPU, rien à retrancher).
# GPU active: on ANY error (OOM, mixed numpy/cupy types after a
# transfer failure mid-run, ...) the GPU is disabled and the
# computation is retried on CPU — a partial failure must not make the
# whole visualization fail. In CPU mode, the original error is
# re-raised (already on CPU, nothing to retry).
if _cp is not None:
logger.warning(f"Erreur GPU ({e.__class__.__name__}: {e}), "
f"retry en CPU...")
logger.warning(f"GPU error ({e.__class__.__name__}: {e}), "
f"retrying on CPU...")
disable_gpu()
cpu_args = tuple(to_cpu(a) for a in args)
cpu_kwargs = {k: to_cpu(v) for k, v in kwargs.items()}