From 5ef0232c80d6cf414fa97f9c6d9f2d86477320c2 Mon Sep 17 00:00:00 2001 From: Antoine Jacquin Date: Sat, 19 Sep 2026 21:58:47 +0200 Subject: [PATCH] =?UTF-8?q?Workers=20parall=C3=A8les=20adaptatifs=20:=20-w?= =?UTF-8?q?=20auto=20=3D=20c=C5=93urs=20-=202,=20r=C3=A9solu=20au=20lancem?= =?UTF-8?q?ent=20de=20chaque=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.worker.yml | 2 +- lidar_pipeline/cli.py | 10 ++++++---- lidar_pipeline/pipeline.py | 20 ++++++++++++++++++++ lidar_pipeline/tests/test_cli.py | 4 ++-- lidar_pipeline/tests/test_pipeline.py | 23 +++++++++++++++++++++++ lidar_pipeline/webapp.py | 8 +++++++- 6 files changed, 59 insertions(+), 8 deletions(-) diff --git a/docker-compose.worker.yml b/docker-compose.worker.yml index 2520cf6..2821e3c 100644 --- a/docker-compose.worker.yml +++ b/docker-compose.worker.yml @@ -33,7 +33,7 @@ services: - LIDAR_OUTPUT_DIR=/data/output # Les générations lancées depuis une webapp distante utilisent le GPU - LIDAR_GPU=1 - - LIDAR_WORKERS=10 + - LIDAR_WORKERS=auto # Protéger l'API si le réseau n'est pas de confiance : même valeur que # LIDAR_REMOTE_TOKEN sur chaque webapp distante (sinon, laisser commenté) # - LIDAR_API_TOKEN=change-moi diff --git a/lidar_pipeline/cli.py b/lidar_pipeline/cli.py index 585ffe5..5b70320 100644 --- a/lidar_pipeline/cli.py +++ b/lidar_pipeline/cli.py @@ -8,6 +8,7 @@ import logging import signal import sys +from .pipeline import resolve_workers from .pipeline import LidarArchaeoPipeline from .gpu import log_gpu_status @@ -117,9 +118,10 @@ def main(): ) parser.add_argument( "-w", "--workers", - type=int, - default=1, - help="Nombre de workers pour traitement parallèle (défaut: 1)" + type=str, + default="auto", + help="Nombre de workers pour traitement parallèle, ou « auto » = " + "cœurs - 2 évalué au lancement du run (défaut: auto)" ) parser.add_argument( "-g", "--gpu", @@ -361,7 +363,7 @@ def main(): input_dir=args.input, output_dir=args.output, resolution=args.resolution, - workers=args.workers, + workers=resolve_workers(args.workers), force=args.force, ground_method=args.ground_classification, ign_classes=args.ign_classes, diff --git a/lidar_pipeline/pipeline.py b/lidar_pipeline/pipeline.py index 04f212a..7321110 100644 --- a/lidar_pipeline/pipeline.py +++ b/lidar_pipeline/pipeline.py @@ -9,6 +9,7 @@ LidarArchaeoPipeline coordinates the full processing chain: import logging import multiprocessing +import os import shutil import time from concurrent.futures import ProcessPoolExecutor, as_completed, TimeoutError as FuturesTimeoutError @@ -25,6 +26,25 @@ except RuntimeError: logger = logging.getLogger("lidar") +def resolve_workers(value): + """Traduit l'option -w en nombre effectif de workers. + + Résolu au lancement de chaque run (pas au démarrage du serveur, ni par + tuile : le pool de processus vit le temps du run). « auto » = cœurs + logiques - 2 (un pour l'OS/serveur, un pour les phases I/O et + l'indexation), borné [2, 16] — chaque worker traite une tuile et peut + lancer un processus PDAL en flux, le compte reste raisonnable même sur + une grosse machine. + """ + if isinstance(value, str) and value.strip().lower() == "auto": + cpus = os.cpu_count() or 4 + return max(2, min(cpus - 2, 16)) + try: + return max(1, int(value)) + except (TypeError, ValueError): + return 1 + + def _file_basename(path): """Extract base name from a LAZ/LAS file, removing all known extensions. diff --git a/lidar_pipeline/tests/test_cli.py b/lidar_pipeline/tests/test_cli.py index 73b4908..6b8f030 100644 --- a/lidar_pipeline/tests/test_cli.py +++ b/lidar_pipeline/tests/test_cli.py @@ -14,7 +14,7 @@ class TestCLIParsing: parser.add_argument("input", help="Input directory") parser.add_argument("-o", "--output", default="/data/output") parser.add_argument("-r", "--resolution", type=float, default=0.5) - parser.add_argument("-w", "--workers", type=int, default=1) + parser.add_argument("-w", "--workers", type=str, default="auto") parser.add_argument("-f", "--force", action="store_true") parser.add_argument("--file", nargs="+", type=str, default=None) parser.add_argument("--keep-tif", action="store_true") @@ -23,7 +23,7 @@ class TestCLIParsing: assert args.input == "./input" assert args.output == "/data/output" assert args.resolution == 0.5 - assert args.workers == 1 + assert args.workers == "auto" assert args.force is False assert args.file is None assert args.keep_tif is False diff --git a/lidar_pipeline/tests/test_pipeline.py b/lidar_pipeline/tests/test_pipeline.py index 2cb19c7..c1e02c8 100644 --- a/lidar_pipeline/tests/test_pipeline.py +++ b/lidar_pipeline/tests/test_pipeline.py @@ -254,3 +254,26 @@ class TestEffectiveGroundMethod: p = LidarArchaeoPipeline(tmpdir, tmpdir, ground_method='smrf', ign_classes="sol,unclassified") assert p._effective_ground_method() == "smrf" + + +class TestResolveWorkers: + def test_auto_scales_with_cpus(self): + from lidar_pipeline.pipeline import resolve_workers + import os + w = resolve_workers('auto') + assert w == max(2, min((os.cpu_count() or 4) - 2, 16)) + + def test_auto_bounded(self): + from lidar_pipeline.pipeline import resolve_workers + # Bornes : jamais sous 2, jamais au-dessus de 16 + assert 2 <= resolve_workers('auto') <= 16 + + def test_explicit_int(self): + from lidar_pipeline.pipeline import resolve_workers + assert resolve_workers('4') == 4 + assert resolve_workers(7) == 7 + + def test_invalid_falls_back_to_one(self): + from lidar_pipeline.pipeline import resolve_workers + assert resolve_workers('abc') == 1 + assert resolve_workers(None) == 1 diff --git a/lidar_pipeline/webapp.py b/lidar_pipeline/webapp.py index 6c14bb6..940b337 100644 --- a/lidar_pipeline/webapp.py +++ b/lidar_pipeline/webapp.py @@ -1280,7 +1280,13 @@ def _build_command(tiles, regenerate=False, ground_class="ign", ign_classes="sol if reclassify: cmd += ["--force-classification"] if os.environ.get("LIDAR_GPU", "") == "1": - cmd += ["-g", "all", "-w", os.environ.get("LIDAR_WORKERS", "10")] + cmd += ["-g", "all"] + # Workers : entier explicite via LIDAR_WORKERS, sinon « auto » — + # résolu par le pipeline au départ de chaque run (cœurs de la + # machine au moment du lancement). + workers = os.environ.get("LIDAR_WORKERS", "auto").strip() + if workers.lower() not in ("auto", ""): + cmd += ["-w", workers] # Carte régénérée après chaque tuile terminée : la webapp l'affiche en # direct via /api/tiles pendant le run. cmd += ["--incremental-index"]