Rendre la carte autonome, télécharger la pyramide en fond, refermer les trous de zoom et borner les workers GPU par la VRAM
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Antoine Jacquin
@ -199,4 +199,57 @@ def test_safe_gpu_call_retries_on_cpu_for_any_error(monkeypatch):
|
||||
gpu.safe_gpu_call(g, 1)
|
||||
raise AssertionError("devait relancer l'erreur")
|
||||
except ValueError:
|
||||
pass
|
||||
pass
|
||||
|
||||
def test_gpu_worker_slots_bounded_by_free_vram(monkeypatch):
|
||||
"""Places GPU par la VRAM libre : l'excédent de workers passe en CPU.
|
||||
|
||||
Cas réel : LIDAR_WORKERS=auto = 12 workers sur 2 RTX 5060 (8 Go) —
|
||||
6 workers par GPU, soit bien plus que la VRAM libre ne tient au pic
|
||||
(calage des lignes + comblement GPU) : OOM.
|
||||
"""
|
||||
from lidar_pipeline import gpu
|
||||
monkeypatch.setattr(gpu, "GPU_WORKER_MIB", 2000)
|
||||
monkeypatch.setattr(gpu, "GPU_RESERVE_MIB", 500)
|
||||
free = {0: 7500, 1: 4600}
|
||||
slots = gpu.gpu_worker_slots([0, 1], 12, free_mib=free)
|
||||
assert len(slots) == 12
|
||||
assert slots.count(0) == 3 # (7500 - 500) // 2000
|
||||
assert slots.count(1) == 2 # (4600 - 500) // 2000
|
||||
assert slots.count(-1) == 7 # reste en CPU
|
||||
# GPU d'abord, entrelacés : les premiers workers créés se répartissent
|
||||
assert slots[:4] == [0, 1, 0, 1]
|
||||
# Moins de workers que de places : aucun CPU forcé
|
||||
assert gpu.gpu_worker_slots([0, 1], 3, free_mib=free) == [0, 1, 0]
|
||||
|
||||
|
||||
def test_gpu_worker_slots_without_gpu_or_vram_info():
|
||||
"""Sans GPU : tout en CPU implicite (None). VRAM inconnue : round-robin
|
||||
historique (pas de bornage sans mesure)."""
|
||||
from lidar_pipeline import gpu
|
||||
assert gpu.gpu_worker_slots([], 4, free_mib={}) == [None] * 4
|
||||
assert gpu.gpu_worker_slots([0, 1], 4, free_mib={}) == [0, 1, 0, 1]
|
||||
|
||||
|
||||
def test_gpu_worker_slots_keeps_one_gpu_worker_when_tight(monkeypatch):
|
||||
"""GPU presque plein : au moins une place pour que le GPU serve encore
|
||||
(le repli CPU de safe_gpu_call couvre l'OOM éventuel)."""
|
||||
from lidar_pipeline import gpu
|
||||
monkeypatch.setattr(gpu, "GPU_WORKER_MIB", 2000)
|
||||
monkeypatch.setattr(gpu, "GPU_RESERVE_MIB", 500)
|
||||
assert gpu.gpu_worker_slots([0], 3, free_mib={0: 900}) == [0, -1, -1]
|
||||
|
||||
|
||||
def test_force_cpu_disables_gpu_selection(monkeypatch):
|
||||
"""force_cpu() : aucun candidat GPU, CuPy jamais initialisé."""
|
||||
import os
|
||||
from lidar_pipeline import gpu
|
||||
monkeypatch.setattr(gpu, "_restricted_gpu_ids", None)
|
||||
monkeypatch.setattr(gpu, "_gpu_candidates", [(0, "X", "12.0", 8000, 1, 12)])
|
||||
monkeypatch.setattr(gpu, "_gpu_initialized", False)
|
||||
monkeypatch.setattr(gpu, "_env_set_by_init", False)
|
||||
monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising=False)
|
||||
gpu.force_cpu()
|
||||
assert gpu.available_gpu_ids() == []
|
||||
assert gpu.HAS_GPU is False
|
||||
assert os.environ.get("CUDA_VISIBLE_DEVICES") == ""
|
||||
|
||||
@ -1032,3 +1032,140 @@ def test_remote_fed_map_keeps_sources_locally(tmp_path, monkeypatch):
|
||||
while mapserve._bg["prefetch"]:
|
||||
mapserve._bg_process_one()
|
||||
assert all(s_.path.exists() for s_ in [thumb] + quads) and not full.path.exists()
|
||||
|
||||
|
||||
def _bg_reset(monkeypatch, mapserve):
|
||||
import collections
|
||||
for key, val in (("queue", collections.deque()), ("queued", set()),
|
||||
("snapshot", None), ("prefetch", collections.deque()),
|
||||
("stats", {"rendues": 0, "a_jour": 0, "vides": 0, "scans": 0,
|
||||
"dernier_scan": None, "ajouts_dernier_scan": 0})):
|
||||
monkeypatch.setitem(mapserve._bg, key, val)
|
||||
|
||||
|
||||
def test_background_downloads_tiles_from_upstream(tmp_path, monkeypatch):
|
||||
"""Maintenance de fond + LIDAR_MAPS_URL : les tuiles de la pyramide sont
|
||||
TÉLÉCHARGÉES en tâche de fond (pas seulement à l'affichage), sans rendu
|
||||
local — la carte se remplit même sans visite et reste servie ensuite
|
||||
sans l'amont."""
|
||||
import lidar_pipeline.mapserve as mapserve
|
||||
from lidar_pipeline import tiles
|
||||
_setup(tmp_path, monkeypatch)
|
||||
_bg_reset(monkeypatch, mapserve)
|
||||
monkeypatch.setattr(mapserve, "MAPS_URL", "http://amont:8975")
|
||||
AVIF = b"\x00\x00\x00\x20ftypavif-TUILE-AMONT"
|
||||
calls = []
|
||||
monkeypatch.setattr(mapserve, "_fetch_upstream",
|
||||
lambda rel: calls.append(rel) or AVIF)
|
||||
|
||||
def no_render(*a, **k):
|
||||
raise AssertionError("rendu local inutile : la tuile vient de l'amont")
|
||||
|
||||
monkeypatch.setattr(tiles, "get_tile", no_render)
|
||||
z = 13
|
||||
x, y = _tile_of_cell(1054, 6882, z)
|
||||
mapserve._bg_enqueue([("aspect", z, x, y)])
|
||||
assert mapserve._bg_process_one() is True
|
||||
assert calls == [f"tiles/aspect/{z}/{x}/{y}@2x.avif"]
|
||||
cache = tiles.tile_cache_path(tmp_path, "aspect", z, x, y, 2, "avif")
|
||||
assert cache.read_bytes() == AVIF
|
||||
assert mapserve._bg["stats"]["telechargees"] == 1
|
||||
# Fraîche : servie en cache seule sans amont ni rendu
|
||||
data, state = tiles.cached_tile(tmp_path, "aspect", z, x, y, 2, "avif")
|
||||
assert state == "fresh" and data == AVIF
|
||||
|
||||
|
||||
def test_background_renders_locally_when_upstream_down(tmp_path, monkeypatch):
|
||||
"""Amont éteint : la maintenance rend la tuile localement (autonomie)."""
|
||||
import lidar_pipeline.mapserve as mapserve
|
||||
from lidar_pipeline import tiles
|
||||
_setup(tmp_path, monkeypatch)
|
||||
_bg_reset(monkeypatch, mapserve)
|
||||
monkeypatch.setattr(mapserve, "MAPS_URL", "http://amont:8975")
|
||||
monkeypatch.setattr(mapserve, "_fetch_upstream", lambda rel: None)
|
||||
z = 13
|
||||
x, y = _tile_of_cell(1054, 6882, z)
|
||||
mapserve._bg_enqueue([("aspect", z, x, y)])
|
||||
assert mapserve._bg_process_one() is True
|
||||
assert tiles.tile_cache_path(tmp_path, "aspect", z, x, y, 2, "avif").is_file()
|
||||
assert mapserve._bg["stats"]["rendues"] == 1
|
||||
|
||||
|
||||
def test_upstream_breaker_rearms_after_expiry(monkeypatch):
|
||||
"""La suspension expirée, de nouveaux échecs la relancent (sinon chaque
|
||||
tuile repaie le délai réseau d'un amont éteint, indéfiniment)."""
|
||||
import lidar_pipeline.mapserve as mapserve
|
||||
monkeypatch.setattr(mapserve, "_UPSTREAM", {"fails": 0, "until": 0.0})
|
||||
for _ in range(3):
|
||||
mapserve._upstream_mark(False)
|
||||
assert mapserve._upstream_offline() is True
|
||||
mapserve._UPSTREAM["until"] = 1.0 # suspension expirée
|
||||
assert mapserve._upstream_offline() is False
|
||||
mapserve._upstream_mark(False) # nouvel échec : resuspendu
|
||||
assert mapserve._upstream_offline() is True
|
||||
|
||||
|
||||
def test_empty_tile_skips_upstream_when_inventory_is_upstream(tmp_path, monkeypatch):
|
||||
"""Inventaire venu de l'amont (LIDAR_SOURCE_URL) et aucune dalle sous la
|
||||
tuile : l'amont n'en sait pas plus — pas de requête (20 s par tuile vide
|
||||
quand il est éteint)."""
|
||||
import lidar_pipeline.mapserve as mapserve
|
||||
from lidar_pipeline import tiles
|
||||
_setup(tmp_path, monkeypatch)
|
||||
monkeypatch.setattr(mapserve, "MAPS_URL", "http://amont:8975")
|
||||
monkeypatch.setattr(tiles, "REMOTE_SOURCE_URL", "http://amont:8973")
|
||||
monkeypatch.setattr(tiles, "_contributing", lambda *a, **k: [])
|
||||
calls = []
|
||||
monkeypatch.setattr(mapserve, "_fetch_upstream", lambda rel: calls.append(rel))
|
||||
resp = asyncio.run(mapserve.tile("aspect", 15, 1, "1.png"))
|
||||
assert calls == []
|
||||
assert resp.headers["X-Tile-Empty"] == "1"
|
||||
|
||||
|
||||
def test_cache_only_serves_stale_tile_while_pending(tmp_path, monkeypatch):
|
||||
"""Cache seule, tuile périmée (dalle régénérée) et amont muet : l'ancienne
|
||||
tuile reste affichée en attendant la nouvelle — pas un trou transparent."""
|
||||
import os
|
||||
import lidar_pipeline.mapserve as mapserve
|
||||
from lidar_pipeline import tiles
|
||||
_setup(tmp_path, monkeypatch)
|
||||
monkeypatch.setenv("LIDAR_TILE_CACHE_ONLY", "1")
|
||||
monkeypatch.setattr(mapserve, "MAPS_URL", "")
|
||||
z = 13
|
||||
x, y = _tile_of_cell(1054, 6882, z)
|
||||
cache = tiles.tile_cache_path(tmp_path, "aspect", z, x, y, 2, "avif")
|
||||
cache.parent.mkdir(parents=True, exist_ok=True)
|
||||
cache.write_bytes(b"ANCIENNE-TUILE")
|
||||
os.utime(cache, (1_000_000, 1_000_000)) # plus vieille que la dalle
|
||||
assert tiles.cached_tile(tmp_path, "aspect", z, x, y, 2, "avif")[1] == "pending"
|
||||
resp = asyncio.run(mapserve.tile("aspect", z, x, f"{y}@2x.avif"))
|
||||
assert resp.body == b"ANCIENNE-TUILE"
|
||||
assert resp.headers["Cache-Control"] == "no-store" # redemandée plus tard
|
||||
assert resp.headers["X-Tile-Pending"] == "1"
|
||||
assert resp.headers["X-Tile-Empty"] == "0"
|
||||
|
||||
|
||||
def test_background_tiles_before_sources(tmp_path, monkeypatch):
|
||||
"""Tuiles d'abord (légères, visibles tout de suite), sources ensuite :
|
||||
des milliers de sources à rapatrier ne retardent pas la pyramide."""
|
||||
import lidar_pipeline.mapserve as mapserve
|
||||
from lidar_pipeline import tiles
|
||||
_setup(tmp_path, monkeypatch)
|
||||
_bg_reset(monkeypatch, mapserve)
|
||||
monkeypatch.setattr(mapserve, "MAPS_URL", "http://amont:8975")
|
||||
monkeypatch.setattr(mapserve, "_fetch_upstream", lambda rel: b"TUILE")
|
||||
fetched = []
|
||||
|
||||
class Src:
|
||||
def ensure(self):
|
||||
fetched.append(1)
|
||||
return True
|
||||
|
||||
mapserve._bg["prefetch"].append(Src())
|
||||
z = 13
|
||||
x, y = _tile_of_cell(1054, 6882, z)
|
||||
mapserve._bg_enqueue([("aspect", z, x, y)])
|
||||
assert mapserve._bg_process_one() is True
|
||||
assert fetched == [] and mapserve._bg["stats"]["telechargees"] == 1
|
||||
assert mapserve._bg_process_one() is True
|
||||
assert fetched == [1]
|
||||
|
||||
@ -308,3 +308,18 @@ class TestResolveWorkers:
|
||||
from lidar_pipeline.pipeline import resolve_workers
|
||||
assert resolve_workers('abc') == 1
|
||||
assert resolve_workers(None) == 1
|
||||
|
||||
def test_worker_slot_initializer(self, monkeypatch):
|
||||
"""Chaque processus du pool prend UNE place à sa création : GPU
|
||||
(set_active_gpu) ou CPU forcé (-1) ; None ou file vide = libre."""
|
||||
import queue
|
||||
from lidar_pipeline import gpu, pipeline
|
||||
calls = []
|
||||
monkeypatch.setattr(gpu, "set_active_gpu", lambda i: calls.append(("gpu", i)))
|
||||
monkeypatch.setattr(gpu, "force_cpu", lambda: calls.append(("cpu",)))
|
||||
q = queue.Queue()
|
||||
for slot in (1, -1, None):
|
||||
q.put(slot)
|
||||
for _ in range(4): # 4e appel : file vide
|
||||
pipeline._init_worker_slot(q)
|
||||
assert calls == [("gpu", 1), ("cpu",)]
|
||||
|
||||
@ -571,3 +571,119 @@ def test_unstored_level_rendered_on_the_fly_without_disk(tmp_path, monkeypatch):
|
||||
assert data and data[:4] == b"\x89PNG"
|
||||
assert not tiles.tile_cache_path(tmp_path, "aspect", z, x, y, 1, "png").exists()
|
||||
assert tiles.get_tile(tmp_path, "aspect", z, x, y, 1, "png") == data and len(calls) == 1
|
||||
|
||||
|
||||
def test_remote_payload_failure_not_retried_each_call(monkeypatch):
|
||||
"""Amont injoignable sans inventaire connu : l'échec est mémorisé (TTL).
|
||||
|
||||
Sinon chaque appel (plusieurs par /api/map/meta) repaie le délai de
|
||||
connexion : l'interface ne se chargeait plus quand le worker était éteint.
|
||||
"""
|
||||
import urllib.request
|
||||
from lidar_pipeline import tiles
|
||||
monkeypatch.setattr(tiles, "REMOTE_SOURCE_URL", "http://amont:8973")
|
||||
monkeypatch.setattr(tiles, "_remote_cache",
|
||||
{"payload": None, "at": 0.0, "index": None, "root": None})
|
||||
calls = []
|
||||
|
||||
def boom(*a, **k):
|
||||
calls.append(1)
|
||||
raise OSError("hôte injoignable")
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", boom)
|
||||
for _ in range(3):
|
||||
assert tiles._remote_payload() is None
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_fetch_source_offline_breaker(tmp_path, monkeypatch):
|
||||
"""Source amont en échec : les suivantes échouent sans attendre le délai
|
||||
réseau pendant la suspension (la maintenance passe au rendu local)."""
|
||||
import urllib.request
|
||||
from lidar_pipeline import tiles
|
||||
monkeypatch.setattr(tiles, "_SOURCE_OFFLINE", {"until": 0.0})
|
||||
calls = []
|
||||
|
||||
def boom(*a, **k):
|
||||
calls.append(1)
|
||||
raise OSError("hôte injoignable")
|
||||
|
||||
monkeypatch.setattr(urllib.request, "urlopen", boom)
|
||||
assert tiles._fetch_source("http://amont/a", tmp_path / "a.avif") is False
|
||||
assert tiles._fetch_source("http://amont/b", tmp_path / "b.avif") is False
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
def test_fetched_source_dated_to_upstream_version(tmp_path, monkeypatch):
|
||||
"""Une source rapatriée porte la date de version amont : quand l'amont
|
||||
s'éteint, l'index local retrouve les mêmes dates et les tuiles déjà
|
||||
faites restent fraîches (pas de pyramide entière à refaire)."""
|
||||
from lidar_pipeline import tiles
|
||||
|
||||
def fake_fetch(url, dest):
|
||||
dest.write_bytes(b"x")
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(tiles, "_fetch_source", fake_fetch)
|
||||
src = tiles._Source(tmp_path / "q.avif", (0, 0, 1, 1), 0.2,
|
||||
url="http://amont/q.avif", version=1700000000000)
|
||||
assert src.ensure() is True
|
||||
assert (tmp_path / "q.avif").stat().st_mtime == 1700000000.0
|
||||
|
||||
|
||||
def test_tile_refreshed_when_older_dalle_appears(tmp_path):
|
||||
"""Dalle entrée dans l'inventaire APRÈS le rendu d'une tuile qui la couvre,
|
||||
mais avec une date de version plus ancienne (écrite avant, inventoriée
|
||||
après) : la tuile doit être périmée — sinon trou permanent à ce niveau,
|
||||
sur le disque, en mémoire et dans le navigateur (stamp inchangé)."""
|
||||
import io
|
||||
import os
|
||||
from PIL import Image
|
||||
from lidar_pipeline import tiles
|
||||
tiles.clear_source_cache()
|
||||
tiles._mem_tiles.clear()
|
||||
z = 10
|
||||
assert _tile_of_cell(1054, 6882, z) == _tile_of_cell(1055, 6882, z)
|
||||
x, y = _tile_of_cell(1054, 6882, z)
|
||||
_make_dalle(tmp_path, 1054, 6882, ["slope"], color=(200, 30, 30))
|
||||
tiles.source_index(tmp_path, force=True)
|
||||
first = tiles.get_tile(tmp_path, "slope", z, x, y)
|
||||
assert first is not None
|
||||
stamp = tiles.tiles_stamp(tmp_path)
|
||||
|
||||
_make_dalle(tmp_path, 1055, 6882, ["slope"], color=(30, 200, 30))
|
||||
for f in tmp_path.rglob("*1055_6882*"):
|
||||
os.utime(f, (1_000_000, 1_000_000)) # version antérieure à la tuile
|
||||
tiles.source_index(tmp_path, force=True)
|
||||
assert tiles.cached_tile(tmp_path, "slope", z, x, y)[1] == "pending"
|
||||
second = tiles.get_tile(tmp_path, "slope", z, x, y)
|
||||
img = Image.open(io.BytesIO(second)).convert("RGBA")
|
||||
assert any(g > 150 and r < 80 and a == 255
|
||||
for r, g, b, a in img.getdata()), "nouvelle dalle absente de la tuile"
|
||||
assert tiles.tiles_stamp(tmp_path) > stamp
|
||||
# Registre persistant : un redémarrage ne réinvalide rien
|
||||
tiles._index_cache.clear()
|
||||
tiles.source_index(tmp_path, force=True)
|
||||
assert tiles.cached_tile(tmp_path, "slope", z, x, y)[1] == "fresh"
|
||||
tiles.clear_source_cache()
|
||||
|
||||
|
||||
def test_existing_cache_without_registry_is_refreshed_once(tmp_path):
|
||||
"""Mise à jour : cache de tuiles présent mais pas de registre — les
|
||||
tuiles existantes (peut-être trouées) sont périmées une seule fois."""
|
||||
from lidar_pipeline import tiles
|
||||
tiles._seen_cache.clear()
|
||||
tiles._mem_tiles.clear()
|
||||
z = 10
|
||||
x, y = _tile_of_cell(1054, 6882, z)
|
||||
_make_dalle(tmp_path, 1054, 6882, ["slope"])
|
||||
tiles.source_index(tmp_path, force=True)
|
||||
assert tiles.get_tile(tmp_path, "slope", z, x, y) is not None
|
||||
(tmp_path / tiles.TILE_DIRNAME / tiles._SEEN_FILE).unlink() # version précédente
|
||||
tiles._seen_cache.clear()
|
||||
tiles.source_index(tmp_path, force=True)
|
||||
assert tiles.cached_tile(tmp_path, "slope", z, x, y)[1] == "pending"
|
||||
tiles.get_tile(tmp_path, "slope", z, x, y)
|
||||
tiles._seen_cache.clear()
|
||||
tiles.source_index(tmp_path, force=True)
|
||||
assert tiles.cached_tile(tmp_path, "slope", z, x, y)[1] == "fresh"
|
||||
|
||||
Reference in New Issue
Block a user