Add web map with zone generation API, side job queue, and restore historical DTM hole rendering
- webapp.py: FastAPI serving the continuous map (port 8973) with /api/preview, /api/generate and /api/status; tiles are downloaded from IGN and processed in a logged subprocess, tracked live in a side "File de génération" panel that survives page reloads - fetch_ign.py: download missing 1 km LiDAR HD tiles from the IGN geoplateforme before processing - index.py: tile thumbnails and 500 m subtiles are now invalidated by mtime so regenerating a tile refreshes its cached images; progress logging per tile - dtm.py: back to the historical gap handling (small gaps filled by fillnodata only, larger holes left as nodata rendered black); lowest-return floor only via --bare-earth, IGN class selection via --ign-classes - cli.py: positional input now optional (--rebuild-index works alone) - docker-compose.yml: serve (GPU, port 8973) and process services; launch via docker compose only (documented in AGENTS.md/AGENTS.md) - tests: 131 passing, incl. regressions for thumbnail staleness, --rebuild-index without input, and nodata rendering
This commit is contained in:
Antoine Jacquin
@ -60,7 +60,11 @@ def test_compute_bbox_empty():
|
||||
|
||||
|
||||
def _make_fake_viz_dir(vis_dir, basename, col, row, viz_keys=('hillshade_multi', 'svf'), ext='webp', res_suffix=''):
|
||||
"""Crée un faux dossier de visualisations avec de petites images."""
|
||||
"""Crée un faux dossier de visualisations avec de petites images.
|
||||
|
||||
Le suffixe de résolution apparaît seulement dans le nom du dossier (miroir
|
||||
du pipeline : les fichiers restent préfixés par le basename nu).
|
||||
"""
|
||||
from PIL import Image as PILImage
|
||||
import numpy as np
|
||||
|
||||
@ -70,7 +74,7 @@ def _make_fake_viz_dir(vis_dir, basename, col, row, viz_keys=('hillshade_multi',
|
||||
for v in viz_keys:
|
||||
arr = np.random.randint(0, 255, (50, 50, 3), dtype=np.uint8)
|
||||
img = PILImage.fromarray(arr)
|
||||
fname = f"LHD_FXX_{col}_{row}_PTS_LAMB93_IGN69{res_suffix}_{v}.{ext}"
|
||||
fname = f"LHD_FXX_{col}_{row}_PTS_LAMB93_IGN69_{v}.{ext}"
|
||||
img.save(str(tile_dir / fname), format='WEBP', quality=80)
|
||||
return tile_dir
|
||||
|
||||
@ -124,6 +128,85 @@ def test_scan_tiles_multi_resolution(tmp_path):
|
||||
assert resolutions == [0.2, 0.5]
|
||||
|
||||
|
||||
def test_res_suffix_str():
|
||||
"""Le suffixe de résolution reflète le nommage du pipeline (miroir)."""
|
||||
from lidar_pipeline.index import _res_suffix_str
|
||||
assert _res_suffix_str(0.5) == ''
|
||||
assert _res_suffix_str(0.2) == '_r0p2'
|
||||
|
||||
|
||||
def test_collect_tile_metadata(tmp_path):
|
||||
"""Les métadonnées lisent la méthode DTM et les dates/tailles des viz."""
|
||||
import os
|
||||
from datetime import datetime
|
||||
from lidar_pipeline.index import _collect_tile_metadata
|
||||
|
||||
basename = "LHD_FXX_1000_6881_PTS_LAMB93_IGN69"
|
||||
tile_dir = tmp_path / "visualisations" / basename
|
||||
tile_dir.mkdir(parents=True)
|
||||
viz_file = tile_dir / f"{basename}_hillshade_multi.webp"
|
||||
viz_file.write_bytes(b"fake")
|
||||
|
||||
dtm_dir = tmp_path / "DTM"
|
||||
dtm_dir.mkdir()
|
||||
method_file = dtm_dir / f"{basename}_dtm_method.txt"
|
||||
method_file.write_text("ign", encoding="utf-8")
|
||||
# Dates déterministes : method.txt plus ancien que la viz
|
||||
os.utime(method_file, (1600000000, 1600000000))
|
||||
os.utime(viz_file, (1700000000, 1700000000))
|
||||
fmt = lambda ts: datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M')
|
||||
|
||||
tile = {
|
||||
'basename': basename, 'resolution': 0.5,
|
||||
'dir_path': str(tile_dir),
|
||||
'viz': {'hillshade_multi': {'filename': viz_file.name, 'ext': 'webp'}},
|
||||
}
|
||||
meta = _collect_tile_metadata(tile, dtm_dir)
|
||||
assert meta['method'] == 'ign'
|
||||
assert meta['generated'] == fmt(1600000000)
|
||||
assert meta['viz']['hillshade_multi']['size'] == 4
|
||||
assert meta['viz']['hillshade_multi']['date'] == fmt(1700000000)
|
||||
|
||||
|
||||
def test_collect_tile_metadata_resolution_suffix(tmp_path):
|
||||
"""Une tuile 0,2 m lit son sidecar _dtm_r0p2_method.txt dédié."""
|
||||
from lidar_pipeline.index import _collect_tile_metadata
|
||||
|
||||
basename = "LHD_FXX_1000_6881_PTS_LAMB93_IGN69"
|
||||
tile_dir = tmp_path / "visualisations" / (basename + "_r0p2")
|
||||
tile_dir.mkdir(parents=True)
|
||||
dtm_dir = tmp_path / "DTM"
|
||||
dtm_dir.mkdir()
|
||||
(dtm_dir / f"{basename}_dtm_r0p2_method.txt").write_text("smrf", encoding="utf-8")
|
||||
|
||||
tile = {'basename': basename, 'resolution': 0.2,
|
||||
'dir_path': str(tile_dir),
|
||||
'viz': {}}
|
||||
meta = _collect_tile_metadata(tile, dtm_dir)
|
||||
assert meta['method'] == 'smrf'
|
||||
# La date vient du sidecar (écrit juste après la création du DTM)
|
||||
assert meta['generated'] is not None
|
||||
assert meta['viz'] == {}
|
||||
|
||||
|
||||
def test_collect_tile_metadata_fallback_date(tmp_path):
|
||||
"""Sans sidecar DTM, la date de génération remonte au plus ancien fichier viz."""
|
||||
from lidar_pipeline.index import _collect_tile_metadata
|
||||
|
||||
basename = "LHD_FXX_1000_6881_PTS_LAMB93_IGN69"
|
||||
tile_dir = tmp_path / "visualisations" / basename
|
||||
tile_dir.mkdir(parents=True)
|
||||
f = tile_dir / f"{basename}_svf.webp"
|
||||
f.write_bytes(b"x")
|
||||
|
||||
tile = {'basename': basename, 'resolution': 0.5,
|
||||
'dir_path': str(tile_dir),
|
||||
'viz': {'svf': {'filename': f.name, 'ext': 'webp'}}}
|
||||
meta = _collect_tile_metadata(tile, tmp_path / "DTM")
|
||||
assert meta['method'] is None
|
||||
assert meta['generated'] is not None
|
||||
|
||||
|
||||
def test_build_index_generates_html(tmp_path):
|
||||
"""build_index génère index.html et les vignettes."""
|
||||
from lidar_pipeline.index import build_index
|
||||
@ -142,11 +225,18 @@ def test_build_index_generates_html(tmp_path):
|
||||
|
||||
content = html_path.read_text(encoding='utf-8')
|
||||
# Vérifie la présence des éléments clés
|
||||
assert "Carte continue LiDAR" in content
|
||||
assert "Carte LiDAR" in content
|
||||
assert "LHD_FXX_1000_6881" in content
|
||||
assert "LHD_FXX_1001_6881" in content
|
||||
# Vérifie que le JSON intégré est valide
|
||||
assert "const DATA" in content
|
||||
assert "const TILES" in content
|
||||
# Vérifie les assets de l'interface (CSS/JS séparés)
|
||||
assets = output_dir / "assets"
|
||||
assert (assets / "app.css").read_text(encoding='utf-8').startswith('/*')
|
||||
app_js = (assets / "app.js").read_text(encoding='utf-8')
|
||||
assert "Couches" in app_js or "layers" in app_js
|
||||
assert 'assets/app.css' in content
|
||||
assert 'assets/app.js' in content
|
||||
# Vérifie les vignettes générées
|
||||
thumb_dir = output_dir / "index_thumbs"
|
||||
assert thumb_dir.is_dir()
|
||||
@ -154,6 +244,68 @@ def test_build_index_generates_html(tmp_path):
|
||||
assert len(thumbs) >= 2 # au moins hillshade pour chaque tuile
|
||||
|
||||
|
||||
def test_build_index_regenerates_stale_thumbnails(tmp_path):
|
||||
"""Une tuile recalculée (source plus récente) régénère sa vignette."""
|
||||
import os
|
||||
import time
|
||||
import numpy as np
|
||||
from PIL import Image as PILImage
|
||||
from lidar_pipeline.index import build_index
|
||||
|
||||
output_dir = tmp_path / "output"
|
||||
vis_dir = output_dir / "visualisations"
|
||||
vis_dir.mkdir(parents=True)
|
||||
tile_dir = _make_fake_viz_dir(vis_dir, "a", 1000, 6881, ('hillshade_multi',))
|
||||
|
||||
assert build_index(output_dir) is not None
|
||||
thumb_path = output_dir / "index_thumbs" / "LHD_FXX_1000_6881_PTS_LAMB93_IGN69_hillshade_multi.jpg"
|
||||
assert thumb_path.exists()
|
||||
m1 = thumb_path.stat().st_mtime
|
||||
|
||||
# Recalcul de la tuile : source réécrite avec une mtime plus récente
|
||||
src = tile_dir / "LHD_FXX_1000_6881_PTS_LAMB93_IGN69_hillshade_multi.webp"
|
||||
arr = np.random.randint(0, 255, (50, 50, 3), dtype=np.uint8)
|
||||
PILImage.fromarray(arr).save(str(src), format='WEBP', quality=80)
|
||||
os.utime(src, (m1 + 5, m1 + 5))
|
||||
|
||||
assert build_index(output_dir) is not None
|
||||
m2 = thumb_path.stat().st_mtime
|
||||
assert m2 > m1 # vignette régénérée
|
||||
|
||||
# Source non modifiée depuis → pas de régénération inutile
|
||||
os.utime(src, (time.time() - 10, time.time() - 10))
|
||||
assert build_index(output_dir) is not None
|
||||
assert thumb_path.stat().st_mtime == m2
|
||||
|
||||
|
||||
def test_build_subtiles_regenerates_stale_crops(tmp_path):
|
||||
"""Une dalle 0,2 m recalculée régénère ses sous-tuiles (par visualisation)."""
|
||||
import os
|
||||
from lidar_pipeline.index import build_index
|
||||
|
||||
output_dir = tmp_path / "output"
|
||||
vis_dir = output_dir / "visualisations"
|
||||
vis_dir.mkdir(parents=True)
|
||||
tile_dir = _make_fake_viz_dir(vis_dir, "a", 1000, 6881,
|
||||
('hillshade_multi', 'aspect'), res_suffix='_r0p2')
|
||||
|
||||
assert build_index(output_dir) is not None
|
||||
sub_dir = output_dir / "index_subtiles"
|
||||
hill_avif = sub_dir / "LHD_FXX_1000_6881_PTS_LAMB93_IGN69_r0p2_hillshade_multi_0_0.avif"
|
||||
aspect_avif = sub_dir / "LHD_FXX_1000_6881_PTS_LAMB93_IGN69_r0p2_aspect_0_0.avif"
|
||||
assert hill_avif.exists() and aspect_avif.exists()
|
||||
m_hill_1 = hill_avif.stat().st_mtime
|
||||
m_aspect_1 = aspect_avif.stat().st_mtime
|
||||
|
||||
# Recalcul : seule la source hillshade est plus récente
|
||||
src = tile_dir / "LHD_FXX_1000_6881_PTS_LAMB93_IGN69_hillshade_multi.webp"
|
||||
os.utime(src, (m_hill_1 + 5, m_hill_1 + 5))
|
||||
|
||||
assert build_index(output_dir) is not None
|
||||
assert hill_avif.stat().st_mtime > m_hill_1 # sous-tuiles hillshade régénérées
|
||||
assert aspect_avif.stat().st_mtime == m_aspect_1 # aspect intact
|
||||
|
||||
|
||||
def test_build_index_empty_returns_none(tmp_path):
|
||||
"""Aucune tuile → build_index retourne None sans crash."""
|
||||
from lidar_pipeline.index import build_index
|
||||
@ -175,26 +327,66 @@ def test_build_index_embeds_valid_json(tmp_path):
|
||||
|
||||
build_index(output_dir)
|
||||
content = (output_dir / "index.html").read_text(encoding='utf-8')
|
||||
# Extrait le JSON entre "const DATA = " et ";"
|
||||
start = content.index("const DATA = ") + len("const DATA = ")
|
||||
# Extrait le JSON entre "const TILES = " et la fin de déclaration
|
||||
start = content.index("const TILES = ") + len("const TILES = ")
|
||||
# Trouve le ; de fin de déclaration
|
||||
depth = 0
|
||||
end = start
|
||||
for i, ch in enumerate(content[start:], start):
|
||||
if ch == '{':
|
||||
if ch in ('{', '['):
|
||||
depth += 1
|
||||
elif ch == '}':
|
||||
elif ch in ('}', ']'):
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
end = i + 1
|
||||
break
|
||||
data = json.loads(content[start:end])
|
||||
assert 'tiles' in data
|
||||
assert 'bbox' in data
|
||||
assert 'vizList' in data
|
||||
assert len(data['tiles']) == 1
|
||||
assert data['tiles'][0]['col'] == 1000
|
||||
assert data['tiles'][0]['row'] == 6881
|
||||
assert len(data) > 0
|
||||
assert data[0]['col'] == 1000
|
||||
assert data[0]['row'] == 6881
|
||||
|
||||
|
||||
def test_attach_gps_bounds():
|
||||
"""attach_gps_bounds ajoute des bounds GPS ordonnées (France métropolitaine)."""
|
||||
from lidar_pipeline.index import attach_gps_bounds
|
||||
tiles = [{'col': 1000, 'row': 6881}, {'col': 1042, 'row': 6900}]
|
||||
attach_gps_bounds(tiles)
|
||||
for t in tiles:
|
||||
assert 'bounds' in t
|
||||
(lat_s, lon_w), (lat_n, lon_e) = t['bounds']
|
||||
assert lat_n > lat_s
|
||||
assert lon_e > lon_w
|
||||
# France métropolitaine
|
||||
assert 41 < lat_s < 51
|
||||
assert -5 < lon_w < 10
|
||||
|
||||
|
||||
def test_attach_gps_bounds_row_is_north_edge():
|
||||
"""Le numéro de ligne du fichier = bord NORD (convention LiDAR HD IGN).
|
||||
|
||||
Vérifié sur les bounds des DTM : X ∈ [col, col+1] km, Y ∈ [row-1, row] km.
|
||||
La régression historique plaçait Y ∈ [row, row+1] (1 km trop au nord).
|
||||
"""
|
||||
from rasterio.warp import transform as warp_transform
|
||||
from lidar_pipeline.index import attach_gps_bounds
|
||||
|
||||
col, row = 1054, 6882
|
||||
tiles = [{'col': col, 'row': row}]
|
||||
attach_gps_bounds(tiles)
|
||||
corners = tiles[0]['corners']
|
||||
|
||||
# Référence exacte de la vraie cellule : SW, SE, NE, NW
|
||||
xs = [col * 1000, (col + 1) * 1000, (col + 1) * 1000, col * 1000]
|
||||
ys = [(row - 1) * 1000, (row - 1) * 1000, row * 1000, row * 1000]
|
||||
lons, lats = warp_transform('EPSG:2154', 'EPSG:4326', xs, ys)
|
||||
for k in range(4):
|
||||
assert abs(corners[k][0] - lats[k]) < 1e-9
|
||||
assert abs(corners[k][1] - lons[k]) < 1e-9
|
||||
|
||||
# L'ancienne convention (row = bord sud) serait décalée d'environ 1 km
|
||||
lat_n = max(c[0] for c in corners)
|
||||
assert abs(lat_n - max(lats)) < 1e-9 # bord nord = Y = row×1000
|
||||
|
||||
|
||||
|
||||
def test_pick_display_viz_prefers_hillshade():
|
||||
@ -203,3 +395,30 @@ def test_pick_display_viz_prefers_hillshade():
|
||||
assert _pick_display_viz(['svf', 'hillshade_multi', 'slope']) == 'hillshade_multi'
|
||||
assert _pick_display_viz(['svf', 'slope']) == 'svf'
|
||||
assert _pick_display_viz(['topo']) == 'topo'
|
||||
|
||||
|
||||
def test_subdivision_k():
|
||||
"""0,5 m/px (2000 px) reste entier ; 0,2 m/px (5000 px) est découpé en 2×2."""
|
||||
from lidar_pipeline.index import _subdivision_k
|
||||
assert _subdivision_k(0.5) == 1
|
||||
assert _subdivision_k(0.2) == 2
|
||||
assert _subdivision_k(1.0) == 1
|
||||
|
||||
|
||||
def test_subtile_corners_grid():
|
||||
"""Les sous-tuiles reconstruisent exactement la grille de la dalle."""
|
||||
from lidar_pipeline.index import _subtile_corners
|
||||
corners = [[10.0, 2.0], [10.0, 3.0], [11.0, 3.0], [11.0, 2.0]] # SW SE NE NW
|
||||
k = 2
|
||||
sw_quad = _subtile_corners(corners, 0, 0, k) # quadrant sud-ouest
|
||||
ne_quad = _subtile_corners(corners, 1, 1, k) # quadrant nord-est
|
||||
# Le quadrant SW partage le coin SW de la dalle
|
||||
assert sw_quad[0] == corners[0]
|
||||
# Le quadrant NE partage le coin NE de la dalle
|
||||
assert ne_quad[2] == corners[2]
|
||||
# Le quadrant SW a son coin NE au centre de la dalle
|
||||
assert sw_quad[2] == [10.5, 2.5]
|
||||
# Adjacence : bord est du SW = bord ouest du SE (0,0)-(1,0)
|
||||
se_quad = _subtile_corners(corners, 1, 0, k)
|
||||
assert sw_quad[1] == se_quad[0]
|
||||
assert sw_quad[2] == se_quad[3]
|
||||
|
||||
Reference in New Issue
Block a user