fb892ea9f2
Comments, docstrings, logs, CLI help, map UI, legends, PDF sheet, scripts, compose files and AGENTS.md are now English. Data keys stay unchanged (relief_oriente, densite_sol, visualisations/, API JSON keys, link params). Wrong comments and help defaults found along the way are corrected. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
290 lines
12 KiB
Python
290 lines
12 KiB
Python
"""Download of IGN LiDAR HD tiles for tiles not yet generated.
|
||
|
||
STAC catalogue (current) : https://browser.stac.teledetection.fr/collections/lidarhd
|
||
API : https://api.stac.teledetection.fr/collections/lidarhd/items
|
||
Files (Géoplateforme) : https://data.geopf.fr/telechargement/download/...
|
||
|
||
Each tile covers 1 km × 1 km in Lambert 93 and is named after its north-west
|
||
corner: LHD_FXX_{col}_{row}_PTS_LAMB93_IGN69.copc.laz
|
||
(col = west X in km, row = north Y in km, see the STAC property
|
||
"lidarhd:coordonnees_NW" in the "0816-6847" format).
|
||
"""
|
||
|
||
import json
|
||
import logging
|
||
import time
|
||
import urllib.parse
|
||
import urllib.request
|
||
from pathlib import Path
|
||
|
||
logger = logging.getLogger("lidar")
|
||
|
||
_STAC_ITEMS_URL = "https://api.stac.teledetection.fr/collections/lidarhd/items"
|
||
_HEADERS = {"User-Agent": "Mozilla/5.0 (lidar-archeo-pipeline)"}
|
||
|
||
|
||
def parse_tile_specs(args):
|
||
"""Convert "col,row" or "col:row" specifications into a list of tuples.
|
||
|
||
Args:
|
||
args: list of strings (e.g. ["1055,6882", "1056:6883"]).
|
||
|
||
Returns:
|
||
List of (col, row) tuples.
|
||
|
||
Raises:
|
||
ValueError: if a specification is malformed.
|
||
"""
|
||
specs = []
|
||
for raw in args:
|
||
text = raw.strip().replace(":", ",").replace(";", ",")
|
||
parts = [p.strip() for p in text.split(",") if p.strip()]
|
||
if len(parts) != 2:
|
||
raise ValueError(f"Invalid tile specification: {raw!r} (expected: col,row)")
|
||
try:
|
||
col, row = int(parts[0]), int(parts[1])
|
||
except ValueError:
|
||
raise ValueError(f"Invalid tile specification: {raw!r} (col and row must be integers)")
|
||
specs.append((col, row))
|
||
return specs
|
||
|
||
|
||
def tile_filename(col, row):
|
||
"""Standard LAZ file name of a (col, row) tile."""
|
||
return f"LHD_FXX_{col:04d}_{row:04d}_PTS_LAMB93_IGN69.copc.laz"
|
||
|
||
|
||
def _report_download(output_dir, tile, state, detail=None):
|
||
"""Emit a download event for the generation queue (best-effort)."""
|
||
if output_dir is None:
|
||
return
|
||
from .progress import report_event
|
||
report_event(output_dir, tile, "download", state, detail=detail)
|
||
|
||
|
||
def _bbox_wgs84(col, row):
|
||
"""WGS84 bbox of the (col,row) tile for the STAC query (padded)."""
|
||
try:
|
||
from rasterio.warp import transform as warp_transform
|
||
xs = [col * 1000, (col + 1) * 1000, col * 1000, (col + 1) * 1000]
|
||
ys = [(row - 1) * 1000] * 2 + [row * 1000] * 2
|
||
lons, lats = warp_transform('EPSG:2154', 'EPSG:4326', xs, ys)
|
||
except Exception:
|
||
lons = None
|
||
if lons is None:
|
||
try: # lightweight (map) image: pyproj without rasterio
|
||
from pyproj import Transformer
|
||
tr = Transformer.from_crs("EPSG:2154", "EPSG:4326", always_xy=True)
|
||
xs = [col * 1000, (col + 1) * 1000, col * 1000, (col + 1) * 1000]
|
||
ys = [(row - 1) * 1000] * 2 + [row * 1000] * 2
|
||
lons, lats = tr.transform(xs, ys)
|
||
except Exception:
|
||
lons = None
|
||
if lons is None:
|
||
from .index import _approx_l93_to_wgs84
|
||
pts = [_approx_l93_to_wgs84(x, y)
|
||
for x in (col * 1000, (col + 1) * 1000)
|
||
for y in ((row - 1) * 1000, row * 1000)]
|
||
lons = [p[0] for p in pts]
|
||
lats = [p[1] for p in pts]
|
||
pad = 0.005 # ~500 m margin to absorb rounding errors at the edges
|
||
return (min(lons) - pad, min(lats) - pad, max(lons) + pad, max(lats) + pad)
|
||
|
||
|
||
def match_feature(features, col, row):
|
||
"""Return the STAC item matching the (col,row) tile, else None."""
|
||
want = f"{col:04d}-{row:04d}"
|
||
for feature in features:
|
||
props = feature.get("properties", {})
|
||
if props.get("lidarhd:coordonnees_NW") == want:
|
||
return feature
|
||
return None
|
||
|
||
|
||
def find_tile_url(col, row, timeout=20, max_pages=5):
|
||
"""Look up the download URL of the (col,row) tile in the STAC catalogue.
|
||
|
||
Returns:
|
||
URL (str), or None if the tile is not (yet) published by IGN.
|
||
"""
|
||
feature = find_tile_feature(col, row, timeout=timeout, max_pages=max_pages)
|
||
return feature.get("assets", {}).get("data", {}).get("href") if feature else None
|
||
|
||
|
||
def tile_ign_metadata(col, row, timeout=10):
|
||
"""IGN record of a tile: acquisition, sensors, edition, download.
|
||
|
||
Returns:
|
||
Normalised dict (found=False if the tile is not published).
|
||
"""
|
||
feature = find_tile_feature(col, row, timeout=timeout)
|
||
if not feature:
|
||
return {"found": False}
|
||
p = feature.get("properties", {})
|
||
self_link = next((link.get("href") for link in feature.get("links", [])
|
||
if link.get("rel") == "self"), None)
|
||
return {
|
||
"found": True,
|
||
"download_url": feature.get("assets", {}).get("data", {}).get("href"),
|
||
"acquisition_start": p.get("start_datetime") or p.get("lidarhd:date_debut_acquisition"),
|
||
"acquisition_end": p.get("end_datetime") or p.get("lidarhd:date_fin_acquisition"),
|
||
"sensors": p.get("lidarhd:capteur") or [],
|
||
"mission": p.get("lidarhd:code_mission"),
|
||
"acquisition_operator": p.get("lidarhd:moe_acquisition"),
|
||
"acquisition_owner": p.get("lidarhd:moa_acquisition"),
|
||
"edition_date": p.get("lidarhd:date_edition"),
|
||
"classification_process": p.get("lidarhd:procede_classement"),
|
||
"points": p.get("lidarhd:nombre_points") or p.get("pc:count"),
|
||
"altimetry": p.get("lidarhd:systeme_altimetrique"),
|
||
"stac_url": self_link,
|
||
}
|
||
|
||
|
||
def find_tile_feature(col, row, timeout=20, max_pages=5):
|
||
"""STAC item of the (col,row) tile, or None if not published.
|
||
|
||
Follows the catalogue pagination (rel=next link): without it, a tile
|
||
beyond the first page of results appeared "not found".
|
||
"""
|
||
w, s, e, n = _bbox_wgs84(col, row)
|
||
query = urllib.parse.urlencode({"bbox": f"{w:.6f},{s:.6f},{e:.6f},{n:.6f}", "limit": 50})
|
||
url = f"{_STAC_ITEMS_URL}?{query}"
|
||
for _ in range(max_pages):
|
||
req = urllib.request.Request(url, headers=_HEADERS)
|
||
with urllib.request.urlopen(req, timeout=timeout) as response:
|
||
data = json.loads(response.read().decode("utf-8"))
|
||
feature = match_feature(data.get("features", []), col, row)
|
||
if feature:
|
||
return feature
|
||
nxt = next((link.get("href") for link in data.get("links", [])
|
||
if link.get("rel") == "next" and link.get("href")), None)
|
||
if not nxt:
|
||
return None
|
||
url = nxt
|
||
return None
|
||
|
||
|
||
def download_file(url, dest_path, timeout=120, chunk=1024 * 1024):
|
||
"""Stream url to dest_path (atomic write).
|
||
|
||
Writes to a .part file then renames it: an interrupted run (SIGKILL,
|
||
crash) never leaves a partial LAZ that a later run would treat as
|
||
already downloaded. Returns the size in bytes.
|
||
"""
|
||
import os
|
||
dest_path = Path(dest_path)
|
||
tmp = dest_path.with_name(dest_path.name + ".part")
|
||
req = urllib.request.Request(url, headers=_HEADERS)
|
||
t0 = time.time()
|
||
try:
|
||
with urllib.request.urlopen(req, timeout=timeout) as response, open(tmp, "wb") as out:
|
||
done = 0
|
||
while True:
|
||
block = response.read(chunk)
|
||
if not block:
|
||
break
|
||
out.write(block)
|
||
done += len(block)
|
||
elapsed = time.time() - t0
|
||
logger.info(f" {done / 1e6:.0f} MB in {elapsed:.0f}s"
|
||
f" ({done / 1e6 / max(elapsed, 0.1):.1f} MB/s)")
|
||
os.replace(tmp, dest_path)
|
||
except BaseException:
|
||
tmp.unlink(missing_ok=True)
|
||
raise
|
||
return done
|
||
|
||
|
||
def fetch_tiles(input_dir, specs, output_dir=None, only_viz=None, resolutions=(0.5,), force=False):
|
||
"""Download the given IGN tiles, except those not needed for processing.
|
||
|
||
Args:
|
||
input_dir: LAZ files directory (must be writable).
|
||
specs: list of (col, row) tuples.
|
||
output_dir: output directory (optional) — used to skip tiles that
|
||
are already complete.
|
||
only_viz: step names of the requested visualisations (e.g. ['aspect']).
|
||
If given, a tile is skipped only if it already has all
|
||
these visualisations at the requested resolutions —
|
||
otherwise its LAZ is (re)downloaded to fill in the
|
||
missing ones. If not given, a tile is skipped as soon as
|
||
any visualisation exists for it.
|
||
resolutions: expected resolutions (m/px) for a tile to count as
|
||
complete when only_viz is given.
|
||
force: True to download even complete tiles (regeneration: their
|
||
LAZ is needed to reprocess with --force).
|
||
|
||
Returns:
|
||
List of downloaded paths.
|
||
"""
|
||
from .dtm import EDGE_NEIGHBORS_DIRNAME
|
||
input_dir = Path(input_dir)
|
||
complete = None
|
||
if output_dir is not None and only_viz and not force:
|
||
from .index import cells_with_all_viz, step_to_keyword
|
||
keys = [step_to_keyword(v) for v in only_viz]
|
||
complete = cells_with_all_viz(Path(output_dir) / "visualisations",
|
||
keys, resolutions)
|
||
# Explicit: never promote when downloading directly INTO edge_neighbors/
|
||
# (_fetch_edge_neighbors call with input_dir=edge_dir) — otherwise a
|
||
# neighbour tile could end up moved onto itself.
|
||
is_edge_dir = input_dir.name == EDGE_NEIGHBORS_DIRNAME
|
||
downloaded = []
|
||
for col, row in specs:
|
||
name = tile_filename(col, row)
|
||
tile = name[:-len(".copc.laz")] if name.endswith(".copc.laz") else name
|
||
dest = input_dir / name
|
||
if dest.exists():
|
||
logger.info(f" {name}: already in input/ — no download")
|
||
_report_download(output_dir, tile, "skip", "already in input/")
|
||
continue
|
||
if not is_edge_dir:
|
||
# dest.exists() above only checks the exact name: a neighbouring
|
||
# ".part" never matches it, so .exists() is enough to rule out a
|
||
# neighbour download still in progress.
|
||
neighbor_copy = input_dir / EDGE_NEIGHBORS_DIRNAME / name
|
||
if neighbor_copy.exists():
|
||
import os
|
||
os.replace(neighbor_copy, dest)
|
||
logger.info(f" {name}: already downloaded as a neighbour "
|
||
f"— moved to input/")
|
||
_report_download(output_dir, tile, "skip",
|
||
"already downloaded as a neighbour — moved to input/")
|
||
continue
|
||
if output_dir is not None and not force:
|
||
if complete is not None:
|
||
if (col, row) in complete:
|
||
logger.info(f" {name}: visualisations already complete — skipped")
|
||
_report_download(output_dir, tile, "skip", "visualisations already complete")
|
||
continue
|
||
else:
|
||
vis_dir = Path(output_dir) / "visualisations"
|
||
if list(vis_dir.glob(f"LHD_FXX_{col:04d}_{row:04d}_PTS*")):
|
||
logger.info(f" {name}: visualisations already generated — skipped")
|
||
_report_download(output_dir, tile, "skip", "visualisations already generated")
|
||
continue
|
||
logger.info(f" {name}: searching the IGN catalogue...")
|
||
_report_download(output_dir, tile, "start", "IGN catalogue")
|
||
try:
|
||
url = find_tile_url(col, row)
|
||
except Exception as e:
|
||
logger.warning(f" ✗ {name}: catalogue error ({e})")
|
||
_report_download(output_dir, tile, "fail", f"catalogue error: {e}")
|
||
continue
|
||
if not url:
|
||
logger.warning(f" ✗ {name}: not found in the IGN catalogue (area not published?)")
|
||
_report_download(output_dir, tile, "fail", "not found in the IGN catalogue")
|
||
continue
|
||
logger.info(f" {name}: downloading from the Géoplateforme...")
|
||
_report_download(output_dir, tile, "start", "Géoplateforme")
|
||
try:
|
||
download_file(url, dest)
|
||
logger.info(f" ✓ {name} downloaded")
|
||
_report_download(output_dir, tile, "ok")
|
||
downloaded.append(dest)
|
||
except Exception as e:
|
||
dest.unlink(missing_ok=True)
|
||
logger.warning(f" ✗ {name}: download failed ({e})")
|
||
_report_download(output_dir, tile, "fail", f"download failed: {e}")
|
||
return downloaded
|