feat(e2e): add distributed E2E test framework with parametric traffic generation

Add run-e2e-test.sh with CLI parameters (--hits, --http-ratio, --dns, --tls,
--src-ips, --keep-analysis, --up) for configurable traffic generation. Traffic
runs from VM endpoints with multiple source IPs (alias IPs on eth0) to produce
distinct sessions for the ML pipeline. Fix curl TLS flags (--tlsv1.2 instead
of --tls-v1-2), skip redundant local verification in distributed mode, and
fix dashboard is_available() cache that never retried after ClickHouse recovery.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

services/dashboard/backend/database.py

@@ -6,17 +6,43 @@ from typing import Any
 
 import clickhouse_connect
 from clickhouse_connect.driver.client import Client
+from clickhouse_connect.driver.exceptions import DatabaseError
 
 from backend.config import CLICKHOUSE_HOST, CLICKHOUSE_PORT, CLICKHOUSE_USER, CLICKHOUSE_PASSWORD
 
 logger = logging.getLogger(__name__)
 
 _client: Client | None = None
+_available: bool | None = None  # None = not tested yet
+
+
+class ClickHouseUnavailable(Exception):
+    """Raised when ClickHouse is not reachable."""
+
+
+def is_available() -> bool:
+    """Check ClickHouse connectivity (retries on every call if previously failed)."""
+    global _client, _available
+    try:
+        # Force re-creation if previously marked unavailable
+        if _available is False:
+            _client = None
+        get_client()
+        _available = True
+        return True
+    except Exception:
+        _available = False
+        _client = None
+        logger.warning("ClickHouse unavailable at %s:%s", CLICKHOUSE_HOST, CLICKHOUSE_PORT)
+        return False
 
 
 def get_client() -> Client:
-    """Return a lazily-initialised ClickHouse client (singleton)."""
-    global _client
+    """Return a lazily-initialised ClickHouse client (singleton).
+
+    Resets the singleton on connection failure so the next call retries.
+    """
+    global _client, _available
     if _client is None:
         _client = clickhouse_connect.get_client(
             host=CLICKHOUSE_HOST,
@@ -25,9 +51,17 @@ def get_client() -> Client:
             password=CLICKHOUSE_PASSWORD,
         )
         logger.info("Connected to ClickHouse at %s:%s", CLICKHOUSE_HOST, CLICKHOUSE_PORT)
+        _available = True
     return _client
 
 
+def _mark_unavailable() -> None:
+    """Reset client and mark ClickHouse as unavailable."""
+    global _client, _available
+    _client = None
+    _available = False
+
+
 def _normalise_value(v: Any) -> Any:
     """Convert ClickHouse-specific types to JSON-friendly Python types."""
     if isinstance(v, (ipaddress.IPv4Address, ipaddress.IPv6Address)):
@@ -41,26 +75,47 @@ def _normalise_value(v: Any) -> Any:
 
 
 def query(sql: str, params: dict | None = None) -> list[dict[str, Any]]:
-    """Execute *sql* and return a list of row-dicts."""
-    client = get_client()
-    result = client.query(sql, parameters=params or {})
-    columns = result.column_names
-    rows: list[dict[str, Any]] = []
-    for row in result.result_rows:
-        rows.append({col: _normalise_value(val) for col, val in zip(columns, row)})
-    return rows
+    """Execute *sql* and return a list of row-dicts.
+
+    Raises ClickHouseUnavailable if the server is not reachable.
+    """
+    try:
+        client = get_client()
+        result = client.query(sql, parameters=params or {})
+        columns = result.column_names
+        rows: list[dict[str, Any]] = []
+        for row in result.result_rows:
+            rows.append({col: _normalise_value(val) for col, val in zip(columns, row)})
+        return rows
+    except (DatabaseError, ConnectionError, OSError) as exc:
+        _mark_unavailable()
+        raise ClickHouseUnavailable(str(exc)) from exc
 
 
 def query_scalar(sql: str, params: dict | None = None) -> Any:
-    """Execute *sql* and return the single scalar value."""
-    client = get_client()
-    result = client.query(sql, parameters=params or {})
-    if result.result_rows:
-        return _normalise_value(result.result_rows[0][0])
-    return None
+    """Execute *sql* and return the single scalar value.
+
+    Raises ClickHouseUnavailable if the server is not reachable.
+    """
+    try:
+        client = get_client()
+        result = client.query(sql, parameters=params or {})
+        if result.result_rows:
+            return _normalise_value(result.result_rows[0][0])
+        return None
+    except (DatabaseError, ConnectionError, OSError) as exc:
+        _mark_unavailable()
+        raise ClickHouseUnavailable(str(exc)) from exc
 
 
 def execute(sql: str, params: dict | None = None) -> None:
-    """Execute a DDL / DML statement that returns no rows."""
-    client = get_client()
-    client.command(sql, parameters=params or {})
+    """Execute a DDL / DML statement that returns no rows.
+
+    Raises ClickHouseUnavailable if the server is not reachable.
+    """
+    try:
+        client = get_client()
+        client.command(sql, parameters=params or {})
+    except (DatabaseError, ConnectionError, OSError) as exc:
+        _mark_unavailable()
+        raise ClickHouseUnavailable(str(exc)) from exc