feat(e2e): add distributed E2E test framework with parametric traffic generation

Add run-e2e-test.sh with CLI parameters (--hits, --http-ratio, --dns, --tls, --src-ips, --keep-analysis, --up) for configurable traffic generation. Traffic runs from VM endpoints with multiple source IPs (alias IPs on eth0) to produce distinct sessions for the ML pipeline. Fix curl TLS flags (--tlsv1.2 instead of --tls-v1-2), skip redundant local verification in distributed mode, and fix dashboard is_available() cache that never retried after ClickHouse recovery. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-04-15 00:09:32 +02:00
parent 7894d39f1c
commit f88b739992
40 changed files with 2154 additions and 337 deletions
--- a/tests/vm/run-e2e-test.sh
+++ b/tests/vm/run-e2e-test.sh
@ -0,0 +1,795 @@
+#!/usr/bin/env bash
+# =============================================================================
+# run-e2e-test.sh — Test E2E distribué ja4-platform
+#
+# Architecture :
+#   3 VMs endpoint (centos8/rocky9/rocky10) : nginx + ja4ebpf
+#   1 VM analysis (192.168.42.10) : ClickHouse + bot-detector + dashboard
+#   Host : orchestrateur + génération de trafic
+#
+# Pipeline testé :
+#   trafic host → endpoints → ja4ebpf → ClickHouse central →
+#   MV agrégation → bot-detector ML → dashboard API
+#
+# Usage :
+#   make e2e-up && make test-e2e
+#   TRAFFIC_COUNT=100 make test-e2e-quick
+#   ./run-e2e-test.sh --hits 1000 --http-ratio 0.3 --dns 2 --tls 1.2,1.3
+#
+# Paramètres :
+#   --hits N          Nombre total de requêtes par VM (déf. 500)
+#   --http-ratio R    Ratio HTTP/HTTPS : R = part HTTP (0.0–1.0, déf. 0.2)
+#   --dns N           Nombre de hostnames SNI utilisés (1–4, déf. 4)
+#   --tls VERS        Versions TLS à utiliser, séparées par virgules (déf. 1.2,1.3)
+#   --src-ips N       Nombre d'IPs sources par VM (alias sur eth0 dans la VM, déf. 1)
+#   --keep-analysis   Conserver la stack analysis (CH + dashboard) après le test
+#   --up              Démarrer la stack analysis et les endpoints, puis s'arrêter
+#                     (pas de trafic ni vérification — utile pour inspection manuelle)
+# =============================================================================
+set -euo pipefail
+
+VM_DIR="$(cd "$(dirname "$0")" && pwd)"
+PROJECT_ROOT="$(cd "$VM_DIR/../.." && pwd)"
+ANALYSIS_IP="192.168.42.10"
+ENDPOINT_VMS="rocky9 rocky10"
+ALL_VMS="rocky9 rocky10 analysis"
+STACK="nginx"
+
+# ── Paramètres par défaut (surchARGEABLES par CLI ou env) ──
+TRAFFIC_COUNT="${TRAFFIC_COUNT:-500}"
+HTTP_RATIO="${HTTP_RATIO:-0.2}"
+DNS_COUNT="${DNS_COUNT:-4}"
+TLS_VERSIONS="${TLS_VERSIONS:-1.2,1.3}"
+SRC_IP_COUNT="${SRC_IP_COUNT:-1}"
+KEEP_ANALYSIS="${KEEP_ANALYSIS:-false}"
+UP_ONLY="${UP_ONLY:-false}"
+
+# ── Parsing des arguments CLI ──
+usage() {
+    echo "Usage: $0 [OPTIONS]"
+    echo ""
+    echo "Options :"
+    echo "  --hits N         Nombre de requêtes par VM (déf. 500)"
+    echo "  --http-ratio R   Ratio HTTP (0=100% HTTPS, 1=100% HTTP, déf. 0.2)"
+    echo "  --dns N          Nombre de hostnames SNI utilisés (1-4, déf. 4)"
+    echo "  --tls VERS       Versions TLS, séparées par virgules (déf. 1.2,1.3)"
+    echo "  --src-ips N      Nombre d'IPs sources par VM (déf. 1)"
+    echo "  --keep-analysis  Conserver la stack analysis (CH + dashboard) après le test"
+    echo "  --up             Démarrer stack analysis + endpoints, puis s'arrêter"
+    echo "  -h, --help       Afficher cette aide"
+    exit 0
+}
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        --hits)        TRAFFIC_COUNT="$2"; shift 2 ;;
+        --http-ratio)  HTTP_RATIO="$2";   shift 2 ;;
+        --dns)         DNS_COUNT="$2";     shift 2 ;;
+        --tls)           TLS_VERSIONS="$2";  shift 2 ;;
+        --src-ips)       SRC_IP_COUNT="$2";  shift 2 ;;
+        --keep-analysis) KEEP_ANALYSIS="true"; shift ;;
+        --up)            UP_ONLY="true"; shift ;;
+        -h|--help)       usage ;;
+        *) echo "Option inconnue : $1"; usage ;;
+    esac
+done
+
+# ── Validation ──
+if ! python3 -c "exit(0 if 0.0 <= ${HTTP_RATIO} <= 1.0 else 1)" 2>/dev/null; then
+    echo "ERREUR: --http-ratio doit être entre 0.0 et 1.0" >&2; exit 1
+fi
+if [ "$DNS_COUNT" -lt 1 ] || [ "$DNS_COUNT" -gt 4 ]; then
+    echo "ERREUR: --dns doit être entre 1 et 4" >&2; exit 1
+fi
+
+# ── Résolution des versions TLS pour curl ──
+CURL_TLS_FLAGS=""
+for v in $(echo "$TLS_VERSIONS" | tr ',' ' '); do
+    case "$v" in
+        1.0) CURL_TLS_FLAGS="$CURL_TLS_FLAGS --tlsv1.0" ;;
+        1.1) CURL_TLS_FLAGS="$CURL_TLS_FLAGS --tlsv1.1" ;;
+        1.2) CURL_TLS_FLAGS="$CURL_TLS_FLAGS --tlsv1.2" ;;
+        1.3) CURL_TLS_FLAGS="$CURL_TLS_FLAGS --tlsv1.3" ;;
+        *)   echo "[e2e] WARN: Version TLS '$v' non reconnue, ignorée" ;;
+    esac
+done
+[ -z "$CURL_TLS_FLAGS" ] && CURL_TLS_FLAGS="--tlsv1.2 --tlsv1.3" && TLS_VERSIONS="1.2,1.3"
+
+# Nombre de requêtes HTTP vs HTTPS dérivés du ratio
+HTTPS_COUNT=$(python3 -c "print(int(${TRAFFIC_COUNT} * (1 - ${HTTP_RATIO})))")
+HTTP_COUNT=$(python3 -c "print(int(${TRAFFIC_COUNT} * ${HTTP_RATIO}))")
+
+GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; CYAN='\033[0;36m'
+RESET='\033[0m'; BOLD='\033[1m'
+PASS_COUNT=0; FAIL_COUNT=0; WARN_COUNT=0
+
+log()  { echo -e "${BOLD}[e2e]${RESET} $(date +%H:%M:%S) $*"; }
+pass() { echo -e "  ${GREEN}PASS${RESET}  $*"; ((PASS_COUNT++)) || true; }
+fail() { echo -e "  ${RED}FAIL${RESET}  $*"; ((FAIL_COUNT++)) || true; }
+warn() { echo -e "  ${YELLOW}WARN${RESET}   $*"; ((WARN_COUNT++)) || true; }
+
+# Requête ClickHouse via HTTP API sur la VM analysis
+ch_query() {
+    curl -sf "http://${ANALYSIS_IP}:8123/" -d "$1" 2>/dev/null || echo ""
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Phase 0 : Setup environnement
+# ═════════════════════════════════════════════════════════════════════════════
+phase0_setup() {
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  Phase 0 : Setup environnement                          ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo ""
+
+    cd "$VM_DIR"
+
+    # Vérifier que les VMs existent
+    for vm in $ALL_VMS; do
+        if ! vagrant status "$vm" 2>/dev/null | grep -q "running"; then
+            log "Démarrage de $vm..."
+            vagrant up "$vm" 2>&1 | tail -5
+        else
+            log "$vm déjà en cours d'exécution"
+        fi
+    done
+
+    # Rsync des fichiers vers toutes les VMs
+    log "Synchronisation des fichiers..."
+    for vm in $ALL_VMS; do
+        vagrant rsync "$vm" 2>&1 | tail -1
+    done
+
+    # Découvrir les IPs eth0 des endpoints et filtrer ceux sans ja4ebpf
+    declare -A VM_IPS
+    ACTIVE_ENDPOINTS=""
+    for vm in $ENDPOINT_VMS; do
+        # Vérifier que ja4ebpf est installé
+        if ! vagrant ssh "$vm" -- 'which ja4ebpf' 2>/dev/null | grep -q ja4ebpf; then
+            warn "$vm ignoré — ja4ebpf non installé"
+            continue
+        fi
+        VM_IPS[$vm]=$(vagrant ssh "$vm" -- 'ip -4 addr show eth0' 2>/dev/null \
+            | awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}')
+        if [ -z "${VM_IPS[$vm]}" ]; then
+            fail "Impossible d'obtenir l'IP eth0 de $vm"
+        else
+            log "  $vm : ${VM_IPS[$vm]}"
+            ACTIVE_ENDPOINTS="$ACTIVE_ENDPOINTS $vm"
+        fi
+    done
+    ENDPOINT_VMS=$(echo $ACTIVE_ENDPOINTS)  # overwrite with active VMs only
+
+    # Exporter les IPs pour les phases suivantes
+    ENDPOINT_IPS_ARRAY=()
+    for vm in $ENDPOINT_VMS; do
+        ENDPOINT_IPS_ARRAY+=("${VM_IPS[$vm]}")
+    done
+
+    pass "Endpoints actifs : ${ENDPOINT_VMS}"
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Phase 1 : Démarrage de la stack analysis
+# ═════════════════════════════════════════════════════════════════════════════
+phase1_analysis() {
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  Phase 1 : Stack analysis (ClickHouse + ML + Dashboard) ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo ""
+
+    cd "$VM_DIR"
+
+    # Purger les volumes pour garantir une DB vide avant le test
+    log "Purge des volumes Docker pour DB vierge..."
+    vagrant ssh analysis -- "cd /ja4-platform/tests/vm/analysis && docker compose down -v --remove-orphans" 2>&1 | tail -5
+
+    # Démarrer la stack analysis via docker compose
+    log "Démarrage de la stack analysis sur la VM..."
+    vagrant ssh analysis -- "cd /ja4-platform/tests/vm/analysis && docker compose up -d --build" 2>&1 | tail -20
+
+    # Attendre ClickHouse
+    log "Attente ClickHouse (max 120s)..."
+    for i in $(seq 1 60); do
+        if curl -sf "http://${ANALYSIS_IP}:8123/ping" >/dev/null 2>&1; then
+            pass "ClickHouse prêt (${i}*2s)"
+            break
+        fi
+        sleep 2
+    done
+
+    # Vérifier les bases
+    local db_count
+    db_count=$(ch_query "SELECT count() FROM system.databases WHERE name IN ('ja4_logs','ja4_processing')" | tr -d ' \n')
+    [ "$db_count" = "2" ] \
+        && pass "Bases ja4_logs + ja4_processing créées" \
+        || fail "Bases manquantes (obtenu: $db_count)"
+
+    # Vérifier que la DB est bien vide (nouvelle instance)
+    local raw_init
+    raw_init=$(ch_query "SELECT count() FROM ja4_logs.http_logs_raw" | tr -d ' \n')
+    [ "${raw_init:-0}" -eq 0 ] 2>/dev/null \
+        && pass "ClickHouse vierge avant test (http_logs_raw=0)" \
+        || warn "ClickHouse non vide (http_logs_raw=${raw_init}) — données résiduelles"
+
+    # Attendre bot-detector
+    log "Attente bot-detector (max 120s)..."
+    for i in $(seq 1 60); do
+        if curl -sf "http://${ANALYSIS_IP}:8080/" >/dev/null 2>&1; then
+            pass "bot-detector prêt (${i}*2s)"
+            break
+        fi
+        sleep 2
+    done
+
+    # Attendre dashboard
+    log "Attente dashboard (max 60s)..."
+    for i in $(seq 1 30); do
+        if curl -sf "http://${ANALYSIS_IP}:8000/health" >/dev/null 2>&1; then
+            pass "dashboard prêt (${i}*2s)"
+            break
+        fi
+        sleep 2
+    done
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Phase 2 : Démarrage des endpoints
+# ═════════════════════════════════════════════════════════════════════════════
+phase2_endpoints() {
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  Phase 2 : Endpoints (nginx + ja4ebpf → analysis CH)    ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo ""
+
+    cd "$VM_DIR"
+
+    # Arrêter les services existants sur les endpoints (session précédente)
+    log "Arrêt des services existants sur les endpoints..."
+    for vm in $ENDPOINT_VMS; do
+        vagrant ssh "$vm" -- "sudo nginx -s stop 2>/dev/null; sudo pkill ja4ebpf 2>/dev/null; sudo pkill httpd 2>/dev/null; sudo pkill hitch 2>/dev/null; sudo pkill varnishd 2>/dev/null" 2>/dev/null || true
+    done
+    sleep 2
+
+    # Démarrer chaque endpoint en arrière-plan
+    # ja4ebpf pointe vers le ClickHouse de la VM analysis (CH_HOST=192.168.42.10)
+    PIDS=()
+    for vm in $ENDPOINT_VMS; do
+        log "Démarrage $vm (nginx + ja4ebpf → ${ANALYSIS_IP})..."
+        vagrant ssh "$vm" -- "sudo rm -f /tmp/ja4ebpf-traffic-done" 2>/dev/null || true
+        vagrant ssh "$vm" -- "sudo CH_HOST=${ANALYSIS_IP} bash /ja4-platform/tests/vm/run-tests-vm.sh ${STACK} start" &
+        PIDS+=($!)
+    done
+
+    # Attendre que les services soient prêts
+    log "Attente démarrage des endpoints (30s)..."
+    sleep 30
+
+    # Vérifier la connectivité HTTP de chaque endpoint
+    for vm in $ENDPOINT_VMS; do
+        local vm_ip
+        vm_ip=$(vagrant ssh "$vm" -- 'ip -4 addr show eth0' 2>/dev/null \
+            | awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}')
+        if curl -sf "http://${vm_ip}/health" >/dev/null 2>&1; then
+            pass "$vm HTTP OK (${vm_ip}:80)"
+        else
+            warn "$vm HTTP injoignable (${vm_ip}:80)"
+        fi
+        if curl -sf -k "https://${vm_ip}/health" >/dev/null 2>&1; then
+            pass "$vm HTTPS OK (${vm_ip}:443)"
+        else
+            warn "$vm HTTPS injoignable (${vm_ip}:443)"
+        fi
+    done
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Phase 3 : Génération de trafic
+# ═════════════════════════════════════════════════════════════════════════════
+phase3_traffic() {
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  Phase 3 : Génération de trafic host → endpoints         ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo ""
+
+    local total_ok=0 total_err=0
+
+    # User-Agents variés pour diversifier les empreintes TLS/HTTP
+    local UA_BROWSER=( \
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36" \
+        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/605.1.15" \
+        "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0" \
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0" \
+    )
+    local UA_BOT=( \
+        "python-requests/2.32.3" \
+        "curl/8.9.1" \
+        "Go-http-client/2.0" \
+        "python-httpx/0.28.1" \
+        "Googlebot/2.1" \
+    )
+
+    # Paths diversifiés pour des fingerprints différents
+    local PATHS=("/" "/health" "/data" "/api/users" "/api/v1/status" "/api/v1/metrics" \
+        "/login" "/logout" "/api/search" "/static/main.js" "/static/style.css" \
+        "/favicon.ico" "/robots.txt" "/sitemap.xml" "/api/v2/data" "/admin")
+
+    # Hostnames pour SNI (cert CN=platform.test, nginx accepte tout via server_name _)
+    local SNI_HOSTS_ALL=("platform.test" "api.platform.test" "www.platform.test" "app.platform.test")
+    local SNI_HOSTS=("${SNI_HOSTS_ALL[@]:0:${DNS_COUNT}}")
+
+    # ── Collecter les IPs de tous les endpoints ──
+    local -A VM_IPS_MAP
+    for vm in $ENDPOINT_VMS; do
+        VM_IPS_MAP[$vm]=$(vagrant ssh "$vm" -- 'ip -4 addr show eth0' 2>/dev/null \
+            | awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}')
+    done
+
+    # ── Ajouter des IPs alias sur chaque VM pour diversifier les sources ──
+    if [ "$SRC_IP_COUNT" -gt 1 ]; then
+        log "Ajout de ${SRC_IP_COUNT} IPs sources sur chaque VM..."
+        for vm in $ENDPOINT_VMS; do
+            local base_ip="${VM_IPS_MAP[$vm]}"
+            local net_prefix
+            net_prefix=$(echo "$base_ip" | awk -F. '{print $1"."$2"."$3}')
+            local base_last
+            base_last=$(echo "$base_ip" | awk -F. '{print $4}')
+            for i in $(seq 1 $((SRC_IP_COUNT - 1))); do
+                local alias_last=$((base_last + 100 + i))
+                [ "$alias_last" -gt 254 ] && alias_last=$((10 + i))
+                local alias_ip="${net_prefix}.${alias_last}"
+                vagrant ssh "$vm" -- "sudo ip addr add ${alias_ip}/24 dev eth0 2>/dev/null || true" 2>/dev/null || true
+            done
+        done
+    fi
+
+    # ── Trafic HTTPS généré depuis les VMs (IPs sources distinctes) ──
+    for src_vm in $ENDPOINT_VMS; do
+        local src_ip="${VM_IPS_MAP[$src_vm]}"
+        log "Génération depuis $src_vm ($src_ip) : ${HTTPS_COUNT} requêtes HTTPS (${SRC_IP_COUNT} IPs src)..."
+
+        # Construire la liste des IPs cibles (toutes les endpoints)
+        local TARGET_IPS=""
+        for target_vm in $ENDPOINT_VMS; do
+            TARGET_IPS="$TARGET_IPS ${VM_IPS_MAP[$target_vm]}"
+        done
+
+        # Script de génération exécuté sur la VM source
+        vagrant ssh "$src_vm" -- "bash -s" <<REMOTE_SCRIPT &
+#!/bin/bash
+set -uo pipefail
+
+HITS=${HTTPS_COUNT}
+TARGET_IPS=(${TARGET_IPS})
+SNI_HOSTS=(${SNI_HOSTS[@]})
+TLS_FLAGS="${CURL_TLS_FLAGS}"
+DNS_COUNT=${DNS_COUNT}
+SRC_IP_COUNT=${SRC_IP_COUNT}
+
+# Collecter les IPs sources disponibles sur eth0
+SRC_IPS=(\$(ip -4 addr show eth0 2>/dev/null | awk '/inet / {sub(/\/.*/, "", \$2); print \$2}'))
+
+UA_BROWSER=(
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36"
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/605.1.15"
+    "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0"
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0"
+)
+UA_BOT=(
+    "python-requests/2.32.3"
+    "curl/8.9.1"
+    "Go-http-client/2.0"
+    "python-httpx/0.28.1"
+    "Googlebot/2.1"
+)
+PATHS=("/" "/health" "/data" "/api/users" "/api/v1/status" "/api/v1/metrics" \
+    "/login" "/logout" "/api/search" "/static/main.js" "/static/style.css" \
+    "/favicon.ico" "/robots.txt" "/sitemap.xml" "/api/v2/data" "/admin")
+
+ok=0
+for i in \$(seq 1 \$HITS); do
+    idx=\$((i - 1))
+    target_ip="\${TARGET_IPS[\$((idx % \${#TARGET_IPS[@]}))]}"
+    sni_host="\${SNI_HOSTS[\$((idx % \${#SNI_HOSTS[@]}))]}"
+    path="\${PATHS[\$((idx % \${#PATHS[@]}))]}"
+
+    case \$((i % 10)) in
+        0|1|2|3|4) method="GET" ;;
+        5|6)       method="POST" ;;
+        7)         method="PUT" ;;
+        8)         method="DELETE" ;;
+        9)         method="HEAD" ;;
+    esac
+
+    if [ \$((i % 10)) -lt 7 ]; then
+        ua="\${UA_BROWSER[\$((idx % \${#UA_BROWSER[@]}))]}"
+    else
+        ua="\${UA_BOT[\$((idx % \${#UA_BOT[@]}))]}"
+    fi
+
+    extra_flags="--resolve \${sni_host}:443:\${target_ip} \$TLS_FLAGS"
+
+    # Alterner entre les IPs sources disponibles
+    if [ \${#SRC_IPS[@]} -gt 1 ]; then
+        src_ip="\${SRC_IPS[\$((idx % \${#SRC_IPS[@]}))]}"
+        extra_flags="\$extra_flags --interface \$src_ip"
+    fi
+
+    case \$method in
+        POST)
+            curl -sf -k \$extra_flags -X POST "https://\${sni_host}\${path}" \
+                -H "User-Agent: \${ua}" -H "Content-Type: application/json" \
+                -d '{"test":1,"seq":'\$i'}' >/dev/null 2>&1 && ok=\$((ok + 1)) || true ;;
+        PUT)
+            curl -sf -k \$extra_flags -X PUT "https://\${sni_host}\${path}" \
+                -H "User-Agent: \${ua}" -d '{}' >/dev/null 2>&1 && ok=\$((ok + 1)) || true ;;
+        DELETE)
+            curl -sf -k \$extra_flags -X DELETE "https://\${sni_host}\${path}" \
+                -H "User-Agent: \${ua}" >/dev/null 2>&1 && ok=\$((ok + 1)) || true ;;
+        HEAD)
+            curl -sf -k \$extra_flags -I "https://\${sni_host}\${path}" \
+                -H "User-Agent: \${ua}" >/dev/null 2>&1 && ok=\$((ok + 1)) || true ;;
+        *)
+            curl -sf -k \$extra_flags -X "\$method" "https://\${sni_host}\${path}" \
+                -H "User-Agent: \${ua}" >/dev/null 2>&1 && ok=\$((ok + 1)) || true ;;
+    esac
+done
+echo "\$ok/\$HITS"
+REMOTE_SCRIPT
+    done
+
+    # Collecter les résultats des processus en arrière-plan
+    for src_vm in $ENDPOINT_VMS; do
+        local result
+        result=$(wait 2>/dev/null || echo "?/?")
+        log "  $src_vm HTTPS : $result requêtes réussies"
+        local ok_count
+        ok_count=$(echo "$result" | cut -d/ -f1)
+        total_ok=$((total_ok + ok_count))
+    done
+
+    # HTTP/2 massif depuis les VMs si httpx est disponible
+    for src_vm in $ENDPOINT_VMS; do
+        if vagrant ssh "$src_vm" -- 'python3 -c "import httpx"' 2>/dev/null; then
+            local src_ip="${VM_IPS_MAP[$src_vm]}"
+            log "Génération HTTP/2 depuis $src_vm (${HTTPS_COUNT} requêtes, TLS=${TLS_VERSIONS}, DNS=${DNS_COUNT})..."
+            local TARGET_IPS_H2=""
+            for target_vm in $ENDPOINT_VMS; do
+                TARGET_IPS_H2="$TARGET_IPS_H2 ${VM_IPS_MAP[$target_vm]}"
+            done
+            vagrant ssh "$src_vm" -- "python3 -c \"
+import httpx, ssl as _ssl, warnings, random
+warnings.filterwarnings('ignore')
+paths = ['/', '/health', '/data', '/api/users', '/api/v1/status', '/login', '/api/search']
+sni_hosts = ${SNI_HOSTS[@]@Q}
+target_ips = '${TARGET_IPS_H2}'.split()
+uas_browser = [
+    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0 Safari/537.36',
+    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Safari/605.1.15',
+    'Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0',
+]
+uas_bot = ['python-httpx/0.28.1', 'Googlebot/2.1', 'Go-http-client/2.0']
+tls_versions = [v.strip() for v in '${TLS_VERSIONS}'.split(',')]
+supported = {'1.2': _ssl.TLSVersion.TLSv1_2, '1.3': _ssl.TLSVersion.TLSv1_3}
+tls_map = [supported[v] for v in tls_versions if v in supported]
+ctx = _ssl.SSLContext(_ssl.PROTOCOL_TLS_CLIENT)
+ctx.check_hostname = False
+ctx.verify_mode = _ssl.CERT_NONE
+if tls_map:
+    ctx.minimum_version = min(tls_map)
+    ctx.maximum_version = max(tls_map)
+with httpx.Client(http2=True, verify=ctx) as c:
+    for i in range(${HTTPS_COUNT}):
+        p = random.choice(paths)
+        target = random.choice(target_ips)
+        h = random.choice(sni_hosts)
+        ua = random.choice(uas_browser if random.random() < 0.7 else uas_bot)
+        try:
+            c.get(f'https://{target}' + p, headers={'User-Agent': ua, 'Host': h})
+        except:
+            pass
+\"" 2>/dev/null || true
+        fi
+    done
+    pass "HTTP/2 généré depuis tous les endpoints"
+
+    # Trafic HTTP (port 80) en plus pour diversifier
+    if [ "${HTTP_COUNT}" -gt 0 ]; then
+        log "Génération HTTP (port 80) depuis les VMs : ${HTTP_COUNT} requêtes/VM..."
+        for src_vm in $ENDPOINT_VMS; do
+            local ok80
+            ok80=$(vagrant ssh "$src_vm" -- "
+                ok=0
+                for i in \$(seq 1 ${HTTP_COUNT}); do
+                    curl -sf http://localhost/health >/dev/null 2>&1 && ok=\$((ok + 1)) || true
+                done
+                echo \$ok
+            " 2>/dev/null || echo "0")
+            log "  $src_vm HTTP : ${ok80}/${HTTP_COUNT} requêtes"
+            total_ok=$((total_ok + ok80))
+        done
+    else
+        log "HTTP (port 80) désactivé (http-ratio=0)"
+    fi
+
+    pass "Trafic total : ${total_ok} requêtes réussies"
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Phase 4 : Attente du pipeline
+# ═════════════════════════════════════════════════════════════════════════════
+phase4_wait() {
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  Phase 4 : Attente du pipeline                           ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo ""
+
+    # Flush ja4ebpf
+    log "Attente flush ja4ebpf (15s)..."
+    sleep 15
+
+    # Signaler aux endpoints que le trafic est terminé
+    cd "$VM_DIR"
+    for vm in $ENDPOINT_VMS; do
+        vagrant ssh "$vm" -- 'sudo touch /tmp/ja4ebpf-traffic-done' 2>/dev/null || true
+    done
+
+    # Attendre que les données brutes arrivent dans ClickHouse
+    log "Attente données dans ClickHouse (max 60s)..."
+    for i in $(seq 1 30); do
+        local raw_count
+        raw_count=$(ch_query "SELECT count() FROM ja4_logs.http_logs_raw" 2>/dev/null | tr -d ' \n')
+        if [ "${raw_count:-0}" -gt 0 ] 2>/dev/null; then
+            pass "Données brutes reçues : ${raw_count} lignes (${i}*2s)"
+            break
+        fi
+        sleep 2
+    done
+
+    # Attendre au moins un cycle bot-detector
+    log "Attente cycle bot-detector (max 120s)..."
+    for i in $(seq 1 60); do
+        local scores
+        scores=$(ch_query "SELECT count() FROM ja4_processing.ml_all_scores" 2>/dev/null | tr -d ' \n')
+        if [ "${scores:-0}" -gt 0 ] 2>/dev/null; then
+            pass "bot-detector a complété au moins 1 cycle ($scores scores)"
+            break
+        fi
+        sleep 2
+    done
+
+    # Attendre les processus endpoint en arrière-plan
+    log "Attente fin des processus endpoint..."
+    for pid in "${PIDS[@]:-}"; do
+        wait "$pid" 2>/dev/null || true
+    done
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Phase 5 : Vérification complète
+# ═════════════════════════════════════════════════════════════════════════════
+phase5_verify() {
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  Phase 5 : Vérification du pipeline complet              ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo ""
+
+    # ── Layer 1 : Données brutes ─────────────────────────────────────────────
+    log "── Layer 1 : Données brutes ──"
+
+    local raw_count
+    raw_count=$(ch_query "SELECT count() FROM ja4_logs.http_logs_raw" | tr -d ' \n')
+    if [ "${raw_count:-0}" -gt 0 ] 2>/dev/null; then
+        pass "http_logs_raw : ${raw_count} lignes"
+    else
+        fail "http_logs_raw vide"
+    fi
+
+    # Multi-source : données de plusieurs endpoints
+    local host_count
+    host_count=$(ch_query "SELECT uniqExact(host) FROM ja4_logs.http_logs" | tr -d ' \n')
+    if [ "${host_count:-0}" -ge 2 ] 2>/dev/null; then
+        pass "Multi-source : ${host_count} hôtes distincts"
+    else
+        warn "Multi-source : ${host_count:-0} hôte(s) distinct(s)"
+    fi
+
+    # ── Layer 2 : Pipeline ClickHouse ────────────────────────────────────────
+    log "── Layer 2 : Pipeline ClickHouse (MVs) ──"
+
+    local ja4_count
+    ja4_count=$(ch_query "SELECT count() FROM ja4_logs.http_logs WHERE ja4 != ''" | tr -d ' \n')
+    [ "${ja4_count:-0}" -gt 0 ] 2>/dev/null \
+        && pass "JA4 fingerprints : ${ja4_count}" \
+        || fail "JA4 fingerprints absents"
+
+    local method_count
+    method_count=$(ch_query "SELECT count() FROM ja4_logs.http_logs WHERE method != ''" | tr -d ' \n')
+    [ "${method_count:-0}" -gt 0 ] 2>/dev/null \
+        && pass "L7 HTTP : ${method_count} requêtes capturées" \
+        || fail "L7 HTTP absent"
+
+    local agg_count
+    agg_count=$(ch_query "SELECT count() FROM ja4_processing.agg_host_ip_ja4_1h" | tr -d ' \n')
+    [ "${agg_count:-0}" -gt 0 ] 2>/dev/null \
+        && pass "Agrégation agg_host_ip_ja4_1h : ${agg_count} entrées" \
+        || warn "Agrégation agg_host_ip_ja4_1h vide (volume insuffisant pour la fenêtre horaire)"
+
+    local feat_count
+    feat_count=$(ch_query "SELECT count() FROM ja4_processing.view_ai_features_1h" | tr -d ' \n')
+    [ "${feat_count:-0}" -gt 0 ] 2>/dev/null \
+        && pass "AI features : ${feat_count} lignes" \
+        || warn "AI features vides (agrégation horaire pas encore calculée)"
+
+    # ── Layer 3 : ML bot-detector ────────────────────────────────────────────
+    log "── Layer 3 : ML bot-detector ──"
+
+    local scores_count
+    scores_count=$(ch_query "SELECT count() FROM ja4_processing.ml_all_scores" | tr -d ' \n')
+    [ "${scores_count:-0}" -gt 0 ] 2>/dev/null \
+        && pass "ml_all_scores : ${scores_count} classifications" \
+        || fail "ml_all_scores vide — bot-detector n'a pas produit de résultats"
+
+    local anomaly_count
+    anomaly_count=$(ch_query "SELECT count() FROM ja4_processing.ml_detected_anomalies" | tr -d ' \n')
+    if [ "${anomaly_count:-0}" -gt 0 ] 2>/dev/null; then
+        pass "ml_detected_anomalies : ${anomaly_count} anomalies détectées"
+    else
+        warn "ml_detected_anomalies vide (pas d'anomalies dans le trafic de test)"
+    fi
+
+    # bot-detector health
+    if curl -sf "http://${ANALYSIS_IP}:8080/" >/dev/null 2>&1; then
+        pass "bot-detector health OK"
+    else
+        fail "bot-detector health KO"
+    fi
+
+    # ── Layer 4 : Dashboard ──────────────────────────────────────────────────
+    log "── Layer 4 : Dashboard ──"
+
+    if curl -sf "http://${ANALYSIS_IP}:8000/health" >/dev/null 2>&1; then
+        pass "dashboard /health OK"
+    else
+        fail "dashboard /health KO"
+    fi
+
+    local overview
+    overview=$(curl -sf "http://${ANALYSIS_IP}:8000/api/overview" 2>/dev/null || echo "")
+    if [ -n "$overview" ] && [ "$overview" != "{}" ] && [ "$overview" != "null" ]; then
+        pass "dashboard /api/overview retourne des données"
+    else
+        warn "dashboard /api/overview vide"
+    fi
+
+    local detections
+    detections=$(curl -sf "http://${ANALYSIS_IP}:8000/api/detections" 2>/dev/null || echo "")
+    if [ -n "$detections" ]; then
+        pass "dashboard /api/detections accessible"
+    else
+        warn "dashboard /api/detections pas de réponse"
+    fi
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Résumé
+# ═════════════════════════════════════════════════════════════════════════════
+phase_summary() {
+    local total=$((PASS_COUNT + FAIL_COUNT + WARN_COUNT))
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  RÉSULTATS E2E                                           ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo -e "  ${GREEN}Réussis : $PASS_COUNT${RESET} / $total"
+    [ "$WARN_COUNT" -gt 0 ] && echo -e "  ${YELLOW}Avertissements : $WARN_COUNT${RESET} / $total"
+    [ "$FAIL_COUNT" -gt 0 ] && echo -e "  ${RED}Échoués : $FAIL_COUNT${RESET} / $total"
+    echo ""
+
+    if [ "$FAIL_COUNT" -eq 0 ]; then
+        echo -e "  ${GREEN}${BOLD}Test E2E distribué : SUCCÈS${RESET}"
+    else
+        echo -e "  ${RED}${BOLD}Test E2E distribué : $FAIL_COUNT ÉCHECS${RESET}"
+    fi
+    echo ""
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Nettoyage
+# ═════════════════════════════════════════════════════════════════════════════
+cleanup() {
+    # Supprimer les IPs alias sur les VMs
+    if [ "${SRC_IP_COUNT:-1}" -gt 1 ]; then
+        log "Suppression des IPs alias sur les VMs..."
+        for vm in $ENDPOINT_VMS; do
+            vagrant ssh "$vm" -- "
+                base_ip=\$(ip -4 addr show eth0 | awk '/inet / {sub(/\/.*/, \"\", \\\$2); print \\\$2; exit}')
+                net_prefix=\$(echo \$base_ip | awk -F. '{print \\\$1\".\"\\\$2\".\"\\\$3}')
+                base_last=\$(echo \$base_ip | awk -F. '{print \\\$4}')
+                for i in \$(seq 1 $((SRC_IP_COUNT - 1))); do
+                    alias_last=\$((base_last + 100 + i))
+                    [ \$alias_last -gt 254 ] && alias_last=\$((10 + i))
+                    sudo ip addr del \${net_prefix}.\${alias_last}/24 dev eth0 2>/dev/null || true
+                done
+            " 2>/dev/null || true
+        done
+    fi
+
+    # Toujours arrêter les endpoints (sauf --keep global)
+    if [ "${KEEP_RUNNING:-false}" != "true" ]; then
+        log "Nettoyage des endpoints..."
+        cd "$VM_DIR"
+        for vm in $ENDPOINT_VMS; do
+            vagrant ssh "$vm" -- "sudo pkill ja4ebpf 2>/dev/null; sudo nginx -s stop 2>/dev/null; sudo pkill httpd 2>/dev/null" 2>/dev/null || true
+        done
+    fi
+
+    # Stack analysis : conserver si --keep-analysis ou --keep, sinon arrêter
+    if [ "${KEEP_RUNNING:-false}" = "true" ] || [ "$KEEP_ANALYSIS" = "true" ]; then
+        log "Stack analysis conservée :"
+        log "  ClickHouse : http://${ANALYSIS_IP}:8123/play"
+        log "  Dashboard  : http://${ANALYSIS_IP}:8000"
+        log "  Bot-detector : http://${ANALYSIS_IP}:8080"
+    else
+        log "Arrêt de la stack analysis..."
+        vagrant ssh analysis -- "cd /ja4-platform/tests/vm/analysis && docker compose down -v" 2>/dev/null || true
+    fi
+
+    if [ "${KEEP_RUNNING:-false}" = "true" ]; then
+        log "VMs endpoints conservées :"
+        for vm in $ENDPOINT_VMS; do
+            local vm_ip
+            vm_ip=$(vagrant ssh "$vm" -- 'ip -4 addr show eth0' 2>/dev/null \
+                | awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}')
+            log "  $vm : ${vm_ip}"
+        done
+    fi
+}
+
+# ═════════════════════════════════════════════════════════════════════════════
+# Main
+# ═════════════════════════════════════════════════════════════════════════════
+
+KEEP_RUNNING="${KEEP_RUNNING:-false}"
+
+trap cleanup EXIT
+
+phase0_setup
+phase1_analysis
+phase2_endpoints
+
+if [ "$UP_ONLY" = "true" ]; then
+    echo ""
+    echo "╔══════════════════════════════════════════════════════════╗"
+    echo "║  Stack prête — mode --up (pas de trafic ni vérification) ║"
+    echo "╚══════════════════════════════════════════════════════════╝"
+    echo ""
+    log "ClickHouse  : http://${ANALYSIS_IP}:8123/play"
+    log "Dashboard   : http://${ANALYSIS_IP}:8000"
+    log "Bot-detector : http://${ANALYSIS_IP}:8080"
+    for vm in $ENDPOINT_VMS; do
+        local vm_ip
+        vm_ip=$(vagrant ssh "$vm" -- 'ip -4 addr show eth0' 2>/dev/null \
+            | awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}')
+        log "$vm HTTP  : http://${vm_ip}:80"
+        log "$vm HTTPS : https://${vm_ip}:443"
+    done
+    log ""
+    log "Pour générer du trafic :"
+    log "  ./run-e2e-test.sh --hits 200 --dns 3 --tls 1.2,1.3 --keep-analysis"
+    log "  curl -sk https://<endpoint_ip>/health"
+    KEEP_ANALYSIS=true
+    exit 0
+fi
+
+phase3_traffic
+phase4_wait
+phase5_verify
+phase_summary
+
+[ "$FAIL_COUNT" -eq 0 ] && exit 0 || exit 1