ja4-platform/tests/vm/run-tests-vm.sh

#!/usr/bin/env bash
# =============================================================================
# run-tests-vm.sh — Lance la stack de test complète dans la VM Rocky Linux 9
#
# Ce script s'exécute DANS la VM (via vagrant ssh ou vagrant provision).
# Il ne peut pas tourner dans Docker — il requiert un vrai kernel pour eBPF.
#
# Usage (depuis le host) :
#   vagrant ssh -- 'bash /ja4-platform/tests/vm/run-tests-vm.sh nginx'
#   vagrant ssh -- 'bash /ja4-platform/tests/vm/run-tests-vm.sh all'
#
# Variables d'environnement :
#   STACK          : stack à tester (nginx|apache|nginx-varnish|hitch-varnish|all)
#   KEEP_RUNNING   : si "true", ne pas arrêter la stack après le test (défaut: false)
# =============================================================================
set -euo pipefail

STACK="${1:-nginx}"
KEEP_RUNNING="${KEEP_RUNNING:-false}"
PROJECT="/ja4-platform"
RESULTS_DIR="/tmp/ja4-test-results"

# ── Couleurs ─────────────────────────────────────────────────────────────────
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; RESET='\033[0m'
BOLD='\033[1m'

log()  { echo -e "${BOLD}[$STACK]${RESET} $(date +%H:%M:%S) $*"; }
pass() { echo -e "  ${GREEN}✅${RESET}  $*"; ((PASS_COUNT++)) || true; }
fail() { echo -e "  ${RED}❌${RESET}  $*"; ((FAIL_COUNT++)) || true; }
warn() { echo -e "  ${YELLOW}⚠️${RESET}   $*"; ((WARN_COUNT++)) || true; }

PASS_COUNT=0; FAIL_COUNT=0; WARN_COUNT=0

# ── Vérification prérequis ────────────────────────────────────────────────────
check_prerequisites() {
    log "Vérification des prérequis..."
    
    # eBPF capabilities
    if [ ! -d /sys/kernel/tracing ]; then
        fail "tracefs non monté — exécuter: sudo mount -t tracefs tracefs /sys/kernel/tracing"
        exit 1
    fi
    if [ ! -d /sys/kernel/debug ]; then
        fail "debugfs non monté"
        exit 1
    fi
    
    command -v ja4ebpf >/dev/null 2>&1 || {
        log "Rebuild ja4ebpf..."
        cd "$PROJECT/services/ja4ebpf"
        export PATH="/usr/local/go/bin:$PATH"
        GOWORK=off go generate ./internal/loader/ 2>&1 | tail -3
        GOWORK=off CGO_ENABLED=0 go build -o /usr/local/bin/ja4ebpf ./cmd/ja4ebpf/
    }
    
    command -v docker >/dev/null 2>&1 || { fail "Docker non installé"; exit 1; }
    command -v nginx >/dev/null 2>&1 || { fail "nginx non installé"; exit 1; }
    pass "Prérequis OK"
}

# ── Démarrage ClickHouse ──────────────────────────────────────────────────────
start_clickhouse() {
    log "Démarrage ClickHouse..."
    
    docker rm -f ja4-clickhouse 2>/dev/null || true
    
    docker run -d --name ja4-clickhouse \
        -p 8123:8123 -p 9000:9000 \
        -e CLICKHOUSE_DB=ja4_processing \
        -e CLICKHOUSE_USER=default \
        -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
        -v "$PROJECT/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh" \
        $(for f in "$PROJECT/shared/clickhouse/"*.sql; do
            echo "-v $f:/initdb-src/$(basename $f):ro"
          done) \
        clickhouse/clickhouse-server:24.8 2>&1 | tail -1
    
    # Attendre que ClickHouse soit prêt
    log "Attente ClickHouse (max 120s)..."
    for i in $(seq 1 60); do
        if curl -sf "http://localhost:8123/ping" >/dev/null 2>&1; then
            pass "ClickHouse prêt"
            return 0
        fi
        sleep 2
    done
    fail "ClickHouse timeout"; exit 1
}

# ── Configuration nginx ────────────────────────────────────────────────────────
setup_nginx() {
    log "Configuration nginx avec TLS..."
    
    # Certificat auto-signé
    openssl req -x509 -nodes -days 365 \
        -subj "/CN=platform.test" \
        -newkey rsa:2048 \
        -keyout /etc/pki/tls/private/nginx.key \
        -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
    
    # Copier la configuration de test
    cp "$PROJECT/tests/integration/nginx/platform/nginx.conf" /etc/nginx/nginx.conf
    
    # Créer les fichiers de test
    mkdir -p /var/www/html
    echo '{"status":"ok","stack":"nginx-vm"}' > /var/www/html/health
    for p in data api/users api/data/test; do
        mkdir -p "/var/www/html/$(dirname $p)"
        echo '{"ok":true}' > "/var/www/html/$p"
    done
    
    nginx -t && nginx
    
    # Attendre nginx
    for i in $(seq 1 20); do
        curl -sf http://localhost/health >/dev/null 2>&1 && break
        sleep 0.5
    done
    pass "nginx démarré"
}

# ── Démarrage ja4ebpf ─────────────────────────────────────────────────────────
start_ja4ebpf() {
    log "Démarrage ja4ebpf..."
    
    pkill ja4ebpf 2>/dev/null || true
    sleep 1
    
    # Créer la config
    cat > /tmp/ja4ebpf.yml << 'EOF'
interface: eth0
ssl_lib_path: "/usr/lib64/libssl.so.3"
clickhouse:
  dsn: "clickhouse://default:@localhost:9000/ja4_logs"
  batch_size: 100
  flush_secs: 1
correlation:
  timeout_ms: 500
  slowloris_ms: 10000
log:
  level: "info"
  format: "json"
EOF
    
    # Lancer avec les capabilities nécessaires
    # Dans la VM (root), on peut lancer directement
    ja4ebpf -config /tmp/ja4ebpf.yml > /tmp/ja4ebpf.log 2>&1 &
    JA4EBPF_PID=$!
    
    sleep 3
    if ! kill -0 "$JA4EBPF_PID" 2>/dev/null; then
        fail "ja4ebpf s'est arrêté immédiatement"
        cat /tmp/ja4ebpf.log | tail -10
        return 1
    fi
    
    log "ja4ebpf démarré (PID $JA4EBPF_PID)"
    
    # Vérifier les uprobes dans tracefs
    sleep 1
    if grep -q "ssl" /sys/kernel/tracing/uprobe_events 2>/dev/null; then
        pass "Uprobes SSL attachés dans tracefs"
    else
        warn "Uprobes non visibles dans tracefs (peuvent être actifs quand même)"
    fi
    
    # Vérifier accept4 tracepoint
    if grep -q "accept4" /sys/kernel/tracing/events/syscalls 2>/dev/null; then
        pass "Tracepoints accept4 disponibles"
    else
        warn "Tracepoints accept4 non trouvés"
    fi
}

# ── Génération de trafic ───────────────────────────────────────────────────────
generate_traffic() {
    log "Génération du trafic (HTTP/1.0 + HTTP/1.1 + HTTP/2)..."
    
    # Trafic HTTP/1.1 (HTTP)
    for path in / /health /data /api/users; do
        curl -sf "http://localhost$path" >/dev/null 2>&1 || true
        curl -sf -X POST "http://localhost/api/data" -d '{"test":1}' >/dev/null 2>&1 || true
    done
    
    # Trafic HTTPS/1.1
    for path in / /health /data /api/users; do
        curl -sf -k "https://localhost$path" >/dev/null 2>&1 || true
        curl -sf -k -X POST "https://localhost/api/data" -d '{"test":1}' >/dev/null 2>&1 || true
        curl -sf -k -X PUT "https://localhost/data" >/dev/null 2>&1 || true
        curl -sf -k -X DELETE "https://localhost/data/1" >/dev/null 2>&1 || true
        curl -sf -k -X HEAD "https://localhost$path" >/dev/null 2>&1 || true
    done
    
    # Trafic HTTP/2
    if command -v python3 >/dev/null 2>&1 && python3 -c "import httpx" 2>/dev/null; then
        python3 << 'PYEOF'
import httpx, ssl, warnings
warnings.filterwarnings("ignore")
ctx = ssl.create_default_context()
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
with httpx.Client(http2=True, verify=False) as client:
    for path in ["/", "/health", "/data"]:
        try: client.get(f"https://localhost{path}")
        except: pass
    try: client.post("https://localhost/api/data", json={"test": "h2"})
    except: pass
PYEOF
        pass "Trafic HTTP/2 généré"
    fi
    
    # Attendre le flush ja4ebpf → ClickHouse
    log "Attente flush ja4ebpf (15s)..."
    sleep 15
    pass "Trafic généré"
}

# ── Vérification ClickHouse ────────────────────────────────────────────────────
verify_db() {
    log "Vérification des données dans ClickHouse..."
    
    ch_query() {
        curl -sf "http://localhost:8123/" \
            --data-urlencode "query=$1" \
            --data-urlencode "database=ja4_logs" \
            -o /dev/null -w '%{http_code}' 2>/dev/null || echo "0"
    }
    
    ch_val() {
        curl -sf "http://localhost:8123/?database=ja4_logs" \
            --data-urlencode "query=$1" 2>/dev/null | tr -d ' \n' || echo "0"
    }
    
    # L3/L4
    ttl=$(ch_val "SELECT count() FROM http_logs WHERE ip_meta_ttl > 0")
    [ "${ttl:-0}" -gt 0 ] && pass "L3/L4 TTL capturé ($ttl lignes)" || fail "L3/L4 TTL absent"
    
    mss=$(ch_val "SELECT count() FROM http_logs WHERE tcp_meta_mss > 0")
    [ "${mss:-0}" -gt 0 ] && pass "TCP MSS capturé ($mss lignes)" || fail "TCP MSS absent"
    
    # TLS
    ja4=$(ch_val "SELECT count() FROM http_logs WHERE ja4 != ''")
    [ "${ja4:-0}" -gt 0 ] && pass "JA4 fingerprint capturé ($ja4 lignes)" || fail "JA4 absent"
    
    sni=$(ch_val "SELECT count() FROM http_logs WHERE tls_sni != ''")
    [ "${sni:-0}" -gt 0 ] && pass "TLS SNI capturé ($sni lignes)" || warn "TLS SNI absent"
    
    # L7 HTTP — c'est ici que ça devrait marcher dans la VM
    method=$(ch_val "SELECT count() FROM http_logs WHERE method != ''")
    [ "${method:-0}" -gt 0 ] && pass "L7 méthodes HTTP capturées ($method lignes)" \
                               || fail "L7 méthodes HTTP ABSENT — uprobe SSL_read ne fonctionne pas"
    
    path=$(ch_val "SELECT count() FROM http_logs WHERE path != ''")
    [ "${path:-0}" -gt 0 ] && pass "L7 path HTTP capturé ($path lignes)" || fail "L7 path absent"
    
    status=$(ch_val "SELECT count() FROM http_logs WHERE status_code > 0")
    [ "${status:-0}" -gt 0 ] && pass "status_code capturé ($status lignes)" || warn "status_code absent"
    
    sig=$(ch_val "SELECT count() FROM http_logs WHERE header_order_signature != ''")
    [ "${sig:-0}" -gt 0 ] && pass "header_order_signature capturé ($sig lignes)" || warn "header_order_sig absent"
    
    # Méthodes HTTP distinctes
    methods=$(ch_val "SELECT groupArray(method) FROM (SELECT DISTINCT method FROM http_logs WHERE method != '')")
    log "Méthodes HTTP vues : $methods"
    
    # Lignes totales
    total=$(ch_val "SELECT count() FROM http_logs")
    pass "Total lignes http_logs : $total"
}

# ── Nettoyage ─────────────────────────────────────────────────────────────────
cleanup() {
    if [ "$KEEP_RUNNING" != "true" ]; then
        log "Nettoyage..."
        pkill ja4ebpf 2>/dev/null || true
        nginx -s stop 2>/dev/null || true
        docker rm -f ja4-clickhouse 2>/dev/null || true
    fi
}
trap cleanup EXIT

# ── Main ──────────────────────────────────────────────────────────────────────
mkdir -p "$RESULTS_DIR"

echo ""
echo "╔══════════════════════════════════════════╗"
echo "║  ja4ebpf VM Test Suite — Rocky Linux 9   ║"
echo "╚══════════════════════════════════════════╝"
echo ""

check_prerequisites
start_clickhouse
setup_nginx
start_ja4ebpf
generate_traffic
verify_db

echo ""
echo "════════════════════════════════════════════"
echo -e "  ${GREEN}OK${RESET}: $PASS_COUNT  ${YELLOW}WARN${RESET}: $WARN_COUNT  ${RED}FAIL${RESET}: $FAIL_COUNT"
if [ "$FAIL_COUNT" -eq 0 ]; then
    echo -e "  ${GREEN}${BOLD}Tous les tests réussis !${RESET}"
    exit 0
else
    echo -e "  ${RED}${BOLD}$FAIL_COUNT tests échoués.${RESET}"
    echo "Logs ja4ebpf :"
    tail -20 /tmp/ja4ebpf.log 2>/dev/null || true
    exit 1
fi