feat: pipeline L7 HTTP complet + infrastructure tests VM
Correctifs pipeline L7 (uprobe SSL_read) :
- uprobe_ssl.c : ssl_set_fd ne retourne plus tôt quand fd_conn_map est
vide (accept4 non disponible en Docker). Sauvegarde ssl_ptr→{fd,0,0}
pour permettre le fallback /proc côté Go.
- main.go : consumeSSLEvents reécrit avec routeur magic-bytes complet :
* HTTP/2 preface → extraction SETTINGS + conversion correlation.HTTP2Settings
* HTTP/1.x requête → method, path, query, headers, header_order_sig
* HTTP/1.x réponse → status_code
* Fallback /proc/<tgid>/fd/<fd> quand src_ip=0 (accept4 absent)
- writer/clickhouse.go : export header_order_signature ajouté
Nouveaux packages :
- internal/parser/http1.go : parseur HTTP/1.x (IsHTTP1Request,
ParseHTTP1Request, IsHTTP1Response, ParseHTTP1Response)
- internal/parser/http1_test.go : 11 tests unitaires (28 total passent)
- internal/procutil/proc_lookup.go : résolution fd→IP via /proc avec cache
TTL 5s (FDCache). Supporte /proc/PID/net/tcp et tcp6, IPv4-mappé IPv6.
Infrastructure tests VM (tests/vm/) :
- Vagrantfile : VM Rocky Linux 9 KVM, 4 CPU / 4 GB RAM
- provision.sh : installation toolchain eBPF + Go + Docker + nginx
- run-tests-vm.sh : suite de test complète dans la VM (L3/L4+TLS+L7)
- README.md : guide d'installation et d'utilisation
- Makefile : cibles vm-up, vm-down, vm-ssh, test-vm-nginx, test-vm-all,
vm-rebuild-ja4ebpf
Corrections stack Docker :
- Dockerfiles nginx/apache/nginx-varnish/hitch-varnish : suppression des
références à shared/go/ja4common/ (répertoire supprimé)
- clickhouse-init.sh : restauré depuis git, seed anubis_ua_rules obsolète
supprimé (table REGEXP_TREE supprimée du schéma)
- traffic-gen : ajout HTTP/1.0 (http.client) et HTTP/2 (httpx)
- verify_db.py : script de vérification 35 checks (L3/L4/TLS/L7/corrélation)
- run-stack-tests.sh : phase 6 verify_db ajoutée
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
309
tests/vm/run-tests-vm.sh
Executable file
309
tests/vm/run-tests-vm.sh
Executable file
@ -0,0 +1,309 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# run-tests-vm.sh — Lance la stack de test complète dans la VM Rocky Linux 9
|
||||
#
|
||||
# Ce script s'exécute DANS la VM (via vagrant ssh ou vagrant provision).
|
||||
# Il ne peut pas tourner dans Docker — il requiert un vrai kernel pour eBPF.
|
||||
#
|
||||
# Usage (depuis le host) :
|
||||
# vagrant ssh -- 'bash /ja4-platform/tests/vm/run-tests-vm.sh nginx'
|
||||
# vagrant ssh -- 'bash /ja4-platform/tests/vm/run-tests-vm.sh all'
|
||||
#
|
||||
# Variables d'environnement :
|
||||
# STACK : stack à tester (nginx|apache|nginx-varnish|hitch-varnish|all)
|
||||
# KEEP_RUNNING : si "true", ne pas arrêter la stack après le test (défaut: false)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
STACK="${1:-nginx}"
|
||||
KEEP_RUNNING="${KEEP_RUNNING:-false}"
|
||||
PROJECT="/ja4-platform"
|
||||
RESULTS_DIR="/tmp/ja4-test-results"
|
||||
|
||||
# ── Couleurs ─────────────────────────────────────────────────────────────────
|
||||
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; RESET='\033[0m'
|
||||
BOLD='\033[1m'
|
||||
|
||||
log() { echo -e "${BOLD}[$STACK]${RESET} $(date +%H:%M:%S) $*"; }
|
||||
pass() { echo -e " ${GREEN}✅${RESET} $*"; ((PASS_COUNT++)) || true; }
|
||||
fail() { echo -e " ${RED}❌${RESET} $*"; ((FAIL_COUNT++)) || true; }
|
||||
warn() { echo -e " ${YELLOW}⚠️${RESET} $*"; ((WARN_COUNT++)) || true; }
|
||||
|
||||
PASS_COUNT=0; FAIL_COUNT=0; WARN_COUNT=0
|
||||
|
||||
# ── Vérification prérequis ────────────────────────────────────────────────────
|
||||
check_prerequisites() {
|
||||
log "Vérification des prérequis..."
|
||||
|
||||
# eBPF capabilities
|
||||
if [ ! -d /sys/kernel/tracing ]; then
|
||||
fail "tracefs non monté — exécuter: sudo mount -t tracefs tracefs /sys/kernel/tracing"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -d /sys/kernel/debug ]; then
|
||||
fail "debugfs non monté"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
command -v ja4ebpf >/dev/null 2>&1 || {
|
||||
log "Rebuild ja4ebpf..."
|
||||
cd "$PROJECT/services/ja4ebpf"
|
||||
export PATH="/usr/local/go/bin:$PATH"
|
||||
GOWORK=off go generate ./internal/loader/ 2>&1 | tail -3
|
||||
GOWORK=off CGO_ENABLED=0 go build -o /usr/local/bin/ja4ebpf ./cmd/ja4ebpf/
|
||||
}
|
||||
|
||||
command -v docker >/dev/null 2>&1 || { fail "Docker non installé"; exit 1; }
|
||||
command -v nginx >/dev/null 2>&1 || { fail "nginx non installé"; exit 1; }
|
||||
pass "Prérequis OK"
|
||||
}
|
||||
|
||||
# ── Démarrage ClickHouse ──────────────────────────────────────────────────────
|
||||
start_clickhouse() {
|
||||
log "Démarrage ClickHouse..."
|
||||
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
|
||||
docker run -d --name ja4-clickhouse \
|
||||
-p 8123:8123 -p 9000:9000 \
|
||||
-e CLICKHOUSE_DB=ja4_processing \
|
||||
-e CLICKHOUSE_USER=default \
|
||||
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
|
||||
-v "$PROJECT/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh" \
|
||||
$(for f in "$PROJECT/shared/clickhouse/"*.sql; do
|
||||
echo "-v $f:/initdb-src/$(basename $f):ro"
|
||||
done) \
|
||||
clickhouse/clickhouse-server:24.8 2>&1 | tail -1
|
||||
|
||||
# Attendre que ClickHouse soit prêt
|
||||
log "Attente ClickHouse (max 120s)..."
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf "http://localhost:8123/ping" >/dev/null 2>&1; then
|
||||
pass "ClickHouse prêt"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
fail "ClickHouse timeout"; exit 1
|
||||
}
|
||||
|
||||
# ── Configuration nginx ────────────────────────────────────────────────────────
|
||||
setup_nginx() {
|
||||
log "Configuration nginx avec TLS..."
|
||||
|
||||
# Certificat auto-signé
|
||||
openssl req -x509 -nodes -days 365 \
|
||||
-subj "/CN=platform.test" \
|
||||
-newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/nginx.key \
|
||||
-out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
||||
|
||||
# Copier la configuration de test
|
||||
cp "$PROJECT/tests/integration/nginx/platform/nginx.conf" /etc/nginx/nginx.conf
|
||||
|
||||
# Créer les fichiers de test
|
||||
mkdir -p /var/www/html
|
||||
echo '{"status":"ok","stack":"nginx-vm"}' > /var/www/html/health
|
||||
for p in data api/users api/data/test; do
|
||||
mkdir -p "/var/www/html/$(dirname $p)"
|
||||
echo '{"ok":true}' > "/var/www/html/$p"
|
||||
done
|
||||
|
||||
nginx -t && nginx
|
||||
|
||||
# Attendre nginx
|
||||
for i in $(seq 1 20); do
|
||||
curl -sf http://localhost/health >/dev/null 2>&1 && break
|
||||
sleep 0.5
|
||||
done
|
||||
pass "nginx démarré"
|
||||
}
|
||||
|
||||
# ── Démarrage ja4ebpf ─────────────────────────────────────────────────────────
|
||||
start_ja4ebpf() {
|
||||
log "Démarrage ja4ebpf..."
|
||||
|
||||
pkill ja4ebpf 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Créer la config
|
||||
cat > /tmp/ja4ebpf.yml << 'EOF'
|
||||
interface: eth0
|
||||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||||
clickhouse:
|
||||
dsn: "clickhouse://default:@localhost:9000/ja4_logs"
|
||||
batch_size: 100
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
timeout_ms: 500
|
||||
slowloris_ms: 10000
|
||||
log:
|
||||
level: "info"
|
||||
format: "json"
|
||||
EOF
|
||||
|
||||
# Lancer avec les capabilities nécessaires
|
||||
# Dans la VM (root), on peut lancer directement
|
||||
ja4ebpf -config /tmp/ja4ebpf.yml > /tmp/ja4ebpf.log 2>&1 &
|
||||
JA4EBPF_PID=$!
|
||||
|
||||
sleep 3
|
||||
if ! kill -0 "$JA4EBPF_PID" 2>/dev/null; then
|
||||
fail "ja4ebpf s'est arrêté immédiatement"
|
||||
cat /tmp/ja4ebpf.log | tail -10
|
||||
return 1
|
||||
fi
|
||||
|
||||
log "ja4ebpf démarré (PID $JA4EBPF_PID)"
|
||||
|
||||
# Vérifier les uprobes dans tracefs
|
||||
sleep 1
|
||||
if grep -q "ssl" /sys/kernel/tracing/uprobe_events 2>/dev/null; then
|
||||
pass "Uprobes SSL attachés dans tracefs"
|
||||
else
|
||||
warn "Uprobes non visibles dans tracefs (peuvent être actifs quand même)"
|
||||
fi
|
||||
|
||||
# Vérifier accept4 tracepoint
|
||||
if grep -q "accept4" /sys/kernel/tracing/events/syscalls 2>/dev/null; then
|
||||
pass "Tracepoints accept4 disponibles"
|
||||
else
|
||||
warn "Tracepoints accept4 non trouvés"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Génération de trafic ───────────────────────────────────────────────────────
|
||||
generate_traffic() {
|
||||
log "Génération du trafic (HTTP/1.0 + HTTP/1.1 + HTTP/2)..."
|
||||
|
||||
# Trafic HTTP/1.1 (HTTP)
|
||||
for path in / /health /data /api/users; do
|
||||
curl -sf "http://localhost$path" >/dev/null 2>&1 || true
|
||||
curl -sf -X POST "http://localhost/api/data" -d '{"test":1}' >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
# Trafic HTTPS/1.1
|
||||
for path in / /health /data /api/users; do
|
||||
curl -sf -k "https://localhost$path" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X POST "https://localhost/api/data" -d '{"test":1}' >/dev/null 2>&1 || true
|
||||
curl -sf -k -X PUT "https://localhost/data" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X DELETE "https://localhost/data/1" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X HEAD "https://localhost$path" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
# Trafic HTTP/2
|
||||
if command -v python3 >/dev/null 2>&1 && python3 -c "import httpx" 2>/dev/null; then
|
||||
python3 << 'PYEOF'
|
||||
import httpx, ssl, warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
with httpx.Client(http2=True, verify=False) as client:
|
||||
for path in ["/", "/health", "/data"]:
|
||||
try: client.get(f"https://localhost{path}")
|
||||
except: pass
|
||||
try: client.post("https://localhost/api/data", json={"test": "h2"})
|
||||
except: pass
|
||||
PYEOF
|
||||
pass "Trafic HTTP/2 généré"
|
||||
fi
|
||||
|
||||
# Attendre le flush ja4ebpf → ClickHouse
|
||||
log "Attente flush ja4ebpf (15s)..."
|
||||
sleep 15
|
||||
pass "Trafic généré"
|
||||
}
|
||||
|
||||
# ── Vérification ClickHouse ────────────────────────────────────────────────────
|
||||
verify_db() {
|
||||
log "Vérification des données dans ClickHouse..."
|
||||
|
||||
ch_query() {
|
||||
curl -sf "http://localhost:8123/" \
|
||||
--data-urlencode "query=$1" \
|
||||
--data-urlencode "database=ja4_logs" \
|
||||
-o /dev/null -w '%{http_code}' 2>/dev/null || echo "0"
|
||||
}
|
||||
|
||||
ch_val() {
|
||||
curl -sf "http://localhost:8123/?database=ja4_logs" \
|
||||
--data-urlencode "query=$1" 2>/dev/null | tr -d ' \n' || echo "0"
|
||||
}
|
||||
|
||||
# L3/L4
|
||||
ttl=$(ch_val "SELECT count() FROM http_logs WHERE ip_meta_ttl > 0")
|
||||
[ "${ttl:-0}" -gt 0 ] && pass "L3/L4 TTL capturé ($ttl lignes)" || fail "L3/L4 TTL absent"
|
||||
|
||||
mss=$(ch_val "SELECT count() FROM http_logs WHERE tcp_meta_mss > 0")
|
||||
[ "${mss:-0}" -gt 0 ] && pass "TCP MSS capturé ($mss lignes)" || fail "TCP MSS absent"
|
||||
|
||||
# TLS
|
||||
ja4=$(ch_val "SELECT count() FROM http_logs WHERE ja4 != ''")
|
||||
[ "${ja4:-0}" -gt 0 ] && pass "JA4 fingerprint capturé ($ja4 lignes)" || fail "JA4 absent"
|
||||
|
||||
sni=$(ch_val "SELECT count() FROM http_logs WHERE tls_sni != ''")
|
||||
[ "${sni:-0}" -gt 0 ] && pass "TLS SNI capturé ($sni lignes)" || warn "TLS SNI absent"
|
||||
|
||||
# L7 HTTP — c'est ici que ça devrait marcher dans la VM
|
||||
method=$(ch_val "SELECT count() FROM http_logs WHERE method != ''")
|
||||
[ "${method:-0}" -gt 0 ] && pass "L7 méthodes HTTP capturées ($method lignes)" \
|
||||
|| fail "L7 méthodes HTTP ABSENT — uprobe SSL_read ne fonctionne pas"
|
||||
|
||||
path=$(ch_val "SELECT count() FROM http_logs WHERE path != ''")
|
||||
[ "${path:-0}" -gt 0 ] && pass "L7 path HTTP capturé ($path lignes)" || fail "L7 path absent"
|
||||
|
||||
status=$(ch_val "SELECT count() FROM http_logs WHERE status_code > 0")
|
||||
[ "${status:-0}" -gt 0 ] && pass "status_code capturé ($status lignes)" || warn "status_code absent"
|
||||
|
||||
sig=$(ch_val "SELECT count() FROM http_logs WHERE header_order_signature != ''")
|
||||
[ "${sig:-0}" -gt 0 ] && pass "header_order_signature capturé ($sig lignes)" || warn "header_order_sig absent"
|
||||
|
||||
# Méthodes HTTP distinctes
|
||||
methods=$(ch_val "SELECT groupArray(method) FROM (SELECT DISTINCT method FROM http_logs WHERE method != '')")
|
||||
log "Méthodes HTTP vues : $methods"
|
||||
|
||||
# Lignes totales
|
||||
total=$(ch_val "SELECT count() FROM http_logs")
|
||||
pass "Total lignes http_logs : $total"
|
||||
}
|
||||
|
||||
# ── Nettoyage ─────────────────────────────────────────────────────────────────
|
||||
cleanup() {
|
||||
if [ "$KEEP_RUNNING" != "true" ]; then
|
||||
log "Nettoyage..."
|
||||
pkill ja4ebpf 2>/dev/null || true
|
||||
nginx -s stop 2>/dev/null || true
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════╗"
|
||||
echo "║ ja4ebpf VM Test Suite — Rocky Linux 9 ║"
|
||||
echo "╚══════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
check_prerequisites
|
||||
start_clickhouse
|
||||
setup_nginx
|
||||
start_ja4ebpf
|
||||
generate_traffic
|
||||
verify_db
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════"
|
||||
echo -e " ${GREEN}OK${RESET}: $PASS_COUNT ${YELLOW}WARN${RESET}: $WARN_COUNT ${RED}FAIL${RESET}: $FAIL_COUNT"
|
||||
if [ "$FAIL_COUNT" -eq 0 ]; then
|
||||
echo -e " ${GREEN}${BOLD}Tous les tests réussis !${RESET}"
|
||||
exit 0
|
||||
else
|
||||
echo -e " ${RED}${BOLD}$FAIL_COUNT tests échoués.${RESET}"
|
||||
echo "Logs ja4ebpf :"
|
||||
tail -20 /tmp/ja4ebpf.log 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user