code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(e2e): add distributed E2E test framework with parametric traffic generation

Browse Source 
Add run-e2e-test.sh with CLI parameters (--hits, --http-ratio, --dns, --tls,
--src-ips, --keep-analysis, --up) for configurable traffic generation. Traffic
runs from VM endpoints with multiple source IPs (alias IPs on eth0) to produce
distinct sessions for the ML pipeline. Fix curl TLS flags (--tlsv1.2 instead
of --tls-v1-2), skip redundant local verification in distributed mode, and
fix dashboard is_available() cache that never retried after ClickHouse recovery.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
2026-04-15 00:09:32 +02:00
parent 7894d39f1c
commit f88b739992
40 changed files with 2154 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
tests/vm/run-tests-vm.sh
View File

@ -68,6 +68,17 @@ get_eth0_ip() {
# ── ClickHouse ────────────────────────────────────────────────────────────────
start_clickhouse() {
# Si un ClickHouse externe est configuré, ne pas démarrer le conteneur local
if [ -n "${CH_HOST:-}" ] && [ "$CH_HOST" != "127.0.0.1" ] && [ "$CH_HOST" != "localhost" ]; then
log "ClickHouse externe ($CH_HOST) — démarrage local ignoré"
# Vérifier que le ClickHouse distant est accessible
for i in $(seq 1 30); do
curl -sf "http://${CH_HOST}:8123/ping" >/dev/null 2>&1 && { pass "ClickHouse distant prêt"; return 0; }
sleep 2
done
fail "ClickHouse distant ($CH_HOST) inaccessible"; return 1
fi
log "Démarrage ClickHouse..."
docker rm -f ja4-clickhouse 2>/dev/null || true
@ -104,11 +115,12 @@ start_ja4ebpf() {
done
[ -z "$ssl_lib" ] && ssl_lib="/usr/lib64/libssl.so.3"
local ch_addr="${CH_HOST:-127.0.0.1}"
cat > /tmp/ja4ebpf.yml << EOF
interface: eth0
ssl_lib_path: "${ssl_lib}"
clickhouse:
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
dsn: "clickhouse://default:@${ch_addr}:9000/ja4_logs"
batch_size: 100
flush_secs: 1
correlation:
@ -287,21 +299,47 @@ verify_db() {
log "Vérification des données dans ClickHouse..."
ch_val() {
curl -sf "http://localhost:8123/?database=ja4_logs" \
local ch_http_host="${CH_HOST:-localhost}"
curl -sf "http://${ch_http_host}:8123/?database=ja4_logs" \
--data-urlencode "query=$1" 2>/dev/null | tr -d ' \n' || echo "0"
}
# http_logs_raw (données brutes avant MV)
local raw_count
raw_count=$(ch_val "SELECT count() FROM http_logs_raw")
if [ "${raw_count:-0}" -gt 0 ] 2>/dev/null; then
pass "http_logs_raw : $raw_count lignes"
else
# Attendre que http_logs_raw contienne des données (max 30s)
local raw_ok=false
log " Attente données brutes dans ClickHouse..."
for i in $(seq 1 15); do
local raw_count
raw_count=$(ch_val "SELECT count() FROM http_logs_raw")
if [ "${raw_count:-0}" -gt 0 ] 2>/dev/null; then
pass "http_logs_raw : $raw_count lignes (${i}*2s)"
raw_ok=true
break
fi
sleep 2
done
if [ "$raw_ok" = "false" ]; then
fail "http_logs_raw vide — ja4ebpf n'a rien capturé"
log " Logs ja4ebpf :"
tail -10 /tmp/ja4ebpf.log 2>/dev/null | sed 's/^/ /'
fi
# Attendre que la MV http_logs se remplisse (max 30s)
local logs_ok=false
log " Attente MV http_logs..."
for i in $(seq 1 15); do
local logs_count
logs_count=$(ch_val "SELECT count() FROM http_logs")
if [ "${logs_count:-0}" -gt 0 ] 2>/dev/null; then
logs_ok=true
break
fi
sleep 2
done
if [ "$logs_ok" = "false" ]; then
warn "MV http_logs vide après 30s — vérification partielle uniquement"
fi
# L3/L4
ttl=$(ch_val "SELECT count() FROM http_logs WHERE ip_meta_ttl > 0")
[ "${ttl:-0}" -gt 0 ] 2>/dev/null && pass "L3/L4 TTL ($ttl)" || fail "L3/L4 TTL absent"
@ -343,7 +381,10 @@ stop_stack() {
apache) stop_apache ;;
hitch-varnish) stop_hitch_varnish ;;
esac
docker rm -f ja4-clickhouse 2>/dev/null || true
# Ne pas supprimer le ClickHouse s'il est externe (VM analysis)
if [ -z "${CH_HOST:-}" ] || [ "$CH_HOST" = "127.0.0.1" ] || [ "$CH_HOST" = "localhost" ]; then
docker rm -f ja4-clickhouse 2>/dev/null || true
fi
}
cleanup() {
@ -371,7 +412,10 @@ do_start() {
GOWORK=off go generate ./internal/loader/ 2>&1 | tail -3
GOWORK=off CGO_ENABLED=0 go build -o /tmp/ja4ebpf_new ./cmd/ja4ebpf/ && mv /tmp/ja4ebpf_new /usr/local/bin/ja4ebpf
}
command -v docker >/dev/null 2>&1 || { fail "Docker non installé"; exit 1; }
# Docker n'est nécessaire que pour un ClickHouse local
if [ -z "${CH_HOST:-}" ] || [ "$CH_HOST" = "127.0.0.1" ] || [ "$CH_HOST" = "localhost" ]; then
command -v docker >/dev/null 2>&1 || { fail "Docker non installé"; exit 1; }
fi
start_clickhouse
@ -434,7 +478,21 @@ case "$MODE" in
[ -f /tmp/ja4ebpf-traffic-done ] && break
sleep 1
done
do_verify
# En mode ClickHouse externe (E2E distribué), la vérification est faite
# par le script orchestrateur (run-e2e-test.sh Phase 5). On saute la
# vérification locale car les MV peuvent ne pas encore être peuplées.
if [ -n "${CH_HOST:-}" ] && [ "$CH_HOST" != "127.0.0.1" ] && [ "$CH_HOST" != "localhost" ]; then
log "ClickHouse externe — vérification locale ignorée (gérée par l'orchestrateur)"
log "Logs ja4ebpf :"
tail -5 /tmp/ja4ebpf.log 2>/dev/null | sed 's/^/ /'
pass "ja4ebpf actif (ClickHouse externe)"
else
# Laisser le temps au pipeline ClickHouse de traiter les données brutes
# (http_logs_raw → MV http_logs) avant de vérifier
log "Attente pipeline ClickHouse (20s)..."
sleep 20
do_verify
fi
;;
verify)
do_verify