code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(e2e): add multi-IP endpoint architecture with dedicated traffic VM

Browse Source 
Replace single-service-per-endpoint with all-ips mode running nginx, apache,
and hitch+varnish simultaneously on 3 dedicated IPs per VM (eth1 alias IPs).
Add a dedicated traffic VM with curl-impersonate for realistic TLS fingerprints,
parallelized traffic generation, and paired SNI_HOSTS/TARGET_IPS lists for
per-VM per-service hostname identification (e.g. rocky9-nginx-platform.test).

Key changes:
- run-tests-vm.sh: add setup_all_ips(), IP-specific Listen/bind directives
  with reset-before-apply pattern, graceful service availability checks
- run-e2e-test.sh: traffic VM architecture, all-ips mode, eth1 network,
  paired IP/SNI lists, updated cleanup for alias IPs
- generate-traffic.sh: parallel background jobs, curl-impersonate detection,
  auto source interface detection via ip route get, Host header in HTTP traffic
- Vagrantfile: add traffic VM with provision-traffic.sh
- provision-traffic.sh: install curl-impersonate and httpx for traffic gen
- test-rpm.sh: multi-interface TC check, updated ja4ebpf config
- clickhouse-init.sh: load CSV stubs for Anubis/bot-networks dictionaries
- Remove obsolete correlator/sentinel/mod-reqin-log docs
- Add h2_settings_ack column to http_logs schema
- Upgrade Go toolchain to 1.25.0

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
2026-04-16 14:25:24 +02:00
parent f0c8fe81c6
commit 36b5065a0a
17 changed files with 674 additions and 924 deletions
Download Patch File Download Diff File Expand all files Collapse all files

243
tests/vm/run-tests-vm.sh
View File

@ -11,6 +11,7 @@
# nginx — nginx avec TLS (HTTP/1.1 + HTTP/2)
# apache — Apache httpd avec TLS (HTTP/1.1 + HTTP/2)
# hitch-varnish — hitch (TLS) → Varnish (cache/H2) → backend Python
# all-ips — 3 services simultanés, 1 IP chacun (nginx IP1, apache IP2, hitch+varnish IP3)
# all — exécute les 3 stacks séquentiellement
#
# Modes :
@ -45,6 +46,61 @@ PASS_COUNT=0; FAIL_COUNT=0; WARN_COUNT=0
# ── Helpers communs ──────────────────────────────────────────────────────────
# IPs des services (positionnées par setup_all_ips ou defaults à l'IP eth0)
IP1="" # nginx
IP2="" # apache
IP3="" # hitch+varnish
setup_all_ips() {
local eth0_ip
eth0_ip=$(get_eth0_ip)
# Utiliser eth1 (réseau ja4-e2e, 192.168.42.0/24) pour les 3 IPs de service.
# eth0 est le réseau vagrant-libvirt (DHCP, IPs dynamiques) — les alias IPs
# ne sont pas routés par le dnsmasq de libvirt et sont injoignables depuis le host.
# eth1 est le réseau ja4-e2e dédié — accessible par toutes les VMs et le host.
local eth1_ip
eth1_ip=$(ip -4 addr show eth1 2>/dev/null | awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}')
if [ -z "$eth1_ip" ]; then
# Fallback: utiliser eth0 avec des alias si eth1 n'existe pas
local net_prefix
net_prefix=$(echo "$eth0_ip" | awk -F. '{print $1"."$2"."$3}')
local base_last
base_last=$(echo "$eth0_ip" | awk -F. '{print $4}')
IP1="$eth0_ip"
IP2="${net_prefix}.$((base_last + 100))"
IP3="${net_prefix}.$((base_last + 101))"
ip addr add "${IP2}/24" dev eth0 2>/dev/null || true
ip addr add "${IP3}/24" dev eth0 2>/dev/null || true
else
# Utiliser eth1 (réseau ja4-e2e) pour les 3 services
local net_prefix
net_prefix=$(echo "$eth1_ip" | awk -F. '{print $1"."$2"."$3}')
local base_last
base_last=$(echo "$eth1_ip" | awk -F. '{print $4}')
IP1="$eth1_ip"
IP2="${net_prefix}.$((base_last + 50))"
IP3="${net_prefix}.$((base_last + 51))"
# Ajouter les alias IPs sur eth1 (idempotent)
ip addr add "${IP2}/24" dev eth1 2>/dev/null || true
ip addr add "${IP3}/24" dev eth1 2>/dev/null || true
fi
log "IPs services : IP1=${IP1} (nginx) IP2=${IP2} (apache) IP3=${IP3} (hitch+varnish)"
}
# Écrire les IPs dans /tmp pour que l'orchestrateur puisse les lire
write_ip_manifest() {
cat > /tmp/e2e-endpoint-ips.json << EOF
{"ip1":"${IP1}","ip2":"${IP2}","ip3":"${IP3}"}
EOF
}
gen_tls_cert() {
local name="$1"
openssl req -x509 -nodes -days 365 -subj "/CN=platform.test" \
@ -54,8 +110,9 @@ gen_tls_cert() {
}
setup_docroot() {
local stack_name="${1:-$STACK}"
mkdir -p /var/www/html
echo '{"status":"ok","stack":"'"$STACK"'"}' > /var/www/html/health
echo '{"status":"ok","stack":"'"$stack_name"'"}' > /var/www/html/health
for p in data api/users api/data/test; do
mkdir -p "/var/www/html/$(dirname $p)"
echo '{"ok":true}' > "/var/www/html/$p"
@ -117,10 +174,14 @@ start_ja4ebpf() {
local ch_addr="${CH_HOST:-127.0.0.1}"
cat > /tmp/ja4ebpf.yml << EOF
interface: eth0
interfaces:
- any
ssl_lib_path: "${ssl_lib}"
listen_ports:
- 80
- 443
clickhouse:
dsn: "clickhouse://default:@${ch_addr}:9000/ja4_logs"
dsn: "clickhouse://default:@${ch_addr}:9000/ja4_logs?async_insert=0"
batch_size: 100
flush_secs: 1
correlation:
@ -143,14 +204,18 @@ EOF
log "ja4ebpf démarré (PID $JA4EBPF_PID)"
# Vérifier XDP
if ip link show dev eth0 2>/dev/null | grep -q "xdp"; then
local xdp_info
xdp_info=$(ip link show dev eth0 | grep "prog/xdp" | sed 's/^[[:space:]]*//')
pass "XDP attaché : $xdp_info"
# Vérifier TC ingress sur les interfaces
local TC_IFACES=0
for IFACE in $(ls /sys/class/net/ 2>/dev/null | grep -v lo); do
if tc filter show dev "$IFACE" ingress 2>/dev/null | grep -qi "bpf\|direct-action"; then
TC_IFACES=$((TC_IFACES + 1))
fi
done
if [ "$TC_IFACES" -gt 0 ]; then
pass "TC ingress attaché sur $TC_IFACES interface(s)"
else
warn "Aucun XDP sur eth0"
bpftool prog show name capture_xdp 2>/dev/null || true
warn "Aucun TC ingress détecté"
bpftool prog show name capture_tc 2>/dev/null || true
fi
}
@ -160,15 +225,27 @@ EOF
setup_nginx() {
log "Configuration nginx avec TLS..."
gen_tls_cert nginx
setup_docroot
setup_docroot nginx
cp "$PROJECT/tests/integration/nginx/platform/nginx.conf" /etc/nginx/nginx.conf
# Binder sur IP1 si en mode multi-IP
local bind_addr="${IP1:-}"
if [ -n "$bind_addr" ]; then
# Reset : remettre les directives listen à leur valeur par défaut avant de binder
# (si un run précédent a déjà remplacé par une IP, le sed suivant ne matcherait pas)
sed -i 's/^listen [0-9.]*:80;/listen 80;/' /etc/nginx/nginx.conf
sed -i 's/^listen [0-9.]*:443 ssl http2;/listen 443 ssl http2;/' /etc/nginx/nginx.conf
sed -i "s/listen 80;/listen ${bind_addr}:80;/" /etc/nginx/nginx.conf
sed -i "s/listen 443 ssl http2;/listen ${bind_addr}:443 ssl http2;/" /etc/nginx/nginx.conf
fi
mkdir -p /run/nginx
nginx -t && nginx
for i in $(seq 1 20); do
curl -sf http://localhost/health >/dev/null 2>&1 && break
curl -sf "http://${IP1:-localhost}/health" >/dev/null 2>&1 && break
sleep 0.5
done
pass "nginx démarré"
pass "nginx démarré (IP ${IP1:-*})"
}
stop_nginx() { nginx -s stop 2>/dev/null || true; }
@ -177,9 +254,15 @@ stop_nginx() { nginx -s stop 2>/dev/null || true; }
# Stack : apache
# ═════════════════════════════════════════════════════════════════════════════
setup_apache() {
# Vérifier que httpd est disponible
if ! command -v httpd >/dev/null 2>&1; then
warn "httpd non disponible — apache ignoré"
return 0
fi
log "Configuration Apache httpd avec TLS..."
gen_tls_cert apache
setup_docroot
setup_docroot apache
if command -v httpd >/dev/null 2>&1; then
if ! httpd -M 2>/dev/null | grep -q http2_module; then
@ -192,13 +275,32 @@ setup_apache() {
cp "$PROJECT/tests/integration/apache/platform/httpd-ssl.conf" \
/etc/httpd/conf.d/ssl.conf 2>/dev/null || true
httpd -t 2>&1 && httpd -DFOREGROUND &
# Binder sur IP2 si en mode multi-IP
local bind_addr="${IP2:-}"
if [ -n "$bind_addr" ]; then
# Reset : remettre les directives Listen/VirtualHost à leur valeur par défaut
# (si un run précédent a déjà remplacé par une IP, le sed suivant ne matcherait pas)
sed -i 's/^Listen [0-9.]*:80$/Listen 80/' /etc/httpd/conf/httpd.conf
sed -i 's/^Listen [0-9.]*:443 https$/Listen 443 https/' /etc/httpd/conf.d/ssl.conf
sed -i 's/<VirtualHost [0-9.]*:80>/<VirtualHost *:80>/' /etc/httpd/conf.d/ssl.conf
sed -i 's/<VirtualHost [0-9.]*:443>/<VirtualHost _default_:443>/' /etc/httpd/conf.d/ssl.conf
# Appliquer les bindings IP2
sed -i "s/^Listen 80$/Listen ${bind_addr}:80/" /etc/httpd/conf/httpd.conf
sed -i "s/^Listen 443 https/Listen ${bind_addr}:443 https/" /etc/httpd/conf.d/ssl.conf
sed -i "s/<VirtualHost _default_:443>/<VirtualHost ${bind_addr}:443>/" /etc/httpd/conf.d/ssl.conf
sed -i "s/<VirtualHost \*:80>/<VirtualHost ${bind_addr}:80>/" /etc/httpd/conf.d/ssl.conf 2>/dev/null || true
# S'assurer qu'il n'y a pas de Listen IP2:80 en double dans ssl.conf
# (le Listen 80 est déjà dans httpd.conf, pas besoin de le remettre dans ssl.conf)
sed -i "/^Listen ${bind_addr}:80$/d" /etc/httpd/conf.d/ssl.conf
fi
httpd -t 2>&1 && httpd
sleep 2
for i in $(seq 1 20); do
curl -sf http://localhost/health >/dev/null 2>&1 && break
curl -sf "http://${IP2:-localhost}/health" >/dev/null 2>&1 && break
sleep 0.5
done
pass "Apache httpd démarré"
pass "Apache httpd démarré (IP ${IP2:-*})"
}
stop_apache() { pkill httpd 2>/dev/null || true; }
@ -207,14 +309,22 @@ stop_apache() { pkill httpd 2>/dev/null || true; }
# Stack : hitch + varnish
# ═════════════════════════════════════════════════════════════════════════════
setup_hitch_varnish() {
# Vérifier que hitch est disponible
if ! command -v hitch >/dev/null 2>&1; then
warn "hitch non disponible — hitch+varnish ignoré"
return 0
fi
log "Configuration hitch + Varnish..."
gen_tls_cert hitch
mkdir -p /etc/hitch
cat /etc/pki/tls/private/hitch.key /etc/pki/tls/certs/hitch.crt \
> /etc/hitch/hitch.pem
cat > /etc/hitch/hitch.conf << 'HCONF'
frontend = "[*]:443"
# Binder hitch sur IP3 si en mode multi-IP, sinon [*]:443
local hitch_bind="${IP3:-*}"
cat > /etc/hitch/hitch.conf << HCONF
frontend = "[${hitch_bind}]:443"
backend = "[127.0.0.1]:6081"
pem-file = "/etc/hitch/hitch.pem"
write-proxy-v1 = on
@ -241,7 +351,7 @@ sub vcl_deliver {
VCL
}
setup_docroot
setup_docroot hitch-varnish
# Backend HTTP (port 8080)
python3 -c "
@ -269,27 +379,47 @@ with socketserver.TCPServer(('127.0.0.1', 8080), H) as s:
" &
sleep 1
varnishd -F -f /etc/varnish/default.vcl \
# HTTP sur IP3:80 (backend dédié pour le trafic HTTP en clair)
if [ -n "${IP3:-}" ]; then
python3 -c "
import http.server, socketserver, json
class H(http.server.BaseHTTPRequestHandler):
def log_message(self, *a): pass
def do_GET(self):
body = json.dumps({'status':'ok','stack':'hitch-varnish','path':self.path}).encode()
self.send_response(200)
self.send_header('Content-Type','application/json')
self.send_header('Content-Length',len(body))
self.end_headers()
self.wfile.write(body)
with socketserver.TCPServer(('${IP3}', 80), H) as s:
s.serve_forever()
" &
sleep 1
fi
varnishd -f /etc/varnish/default.vcl \
-a "127.0.0.1:6081,PROXY" \
-p feature=+http2 \
-s malloc,64m \
-T 127.0.0.1:6082 &
-T 127.0.0.1:6082 2>/dev/null
sleep 2
hitch --config=/etc/hitch/hitch.conf &
nohup hitch --config=/etc/hitch/hitch.conf >/dev/null 2>&1 &
sleep 2
for i in $(seq 1 20); do
curl -skf https://localhost/health >/dev/null 2>&1 && break
curl -skf "https://${IP3:-localhost}/health" >/dev/null 2>&1 && break
sleep 0.5
done
pass "hitch + Varnish démarrés"
pass "hitch + Varnish démarrés (IP ${IP3:-*})"
}
stop_hitch_varnish() {
pkill hitch 2>/dev/null || true
pkill varnishd 2>/dev/null || true
pkill -f "TCPServer.*8080" 2>/dev/null || true
pkill -f "TCPServer.*':80'" 2>/dev/null || true
}
# ═════════════════════════════════════════════════════════════════════════════
@ -380,6 +510,7 @@ stop_stack() {
nginx) stop_nginx ;;
apache) stop_apache ;;
hitch-varnish) stop_hitch_varnish ;;
all-ips) stop_nginx; stop_apache; stop_hitch_varnish; remove_alias_ips ;;
esac
# Ne pas supprimer le ClickHouse s'il est externe (VM analysis)
if [ -z "${CH_HOST:-}" ] || [ "$CH_HOST" = "127.0.0.1" ] || [ "$CH_HOST" = "localhost" ]; then
@ -387,10 +518,33 @@ stop_stack() {
fi
}
remove_alias_ips() {
# Déterminer l'interface des IPs alias (eth1 si réseau ja4-e2e, eth0 sinon)
local iface="eth0"
if [ -n "${IP2:-}" ]; then
# Si IP2 commence par 192.168.42, c'est sur eth1
case "$IP2" in
192.168.42.*) iface="eth1" ;;
esac
fi
if [ -n "${IP2:-}" ]; then
ip addr del "${IP2}/24" dev "$iface" 2>/dev/null || true
fi
if [ -n "${IP3:-}" ]; then
ip addr del "${IP3}/24" dev "$iface" 2>/dev/null || true
fi
}
cleanup() {
if [ "$KEEP_RUNNING" != "true" ]; then
log "Nettoyage..."
stop_stack
# En mode E2E distribué (CH_HOST externe), l'orchestrateur gère le nettoyage.
# On ne nettoie que si le script est lancé en mode standalone.
if [ -n "${CH_HOST:-}" ] && [ "$CH_HOST" != "127.0.0.1" ] && [ "$CH_HOST" != "localhost" ]; then
log "Nettoyage ignoré (mode distribué — géré par l'orchestrateur)"
else
log "Nettoyage..."
stop_stack
fi
fi
}
trap cleanup EXIT
@ -423,6 +577,7 @@ do_start() {
nginx) setup_nginx ;;
apache) setup_apache ;;
hitch-varnish) setup_hitch_varnish ;;
all-ips) setup_all_ips; setup_nginx; setup_apache; setup_hitch_varnish; write_ip_manifest ;;
*) fail "Stack inconnue: $STACK"; exit 1 ;;
esac
@ -432,12 +587,21 @@ do_start() {
local eth0_ip
eth0_ip=$(get_eth0_ip)
echo ""
echo " ┌─────────────────────────────────────────┐"
echo " │ Services prêts ! │"
echo " │ IP eth0 : $eth0_ip"
echo " │ HTTP : http://$eth0_ip:80"
echo " │ HTTPS : https://$eth0_ip:443"
echo " └─────────────────────────────────────────┘"
if [ "$STACK" = "all-ips" ]; then
echo " ┌─────────────────────────────────────────────┐"
echo " │ Services prêts ! │"
echo " │ nginx : http://${IP1}:80 https://${IP1}:443"
echo " │ apache : http://${IP2}:80 https://${IP2}:443"
echo " │ hitch+varnish : http://${IP3}:80 https://${IP3}:443"
echo " └─────────────────────────────────────────────┘"
else
echo " ┌─────────────────────────────────────────┐"
echo " │ Services prêts ! │"
echo " │ IP eth0 : $eth0_ip"
echo " │ HTTP : http://$eth0_ip:80"
echo " │ HTTPS : https://$eth0_ip:443"
echo " └─────────────────────────────────────────┘"
fi
echo ""
}
@ -474,10 +638,15 @@ case "$MODE" in
echo " En attente de trafic depuis le host..."
# Attendre que le host génère le trafic
# Le fichier /tmp/ja4ebpf-traffic-done est créé par le host après le trafic
for i in $(seq 1 120); do
[ -f /tmp/ja4ebpf-traffic-done ] && break
sleep 1
done
# En mode E2E distribué (CH_HOST externe), on attend sans limite de temps
if [ -n "${CH_HOST:-}" ] && [ "$CH_HOST" != "127.0.0.1" ] && [ "$CH_HOST" != "localhost" ]; then
while [ ! -f /tmp/ja4ebpf-traffic-done ]; do sleep 2; done
else
for i in $(seq 1 120); do
[ -f /tmp/ja4ebpf-traffic-done ] && break
sleep 1
done
fi
# En mode ClickHouse externe (E2E distribué), la vérification est faite
# par le script orchestrateur (run-e2e-test.sh Phase 5). On saute la
# vérification locale car les MV peuvent ne pas encore être peuplées.