feat: multi-distro VM tests, ja4ebpf eBPF improvements, bot-detector scoring
ja4ebpf: - Refactor BPF TC capture with improved SYN offset handling and TCP option parsing - Enhance TLS uprobe SSL hooking for better key extraction - Add ClickHouse writer improvements for HTTP log materialized views - Update RPM spec for Rocky Linux 8/9/10, fix systemd service - Simplify loader with cleaner bpf2go integration bot-detector: - Add H2 SETTINGS per-parameter comparison in browser_matcher - Enhance browser signatures and scoring pipeline - Improve preprocessing and cycle detection infra: - Multi-distro Vagrantfile (centos8, rocky9, rocky10) with per-distro provisioning - New Makefile targets: vm-up-all, test-vm-matrix, test-vm-centos8/rocky10 - Add debug helpers and run-test-from-host.sh for host-driven VM testing - Update run-tests-vm.sh for cross-distro compatibility - Remove accidental binary blob (\004) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
111
tests/vm/debug-xdp.sh
Normal file
111
tests/vm/debug-xdp.sh
Normal file
@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
# debug-xdp.sh — Test XDP + host traffic en une seule session SSH
|
||||
# Usage: vagrant ssh rocky9 -- 'sudo bash -c "PATH=/usr/local/bin:$PATH /ja4-platform/tests/vm/debug-xdp.sh"'
|
||||
set -euo pipefail
|
||||
export PATH=/usr/local/bin:/usr/local/go/bin:$PATH
|
||||
STACK="${1:-nginx}"
|
||||
|
||||
# === Start ClickHouse ===
|
||||
echo "[1] Starting ClickHouse..."
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
docker run -d --name ja4-clickhouse -p 8123:8123 -p 9000:9000 \
|
||||
-e CLICKHOUSE_DB=ja4_processing -e CLICKHOUSE_USER=default \
|
||||
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
|
||||
-v /ja4-platform/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh \
|
||||
-v /ja4-platform/tests/integration/platform/csv-stubs:/var/lib/clickhouse/user_files \
|
||||
-v /ja4-platform/shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/10_perf_indexes.sql:/initdb-src/10_perf_indexes.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/11_views.sql:/initdb-src/11_views.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/12_thesis_features.sql:/initdb-src/12_thesis_features.sql:ro \
|
||||
clickhouse/clickhouse-server:24.8 >/dev/null
|
||||
for i in $(seq 1 30); do curl -sf http://localhost:8123/ping >/dev/null 2>&1 && break; sleep 2; done
|
||||
echo " ClickHouse ready"
|
||||
|
||||
# === Start nginx ===
|
||||
echo "[2] Starting nginx..."
|
||||
nginx -s stop 2>/dev/null || true; sleep 1
|
||||
mkdir -p /run/nginx /var/www/html
|
||||
echo '{"ok":true}' > /var/www/html/health
|
||||
cp /ja4-platform/tests/integration/nginx/platform/nginx.conf /etc/nginx/nginx.conf
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/nginx.key -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
||||
nginx && echo " nginx ready"
|
||||
|
||||
# === Start ja4ebpf ===
|
||||
echo "[3] Starting ja4ebpf..."
|
||||
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
||||
cat > /tmp/ja4.yml << 'YEOF'
|
||||
interface: eth0
|
||||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||||
clickhouse:
|
||||
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
||||
batch_size: 50
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
timeout_ms: 500
|
||||
slowloris_ms: 10000
|
||||
log:
|
||||
level: "debug"
|
||||
format: "json"
|
||||
YEOF
|
||||
JA4EBPF_CONFIG=/tmp/ja4.yml ja4ebpf > /tmp/ja4.log 2>&1 &
|
||||
sleep 3
|
||||
JA4PID=$(pgrep ja4ebpf || echo NONE)
|
||||
if [ "$JA4PID" = "NONE" ]; then
|
||||
echo " ja4ebpf DEAD!"; cat /tmp/ja4.log; exit 1
|
||||
fi
|
||||
echo " ja4ebpf PID=$JA4PID"
|
||||
|
||||
# Verify XDP
|
||||
XDP_INFO=$(ip link show dev eth0 | grep "prog/xdp" || echo NONE)
|
||||
echo " XDP: $XDP_INFO"
|
||||
|
||||
# Show eth0 IP
|
||||
ETH0_IP=$(ip -4 addr show eth0 | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════╗"
|
||||
echo "║ Services prêts — IP: $ETH0_IP"
|
||||
echo "║ Attente trafic host (60s max)..."
|
||||
echo "╚══════════════════════════════════════╝"
|
||||
|
||||
# Wait for host traffic signal
|
||||
for i in $(seq 1 60); do
|
||||
[ -f /tmp/traffic-done ] && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Check prog run count
|
||||
echo "[4] Checking results..."
|
||||
echo " ja4ebpf: $(pgrep ja4ebpf && echo alive || echo DEAD)"
|
||||
bpftool prog show name capture_xdp 2>/dev/null | head -5
|
||||
|
||||
# Check raw data
|
||||
RAW=$(curl -sf "http://localhost:8123/?database=ja4_logs" --data-urlencode "query=SELECT count() FROM http_logs_raw" 2>/dev/null || echo "0")
|
||||
echo " http_logs_raw: $RAW lignes"
|
||||
|
||||
# ja4ebpf logs
|
||||
echo " Logs:"
|
||||
tail -5 /tmp/ja4.log | sed 's/^/ /'
|
||||
|
||||
# Cleanup
|
||||
pkill ja4ebpf 2>/dev/null; nginx -s stop 2>/dev/null
|
||||
docker rm -f ja4-clickhouse 2>/dev/null
|
||||
|
||||
if [ "${RAW:-0}" -gt 0 ] 2>/dev/null; then
|
||||
echo ""
|
||||
echo " SUCCESS: $RAW rows captured"
|
||||
exit 0
|
||||
else
|
||||
echo ""
|
||||
echo " FAIL: 0 rows captured"
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user