d75825278e
ja4ebpf: - Refactor BPF TC capture with improved SYN offset handling and TCP option parsing - Enhance TLS uprobe SSL hooking for better key extraction - Add ClickHouse writer improvements for HTTP log materialized views - Update RPM spec for Rocky Linux 8/9/10, fix systemd service - Simplify loader with cleaner bpf2go integration bot-detector: - Add H2 SETTINGS per-parameter comparison in browser_matcher - Enhance browser signatures and scoring pipeline - Improve preprocessing and cycle detection infra: - Multi-distro Vagrantfile (centos8, rocky9, rocky10) with per-distro provisioning - New Makefile targets: vm-up-all, test-vm-matrix, test-vm-centos8/rocky10 - Add debug helpers and run-test-from-host.sh for host-driven VM testing - Update run-tests-vm.sh for cross-distro compatibility - Remove accidental binary blob (\004) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
112 lines
4.5 KiB
Bash
112 lines
4.5 KiB
Bash
#!/usr/bin/env bash
|
|
# debug-xdp.sh — Test XDP + host traffic en une seule session SSH
|
|
# Usage: vagrant ssh rocky9 -- 'sudo bash -c "PATH=/usr/local/bin:$PATH /ja4-platform/tests/vm/debug-xdp.sh"'
|
|
set -euo pipefail
|
|
export PATH=/usr/local/bin:/usr/local/go/bin:$PATH
|
|
STACK="${1:-nginx}"
|
|
|
|
# === Start ClickHouse ===
|
|
echo "[1] Starting ClickHouse..."
|
|
docker rm -f ja4-clickhouse 2>/dev/null || true
|
|
docker run -d --name ja4-clickhouse -p 8123:8123 -p 9000:9000 \
|
|
-e CLICKHOUSE_DB=ja4_processing -e CLICKHOUSE_USER=default \
|
|
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
|
|
-v /ja4-platform/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh \
|
|
-v /ja4-platform/tests/integration/platform/csv-stubs:/var/lib/clickhouse/user_files \
|
|
-v /ja4-platform/shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/10_perf_indexes.sql:/initdb-src/10_perf_indexes.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/11_views.sql:/initdb-src/11_views.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/12_thesis_features.sql:/initdb-src/12_thesis_features.sql:ro \
|
|
clickhouse/clickhouse-server:24.8 >/dev/null
|
|
for i in $(seq 1 30); do curl -sf http://localhost:8123/ping >/dev/null 2>&1 && break; sleep 2; done
|
|
echo " ClickHouse ready"
|
|
|
|
# === Start nginx ===
|
|
echo "[2] Starting nginx..."
|
|
nginx -s stop 2>/dev/null || true; sleep 1
|
|
mkdir -p /run/nginx /var/www/html
|
|
echo '{"ok":true}' > /var/www/html/health
|
|
cp /ja4-platform/tests/integration/nginx/platform/nginx.conf /etc/nginx/nginx.conf
|
|
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
|
-keyout /etc/pki/tls/private/nginx.key -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
|
nginx && echo " nginx ready"
|
|
|
|
# === Start ja4ebpf ===
|
|
echo "[3] Starting ja4ebpf..."
|
|
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
|
cat > /tmp/ja4.yml << 'YEOF'
|
|
interface: eth0
|
|
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
|
clickhouse:
|
|
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
|
batch_size: 50
|
|
flush_secs: 1
|
|
correlation:
|
|
timeout_ms: 500
|
|
slowloris_ms: 10000
|
|
log:
|
|
level: "debug"
|
|
format: "json"
|
|
YEOF
|
|
JA4EBPF_CONFIG=/tmp/ja4.yml ja4ebpf > /tmp/ja4.log 2>&1 &
|
|
sleep 3
|
|
JA4PID=$(pgrep ja4ebpf || echo NONE)
|
|
if [ "$JA4PID" = "NONE" ]; then
|
|
echo " ja4ebpf DEAD!"; cat /tmp/ja4.log; exit 1
|
|
fi
|
|
echo " ja4ebpf PID=$JA4PID"
|
|
|
|
# Verify XDP
|
|
XDP_INFO=$(ip link show dev eth0 | grep "prog/xdp" || echo NONE)
|
|
echo " XDP: $XDP_INFO"
|
|
|
|
# Show eth0 IP
|
|
ETH0_IP=$(ip -4 addr show eth0 | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
|
|
echo ""
|
|
echo "╔══════════════════════════════════════╗"
|
|
echo "║ Services prêts — IP: $ETH0_IP"
|
|
echo "║ Attente trafic host (60s max)..."
|
|
echo "╚══════════════════════════════════════╝"
|
|
|
|
# Wait for host traffic signal
|
|
for i in $(seq 1 60); do
|
|
[ -f /tmp/traffic-done ] && break
|
|
sleep 1
|
|
done
|
|
|
|
# Check prog run count
|
|
echo "[4] Checking results..."
|
|
echo " ja4ebpf: $(pgrep ja4ebpf && echo alive || echo DEAD)"
|
|
bpftool prog show name capture_xdp 2>/dev/null | head -5
|
|
|
|
# Check raw data
|
|
RAW=$(curl -sf "http://localhost:8123/?database=ja4_logs" --data-urlencode "query=SELECT count() FROM http_logs_raw" 2>/dev/null || echo "0")
|
|
echo " http_logs_raw: $RAW lignes"
|
|
|
|
# ja4ebpf logs
|
|
echo " Logs:"
|
|
tail -5 /tmp/ja4.log | sed 's/^/ /'
|
|
|
|
# Cleanup
|
|
pkill ja4ebpf 2>/dev/null; nginx -s stop 2>/dev/null
|
|
docker rm -f ja4-clickhouse 2>/dev/null
|
|
|
|
if [ "${RAW:-0}" -gt 0 ] 2>/dev/null; then
|
|
echo ""
|
|
echo " SUCCESS: $RAW rows captured"
|
|
exit 0
|
|
else
|
|
echo ""
|
|
echo " FAIL: 0 rows captured"
|
|
exit 1
|
|
fi
|