d75825278e
ja4ebpf: - Refactor BPF TC capture with improved SYN offset handling and TCP option parsing - Enhance TLS uprobe SSL hooking for better key extraction - Add ClickHouse writer improvements for HTTP log materialized views - Update RPM spec for Rocky Linux 8/9/10, fix systemd service - Simplify loader with cleaner bpf2go integration bot-detector: - Add H2 SETTINGS per-parameter comparison in browser_matcher - Enhance browser signatures and scoring pipeline - Improve preprocessing and cycle detection infra: - Multi-distro Vagrantfile (centos8, rocky9, rocky10) with per-distro provisioning - New Makefile targets: vm-up-all, test-vm-matrix, test-vm-centos8/rocky10 - Add debug helpers and run-test-from-host.sh for host-driven VM testing - Update run-tests-vm.sh for cross-distro compatibility - Remove accidental binary blob (\004) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
97 lines
3.8 KiB
Bash
97 lines
3.8 KiB
Bash
#!/usr/bin/env bash
|
|
# Debug script — start everything and check XDP stats
|
|
set -euo pipefail
|
|
export PATH=/usr/local/bin:/usr/local/go/bin:$PATH
|
|
|
|
echo "=== Starting ClickHouse ==="
|
|
docker rm -f ja4-clickhouse 2>/dev/null || true
|
|
docker run -d --name ja4-clickhouse -p 8123:8123 -p 9000:9000 \
|
|
-e CLICKHOUSE_DB=ja4_processing -e CLICKHOUSE_USER=default -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
|
|
-v /ja4-platform/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh \
|
|
-v /ja4-platform/tests/integration/platform/csv-stubs:/var/lib/clickhouse/user_files \
|
|
-v /ja4-platform/shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/10_perf_indexes.sql:/initdb-src/10_perf_indexes.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/11_views.sql:/initdb-src/11_views.sql:ro \
|
|
-v /ja4-platform/shared/clickhouse/12_thesis_features.sql:/initdb-src/12_thesis_features.sql:ro \
|
|
clickhouse/clickhouse-server:24.8
|
|
|
|
for i in $(seq 1 30); do curl -sf http://localhost:8123/ping >/dev/null 2>&1 && break; sleep 2; done
|
|
echo "CH ready: $?"
|
|
|
|
echo "=== Starting nginx ==="
|
|
pkill nginx 2>/dev/null || true; sleep 1
|
|
mkdir -p /run/nginx /var/www/html
|
|
echo '{"ok":true}' > /var/www/html/health
|
|
cp /ja4-platform/tests/integration/nginx/platform/nginx.conf /etc/nginx/nginx.conf
|
|
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
|
-keyout /etc/pki/tls/private/nginx.key -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
|
nginx && echo "nginx OK"
|
|
|
|
echo "=== Starting ja4ebpf ==="
|
|
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
|
|
|
cat > /tmp/ja4.yml << 'YEOF'
|
|
interface: eth0
|
|
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
|
clickhouse:
|
|
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
|
batch_size: 50
|
|
flush_secs: 1
|
|
correlation:
|
|
timeout_ms: 500
|
|
slowloris_ms: 10000
|
|
log:
|
|
level: "debug"
|
|
format: "json"
|
|
YEOF
|
|
|
|
JA4EBPF_CONFIG=/tmp/ja4.yml ja4ebpf > /tmp/ja4.log 2>&1 &
|
|
sleep 4
|
|
|
|
JA4PID=$(pgrep ja4ebpf || echo NONE)
|
|
echo "ja4ebpf PID: $JA4PID"
|
|
|
|
if [ "$JA4PID" = "NONE" ]; then
|
|
echo "DEAD! Logs:"
|
|
cat /tmp/ja4.log
|
|
exit 1
|
|
fi
|
|
|
|
echo "=== XDP status ==="
|
|
ip link show dev eth0 | grep -i xdp
|
|
echo "=== Prog stats ==="
|
|
bpftool prog show name capture_xdp 2>/dev/null || echo "no prog"
|
|
|
|
echo ""
|
|
echo "=== Waiting for external traffic ==="
|
|
echo "Send traffic from host to $(ip -4 addr show eth0 | awk '/inet /{sub(/\/.*/, "", $2); print $2}')"
|
|
echo "After sending, press Enter or wait 60s..."
|
|
|
|
# Wait for signal or timeout
|
|
for i in $(seq 1 60); do
|
|
[ -f /tmp/traffic-done ] && break
|
|
sleep 1
|
|
done
|
|
|
|
echo "=== After traffic ==="
|
|
pgrep ja4ebpf && echo "ja4ebpf still alive" || echo "ja4ebpf DEAD"
|
|
bpftool prog show name capture_xdp 2>/dev/null || echo "no prog"
|
|
|
|
echo "=== Raw data count ==="
|
|
curl -sf "http://localhost:8123/?database=ja4_logs" --data-urlencode "query=SELECT count() FROM http_logs_raw" 2>/dev/null || echo "0"
|
|
|
|
echo "=== ja4ebpf logs ==="
|
|
cat /tmp/ja4.log
|
|
|
|
# Cleanup
|
|
pkill ja4ebpf 2>/dev/null; nginx -s stop 2>/dev/null; docker rm -f ja4-clickhouse 2>/dev/null
|