feat: multi-distro VM tests, ja4ebpf eBPF improvements, bot-detector scoring
ja4ebpf: - Refactor BPF TC capture with improved SYN offset handling and TCP option parsing - Enhance TLS uprobe SSL hooking for better key extraction - Add ClickHouse writer improvements for HTTP log materialized views - Update RPM spec for Rocky Linux 8/9/10, fix systemd service - Simplify loader with cleaner bpf2go integration bot-detector: - Add H2 SETTINGS per-parameter comparison in browser_matcher - Enhance browser signatures and scoring pipeline - Improve preprocessing and cycle detection infra: - Multi-distro Vagrantfile (centos8, rocky9, rocky10) with per-distro provisioning - New Makefile targets: vm-up-all, test-vm-matrix, test-vm-centos8/rocky10 - Add debug helpers and run-test-from-host.sh for host-driven VM testing - Update run-tests-vm.sh for cross-distro compatibility - Remove accidental binary blob (\004) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
@ -1 +1 @@
|
||||
{"dependencies":[["racc",["~> 1.4"]],["nokogiri",["~> 1.6"]],["diffy",[">= 0"]],["rexml",[">= 0"]],["xml-simple",[">= 0"]],["logger",[">= 0"]],["mime-types-data",["~> 3.2025",">= 3.2025.0507"]],["mime-types",[">= 0"]],["io-console",["~> 0.5"]],["reline",[">= 0"]],["formatador",[">= 0.2","< 2.0"]],["excon",["~> 1.0"]],["builder",[">= 0"]],["fog-core",["~> 2"]],["ruby-libvirt",[">= 0.7.0"]],["json",[">= 0"]],["fog-xml",["~> 0.1.1"]],["multi_json",["~> 1.10"]],["fog-json",[">= 0"]],["fog-libvirt",[">= 0.6.0"]],["vagrant-libvirt",["= 0.12.2"]],["vagrant-qemu",["= 0.3.12"]]],"checksum":"8812dc95b590d4059a84fe716eaa6eea39b29aecb1c994c959de405ba3705361","vagrant_version":"2.4.9"}
|
||||
{"dependencies":[["racc",["~> 1.4"]],["nokogiri",["~> 1.6"]],["diffy",[">= 0"]],["rexml",[">= 0"]],["xml-simple",[">= 0"]],["logger",[">= 0"]],["mime-types-data",["~> 3.2025",">= 3.2025.0507"]],["mime-types",[">= 0"]],["io-console",["~> 0.5"]],["reline",[">= 0"]],["formatador",[">= 0.2","< 2.0"]],["excon",["~> 1.0"]],["builder",[">= 0"]],["fog-core",["~> 2"]],["ruby-libvirt",[">= 0.7.0"]],["json",[">= 0"]],["fog-xml",["~> 0.1.1"]],["multi_json",["~> 1.10"]],["fog-json",[">= 0"]],["fog-libvirt",[">= 0.6.0"]],["vagrant-libvirt",["= 0.12.2"]]],"checksum":"b69e3c206e3d26fb25b062fbb15a80865764c5efb5e9cce85cfac1f745449033","vagrant_version":"2.4.9"}
|
||||
73
tests/vm/Vagrantfile
vendored
73
tests/vm/Vagrantfile
vendored
@ -1,7 +1,12 @@
|
||||
# -*- mode: ruby -*-
|
||||
# vi: set ft=ruby :
|
||||
# =============================================================================
|
||||
# Vagrantfile — VM de test ja4ebpf sur Rocky Linux 9
|
||||
# Vagrantfile — VMs de test ja4ebpf multi-distro
|
||||
#
|
||||
# 3 VMs pour les tests unitaires eBPF sur kernel réel :
|
||||
# - centos8 : CentOS 8 (el8)
|
||||
# - rocky9 : Rocky Linux 9 (el9)
|
||||
# - rocky10 : Rocky Linux 10 (el10)
|
||||
#
|
||||
# Fournit un environnement kernel complet pour les tests eBPF :
|
||||
# - tracefs / debugfs montés
|
||||
@ -14,47 +19,69 @@
|
||||
# sudo usermod -aG libvirt,kvm $USER # puis se reconnecter
|
||||
#
|
||||
# Utilisation :
|
||||
# vagrant up # créer + provisionner (~5 min)
|
||||
# vagrant ssh # connexion SSH
|
||||
# make test-vm-nginx # lancer les tests depuis le host
|
||||
# vagrant destroy -f # détruire la VM
|
||||
# vagrant up # créer + provisionner toutes les VMs
|
||||
# vagrant up rocky9 # créer une seule VM
|
||||
# vagrant ssh rocky9 # connexion SSH
|
||||
# make test-vm-nginx # test nginx sur Rocky 9 (défaut)
|
||||
# make test-vm-all # tous les tests sur Rocky 9
|
||||
# ./tests/vm/run-all-vms.sh # tests sur les 3 VMs
|
||||
# vagrant destroy -f # détruire toutes les VMs
|
||||
# =============================================================================
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
|
||||
# ── Box Rocky Linux 9 avec provider libvirt (image qcow2) ─────────────────
|
||||
config.vm.box = "generic/rocky9"
|
||||
|
||||
# ── Désactiver synced_folder par défaut (utiliser rsync explicitement) ─────
|
||||
# ── Désactiver synced_folder par défaut ─────────────────────────────────────
|
||||
config.vm.synced_folder ".", "/vagrant", disabled: true
|
||||
|
||||
# ── Provider libvirt ───────────────────────────────────────────────────────
|
||||
# ── Provider libvirt commun ─────────────────────────────────────────────────
|
||||
config.vm.provider :libvirt do |v|
|
||||
v.cpus = 4
|
||||
v.memory = 4096
|
||||
v.nested = false
|
||||
v.cpu_mode = "host-passthrough" # expose les capacités CPU hôte → KVM perf
|
||||
v.cpu_mode = "host-passthrough"
|
||||
v.driver = "kvm"
|
||||
v.disk_bus = "virtio"
|
||||
v.nic_model_type = "virtio"
|
||||
end
|
||||
|
||||
# ── Synchronisation du projet via rsync ────────────────────────────────────
|
||||
# ── Synchronisation du projet via rsync ─────────────────────────────────────
|
||||
config.vm.synced_folder "../..", "/ja4-platform",
|
||||
type: "rsync",
|
||||
rsync__exclude: [".git/", "old/", "*.rpm", "dist/"]
|
||||
|
||||
# ── Provisioning ───────────────────────────────────────────────────────────
|
||||
config.vm.provision "shell", path: "provision.sh"
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# VM 1 : CentOS 8 (el8)
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
config.vm.define "centos8", autostart: false do |node|
|
||||
node.vm.box = "centos/8"
|
||||
node.vm.provision "shell", path: "provision-el8.sh"
|
||||
node.vm.post_up_message = "VM centos8 prête ! Tests : make test-vm-centos8"
|
||||
end
|
||||
|
||||
# ── Message post-démarrage ─────────────────────────────────────────────────
|
||||
config.vm.post_up_message = <<~MSG
|
||||
VM ja4ebpf prête !
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# VM 2 : Rocky Linux 9 (el9) — VM par défaut
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
config.vm.define "rocky9", primary: true do |node|
|
||||
node.vm.box = "generic/rocky9"
|
||||
node.vm.provision "shell", path: "provision.sh"
|
||||
node.vm.post_up_message = <<~MSG
|
||||
VM rocky9 prête !
|
||||
|
||||
Depuis la racine du projet :
|
||||
make vm-ssh # connexion interactive
|
||||
make test-vm-nginx # test nginx complet (L3/L4 + TLS + L7)
|
||||
make test-vm-all # tous les tests
|
||||
make vm-rebuild-ja4ebpf # resynchroniser + recompiler après modif
|
||||
MSG
|
||||
end
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
# VM 3 : Rocky Linux 10 (el10)
|
||||
# ═══════════════════════════════════════════════════════════════════════════
|
||||
config.vm.define "rocky10", autostart: false do |node|
|
||||
node.vm.box = "almalinux/10"
|
||||
node.vm.provision "shell", path: "provision.sh"
|
||||
node.vm.post_up_message = "VM rocky10 prête ! Tests : make test-vm-rocky10"
|
||||
end
|
||||
|
||||
Depuis la racine du projet :
|
||||
make vm-ssh # connexion interactive
|
||||
make test-vm-nginx # test nginx complet (L3/L4 + TLS + L7)
|
||||
make test-vm-all # tous les tests
|
||||
make vm-rebuild-ja4ebpf # resynchroniser + recompiler après modif
|
||||
MSG
|
||||
end
|
||||
|
||||
107
tests/vm/debug-mode-host.sh
Normal file
107
tests/vm/debug-mode-host.sh
Normal file
@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env bash
|
||||
# debug-mode-host.sh — Test debug ja4ebpf avec trafic host→VM
|
||||
# Usage: ./debug-mode-host.sh rocky9
|
||||
set -euo pipefail
|
||||
|
||||
VM="${1:-rocky9}"
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
echo "=== [1] Setup VM: nginx + ja4ebpf debug ==="
|
||||
vagrant ssh "$VM" -- "sudo bash -c '
|
||||
PATH=/usr/local/bin:\$PATH
|
||||
# Install debug binary
|
||||
cp /tmp/ja4ebpf-debug /usr/local/bin/ja4ebpf
|
||||
chmod +x /usr/local/bin/ja4ebpf
|
||||
|
||||
# Start nginx
|
||||
nginx -s stop 2>/dev/null || true; sleep 1
|
||||
mkdir -p /run/nginx /var/www/html
|
||||
echo {\"ok\":true} > /var/www/html/health
|
||||
cat > /etc/nginx/nginx.conf << \"NEOF\"
|
||||
worker_processes 1;
|
||||
events { worker_connections 64; }
|
||||
http {
|
||||
server {
|
||||
listen 80;
|
||||
listen 443 ssl;
|
||||
ssl_certificate /etc/pki/tls/certs/nginx.crt;
|
||||
ssl_certificate_key /etc/pki/tls/private/nginx.key;
|
||||
root /var/www/html;
|
||||
}
|
||||
}
|
||||
NEOF
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/nginx.key -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
||||
nginx
|
||||
|
||||
# Start ja4ebpf debug
|
||||
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
||||
cat > /tmp/ja4-debug.yml << \"YEOF\"
|
||||
interface: eth0
|
||||
ssl_lib_path: \"/usr/lib64/libssl.so.3\"
|
||||
debug: true
|
||||
clickhouse:
|
||||
dsn: \"clickhouse://default:@127.0.0.1:9000/ja4_logs\"
|
||||
batch_size: 50
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
timeout_ms: 500
|
||||
slowloris_ms: 10000
|
||||
log:
|
||||
level: \"debug\"
|
||||
format: \"text\"
|
||||
YEOF
|
||||
JA4EBPF_CONFIG=/tmp/ja4-debug.yml ja4ebpf > /tmp/ja4-debug.log 2>&1 &
|
||||
sleep 3
|
||||
PID=\$(pgrep ja4ebpf || echo NONE)
|
||||
echo \" ja4ebpf PID=\$PID\"
|
||||
if [ \"\$PID\" = \"NONE\" ]; then cat /tmp/ja4-debug.log; exit 1; fi
|
||||
|
||||
# Open firewall
|
||||
firewall-cmd --add-service=http --add-service=https 2>/dev/null || true
|
||||
|
||||
# Show eth0 IP
|
||||
ip -4 addr show eth0 | awk \"/inet /{sub(/\\/.*/,\"\",\\\$2); print \\\" eth0 IP: \\\"\\\$2; exit}\"
|
||||
'" 2>&1
|
||||
|
||||
echo ""
|
||||
echo "=== [2] Get VM IP ==="
|
||||
VM_IP=$(vagrant ssh "$VM" -- "ip -4 addr show eth0" 2>/dev/null | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
|
||||
echo " VM IP: $VM_IP"
|
||||
|
||||
if [ -z "$VM_IP" ]; then
|
||||
echo " ERROR: no eth0 IP found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== [3] Generate traffic from HOST to VM ==="
|
||||
for i in $(seq 1 3); do
|
||||
curl -sf "http://$VM_IP/health" -o /dev/null -w " HTTP $i: %{http_code}\n" 2>&1 || echo " HTTP $i: FAIL"
|
||||
curl -skf "https://$VM_IP/health" -o /dev/null -w " HTTPS $i: %{http_code}\n" 2>&1 || echo " HTTPS $i: FAIL"
|
||||
done
|
||||
|
||||
echo ""
|
||||
echo "=== [4] Wait for debug dump (8s) ==="
|
||||
sleep 8
|
||||
|
||||
echo ""
|
||||
echo "=== [5] Collect results ==="
|
||||
vagrant ssh "$VM" -- "sudo bash -c '
|
||||
echo \" ja4ebpf: \$(pgrep ja4ebpf > /dev/null && echo alive || echo DEAD)\"
|
||||
echo \"\"
|
||||
echo \" === BPF stats ===\"
|
||||
STATS_MAP_ID=\$(bpftool map show name xdp_stats 2>/dev/null | grep -oP \"id \K\d+\" || echo NONE)
|
||||
if [ \"\$STATS_MAP_ID\" != \"NONE\" ]; then
|
||||
bpftool map dump id \$STATS_MAP_ID 2>/dev/null | sed \"s/^/ /\"
|
||||
else
|
||||
echo \" xdp_stats map not found!\"
|
||||
fi
|
||||
echo \"\"
|
||||
echo \" === Log tail ===\"
|
||||
tail -30 /tmp/ja4-debug.log | sed \"s/^/ /\"
|
||||
|
||||
# Cleanup
|
||||
pkill ja4ebpf 2>/dev/null || true
|
||||
nginx -s stop 2>/dev/null || true
|
||||
'" 2>&1
|
||||
98
tests/vm/debug-mode.sh
Normal file
98
tests/vm/debug-mode.sh
Normal file
@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env bash
|
||||
# debug-mode.sh — Test rapide du mode debug ja4ebpf sur une VM
|
||||
# Usage: vagrant upload /ja4-platform/tests/vm/debug-mode.sh /tmp/debug-mode.sh rocky9
|
||||
# vagrant ssh rocky9 -- 'sudo bash /tmp/debug-mode.sh'
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== [1] Install debug binary ==="
|
||||
cp /tmp/ja4ebpf-debug /usr/local/bin/ja4ebpf
|
||||
chmod +x /usr/local/bin/ja4ebpf
|
||||
|
||||
echo "=== [2] Start nginx ==="
|
||||
nginx -s stop 2>/dev/null || true; sleep 1
|
||||
mkdir -p /run/nginx /var/www/html
|
||||
echo '{"ok":true}' > /var/www/html/health
|
||||
# Minimal nginx config for TLS
|
||||
cat > /etc/nginx/nginx.conf << 'NEOF'
|
||||
worker_processes 1;
|
||||
events { worker_connections 64; }
|
||||
http {
|
||||
server {
|
||||
listen 80;
|
||||
listen 443 ssl;
|
||||
ssl_certificate /etc/pki/tls/certs/nginx.crt;
|
||||
ssl_certificate_key /etc/pki/tls/private/nginx.key;
|
||||
root /var/www/html;
|
||||
}
|
||||
}
|
||||
NEOF
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/nginx.key -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
||||
nginx && echo " nginx ready"
|
||||
|
||||
echo "=== [3] Start ja4ebpf in DEBUG mode ==="
|
||||
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
||||
|
||||
# Config with debug=true — no ClickHouse needed in debug mode
|
||||
cat > /tmp/ja4-debug.yml << 'YEOF'
|
||||
interface: eth0
|
||||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||||
debug: true
|
||||
clickhouse:
|
||||
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
||||
batch_size: 50
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
timeout_ms: 500
|
||||
slowloris_ms: 10000
|
||||
log:
|
||||
level: "debug"
|
||||
format: "text"
|
||||
YEOF
|
||||
|
||||
JA4EBPF_CONFIG=/tmp/ja4-debug.yml ja4ebpf > /tmp/ja4-debug.log 2>&1 &
|
||||
sleep 3
|
||||
|
||||
JA4PID=$(pgrep ja4ebpf || echo NONE)
|
||||
if [ "$JA4PID" = "NONE" ]; then
|
||||
echo " ja4ebpf DEAD! Log:"
|
||||
cat /tmp/ja4-debug.log
|
||||
exit 1
|
||||
fi
|
||||
echo " ja4ebpf PID=$JA4PID"
|
||||
|
||||
# Verify XDP
|
||||
echo " XDP check:"
|
||||
ip -d link show dev eth0 | grep -i xdp || echo " (no XDP attached)"
|
||||
|
||||
echo "=== [4] Generate traffic ==="
|
||||
ETH0_IP=$(ip -4 addr show eth0 | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
|
||||
echo " eth0 IP: $ETH0_IP"
|
||||
|
||||
# HTTP traffic from localhost via eth0 IP
|
||||
for i in $(seq 1 5); do
|
||||
curl -sf "http://$ETH0_IP/health" -o /dev/null 2>&1 && echo " HTTP $i: OK" || echo " HTTP $i: FAIL"
|
||||
curl -skf "https://$ETH0_IP/health" -o /dev/null 2>&1 && echo " HTTPS $i: OK" || echo " HTTPS $i: FAIL"
|
||||
done
|
||||
|
||||
echo "=== [5] Wait for debug dump (6s) ==="
|
||||
sleep 6
|
||||
|
||||
echo "=== [6] Results ==="
|
||||
echo " ja4ebpf: $(pgrep ja4ebpf > /dev/null && echo alive || echo DEAD)"
|
||||
echo ""
|
||||
echo " === Last 20 lines of log ==="
|
||||
tail -20 /tmp/ja4-debug.log | sed 's/^/ /'
|
||||
|
||||
echo ""
|
||||
echo " === BPF map stats (bpftool) ==="
|
||||
STATS_MAP_ID=$(bpftool map show name xdp_stats 2>/dev/null | grep -oP 'id \K\d+' || echo NONE)
|
||||
if [ "$STATS_MAP_ID" != "NONE" ]; then
|
||||
bpftool map dump id $STATS_MAP_ID 2>/dev/null | head -40 | sed 's/^/ /'
|
||||
else
|
||||
echo " xdp_stats map not found!"
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
pkill ja4ebpf 2>/dev/null || true
|
||||
nginx -s stop 2>/dev/null || true
|
||||
96
tests/vm/debug-test.sh
Normal file
96
tests/vm/debug-test.sh
Normal file
@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
# Debug script — start everything and check XDP stats
|
||||
set -euo pipefail
|
||||
export PATH=/usr/local/bin:/usr/local/go/bin:$PATH
|
||||
|
||||
echo "=== Starting ClickHouse ==="
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
docker run -d --name ja4-clickhouse -p 8123:8123 -p 9000:9000 \
|
||||
-e CLICKHOUSE_DB=ja4_processing -e CLICKHOUSE_USER=default -e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
|
||||
-v /ja4-platform/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh \
|
||||
-v /ja4-platform/tests/integration/platform/csv-stubs:/var/lib/clickhouse/user_files \
|
||||
-v /ja4-platform/shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/10_perf_indexes.sql:/initdb-src/10_perf_indexes.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/11_views.sql:/initdb-src/11_views.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/12_thesis_features.sql:/initdb-src/12_thesis_features.sql:ro \
|
||||
clickhouse/clickhouse-server:24.8
|
||||
|
||||
for i in $(seq 1 30); do curl -sf http://localhost:8123/ping >/dev/null 2>&1 && break; sleep 2; done
|
||||
echo "CH ready: $?"
|
||||
|
||||
echo "=== Starting nginx ==="
|
||||
pkill nginx 2>/dev/null || true; sleep 1
|
||||
mkdir -p /run/nginx /var/www/html
|
||||
echo '{"ok":true}' > /var/www/html/health
|
||||
cp /ja4-platform/tests/integration/nginx/platform/nginx.conf /etc/nginx/nginx.conf
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/nginx.key -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
||||
nginx && echo "nginx OK"
|
||||
|
||||
echo "=== Starting ja4ebpf ==="
|
||||
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
||||
|
||||
cat > /tmp/ja4.yml << 'YEOF'
|
||||
interface: eth0
|
||||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||||
clickhouse:
|
||||
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
||||
batch_size: 50
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
timeout_ms: 500
|
||||
slowloris_ms: 10000
|
||||
log:
|
||||
level: "debug"
|
||||
format: "json"
|
||||
YEOF
|
||||
|
||||
JA4EBPF_CONFIG=/tmp/ja4.yml ja4ebpf > /tmp/ja4.log 2>&1 &
|
||||
sleep 4
|
||||
|
||||
JA4PID=$(pgrep ja4ebpf || echo NONE)
|
||||
echo "ja4ebpf PID: $JA4PID"
|
||||
|
||||
if [ "$JA4PID" = "NONE" ]; then
|
||||
echo "DEAD! Logs:"
|
||||
cat /tmp/ja4.log
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "=== XDP status ==="
|
||||
ip link show dev eth0 | grep -i xdp
|
||||
echo "=== Prog stats ==="
|
||||
bpftool prog show name capture_xdp 2>/dev/null || echo "no prog"
|
||||
|
||||
echo ""
|
||||
echo "=== Waiting for external traffic ==="
|
||||
echo "Send traffic from host to $(ip -4 addr show eth0 | awk '/inet /{sub(/\/.*/, "", $2); print $2}')"
|
||||
echo "After sending, press Enter or wait 60s..."
|
||||
|
||||
# Wait for signal or timeout
|
||||
for i in $(seq 1 60); do
|
||||
[ -f /tmp/traffic-done ] && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "=== After traffic ==="
|
||||
pgrep ja4ebpf && echo "ja4ebpf still alive" || echo "ja4ebpf DEAD"
|
||||
bpftool prog show name capture_xdp 2>/dev/null || echo "no prog"
|
||||
|
||||
echo "=== Raw data count ==="
|
||||
curl -sf "http://localhost:8123/?database=ja4_logs" --data-urlencode "query=SELECT count() FROM http_logs_raw" 2>/dev/null || echo "0"
|
||||
|
||||
echo "=== ja4ebpf logs ==="
|
||||
cat /tmp/ja4.log
|
||||
|
||||
# Cleanup
|
||||
pkill ja4ebpf 2>/dev/null; nginx -s stop 2>/dev/null; docker rm -f ja4-clickhouse 2>/dev/null
|
||||
111
tests/vm/debug-xdp.sh
Normal file
111
tests/vm/debug-xdp.sh
Normal file
@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env bash
|
||||
# debug-xdp.sh — Test XDP + host traffic en une seule session SSH
|
||||
# Usage: vagrant ssh rocky9 -- 'sudo bash -c "PATH=/usr/local/bin:$PATH /ja4-platform/tests/vm/debug-xdp.sh"'
|
||||
set -euo pipefail
|
||||
export PATH=/usr/local/bin:/usr/local/go/bin:$PATH
|
||||
STACK="${1:-nginx}"
|
||||
|
||||
# === Start ClickHouse ===
|
||||
echo "[1] Starting ClickHouse..."
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
docker run -d --name ja4-clickhouse -p 8123:8123 -p 9000:9000 \
|
||||
-e CLICKHOUSE_DB=ja4_processing -e CLICKHOUSE_USER=default \
|
||||
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
|
||||
-v /ja4-platform/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh \
|
||||
-v /ja4-platform/tests/integration/platform/csv-stubs:/var/lib/clickhouse/user_files \
|
||||
-v /ja4-platform/shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/10_perf_indexes.sql:/initdb-src/10_perf_indexes.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/11_views.sql:/initdb-src/11_views.sql:ro \
|
||||
-v /ja4-platform/shared/clickhouse/12_thesis_features.sql:/initdb-src/12_thesis_features.sql:ro \
|
||||
clickhouse/clickhouse-server:24.8 >/dev/null
|
||||
for i in $(seq 1 30); do curl -sf http://localhost:8123/ping >/dev/null 2>&1 && break; sleep 2; done
|
||||
echo " ClickHouse ready"
|
||||
|
||||
# === Start nginx ===
|
||||
echo "[2] Starting nginx..."
|
||||
nginx -s stop 2>/dev/null || true; sleep 1
|
||||
mkdir -p /run/nginx /var/www/html
|
||||
echo '{"ok":true}' > /var/www/html/health
|
||||
cp /ja4-platform/tests/integration/nginx/platform/nginx.conf /etc/nginx/nginx.conf
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/nginx.key -out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
||||
nginx && echo " nginx ready"
|
||||
|
||||
# === Start ja4ebpf ===
|
||||
echo "[3] Starting ja4ebpf..."
|
||||
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
||||
cat > /tmp/ja4.yml << 'YEOF'
|
||||
interface: eth0
|
||||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||||
clickhouse:
|
||||
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
||||
batch_size: 50
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
timeout_ms: 500
|
||||
slowloris_ms: 10000
|
||||
log:
|
||||
level: "debug"
|
||||
format: "json"
|
||||
YEOF
|
||||
JA4EBPF_CONFIG=/tmp/ja4.yml ja4ebpf > /tmp/ja4.log 2>&1 &
|
||||
sleep 3
|
||||
JA4PID=$(pgrep ja4ebpf || echo NONE)
|
||||
if [ "$JA4PID" = "NONE" ]; then
|
||||
echo " ja4ebpf DEAD!"; cat /tmp/ja4.log; exit 1
|
||||
fi
|
||||
echo " ja4ebpf PID=$JA4PID"
|
||||
|
||||
# Verify XDP
|
||||
XDP_INFO=$(ip link show dev eth0 | grep "prog/xdp" || echo NONE)
|
||||
echo " XDP: $XDP_INFO"
|
||||
|
||||
# Show eth0 IP
|
||||
ETH0_IP=$(ip -4 addr show eth0 | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════╗"
|
||||
echo "║ Services prêts — IP: $ETH0_IP"
|
||||
echo "║ Attente trafic host (60s max)..."
|
||||
echo "╚══════════════════════════════════════╝"
|
||||
|
||||
# Wait for host traffic signal
|
||||
for i in $(seq 1 60); do
|
||||
[ -f /tmp/traffic-done ] && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Check prog run count
|
||||
echo "[4] Checking results..."
|
||||
echo " ja4ebpf: $(pgrep ja4ebpf && echo alive || echo DEAD)"
|
||||
bpftool prog show name capture_xdp 2>/dev/null | head -5
|
||||
|
||||
# Check raw data
|
||||
RAW=$(curl -sf "http://localhost:8123/?database=ja4_logs" --data-urlencode "query=SELECT count() FROM http_logs_raw" 2>/dev/null || echo "0")
|
||||
echo " http_logs_raw: $RAW lignes"
|
||||
|
||||
# ja4ebpf logs
|
||||
echo " Logs:"
|
||||
tail -5 /tmp/ja4.log | sed 's/^/ /'
|
||||
|
||||
# Cleanup
|
||||
pkill ja4ebpf 2>/dev/null; nginx -s stop 2>/dev/null
|
||||
docker rm -f ja4-clickhouse 2>/dev/null
|
||||
|
||||
if [ "${RAW:-0}" -gt 0 ] 2>/dev/null; then
|
||||
echo ""
|
||||
echo " SUCCESS: $RAW rows captured"
|
||||
exit 0
|
||||
else
|
||||
echo ""
|
||||
echo " FAIL: 0 rows captured"
|
||||
exit 1
|
||||
fi
|
||||
65
tests/vm/provision-el8.sh
Executable file
65
tests/vm/provision-el8.sh
Executable file
@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# provision-el8.sh — Provisionnement CentOS 8 (dépôts archivés vault)
|
||||
#
|
||||
# CentOS 8 est EOL depuis juin 2024. Les dépôts sont sur vault.centos.org.
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
log() { echo "[provision] $(date +%H:%M:%S) $*"; }
|
||||
|
||||
# ── 1. Rediriger les dépôts vers vault.centos.org ─────────────────────────────
|
||||
log "Configuration des dépôts CentOS 8 vault..."
|
||||
sed -i 's|^mirrorlist=|#mirrorlist=|' /etc/yum.repos.d/CentOS-*.repo 2>/dev/null || true
|
||||
sed -i 's|^#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|' /etc/yum.repos.d/CentOS-*.repo 2>/dev/null || true
|
||||
dnf clean all
|
||||
dnf update -y --quiet
|
||||
|
||||
# ── 2. Toolchain eBPF ────────────────────────────────────────────────────────
|
||||
log "Installation toolchain eBPF..."
|
||||
dnf install -y \
|
||||
clang llvm libbpf-devel bpftool \
|
||||
kernel-devel-$(uname -r) \
|
||||
make git curl tar gzip \
|
||||
epel-release dnf-plugins-core || true
|
||||
|
||||
# ── 3. Go ─────────────────────────────────────────────────────────────────────
|
||||
log "Installation de Go..."
|
||||
GO_VERSION="1.24.3"
|
||||
if ! command -v go &>/dev/null || [[ "$(go version 2>/dev/null | awk '{print $3}')" != "go${GO_VERSION}" ]]; then
|
||||
curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" -o /tmp/go.tar.gz
|
||||
rm -rf /usr/local/go
|
||||
tar -C /usr/local -xzf /tmp/go.tar.gz
|
||||
rm /tmp/go.tar.gz
|
||||
fi
|
||||
|
||||
cat > /etc/profile.d/go.sh << 'EOF'
|
||||
export PATH="/usr/local/go/bin:$PATH"
|
||||
export GOPATH="/home/vagrant/go"
|
||||
EOF
|
||||
|
||||
# ── 4. Serveurs web (nginx + httpd) + TLS + hitch + varnish ────────────────────
|
||||
log "Installation des serveurs web et reverse proxy..."
|
||||
dnf install -y nginx openssl curl
|
||||
dnf install -y httpd mod_ssl || true
|
||||
dnf install -y hitch varnish || true
|
||||
|
||||
# ── 5. Python3 + outils de test ──────────────────────────────────────────────
|
||||
log "Installation Python3 et outils de test..."
|
||||
dnf install -y python3 python3-pip
|
||||
pip3 install --quiet "httpx[http2]" requests 2>/dev/null || pip3 install --quiet httpx requests
|
||||
|
||||
# ── 6. Montage tracefs + debugfs ─────────────────────────────────────────────
|
||||
log "Configuration des pseudo-systèmes de fichiers eBPF..."
|
||||
mount -t tracefs tracefs /sys/kernel/tracing 2>/dev/null || true
|
||||
mount -t debugfs debugfs /sys/kernel/debug 2>/dev/null || true
|
||||
|
||||
# ── 7. Build ja4ebpf ─────────────────────────────────────────────────────────
|
||||
log "Build initial de ja4ebpf..."
|
||||
export PATH="/usr/local/go/bin:$PATH"
|
||||
cd /ja4-platform/services/ja4ebpf
|
||||
GOWORK=off go generate ./internal/loader/ 2>&1 | tail -5 || log "go generate: erreur (normal si vmlinux.h absent)"
|
||||
GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
go build -ldflags="-s -w" -o /usr/local/bin/ja4ebpf ./cmd/ja4ebpf/ 2>&1 | tail -5
|
||||
|
||||
log "Provisionnement CentOS 8 terminé !"
|
||||
@ -50,14 +50,31 @@ EOF
|
||||
log "Installation de Docker..."
|
||||
dnf config-manager --add-repo https://download.docker.com/linux/rhel/docker-ce.repo
|
||||
dnf install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
||||
|
||||
# Sur el10+ (kernel 6.12+), nf_tables a des incompatibilités avec iptables-nft.
|
||||
# Désactiver la gestion iptables par Docker pour éviter l'échec au démarrage.
|
||||
if ! systemctl start docker 2>/dev/null; then
|
||||
log "Docker: fallback iptables=false pour kernel $(uname -r)"
|
||||
mkdir -p /etc/docker
|
||||
echo '{"iptables": false}' > /etc/docker/daemon.json
|
||||
fi
|
||||
|
||||
systemctl enable --now docker
|
||||
usermod -aG docker vagrant
|
||||
# Accès sans sudo pour vagrant
|
||||
chmod 666 /var/run/docker.sock || true
|
||||
|
||||
# ── 5. nginx + openssl ───────────────────────────────────────────────────────
|
||||
log "Installation de nginx..."
|
||||
# ── 5. Serveurs web (nginx + httpd) + TLS + hitch + varnish ─────────────────────
|
||||
log "Installation des serveurs web et reverse proxy..."
|
||||
dnf install -y nginx openssl curl
|
||||
dnf install -y httpd mod_ssl
|
||||
dnf install -y hitch varnish
|
||||
|
||||
# Ouvrir les ports HTTP/HTTPS dans le firewall
|
||||
log "Configuration firewall..."
|
||||
firewall-cmd --add-service=http --add-service=https --permanent 2>/dev/null || true
|
||||
firewall-cmd --add-port=80/tcp --add-port=443/tcp --permanent 2>/dev/null || true
|
||||
firewall-cmd --reload 2>/dev/null || true
|
||||
|
||||
# ── 6. Python3 + outils de test ──────────────────────────────────────────────
|
||||
log "Installation Python3 et outils de test..."
|
||||
|
||||
119
tests/vm/run-test-from-host.sh
Executable file
119
tests/vm/run-test-from-host.sh
Executable file
@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# run-test-from-host.sh — Orchestrateur de test VM depuis le host
|
||||
#
|
||||
# Lance le test complet d'une stack sur une VM :
|
||||
# 1. Rsync les fichiers
|
||||
# 2. Démarre les services dans la VM (en background via SSH)
|
||||
# 3. Génère le trafic depuis le HOST vers l'IP eth0 de la VM
|
||||
# 4. Lance la vérification dans la VM
|
||||
#
|
||||
# Usage :
|
||||
# ./tests/vm/run-test-from-host.sh rocky9 nginx
|
||||
# ./tests/vm/run-test-from-host.sh centos8 apache
|
||||
# make test-vm-nginx
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
VM="${1:-rocky9}"
|
||||
STACK="${2:-nginx}"
|
||||
VM_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; RESET='\033[0m'
|
||||
BOLD='\033[1m'
|
||||
|
||||
log() { echo -e "${BOLD}[$VM/$STACK]${RESET} $(date +%H:%M:%S) $*"; }
|
||||
pass() { echo -e " ${GREEN}PASS${RESET} $*"; }
|
||||
fail() { echo -e " ${RED}FAIL${RESET} $*"; }
|
||||
|
||||
cd "$VM_DIR"
|
||||
|
||||
# ── 1. Synchroniser les fichiers ─────────────────────────────────────────────
|
||||
log "Rsync fichiers vers $VM..."
|
||||
vagrant rsync "$VM"
|
||||
|
||||
# ── 2. Obtenir l'IP eth0 de la VM ────────────────────────────────────────────
|
||||
VM_IP=$(vagrant ssh "$VM" -- 'ip -4 addr show eth0' 2>/dev/null \
|
||||
| awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}')
|
||||
|
||||
if [ -z "$VM_IP" ]; then
|
||||
fail "Impossible d'obtenir l'IP eth0 de $VM"
|
||||
exit 1
|
||||
fi
|
||||
log "IP eth0 : $VM_IP"
|
||||
|
||||
# ── 3. Démarrer les services dans la VM (en background) ──────────────────────
|
||||
log "Démarrage des services dans $VM ($STACK)..."
|
||||
|
||||
# Nettoyer le signal de l'itération précédente
|
||||
vagrant ssh "$VM" -- 'sudo rm -f /tmp/ja4ebpf-traffic-done' 2>/dev/null || true
|
||||
|
||||
# Lancer le script de test en mode "start" dans la VM
|
||||
# Le script attendra le signal /tmp/ja4ebpf-traffic-done
|
||||
vagrant ssh "$VM" -- "sudo bash /ja4-platform/tests/vm/run-tests-vm.sh $STACK start" &
|
||||
VM_PID=$!
|
||||
|
||||
# ── 4. Attendre que les services soient prêts ────────────────────────────────
|
||||
log "Attente démarrage des services (30s)..."
|
||||
sleep 30
|
||||
|
||||
# ── 5. Vérifier que les services répondent ───────────────────────────────────
|
||||
log "Vérification connectivité..."
|
||||
if curl -sf "http://$VM_IP/health" >/dev/null 2>&1; then
|
||||
pass "HTTP $VM_IP:80 OK"
|
||||
else
|
||||
fail "HTTP $VM_IP:80 injoignable"
|
||||
fi
|
||||
if curl -sf -k "https://$VM_IP/health" >/dev/null 2>&1; then
|
||||
pass "HTTPS $VM_IP:443 OK"
|
||||
else
|
||||
fail "HTTPS $VM_IP:443 injoignable"
|
||||
fi
|
||||
|
||||
# ── 6. Générer le trafic depuis le host ──────────────────────────────────────
|
||||
log "Génération du trafic host → $VM_IP..."
|
||||
for path in / /health /data /api/users; do
|
||||
curl -sf -k "https://$VM_IP$path" >/dev/null 2>&1 || true
|
||||
curl -sf "http://$VM_IP$path" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X POST "https://$VM_IP/api/data" -d '{"test":1}' >/dev/null 2>&1 || true
|
||||
curl -sf -k -X PUT "https://$VM_IP/data" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X DELETE "https://$VM_IP/data/1" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X HEAD "https://$VM_IP$path" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
# HTTP/2 via Python si disponible
|
||||
if python3 -c "import httpx" 2>/dev/null; then
|
||||
python3 -c "
|
||||
import httpx, ssl, warnings
|
||||
warnings.filterwarnings('ignore')
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
with httpx.Client(http2=True, verify=False) as c:
|
||||
for p in ['/', '/health', '/data']:
|
||||
try: c.get('https://$VM_IP' + p)
|
||||
except: pass
|
||||
" 2>/dev/null && pass "HTTP/2 généré" || true
|
||||
fi
|
||||
|
||||
log "Attente flush ja4ebpf (15s)..."
|
||||
sleep 15
|
||||
|
||||
# ── 7. Signaler à la VM de lancer la vérification ────────────────────────────
|
||||
log "Signal de vérification..."
|
||||
vagrant ssh "$VM" -- 'sudo touch /tmp/ja4ebpf-traffic-done' 2>/dev/null
|
||||
|
||||
# ── 8. Attendre la fin du processus VM ───────────────────────────────────────
|
||||
log "Attente résultat..."
|
||||
wait $VM_PID 2>/dev/null
|
||||
RESULT=$?
|
||||
|
||||
if [ $RESULT -eq 0 ]; then
|
||||
echo ""
|
||||
echo -e " ${GREEN}${BOLD}$VM/$STACK : SUCCÈS${RESET}"
|
||||
else
|
||||
echo ""
|
||||
echo -e " ${RED}${BOLD}$VM/$STACK : ÉCHEC (code $RESULT)${RESET}"
|
||||
fi
|
||||
|
||||
exit $RESULT
|
||||
@ -1,142 +1,114 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# run-tests-vm.sh — Lance la stack de test complète dans la VM Rocky Linux 9
|
||||
# run-tests-vm.sh — Tests ja4ebpf multi-stack dans une VM Vagrant
|
||||
#
|
||||
# Ce script s'exécute DANS la VM (via vagrant ssh ou vagrant provision).
|
||||
# Il ne peut pas tourner dans Docker — il requiert un vrai kernel pour eBPF.
|
||||
# Architecture :
|
||||
# Phase 1 (dans la VM) : démarrer ClickHouse, serveur web, ja4ebpf
|
||||
# Phase 2 (depuis le host) : générer du trafic vers l'IP eth0 de la VM
|
||||
# Phase 3 (dans la VM) : vérifier les données dans ClickHouse
|
||||
#
|
||||
# Usage (depuis le host) :
|
||||
# vagrant ssh -- 'bash /ja4-platform/tests/vm/run-tests-vm.sh nginx'
|
||||
# vagrant ssh -- 'bash /ja4-platform/tests/vm/run-tests-vm.sh all'
|
||||
# Stacks supportées :
|
||||
# nginx — nginx avec TLS (HTTP/1.1 + HTTP/2)
|
||||
# apache — Apache httpd avec TLS (HTTP/1.1 + HTTP/2)
|
||||
# hitch-varnish — hitch (TLS) → Varnish (cache/H2) → backend Python
|
||||
# all — exécute les 3 stacks séquentiellement
|
||||
#
|
||||
# Variables d'environnement :
|
||||
# STACK : stack à tester (nginx|apache|nginx-varnish|hitch-varnish|all)
|
||||
# KEEP_RUNNING : si "true", ne pas arrêter la stack après le test (défaut: false)
|
||||
# Modes :
|
||||
# start — démarrer les services (Phase 1)
|
||||
# verify — vérifier les données (Phase 3)
|
||||
# (défaut) — start + verify (le trafic doit être généré entre les deux)
|
||||
#
|
||||
# Usage (depuis le host via Makefile) :
|
||||
# make test-vm-nginx
|
||||
# make test-vm-apache
|
||||
# make test-vm-hitch-varnish
|
||||
# make test-vm-matrix
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# S'assurer que /usr/local/bin et go sont dans PATH (nécessaire pour sudo bash)
|
||||
export PATH="/usr/local/bin:/usr/local/go/bin:$PATH"
|
||||
|
||||
STACK="${1:-nginx}"
|
||||
MODE="${2:-full}" # start | verify | full
|
||||
KEEP_RUNNING="${KEEP_RUNNING:-false}"
|
||||
PROJECT="/ja4-platform"
|
||||
RESULTS_DIR="/tmp/ja4-test-results"
|
||||
|
||||
# ── Couleurs ─────────────────────────────────────────────────────────────────
|
||||
GREEN='\033[0;32m'; RED='\033[0;31m'; YELLOW='\033[1;33m'; RESET='\033[0m'
|
||||
BOLD='\033[1m'
|
||||
|
||||
log() { echo -e "${BOLD}[$STACK]${RESET} $(date +%H:%M:%S) $*"; }
|
||||
pass() { echo -e " ${GREEN}✅${RESET} $*"; ((PASS_COUNT++)) || true; }
|
||||
fail() { echo -e " ${RED}❌${RESET} $*"; ((FAIL_COUNT++)) || true; }
|
||||
warn() { echo -e " ${YELLOW}⚠️${RESET} $*"; ((WARN_COUNT++)) || true; }
|
||||
pass() { echo -e " ${GREEN}PASS${RESET} $*"; ((PASS_COUNT++)) || true; }
|
||||
fail() { echo -e " ${RED}FAIL${RESET} $*"; ((FAIL_COUNT++)) || true; }
|
||||
warn() { echo -e " ${YELLOW}WARN${RESET} $*"; ((WARN_COUNT++)) || true; }
|
||||
|
||||
PASS_COUNT=0; FAIL_COUNT=0; WARN_COUNT=0
|
||||
|
||||
# ── Vérification prérequis ────────────────────────────────────────────────────
|
||||
check_prerequisites() {
|
||||
log "Vérification des prérequis..."
|
||||
|
||||
# eBPF capabilities
|
||||
if [ ! -d /sys/kernel/tracing ]; then
|
||||
fail "tracefs non monté — exécuter: sudo mount -t tracefs tracefs /sys/kernel/tracing"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -d /sys/kernel/debug ]; then
|
||||
fail "debugfs non monté"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
command -v ja4ebpf >/dev/null 2>&1 || {
|
||||
log "Rebuild ja4ebpf..."
|
||||
cd "$PROJECT/services/ja4ebpf"
|
||||
export PATH="/usr/local/go/bin:$PATH"
|
||||
GOWORK=off go generate ./internal/loader/ 2>&1 | tail -3
|
||||
GOWORK=off CGO_ENABLED=0 go build -o /tmp/ja4ebpf_new ./cmd/ja4ebpf/ && mv /tmp/ja4ebpf_new /usr/local/bin/ja4ebpf
|
||||
}
|
||||
|
||||
command -v docker >/dev/null 2>&1 || { fail "Docker non installé"; exit 1; }
|
||||
command -v nginx >/dev/null 2>&1 || { fail "nginx non installé"; exit 1; }
|
||||
pass "Prérequis OK"
|
||||
# ── Helpers communs ──────────────────────────────────────────────────────────
|
||||
|
||||
gen_tls_cert() {
|
||||
local name="$1"
|
||||
openssl req -x509 -nodes -days 365 -subj "/CN=platform.test" \
|
||||
-newkey rsa:2048 \
|
||||
-keyout "/etc/pki/tls/private/${name}.key" \
|
||||
-out "/etc/pki/tls/certs/${name}.crt" 2>/dev/null
|
||||
}
|
||||
|
||||
# ── Démarrage ClickHouse ──────────────────────────────────────────────────────
|
||||
setup_docroot() {
|
||||
mkdir -p /var/www/html
|
||||
echo '{"status":"ok","stack":"'"$STACK"'"}' > /var/www/html/health
|
||||
for p in data api/users api/data/test; do
|
||||
mkdir -p "/var/www/html/$(dirname $p)"
|
||||
echo '{"ok":true}' > "/var/www/html/$p"
|
||||
done
|
||||
}
|
||||
|
||||
get_eth0_ip() {
|
||||
ip -4 addr show eth0 | awk '/inet / {sub(/\/.*/, "", $2); print $2; exit}' 2>/dev/null || echo ""
|
||||
}
|
||||
|
||||
# ── ClickHouse ────────────────────────────────────────────────────────────────
|
||||
start_clickhouse() {
|
||||
log "Démarrage ClickHouse..."
|
||||
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
|
||||
|
||||
CSV_DIR="$PROJECT/tests/integration/platform/csv-stubs"
|
||||
docker run -d --name ja4-clickhouse \
|
||||
-p 8123:8123 -p 9000:9000 \
|
||||
-e CLICKHOUSE_DB=ja4_processing \
|
||||
-e CLICKHOUSE_USER=default \
|
||||
-e CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 \
|
||||
-v "$PROJECT/tests/integration/platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh" \
|
||||
-v "$CSV_DIR:/var/lib/clickhouse/user_files" \
|
||||
$(for f in "$PROJECT/shared/clickhouse/"*.sql; do
|
||||
echo "-v $f:/initdb-src/$(basename $f):ro"
|
||||
done) \
|
||||
clickhouse/clickhouse-server:24.8 2>&1 | tail -1
|
||||
|
||||
# Attendre que ClickHouse soit prêt
|
||||
|
||||
log "Attente ClickHouse (max 120s)..."
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf "http://localhost:8123/ping" >/dev/null 2>&1; then
|
||||
pass "ClickHouse prêt"
|
||||
return 0
|
||||
fi
|
||||
curl -sf "http://localhost:8123/ping" >/dev/null 2>&1 && { pass "ClickHouse prêt"; return 0; }
|
||||
sleep 2
|
||||
done
|
||||
fail "ClickHouse timeout"; exit 1
|
||||
}
|
||||
|
||||
# ── Configuration nginx ────────────────────────────────────────────────────────
|
||||
setup_nginx() {
|
||||
log "Configuration nginx avec TLS..."
|
||||
|
||||
# Certificat auto-signé
|
||||
openssl req -x509 -nodes -days 365 \
|
||||
-subj "/CN=platform.test" \
|
||||
-newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/nginx.key \
|
||||
-out /etc/pki/tls/certs/nginx.crt 2>/dev/null
|
||||
|
||||
# Copier la configuration de test
|
||||
cp "$PROJECT/tests/integration/nginx/platform/nginx.conf" /etc/nginx/nginx.conf
|
||||
|
||||
# Créer les fichiers de test
|
||||
mkdir -p /var/www/html
|
||||
# /run/nginx est un tmpfs recréé à chaque boot, nginx en a besoin pour son PID
|
||||
mkdir -p /run/nginx
|
||||
echo '{"status":"ok","stack":"nginx-vm"}' > /var/www/html/health
|
||||
for p in data api/users api/data/test; do
|
||||
mkdir -p "/var/www/html/$(dirname $p)"
|
||||
echo '{"ok":true}' > "/var/www/html/$p"
|
||||
done
|
||||
|
||||
nginx -t && nginx
|
||||
|
||||
# Attendre nginx
|
||||
for i in $(seq 1 20); do
|
||||
curl -sf http://localhost/health >/dev/null 2>&1 && break
|
||||
sleep 0.5
|
||||
done
|
||||
pass "nginx démarré"
|
||||
}
|
||||
|
||||
# ── Démarrage ja4ebpf ─────────────────────────────────────────────────────────
|
||||
# ── ja4ebpf ────────────────────────────────────────────────────────────────────
|
||||
start_ja4ebpf() {
|
||||
log "Démarrage ja4ebpf..."
|
||||
|
||||
pkill ja4ebpf 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Créer la config
|
||||
cat > /tmp/ja4ebpf.yml << 'EOF'
|
||||
|
||||
local ssl_lib=""
|
||||
for lib in /usr/lib64/libssl.so.3 /usr/lib64/libssl.so.1.1 /usr/lib/libssl.so.3 /usr/lib/libssl.so.1.1; do
|
||||
[ -f "$lib" ] && { ssl_lib="$lib"; break; }
|
||||
done
|
||||
[ -z "$ssl_lib" ] && ssl_lib="/usr/lib64/libssl.so.3"
|
||||
|
||||
cat > /tmp/ja4ebpf.yml << EOF
|
||||
interface: eth0
|
||||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||||
ssl_lib_path: "${ssl_lib}"
|
||||
clickhouse:
|
||||
dsn: "clickhouse://default:@localhost:9000/ja4_logs"
|
||||
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
||||
batch_size: 100
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
@ -146,169 +118,340 @@ log:
|
||||
level: "info"
|
||||
format: "json"
|
||||
EOF
|
||||
|
||||
# Lancer avec les capabilities nécessaires
|
||||
# Dans la VM (root), on peut lancer directement
|
||||
|
||||
JA4EBPF_CONFIG=/tmp/ja4ebpf.yml ja4ebpf > /tmp/ja4ebpf.log 2>&1 &
|
||||
JA4EBPF_PID=$!
|
||||
|
||||
sleep 3
|
||||
|
||||
if ! kill -0 "$JA4EBPF_PID" 2>/dev/null; then
|
||||
fail "ja4ebpf s'est arrêté immédiatement"
|
||||
cat /tmp/ja4ebpf.log | tail -10
|
||||
tail -10 /tmp/ja4ebpf.log
|
||||
return 1
|
||||
fi
|
||||
|
||||
|
||||
log "ja4ebpf démarré (PID $JA4EBPF_PID)"
|
||||
|
||||
# Vérifier les uprobes dans tracefs
|
||||
|
||||
# Vérifier XDP
|
||||
if ip link show dev eth0 2>/dev/null | grep -q "xdp"; then
|
||||
local xdp_info
|
||||
xdp_info=$(ip link show dev eth0 | grep "prog/xdp" | sed 's/^[[:space:]]*//')
|
||||
pass "XDP attaché : $xdp_info"
|
||||
else
|
||||
warn "Aucun XDP sur eth0"
|
||||
bpftool prog show name capture_xdp 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Stack : nginx
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
setup_nginx() {
|
||||
log "Configuration nginx avec TLS..."
|
||||
gen_tls_cert nginx
|
||||
setup_docroot
|
||||
cp "$PROJECT/tests/integration/nginx/platform/nginx.conf" /etc/nginx/nginx.conf
|
||||
mkdir -p /run/nginx
|
||||
nginx -t && nginx
|
||||
for i in $(seq 1 20); do
|
||||
curl -sf http://localhost/health >/dev/null 2>&1 && break
|
||||
sleep 0.5
|
||||
done
|
||||
pass "nginx démarré"
|
||||
}
|
||||
|
||||
stop_nginx() { nginx -s stop 2>/dev/null || true; }
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Stack : apache
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
setup_apache() {
|
||||
log "Configuration Apache httpd avec TLS..."
|
||||
gen_tls_cert apache
|
||||
setup_docroot
|
||||
|
||||
if command -v httpd >/dev/null 2>&1; then
|
||||
if ! httpd -M 2>/dev/null | grep -q http2_module; then
|
||||
echo "LoadModule http2_module modules/mod_http2.so" \
|
||||
>> /etc/httpd/conf.modules.d/00-base.conf 2>/dev/null || true
|
||||
fi
|
||||
fi
|
||||
mkdir -p /run/httpd /var/log/httpd
|
||||
|
||||
cp "$PROJECT/tests/integration/apache/platform/httpd-ssl.conf" \
|
||||
/etc/httpd/conf.d/ssl.conf 2>/dev/null || true
|
||||
|
||||
httpd -t 2>&1 && httpd -DFOREGROUND &
|
||||
sleep 2
|
||||
for i in $(seq 1 20); do
|
||||
curl -sf http://localhost/health >/dev/null 2>&1 && break
|
||||
sleep 0.5
|
||||
done
|
||||
pass "Apache httpd démarré"
|
||||
}
|
||||
|
||||
stop_apache() { pkill httpd 2>/dev/null || true; }
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Stack : hitch + varnish
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
setup_hitch_varnish() {
|
||||
log "Configuration hitch + Varnish..."
|
||||
gen_tls_cert hitch
|
||||
mkdir -p /etc/hitch
|
||||
cat /etc/pki/tls/private/hitch.key /etc/pki/tls/certs/hitch.crt \
|
||||
> /etc/hitch/hitch.pem
|
||||
|
||||
cat > /etc/hitch/hitch.conf << 'HCONF'
|
||||
frontend = "[*]:443"
|
||||
backend = "[127.0.0.1]:6081"
|
||||
pem-file = "/etc/hitch/hitch.pem"
|
||||
write-proxy-v1 = on
|
||||
tls-protos = TLSv1.2 TLSv1.3
|
||||
ciphers = "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-RSA-AES128-GCM-SHA256:TLS_AES_256_GCM_SHA384:TLS_AES_128_GCM_SHA256"
|
||||
alpn-protos = "h2,http/1.1"
|
||||
workers = 2
|
||||
user = "nobody"
|
||||
daemon = off
|
||||
log-level = 1
|
||||
syslog = off
|
||||
HCONF
|
||||
|
||||
mkdir -p /etc/varnish
|
||||
cp "$PROJECT/tests/integration/hitch-varnish/platform/varnish.vcl" \
|
||||
/etc/varnish/default.vcl 2>/dev/null || {
|
||||
cat > /etc/varnish/default.vcl << 'VCL'
|
||||
vcl 4.1;
|
||||
backend default { .host = "127.0.0.1"; .port = "8080"; }
|
||||
sub vcl_deliver {
|
||||
set resp.http.Via = "1.1 varnish";
|
||||
set resp.http.X-Client-IP = client.ip;
|
||||
}
|
||||
VCL
|
||||
}
|
||||
|
||||
setup_docroot
|
||||
|
||||
# Backend HTTP (port 8080)
|
||||
python3 -c "
|
||||
import http.server, socketserver, json
|
||||
class H(http.server.BaseHTTPRequestHandler):
|
||||
def log_message(self, *a): pass
|
||||
def do_GET(self):
|
||||
body = json.dumps({'status':'ok','stack':'hitch-varnish','path':self.path}).encode()
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type','application/json')
|
||||
self.send_header('Content-Length',len(body))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
def do_POST(self):
|
||||
n = int(self.headers.get('Content-Length',0))
|
||||
self.rfile.read(n)
|
||||
body = b'{\"result\":\"accepted\"}'
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type','application/json')
|
||||
self.send_header('Content-Length',len(body))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
with socketserver.TCPServer(('127.0.0.1', 8080), H) as s:
|
||||
s.serve_forever()
|
||||
" &
|
||||
sleep 1
|
||||
if grep -q "ssl" /sys/kernel/tracing/uprobe_events 2>/dev/null; then
|
||||
pass "Uprobes SSL attachés dans tracefs"
|
||||
else
|
||||
warn "Uprobes non visibles dans tracefs (peuvent être actifs quand même)"
|
||||
fi
|
||||
|
||||
# Vérifier accept4 tracepoint
|
||||
if grep -q "accept4" /sys/kernel/tracing/events/syscalls 2>/dev/null; then
|
||||
pass "Tracepoints accept4 disponibles"
|
||||
else
|
||||
warn "Tracepoints accept4 non trouvés"
|
||||
fi
|
||||
|
||||
varnishd -F -f /etc/varnish/default.vcl \
|
||||
-a "127.0.0.1:6081,PROXY" \
|
||||
-p feature=+http2 \
|
||||
-s malloc,64m \
|
||||
-T 127.0.0.1:6082 &
|
||||
sleep 2
|
||||
|
||||
hitch --config=/etc/hitch/hitch.conf &
|
||||
sleep 2
|
||||
|
||||
for i in $(seq 1 20); do
|
||||
curl -skf https://localhost/health >/dev/null 2>&1 && break
|
||||
sleep 0.5
|
||||
done
|
||||
pass "hitch + Varnish démarrés"
|
||||
}
|
||||
|
||||
# ── Génération de trafic ───────────────────────────────────────────────────────
|
||||
generate_traffic() {
|
||||
log "Génération du trafic (HTTP/1.0 + HTTP/1.1 + HTTP/2)..."
|
||||
|
||||
# Trafic HTTP/1.1 (HTTP)
|
||||
for path in / /health /data /api/users; do
|
||||
curl -sf "http://localhost$path" >/dev/null 2>&1 || true
|
||||
curl -sf -X POST "http://localhost/api/data" -d '{"test":1}' >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
# Trafic HTTPS/1.1
|
||||
for path in / /health /data /api/users; do
|
||||
curl -sf -k "https://localhost$path" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X POST "https://localhost/api/data" -d '{"test":1}' >/dev/null 2>&1 || true
|
||||
curl -sf -k -X PUT "https://localhost/data" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X DELETE "https://localhost/data/1" >/dev/null 2>&1 || true
|
||||
curl -sf -k -X HEAD "https://localhost$path" >/dev/null 2>&1 || true
|
||||
done
|
||||
|
||||
# Trafic HTTP/2
|
||||
if command -v python3 >/dev/null 2>&1 && python3 -c "import httpx" 2>/dev/null; then
|
||||
python3 << 'PYEOF'
|
||||
import httpx, ssl, warnings
|
||||
warnings.filterwarnings("ignore")
|
||||
ctx = ssl.create_default_context()
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
with httpx.Client(http2=True, verify=False) as client:
|
||||
for path in ["/", "/health", "/data"]:
|
||||
try: client.get(f"https://localhost{path}")
|
||||
except: pass
|
||||
try: client.post("https://localhost/api/data", json={"test": "h2"})
|
||||
except: pass
|
||||
PYEOF
|
||||
pass "Trafic HTTP/2 généré"
|
||||
fi
|
||||
|
||||
# Attendre le flush ja4ebpf → ClickHouse
|
||||
log "Attente flush ja4ebpf (15s)..."
|
||||
sleep 15
|
||||
pass "Trafic généré"
|
||||
stop_hitch_varnish() {
|
||||
pkill hitch 2>/dev/null || true
|
||||
pkill varnishd 2>/dev/null || true
|
||||
pkill -f "TCPServer.*8080" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ── Vérification ClickHouse ────────────────────────────────────────────────────
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Vérification ClickHouse
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
verify_db() {
|
||||
log "Vérification des données dans ClickHouse..."
|
||||
|
||||
ch_query() {
|
||||
curl -sf "http://localhost:8123/" \
|
||||
--data-urlencode "query=$1" \
|
||||
--data-urlencode "database=ja4_logs" \
|
||||
-o /dev/null -w '%{http_code}' 2>/dev/null || echo "0"
|
||||
}
|
||||
|
||||
|
||||
ch_val() {
|
||||
curl -sf "http://localhost:8123/?database=ja4_logs" \
|
||||
--data-urlencode "query=$1" 2>/dev/null | tr -d ' \n' || echo "0"
|
||||
}
|
||||
|
||||
|
||||
# http_logs_raw (données brutes avant MV)
|
||||
local raw_count
|
||||
raw_count=$(ch_val "SELECT count() FROM http_logs_raw")
|
||||
if [ "${raw_count:-0}" -gt 0 ] 2>/dev/null; then
|
||||
pass "http_logs_raw : $raw_count lignes"
|
||||
else
|
||||
fail "http_logs_raw vide — ja4ebpf n'a rien capturé"
|
||||
log " Logs ja4ebpf :"
|
||||
tail -10 /tmp/ja4ebpf.log 2>/dev/null | sed 's/^/ /'
|
||||
fi
|
||||
|
||||
# L3/L4
|
||||
ttl=$(ch_val "SELECT count() FROM http_logs WHERE ip_meta_ttl > 0")
|
||||
[ "${ttl:-0}" -gt 0 ] && pass "L3/L4 TTL capturé ($ttl lignes)" || fail "L3/L4 TTL absent"
|
||||
|
||||
[ "${ttl:-0}" -gt 0 ] 2>/dev/null && pass "L3/L4 TTL ($ttl)" || fail "L3/L4 TTL absent"
|
||||
|
||||
mss=$(ch_val "SELECT count() FROM http_logs WHERE tcp_meta_mss > 0")
|
||||
[ "${mss:-0}" -gt 0 ] && pass "TCP MSS capturé ($mss lignes)" || fail "TCP MSS absent"
|
||||
|
||||
[ "${mss:-0}" -gt 0 ] 2>/dev/null && pass "TCP MSS ($mss)" || fail "TCP MSS absent"
|
||||
|
||||
# TLS
|
||||
ja4=$(ch_val "SELECT count() FROM http_logs WHERE ja4 != ''")
|
||||
[ "${ja4:-0}" -gt 0 ] && pass "JA4 fingerprint capturé ($ja4 lignes)" || fail "JA4 absent"
|
||||
|
||||
[ "${ja4:-0}" -gt 0 ] 2>/dev/null && pass "JA4 fingerprint ($ja4)" || fail "JA4 absent"
|
||||
|
||||
sni=$(ch_val "SELECT count() FROM http_logs WHERE tls_sni != ''")
|
||||
[ "${sni:-0}" -gt 0 ] && pass "TLS SNI capturé ($sni lignes)" || warn "TLS SNI absent"
|
||||
|
||||
# L7 HTTP — c'est ici que ça devrait marcher dans la VM
|
||||
[ "${sni:-0}" -gt 0 ] 2>/dev/null && pass "TLS SNI ($sni)" || warn "TLS SNI absent"
|
||||
|
||||
# L7 HTTP
|
||||
method=$(ch_val "SELECT count() FROM http_logs WHERE method != ''")
|
||||
[ "${method:-0}" -gt 0 ] && pass "L7 méthodes HTTP capturées ($method lignes)" \
|
||||
|| fail "L7 méthodes HTTP ABSENT — uprobe SSL_read ne fonctionne pas"
|
||||
|
||||
[ "${method:-0}" -gt 0 ] 2>/dev/null && pass "L7 HTTP ($method)" || fail "L7 HTTP ABSENT"
|
||||
|
||||
path=$(ch_val "SELECT count() FROM http_logs WHERE path != ''")
|
||||
[ "${path:-0}" -gt 0 ] && pass "L7 path HTTP capturé ($path lignes)" || fail "L7 path absent"
|
||||
|
||||
[ "${path:-0}" -gt 0 ] 2>/dev/null && pass "L7 path ($path)" || fail "L7 path absent"
|
||||
|
||||
status=$(ch_val "SELECT count() FROM http_logs WHERE status_code > 0")
|
||||
[ "${status:-0}" -gt 0 ] && pass "status_code capturé ($status lignes)" || warn "status_code absent"
|
||||
|
||||
sig=$(ch_val "SELECT count() FROM http_logs WHERE header_order_signature != ''")
|
||||
[ "${sig:-0}" -gt 0 ] && pass "header_order_signature capturé ($sig lignes)" || warn "header_order_sig absent"
|
||||
|
||||
# Méthodes HTTP distinctes
|
||||
[ "${status:-0}" -gt 0 ] 2>/dev/null && pass "status_code ($status)" || warn "status_code absent"
|
||||
|
||||
methods=$(ch_val "SELECT groupArray(method) FROM (SELECT DISTINCT method FROM http_logs WHERE method != '')")
|
||||
log "Méthodes HTTP vues : $methods"
|
||||
|
||||
# Lignes totales
|
||||
log "Méthodes HTTP : $methods"
|
||||
|
||||
total=$(ch_val "SELECT count() FROM http_logs")
|
||||
pass "Total lignes http_logs : $total"
|
||||
pass "Total http_logs : $total"
|
||||
}
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Nettoyage
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
stop_stack() {
|
||||
pkill ja4ebpf 2>/dev/null || true
|
||||
case "$STACK" in
|
||||
nginx) stop_nginx ;;
|
||||
apache) stop_apache ;;
|
||||
hitch-varnish) stop_hitch_varnish ;;
|
||||
esac
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
}
|
||||
|
||||
# ── Nettoyage ─────────────────────────────────────────────────────────────────
|
||||
cleanup() {
|
||||
if [ "$KEEP_RUNNING" != "true" ]; then
|
||||
log "Nettoyage..."
|
||||
pkill ja4ebpf 2>/dev/null || true
|
||||
nginx -s stop 2>/dev/null || true
|
||||
docker rm -f ja4-clickhouse 2>/dev/null || true
|
||||
stop_stack
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||||
mkdir -p "$RESULTS_DIR"
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Phase 1 : démarrage des services
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
do_start() {
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════╗"
|
||||
echo "║ Phase 1 : Démarrage — $STACK"
|
||||
echo "╚══════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════╗"
|
||||
echo "║ ja4ebpf VM Test Suite — Rocky Linux 9 ║"
|
||||
echo "╚══════════════════════════════════════════╝"
|
||||
echo ""
|
||||
# Vérifier prérequis
|
||||
command -v ja4ebpf >/dev/null 2>&1 || {
|
||||
log "Rebuild ja4ebpf..."
|
||||
cd "$PROJECT/services/ja4ebpf"
|
||||
GOWORK=off go generate ./internal/loader/ 2>&1 | tail -3
|
||||
GOWORK=off CGO_ENABLED=0 go build -o /tmp/ja4ebpf_new ./cmd/ja4ebpf/ && mv /tmp/ja4ebpf_new /usr/local/bin/ja4ebpf
|
||||
}
|
||||
command -v docker >/dev/null 2>&1 || { fail "Docker non installé"; exit 1; }
|
||||
|
||||
check_prerequisites
|
||||
start_clickhouse
|
||||
setup_nginx
|
||||
start_ja4ebpf
|
||||
generate_traffic
|
||||
verify_db
|
||||
start_clickhouse
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════"
|
||||
echo -e " ${GREEN}OK${RESET}: $PASS_COUNT ${YELLOW}WARN${RESET}: $WARN_COUNT ${RED}FAIL${RESET}: $FAIL_COUNT"
|
||||
if [ "$FAIL_COUNT" -eq 0 ]; then
|
||||
echo -e " ${GREEN}${BOLD}Tous les tests réussis !${RESET}"
|
||||
exit 0
|
||||
else
|
||||
echo -e " ${RED}${BOLD}$FAIL_COUNT tests échoués.${RESET}"
|
||||
echo "Logs ja4ebpf :"
|
||||
tail -20 /tmp/ja4ebpf.log 2>/dev/null || true
|
||||
exit 1
|
||||
fi
|
||||
case "$STACK" in
|
||||
nginx) setup_nginx ;;
|
||||
apache) setup_apache ;;
|
||||
hitch-varnish) setup_hitch_varnish ;;
|
||||
*) fail "Stack inconnue: $STACK"; exit 1 ;;
|
||||
esac
|
||||
|
||||
start_ja4ebpf
|
||||
|
||||
# Afficher l'IP pour le host
|
||||
local eth0_ip
|
||||
eth0_ip=$(get_eth0_ip)
|
||||
echo ""
|
||||
echo " ┌─────────────────────────────────────────┐"
|
||||
echo " │ Services prêts ! │"
|
||||
echo " │ IP eth0 : $eth0_ip"
|
||||
echo " │ HTTP : http://$eth0_ip:80"
|
||||
echo " │ HTTPS : https://$eth0_ip:443"
|
||||
echo " └─────────────────────────────────────────┘"
|
||||
echo ""
|
||||
}
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Phase 3 : vérification
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
do_verify() {
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════╗"
|
||||
echo "║ Phase 3 : Vérification — $STACK"
|
||||
echo "╚══════════════════════════════════════════╝"
|
||||
echo ""
|
||||
|
||||
verify_db
|
||||
|
||||
echo ""
|
||||
echo "════════════════════════════════════════════"
|
||||
echo -e " ${GREEN}OK${RESET}: $PASS_COUNT ${YELLOW}WARN${RESET}: $WARN_COUNT ${RED}FAIL${RESET}: $FAIL_COUNT"
|
||||
if [ "$FAIL_COUNT" -eq 0 ]; then
|
||||
echo -e " ${GREEN}${BOLD}$STACK : Tous les tests réussis !${RESET}"
|
||||
else
|
||||
echo -e " ${RED}${BOLD}$STACK : $FAIL_COUNT tests échoués${RESET}"
|
||||
tail -20 /tmp/ja4ebpf.log 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
# Main
|
||||
# ═════════════════════════════════════════════════════════════════════════════
|
||||
|
||||
case "$MODE" in
|
||||
start)
|
||||
do_start
|
||||
echo " En attente de trafic depuis le host..."
|
||||
# Attendre que le host génère le trafic
|
||||
# Le fichier /tmp/ja4ebpf-traffic-done est créé par le host après le trafic
|
||||
for i in $(seq 1 120); do
|
||||
[ -f /tmp/ja4ebpf-traffic-done ] && break
|
||||
sleep 1
|
||||
done
|
||||
do_verify
|
||||
;;
|
||||
verify)
|
||||
do_verify
|
||||
;;
|
||||
*)
|
||||
# Mode legacy : tout dans la VM (trafic local uniquement)
|
||||
# Note : XDP sur eth0 ne capturera PAS le trafic localhost
|
||||
do_start
|
||||
log "ATTENTION : le trafic localhost n'est pas capturé par XDP/eth0"
|
||||
log "Utilisez 'make test-vm-matrix' pour le test complet avec trafic host"
|
||||
# Générer quand même du trafic pour les uprobes
|
||||
for path in / /health; do
|
||||
curl -sf -k "https://localhost$path" >/dev/null 2>&1 || true
|
||||
done
|
||||
sleep 10
|
||||
do_verify
|
||||
;;
|
||||
esac
|
||||
|
||||
[ "$FAIL_COUNT" -eq 0 ] && exit 0 || exit 1
|
||||
|
||||
Reference in New Issue
Block a user