fix(ja4ebpf): fix TLS capture, SYN offsets, TCP option parsing
- Increase MAX_TLS_PAYLOAD from 512 to 2048 bytes to capture full TLS ClientHellos (modern browsers/curl send 1000-1543 byte ClientHellos) - Fix ParseClientHello to tolerate XDP-truncated payloads: clamp recordLength and chLen to available data instead of returning error - Fix cipher suites, compression, extensions truncation to use clamping - Fix consumeSynEvents struct field offsets: dst_ip (4 bytes at offset 4) was not accounted for, causing all L3/L4 metadata to be read from wrong positions (TTL was actually dst_ip[0], windowSize was dst_port, etc.) - Add parseTCPOptions() to extract MSS and Window Scale from raw TCP options (C code sets defaults of mss=0, window_scale=0xFF, expects Go to parse) - Fix consumeAcceptEvents: skip zero-IP events to avoid phantom sessions - Fix consumeSSLEvents: filter zero-IP/port events when proc fallback fails - Add missing consumeHTTPPlainEvents goroutine (was defined but never called) - Fix race condition: SYN consumer sets Correlated=true if TLS already present - Update tls_hello_event struct offsets in Go consumer (payload_len now at offset 2054, was 518, due to payload array growing from 512 to 2048 bytes) - Remove debug logging from consumers and GC E2E verified: HTTP plain (port 80) and HTTPS (port 443) both produce fully correlated sessions in ClickHouse with correct: - ip_meta_ttl=64, ip_meta_df=true, ip_meta_id - tcp_meta_window_size=64240, tcp_meta_window_scale=10, tcp_meta_mss=1460 - ja4=t13i3010_1d37bd780c83_95d2a80e6515 - tls_alpn=http/1.1 - method=GET, path=/, header_order_signature=Host;User-Agent;Accept - correlated=1 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
54
tests/vm/Vagrantfile
vendored
54
tests/vm/Vagrantfile
vendored
@ -6,16 +6,15 @@
|
||||
# Fournit un environnement kernel complet pour les tests eBPF :
|
||||
# - tracefs / debugfs montés
|
||||
# - perf_kprobe PMU disponible
|
||||
# - uprobes fonctionnels avec accept4 kprobe/tracepoint
|
||||
# - uprobes fonctionnels avec accept4 tracepoints
|
||||
#
|
||||
# Prérequis (host Ubuntu) :
|
||||
# sudo apt-get install -y vagrant libvirt-daemon-system libvirt-clients \
|
||||
# qemu-kvm ruby-libvirt
|
||||
# sudo apt-get install -y libvirt-daemon-system libvirt-clients qemu-kvm libvirt-dev ruby-dev
|
||||
# vagrant plugin install vagrant-libvirt
|
||||
# sudo usermod -aG libvirt,kvm $USER # puis se reconnecter
|
||||
#
|
||||
# Utilisation :
|
||||
# vagrant up # créer + provisionner la VM (première fois ~5 min)
|
||||
# vagrant up # créer + provisionner (~5 min)
|
||||
# vagrant ssh # connexion SSH
|
||||
# make test-vm-nginx # lancer les tests depuis le host
|
||||
# vagrant destroy -f # détruire la VM
|
||||
@ -23,44 +22,39 @@
|
||||
|
||||
Vagrant.configure("2") do |config|
|
||||
|
||||
# ── Box Rocky Linux 9 ──────────────────────────────────────────────────────
|
||||
# ── Box Rocky Linux 9 avec provider libvirt (image qcow2) ─────────────────
|
||||
config.vm.box = "generic/rocky9"
|
||||
|
||||
# ── Réseau : IP privée pour accès depuis le host ───────────────────────────
|
||||
config.vm.network "private_network", ip: "192.168.56.10"
|
||||
# ── Désactiver synced_folder par défaut (utiliser rsync explicitement) ─────
|
||||
config.vm.synced_folder ".", "/vagrant", disabled: true
|
||||
|
||||
# ── Ressources VM ─────────────────────────────────────────────────────────
|
||||
# ── Provider libvirt ───────────────────────────────────────────────────────
|
||||
config.vm.provider :libvirt do |v|
|
||||
v.cpus = 4
|
||||
v.memory = 4096
|
||||
v.nested = false # pas besoin de virtualisation imbriquée
|
||||
# Pour VirtualBox (fallback)
|
||||
v.cpus = 4
|
||||
v.memory = 4096
|
||||
v.nested = false
|
||||
v.cpu_mode = "host-passthrough" # expose les capacités CPU hôte → KVM perf
|
||||
v.driver = "kvm"
|
||||
v.disk_bus = "virtio"
|
||||
v.nic_model_type = "virtio"
|
||||
end
|
||||
|
||||
config.vm.provider :virtualbox do |v|
|
||||
v.cpus = 4
|
||||
v.memory = 4096
|
||||
v.customize ["modifyvm", :id, "--nicpromisc2", "allow-all"]
|
||||
end
|
||||
|
||||
# ── Montage du projet ─────────────────────────────────────────────────────
|
||||
# Le répertoire racine du projet est monté dans /ja4-platform
|
||||
# ── Synchronisation du projet via rsync ────────────────────────────────────
|
||||
config.vm.synced_folder "../..", "/ja4-platform",
|
||||
type: "rsync",
|
||||
rsync__exclude: [".git/", "old/", "*.rpm", "services/*/target/"]
|
||||
rsync__exclude: [".git/", "old/", "*.rpm", "dist/"]
|
||||
|
||||
# ── Provisioning ─────────────────────────────────────────────────────────
|
||||
# ── Provisioning ───────────────────────────────────────────────────────────
|
||||
config.vm.provision "shell", path: "provision.sh"
|
||||
|
||||
# ── Message post-démarrage ────────────────────────────────────────────────
|
||||
# ── Message post-démarrage ─────────────────────────────────────────────────
|
||||
config.vm.post_up_message = <<~MSG
|
||||
VM ja4ebpf prête !
|
||||
|
||||
Depuis le répertoire tests/vm/ :
|
||||
vagrant ssh # connexion interactive
|
||||
make -C ../.. test-vm-nginx # lancer le test nginx
|
||||
make -C ../.. test-vm-matrix # lancer tous les tests
|
||||
|
||||
IP de la VM : 192.168.56.10
|
||||
|
||||
Depuis la racine du projet :
|
||||
make vm-ssh # connexion interactive
|
||||
make test-vm-nginx # test nginx complet (L3/L4 + TLS + L7)
|
||||
make test-vm-all # tous les tests
|
||||
make vm-rebuild-ja4ebpf # resynchroniser + recompiler après modif
|
||||
MSG
|
||||
end
|
||||
|
||||
@ -15,6 +15,9 @@
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
# S'assurer que /usr/local/bin et go sont dans PATH (nécessaire pour sudo bash)
|
||||
export PATH="/usr/local/bin:/usr/local/go/bin:$PATH"
|
||||
|
||||
STACK="${1:-nginx}"
|
||||
KEEP_RUNNING="${KEEP_RUNNING:-false}"
|
||||
PROJECT="/ja4-platform"
|
||||
@ -50,7 +53,7 @@ check_prerequisites() {
|
||||
cd "$PROJECT/services/ja4ebpf"
|
||||
export PATH="/usr/local/go/bin:$PATH"
|
||||
GOWORK=off go generate ./internal/loader/ 2>&1 | tail -3
|
||||
GOWORK=off CGO_ENABLED=0 go build -o /usr/local/bin/ja4ebpf ./cmd/ja4ebpf/
|
||||
GOWORK=off CGO_ENABLED=0 go build -o /tmp/ja4ebpf_new ./cmd/ja4ebpf/ && mv /tmp/ja4ebpf_new /usr/local/bin/ja4ebpf
|
||||
}
|
||||
|
||||
command -v docker >/dev/null 2>&1 || { fail "Docker non installé"; exit 1; }
|
||||
@ -103,6 +106,8 @@ setup_nginx() {
|
||||
|
||||
# Créer les fichiers de test
|
||||
mkdir -p /var/www/html
|
||||
# /run/nginx est un tmpfs recréé à chaque boot, nginx en a besoin pour son PID
|
||||
mkdir -p /run/nginx
|
||||
echo '{"status":"ok","stack":"nginx-vm"}' > /var/www/html/health
|
||||
for p in data api/users api/data/test; do
|
||||
mkdir -p "/var/www/html/$(dirname $p)"
|
||||
@ -144,7 +149,7 @@ EOF
|
||||
|
||||
# Lancer avec les capabilities nécessaires
|
||||
# Dans la VM (root), on peut lancer directement
|
||||
ja4ebpf -config /tmp/ja4ebpf.yml > /tmp/ja4ebpf.log 2>&1 &
|
||||
JA4EBPF_CONFIG=/tmp/ja4ebpf.yml ja4ebpf > /tmp/ja4ebpf.log 2>&1 &
|
||||
JA4EBPF_PID=$!
|
||||
|
||||
sleep 3
|
||||
|
||||
Reference in New Issue
Block a user