Replace single-service-per-endpoint with all-ips mode running nginx, apache, and hitch+varnish simultaneously on 3 dedicated IPs per VM (eth1 alias IPs). Add a dedicated traffic VM with curl-impersonate for realistic TLS fingerprints, parallelized traffic generation, and paired SNI_HOSTS/TARGET_IPS lists for per-VM per-service hostname identification (e.g. rocky9-nginx-platform.test). Key changes: - run-tests-vm.sh: add setup_all_ips(), IP-specific Listen/bind directives with reset-before-apply pattern, graceful service availability checks - run-e2e-test.sh: traffic VM architecture, all-ips mode, eth1 network, paired IP/SNI lists, updated cleanup for alias IPs - generate-traffic.sh: parallel background jobs, curl-impersonate detection, auto source interface detection via ip route get, Host header in HTTP traffic - Vagrantfile: add traffic VM with provision-traffic.sh - provision-traffic.sh: install curl-impersonate and httpx for traffic gen - test-rpm.sh: multi-interface TC check, updated ja4ebpf config - clickhouse-init.sh: load CSV stubs for Anubis/bot-networks dictionaries - Remove obsolete correlator/sentinel/mod-reqin-log docs - Add h2_settings_ack column to http_logs schema - Upgrade Go toolchain to 1.25.0 Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
142 lines
7.3 KiB
Ruby
142 lines
7.3 KiB
Ruby
# -*- mode: ruby -*-
|
|
# vi: set ft=ruby :
|
|
# =============================================================================
|
|
# Vagrantfile — VMs de test ja4ebpf multi-distro
|
|
#
|
|
# 3 VMs pour les tests unitaires eBPF sur kernel réel :
|
|
# - centos8 : CentOS 8 (el8)
|
|
# - rocky9 : Rocky Linux 9 (el9)
|
|
# - rocky10 : Rocky Linux 10 (el10)
|
|
#
|
|
# Fournit un environnement kernel complet pour les tests eBPF :
|
|
# - tracefs / debugfs montés
|
|
# - perf_kprobe PMU disponible
|
|
# - uprobes fonctionnels avec accept4 tracepoints
|
|
#
|
|
# Prérequis (host Ubuntu) :
|
|
# sudo apt-get install -y libvirt-daemon-system libvirt-clients qemu-kvm libvirt-dev ruby-dev
|
|
# vagrant plugin install vagrant-libvirt
|
|
# sudo usermod -aG libvirt,kvm $USER # puis se reconnecter
|
|
#
|
|
# Utilisation :
|
|
# vagrant up # créer + provisionner toutes les VMs
|
|
# vagrant up rocky9 # créer une seule VM
|
|
# vagrant ssh rocky9 # connexion SSH
|
|
# make test-vm-nginx # test nginx sur Rocky 9 (défaut)
|
|
# make test-vm-all # tous les tests sur Rocky 9
|
|
# ./tests/vm/run-all-vms.sh # tests sur les 3 VMs
|
|
# vagrant destroy -f # détruire toutes les VMs
|
|
# =============================================================================
|
|
|
|
Vagrant.configure("2") do |config|
|
|
|
|
# ── Désactiver synced_folder par défaut ─────────────────────────────────────
|
|
config.vm.synced_folder ".", "/vagrant", disabled: true
|
|
|
|
# ── Provider libvirt commun ─────────────────────────────────────────────────
|
|
config.vm.provider :libvirt do |v|
|
|
v.cpus = 4
|
|
v.memory = 4096
|
|
v.nested = false
|
|
v.cpu_mode = "host-passthrough"
|
|
v.driver = "kvm"
|
|
v.disk_bus = "virtio"
|
|
v.nic_model_type = "virtio"
|
|
end
|
|
|
|
# ── Synchronisation du projet via rsync ─────────────────────────────────────
|
|
config.vm.synced_folder "../..", "/ja4-platform",
|
|
type: "rsync",
|
|
rsync__exclude: [".git/", "old/", "*.rpm", "dist/"]
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# VM 1 : CentOS 8 (el8)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
config.vm.define "centos8", autostart: false do |node|
|
|
node.vm.box = "centos/8"
|
|
node.vm.network "private_network",
|
|
libvirt__network_name: "ja4-e2e",
|
|
type: "dhcp"
|
|
node.vm.provision "shell", path: "provision-el8.sh"
|
|
node.vm.post_up_message = "VM centos8 prête ! Tests : make test-vm-centos8"
|
|
end
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# VM 2 : Rocky Linux 9 (el9) — VM par défaut
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
config.vm.define "rocky9", primary: true do |node|
|
|
node.vm.box = "generic/rocky9"
|
|
node.vm.network "private_network",
|
|
libvirt__network_name: "ja4-e2e",
|
|
type: "dhcp"
|
|
node.vm.provision "shell", path: "provision.sh"
|
|
node.vm.post_up_message = <<~MSG
|
|
VM rocky9 prête !
|
|
|
|
Depuis la racine du projet :
|
|
make vm-ssh # connexion interactive
|
|
make test-vm-nginx # test nginx complet (L3/L4 + TLS + L7)
|
|
make test-vm-all # tous les tests
|
|
make vm-rebuild-ja4ebpf # resynchroniser + recompiler après modif
|
|
MSG
|
|
end
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# VM 3 : Rocky Linux 10 (el10)
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
config.vm.define "rocky10", autostart: false do |node|
|
|
node.vm.box = "almalinux/10"
|
|
node.vm.network "private_network",
|
|
libvirt__network_name: "ja4-e2e",
|
|
type: "dhcp"
|
|
node.vm.provision "shell", path: "provision.sh"
|
|
node.vm.post_up_message = "VM rocky10 prête ! Tests : make test-vm-rocky10"
|
|
end
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# VM 5 : Traffic Generator (curl-impersonate + httpx)
|
|
#
|
|
# VM dédiée à la génération de trafic vers les endpoints.
|
|
# Séparée des VMs endpoint pour des TLS fingerprints réalistes
|
|
# et des IPs sources distinctes.
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
config.vm.define "traffic", autostart: false do |node|
|
|
node.vm.box = "generic/rocky9"
|
|
node.vm.network "private_network",
|
|
libvirt__network_name: "ja4-e2e",
|
|
type: "dhcp"
|
|
node.vm.provider :libvirt do |v|
|
|
v.cpus = 2
|
|
v.memory = 1024
|
|
end
|
|
node.vm.provision "shell", path: "provision-traffic.sh"
|
|
node.vm.post_up_message = "VM traffic prête ! Génération de trafic vers les endpoints."
|
|
end
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
# VM 4 : Analysis Server (ClickHouse + bot-detector + dashboard)
|
|
#
|
|
# VM centralisée pour le test E2E distribué. Les endpoints EL8/9/10 envoient
|
|
# leurs logs ja4ebpf vers le ClickHouse de cette VM (192.168.42.10).
|
|
# ═══════════════════════════════════════════════════════════════════════════
|
|
config.vm.define "analysis", autostart: false do |node|
|
|
node.vm.box = "generic/rocky9"
|
|
node.vm.network "private_network", ip: "192.168.42.10",
|
|
libvirt__network_name: "ja4-e2e",
|
|
libvirt__netmask: "255.255.255.0"
|
|
node.vm.provider :libvirt do |v|
|
|
v.cpus = 4
|
|
v.memory = 12288 # 12 Go — torch + isotree build gourmand en RAM
|
|
end
|
|
node.vm.provision "shell", path: "provision-analysis.sh"
|
|
node.vm.post_up_message = <<~MSG
|
|
VM analysis prête !
|
|
|
|
Depuis la racine du projet :
|
|
make test-e2e # test E2E complet (capture + ML + dashboard)
|
|
make test-e2e-quick # test rapide avec trafic réduit
|
|
MSG
|
|
end
|
|
|
|
end
|