b1218a2367
- Increase MAX_TLS_PAYLOAD from 512 to 2048 bytes to capture full TLS ClientHellos (modern browsers/curl send 1000-1543 byte ClientHellos) - Fix ParseClientHello to tolerate XDP-truncated payloads: clamp recordLength and chLen to available data instead of returning error - Fix cipher suites, compression, extensions truncation to use clamping - Fix consumeSynEvents struct field offsets: dst_ip (4 bytes at offset 4) was not accounted for, causing all L3/L4 metadata to be read from wrong positions (TTL was actually dst_ip[0], windowSize was dst_port, etc.) - Add parseTCPOptions() to extract MSS and Window Scale from raw TCP options (C code sets defaults of mss=0, window_scale=0xFF, expects Go to parse) - Fix consumeAcceptEvents: skip zero-IP events to avoid phantom sessions - Fix consumeSSLEvents: filter zero-IP/port events when proc fallback fails - Add missing consumeHTTPPlainEvents goroutine (was defined but never called) - Fix race condition: SYN consumer sets Correlated=true if TLS already present - Update tls_hello_event struct offsets in Go consumer (payload_len now at offset 2054, was 518, due to payload array growing from 512 to 2048 bytes) - Remove debug logging from consumers and GC E2E verified: HTTP plain (port 80) and HTTPS (port 443) both produce fully correlated sessions in ClickHouse with correct: - ip_meta_ttl=64, ip_meta_df=true, ip_meta_id - tcp_meta_window_size=64240, tcp_meta_window_scale=10, tcp_meta_mss=1460 - ja4=t13i3010_1d37bd780c83_95d2a80e6515 - tls_alpn=http/1.1 - method=GET, path=/, header_order_signature=Host;User-Agent;Accept - correlated=1 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
215 lines
8.4 KiB
Makefile
215 lines
8.4 KiB
Makefile
# =============================================================================
|
||
# ja4-platform — Monorepo Makefile
|
||
# Service de capture : ja4ebpf (eBPF CO-RE)
|
||
# =============================================================================
|
||
|
||
VERSION ?= $(shell git describe --tags --always 2>/dev/null || echo dev)
|
||
|
||
.PHONY: help \
|
||
build-all test-all rpm-all dist \
|
||
build-ja4ebpf test-ja4ebpf rpm-ja4ebpf \
|
||
build-bot-detector test-bot-detector \
|
||
build-dashboard test-dashboard \
|
||
test-all-stacks test-nginx test-nginx-varnish test-hitch-varnish test-apache \
|
||
test-matrix \
|
||
test-vm-nginx test-vm-all vm-up vm-down vm-ssh \
|
||
reload-prod-logs init-stack import-prod-data init-and-import \
|
||
purge-db
|
||
|
||
.DEFAULT_GOAL := help
|
||
|
||
help: ## Affiche cette aide
|
||
@echo ""
|
||
@echo " ja4-platform — Makefile"
|
||
@echo " ─────────────────────────────────────────────────────"
|
||
@echo ""
|
||
@echo " Build"
|
||
@echo " make build-all Construit toutes les images Docker"
|
||
@echo " make build-ja4ebpf Image ja4ebpf (agent eBPF CO-RE)"
|
||
@echo " make build-bot-detector Image bot-detector (détection ML)"
|
||
@echo " make build-dashboard Image dashboard (SOC UI)"
|
||
@echo ""
|
||
@echo " Tests unitaires"
|
||
@echo " make test-all Lance tous les tests unitaires"
|
||
@echo " make test-ja4ebpf Tests Go ja4ebpf"
|
||
@echo " make test-bot-detector Tests Python bot-detector"
|
||
@echo " make test-dashboard Tests Python dashboard"
|
||
@echo ""
|
||
@echo " Tests VM (eBPF sur kernel réel — nécessite 'make vm-up' d'abord)"
|
||
@echo " make vm-up Créer la VM Rocky Linux 9 (vagrant up)"
|
||
@echo " make vm-down Détruire la VM (vagrant destroy)"
|
||
@echo " make vm-ssh Connexion SSH à la VM"
|
||
@echo " make test-vm-nginx Test nginx dans la VM (L7 complet)"
|
||
@echo " make test-vm-all Tous les tests dans la VM"
|
||
@echo ""
|
||
@echo " Tests d'intégration (par stack, Docker — L3/L4/TLS uniquement)"
|
||
@echo " make test-all-stacks Toutes les stacks sur Rocky Linux 9"
|
||
@echo " make test-apache Stack Apache + ja4ebpf"
|
||
@echo " make test-nginx Stack nginx + ja4ebpf"
|
||
@echo " make test-nginx-varnish Stack nginx + Varnish + ja4ebpf"
|
||
@echo " make test-hitch-varnish Stack hitch + Varnish + ja4ebpf"
|
||
@echo ""
|
||
@echo " Matrice multi-distro"
|
||
@echo " make test-matrix Toutes stacks × el8/el9/el10"
|
||
@echo " make test-matrix MATRIX_STACKS=nginx,apache MATRIX_DISTROS=el9,el10"
|
||
@echo ""
|
||
@echo " RPM"
|
||
@echo " make rpm-all Construit tous les RPMs ja4ebpf (el8/el9/el10)"
|
||
@echo " make rpm-ja4ebpf RPMs ja4ebpf (el8, el9, el10)"
|
||
@echo " make dist Alias de rpm-all"
|
||
@echo ""
|
||
@echo " Base de données"
|
||
@echo " make init-stack Initialise ClickHouse (schéma complet)"
|
||
@echo " make import-prod-data Importe les données prod (date shift)"
|
||
@echo " make init-and-import init-stack + import-prod-data"
|
||
@echo " make reload-prod-logs Exporte prod → importe en dev"
|
||
@echo " make purge-db Supprime et recrée les bases ja4_*"
|
||
@echo ""
|
||
|
||
# ── Cibles agrégées ──────────────────────────────────────────────────────────
|
||
|
||
build-all: build-ja4ebpf build-bot-detector build-dashboard
|
||
@echo "All services built."
|
||
|
||
test-all: test-ja4ebpf test-bot-detector test-dashboard
|
||
@echo "All unit tests completed."
|
||
|
||
rpm-all: rpm-ja4ebpf
|
||
@echo "All RPMs built."
|
||
|
||
dist: rpm-all
|
||
@echo "RPMs disponibles dans services/ja4ebpf/dist/"
|
||
|
||
# ── ja4ebpf (agent eBPF CO-RE) ───────────────────────────────────────────────
|
||
|
||
build-ja4ebpf:
|
||
docker build \
|
||
-f services/ja4ebpf/Dockerfile \
|
||
--build-arg BUILD_VERSION=$(VERSION) \
|
||
-t ja4-platform/ja4ebpf:latest \
|
||
.
|
||
|
||
test-ja4ebpf:
|
||
docker build \
|
||
-f services/ja4ebpf/Dockerfile.tests \
|
||
-t ja4-platform/ja4ebpf-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/ja4ebpf-tests:latest
|
||
|
||
rpm-ja4ebpf:
|
||
# Build multi-distro : el8 (AlmaLinux 8) + el9 (Rocky 9) + el10 (AlmaLinux 10)
|
||
# Sortie : services/ja4ebpf/dist/el{8,9,10}/ja4ebpf-*.rpm
|
||
docker build \
|
||
-f services/ja4ebpf/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/ja4ebpf/dist \
|
||
--build-arg BUILD_VERSION=$(VERSION) \
|
||
.
|
||
@echo ""
|
||
@echo "RPMs produits :"
|
||
@find services/ja4ebpf/dist -name '*.rpm' | sort | sed 's/^/ /'
|
||
|
||
# ── bot-detector ─────────────────────────────────────────────────────────────
|
||
|
||
build-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile \
|
||
-t ja4-platform/bot-detector:latest \
|
||
.
|
||
|
||
test-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile.tests \
|
||
-t ja4-platform/bot-detector-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/bot-detector-tests:latest
|
||
|
||
# ── dashboard ────────────────────────────────────────────────────────────────
|
||
|
||
build-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile \
|
||
-t ja4-platform/dashboard:latest \
|
||
.
|
||
|
||
test-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile.tests \
|
||
-t ja4-platform/dashboard-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/dashboard-tests:latest
|
||
|
||
# ── Tests d'intégration par stack ────────────────────────────────────────────
|
||
|
||
test-all-stacks: ## Toutes les stacks sur la distro par défaut (Rocky Linux 9)
|
||
cd tests/integration && bash run-all-stacks.sh
|
||
|
||
test-apache:
|
||
cd tests/integration && bash apache/run-tests.sh
|
||
|
||
test-nginx:
|
||
cd tests/integration && bash nginx/run-tests.sh
|
||
|
||
test-nginx-varnish:
|
||
cd tests/integration && bash nginx-varnish/run-tests.sh
|
||
|
||
test-hitch-varnish:
|
||
cd tests/integration && bash hitch-varnish/run-tests.sh
|
||
|
||
# ── Tests VM (kernel réel, eBPF complet) ─────────────────────────────────────
|
||
|
||
# Répertoire Vagrantfile
|
||
VM_DIR := tests/vm
|
||
|
||
vm-up: ## Créer la VM Rocky Linux 9 pour les tests eBPF
|
||
cd $(VM_DIR) && vagrant up
|
||
|
||
vm-down: ## Détruire la VM
|
||
cd $(VM_DIR) && vagrant destroy -f
|
||
|
||
vm-ssh: ## Connexion SSH à la VM
|
||
cd $(VM_DIR) && vagrant ssh
|
||
|
||
vm-rebuild-ja4ebpf: ## Recompiler ja4ebpf dans la VM (après modifications)
|
||
cd $(VM_DIR) && vagrant rsync && vagrant ssh -- \
|
||
'export PATH=/usr/local/go/bin:$$PATH && \
|
||
cd /ja4-platform/services/ja4ebpf && \
|
||
GOWORK=off go generate ./internal/loader/ && \
|
||
GOWORK=off CGO_ENABLED=0 go build -o /tmp/ja4ebpf ./cmd/ja4ebpf/ && \
|
||
sudo mv /tmp/ja4ebpf /usr/local/bin/ja4ebpf && \
|
||
echo "ja4ebpf rebuilt OK"'
|
||
|
||
test-vm-nginx: ## Test nginx dans la VM (L3/L4/TLS/L7 HTTP complet)
|
||
@echo "=== Test VM nginx (kernel réel) ==="
|
||
cd $(VM_DIR) && vagrant rsync && vagrant ssh -- \
|
||
'sudo bash /ja4-platform/tests/vm/run-tests-vm.sh nginx'
|
||
|
||
test-vm-all: ## Tous les tests dans la VM
|
||
@echo "=== Tests VM (toutes stacks) ==="
|
||
cd $(VM_DIR) && vagrant rsync && vagrant ssh -- \
|
||
'sudo bash /ja4-platform/tests/vm/run-tests-vm.sh all'
|
||
|
||
# ── Matrice multi-distro ─────────────────────────────────────────────────────
|
||
|
||
test-matrix: ## Toutes stacks × el8 + el9 + el10
|
||
cd tests/integration && bash run-distro-matrix.sh \
|
||
$${MATRIX_STACKS:+--stacks=$${MATRIX_STACKS}} \
|
||
$${MATRIX_DISTROS:+--distros=$${MATRIX_DISTROS}}
|
||
|
||
# ── Base de données ───────────────────────────────────────────────────────────
|
||
|
||
reload-prod-logs:
|
||
./scripts/reload-prod-logs.sh
|
||
|
||
init-stack:
|
||
./scripts/init-stack.sh
|
||
|
||
import-prod-data:
|
||
./scripts/import-prod-data.sh
|
||
|
||
init-and-import:
|
||
./scripts/init-stack.sh --import-prod
|
||
|
||
purge-db:
|
||
./scripts/init-stack.sh --reset
|