d75825278e
ja4ebpf: - Refactor BPF TC capture with improved SYN offset handling and TCP option parsing - Enhance TLS uprobe SSL hooking for better key extraction - Add ClickHouse writer improvements for HTTP log materialized views - Update RPM spec for Rocky Linux 8/9/10, fix systemd service - Simplify loader with cleaner bpf2go integration bot-detector: - Add H2 SETTINGS per-parameter comparison in browser_matcher - Enhance browser signatures and scoring pipeline - Improve preprocessing and cycle detection infra: - Multi-distro Vagrantfile (centos8, rocky9, rocky10) with per-distro provisioning - New Makefile targets: vm-up-all, test-vm-matrix, test-vm-centos8/rocky10 - Add debug helpers and run-test-from-host.sh for host-driven VM testing - Update run-tests-vm.sh for cross-distro compatibility - Remove accidental binary blob (\004) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
294 lines
12 KiB
Makefile
294 lines
12 KiB
Makefile
# =============================================================================
|
||
# ja4-platform — Monorepo Makefile
|
||
# Service de capture : ja4ebpf (eBPF CO-RE)
|
||
# =============================================================================
|
||
|
||
VERSION ?= $(shell git describe --tags --always 2>/dev/null || echo dev)
|
||
|
||
.PHONY: help \
|
||
build-all test-all rpm-all dist \
|
||
build-ja4ebpf test-ja4ebpf rpm-ja4ebpf \
|
||
build-bot-detector test-bot-detector \
|
||
build-dashboard test-dashboard \
|
||
test-all-stacks test-nginx test-nginx-varnish test-hitch-varnish test-apache \
|
||
test-matrix \
|
||
test-vm-nginx test-vm-all vm-up vm-down vm-ssh \
|
||
reload-prod-logs init-stack import-prod-data init-and-import \
|
||
purge-db
|
||
|
||
.DEFAULT_GOAL := help
|
||
|
||
help: ## Affiche cette aide
|
||
@echo ""
|
||
@echo " ja4-platform — Makefile"
|
||
@echo " ─────────────────────────────────────────────────────"
|
||
@echo ""
|
||
@echo " Build"
|
||
@echo " make build-all Construit toutes les images Docker"
|
||
@echo " make build-ja4ebpf Image ja4ebpf (agent eBPF CO-RE)"
|
||
@echo " make build-bot-detector Image bot-detector (détection ML)"
|
||
@echo " make build-dashboard Image dashboard (SOC UI)"
|
||
@echo ""
|
||
@echo " Tests unitaires"
|
||
@echo " make test-all Lance tous les tests unitaires"
|
||
@echo " make test-ja4ebpf Tests Go ja4ebpf"
|
||
@echo " make test-bot-detector Tests Python bot-detector"
|
||
@echo " make test-dashboard Tests Python dashboard"
|
||
@echo ""
|
||
@echo " Tests VM (eBPF sur kernel réel — nécessite 'make vm-up' d'abord)"
|
||
@echo " make vm-up Créer la VM Rocky Linux 9 (vagrant up)"
|
||
@echo " make vm-up-all Créer les 3 VMs (centos8/rocky9/rocky10)"
|
||
@echo " make vm-down Détruire la VM (vagrant destroy)"
|
||
@echo " make vm-ssh Connexion SSH à la VM"
|
||
@echo " make vm-reprovision Re-provisionner les 3 VMs"
|
||
@echo " make test-vm-nginx Test nginx dans la VM Rocky 9"
|
||
@echo " make test-vm-apache Test apache dans la VM Rocky 9"
|
||
@echo " make test-vm-hitch-varnish Test hitch+varnish dans la VM Rocky 9"
|
||
@echo " make test-vm-all Tous les tests (3 stacks) dans la VM Rocky 9"
|
||
@echo " make test-vm-centos8 Tous les tests dans la VM CentOS 8"
|
||
@echo " make test-vm-rocky10 Tous les tests dans la VM Rocky 10"
|
||
@echo " make test-vm-matrix Matrice complète : 3 stacks × 3 distros"
|
||
@echo ""
|
||
@echo " Tests d'intégration (par stack, Docker — L3/L4/TLS uniquement)"
|
||
@echo " make test-all-stacks Toutes les stacks sur Rocky Linux 9"
|
||
@echo " make test-apache Stack Apache + ja4ebpf"
|
||
@echo " make test-nginx Stack nginx + ja4ebpf"
|
||
@echo " make test-nginx-varnish Stack nginx + Varnish + ja4ebpf"
|
||
@echo " make test-hitch-varnish Stack hitch + Varnish + ja4ebpf"
|
||
@echo ""
|
||
@echo " Matrice multi-distro"
|
||
@echo " make test-matrix Toutes stacks × el8/el9/el10"
|
||
@echo " make test-matrix MATRIX_STACKS=nginx,apache MATRIX_DISTROS=el9,el10"
|
||
@echo ""
|
||
@echo " RPM"
|
||
@echo " make rpm-all Construit tous les RPMs ja4ebpf (el8/el9/el10)"
|
||
@echo " make rpm-ja4ebpf RPMs ja4ebpf (el8, el9, el10)"
|
||
@echo " make dist Alias de rpm-all"
|
||
@echo ""
|
||
@echo " Base de données"
|
||
@echo " make init-stack Initialise ClickHouse (schéma complet)"
|
||
@echo " make import-prod-data Importe les données prod (date shift)"
|
||
@echo " make init-and-import init-stack + import-prod-data"
|
||
@echo " make reload-prod-logs Exporte prod → importe en dev"
|
||
@echo " make purge-db Supprime et recrée les bases ja4_*"
|
||
@echo ""
|
||
|
||
# ── Cibles agrégées ──────────────────────────────────────────────────────────
|
||
|
||
build-all: build-ja4ebpf build-bot-detector build-dashboard
|
||
@echo "All services built."
|
||
|
||
test-all: test-ja4ebpf test-bot-detector test-dashboard
|
||
@echo "All unit tests completed."
|
||
|
||
rpm-all: rpm-ja4ebpf
|
||
@echo "All RPMs built."
|
||
|
||
dist: rpm-all
|
||
@echo "RPMs disponibles dans services/ja4ebpf/dist/"
|
||
|
||
# ── ja4ebpf (agent eBPF CO-RE) ───────────────────────────────────────────────
|
||
|
||
build-ja4ebpf:
|
||
docker build \
|
||
-f services/ja4ebpf/Dockerfile \
|
||
--build-arg BUILD_VERSION=$(VERSION) \
|
||
-t ja4-platform/ja4ebpf:latest \
|
||
.
|
||
|
||
test-ja4ebpf:
|
||
docker build \
|
||
-f services/ja4ebpf/Dockerfile.tests \
|
||
-t ja4-platform/ja4ebpf-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/ja4ebpf-tests:latest
|
||
|
||
rpm-ja4ebpf:
|
||
# Build multi-distro : el8 (AlmaLinux 8) + el9 (Rocky 9) + el10 (AlmaLinux 10)
|
||
# Sortie : services/ja4ebpf/dist/el{8,9,10}/ja4ebpf-*.rpm
|
||
docker build \
|
||
-f services/ja4ebpf/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/ja4ebpf/dist \
|
||
--build-arg BUILD_VERSION=$(VERSION) \
|
||
.
|
||
@echo ""
|
||
@echo "RPMs produits :"
|
||
@find services/ja4ebpf/dist -name '*.rpm' | sort | sed 's/^/ /'
|
||
|
||
# ── bot-detector ─────────────────────────────────────────────────────────────
|
||
|
||
build-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile \
|
||
-t ja4-platform/bot-detector:latest \
|
||
.
|
||
|
||
test-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile.tests \
|
||
-t ja4-platform/bot-detector-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/bot-detector-tests:latest
|
||
|
||
# ── dashboard ────────────────────────────────────────────────────────────────
|
||
|
||
build-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile \
|
||
-t ja4-platform/dashboard:latest \
|
||
.
|
||
|
||
test-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile.tests \
|
||
-t ja4-platform/dashboard-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/dashboard-tests:latest
|
||
|
||
# ── Tests d'intégration par stack ────────────────────────────────────────────
|
||
|
||
test-all-stacks: ## Toutes les stacks sur la distro par défaut (Rocky Linux 9)
|
||
cd tests/integration && bash run-all-stacks.sh
|
||
|
||
test-apache:
|
||
cd tests/integration && bash apache/run-tests.sh
|
||
|
||
test-nginx:
|
||
cd tests/integration && bash nginx/run-tests.sh
|
||
|
||
test-nginx-varnish:
|
||
cd tests/integration && bash nginx-varnish/run-tests.sh
|
||
|
||
test-hitch-varnish:
|
||
cd tests/integration && bash hitch-varnish/run-tests.sh
|
||
|
||
# ── Tests VM (kernel réel, eBPF complet) ─────────────────────────────────────
|
||
|
||
# Répertoire Vagrantfile
|
||
VM_DIR := tests/vm
|
||
VMS := centos8 rocky9 rocky10
|
||
STACKS := nginx apache hitch-varnish
|
||
|
||
vm-up: ## Créer la VM Rocky Linux 9 pour les tests eBPF
|
||
cd $(VM_DIR) && vagrant up rocky9
|
||
|
||
vm-up-all: ## Créer les 3 VMs (centos8, rocky9, rocky10)
|
||
cd $(VM_DIR) && vagrant up centos8 rocky9 rocky10
|
||
|
||
vm-down: ## Détruire la VM
|
||
cd $(VM_DIR) && vagrant destroy -f
|
||
|
||
vm-down-all: ## Détruire toutes les VMs
|
||
cd $(VM_DIR) && vagrant destroy -f
|
||
|
||
vm-ssh: ## Connexion SSH à la VM Rocky 9
|
||
cd $(VM_DIR) && vagrant ssh rocky9
|
||
|
||
vm-rebuild-ja4ebpf: ## Recompiler ja4ebpf dans la VM Rocky 9 (après modifications)
|
||
cd $(VM_DIR) && vagrant rsync rocky9 && vagrant ssh rocky9 -- \
|
||
'export PATH=/usr/local/go/bin:$$PATH && \
|
||
cd /ja4-platform/services/ja4ebpf && \
|
||
GOWORK=off go generate ./internal/loader/ && \
|
||
GOWORK=off CGO_ENABLED=0 go build -o /tmp/ja4ebpf ./cmd/ja4ebpf/ && \
|
||
sudo mv /tmp/ja4ebpf /usr/local/bin/ja4ebpf && \
|
||
echo "ja4ebpf rebuilt OK"'
|
||
|
||
# ── Tests VM : cibles par stack ──────────────────────────────────────────────
|
||
|
||
test-vm-nginx: ## Test nginx dans la VM Rocky 9 (trafic host → VM)
|
||
bash tests/vm/run-test-from-host.sh rocky9 nginx
|
||
|
||
test-vm-apache: ## Test apache dans la VM Rocky 9
|
||
bash tests/vm/run-test-from-host.sh rocky9 apache
|
||
|
||
test-vm-hitch-varnish: ## Test hitch+varnish dans la VM Rocky 9
|
||
bash tests/vm/run-test-from-host.sh rocky9 hitch-varnish
|
||
|
||
test-vm-all: ## Tous les tests (3 stacks) dans la VM Rocky 9
|
||
@for stack in $(STACKS); do \
|
||
bash tests/vm/run-test-from-host.sh rocky9 $$stack || true; \
|
||
done
|
||
|
||
# ── Tests VM : cibles par distro ─────────────────────────────────────────────
|
||
|
||
test-vm-centos8: ## Test nginx dans la VM CentOS 8
|
||
bash tests/vm/run-test-from-host.sh centos8 nginx
|
||
|
||
test-vm-rocky10: ## Test nginx dans la VM Rocky 10
|
||
bash tests/vm/run-test-from-host.sh rocky10 nginx
|
||
|
||
# ── Matrice complète : toutes stacks × toutes distros ────────────────────────
|
||
|
||
test-vm-matrix: ## Toutes stacks × toutes VMs (nginx/apache/hitch-varnish sur centos8/rocky9/rocky10)
|
||
@echo "╔══════════════════════════════════════════════╗"
|
||
@echo "║ Matrice VM : 3 stacks × 3 distros ║"
|
||
@echo "╚══════════════════════════════════════════════╝"
|
||
@TOTAL_FAIL=0; \
|
||
for vm in $(VMS); do \
|
||
for stack in $(STACKS); do \
|
||
bash tests/vm/run-test-from-host.sh $$vm $$stack || TOTAL_FAIL=$$((TOTAL_FAIL + 1)); \
|
||
done; \
|
||
done; \
|
||
echo ""; \
|
||
if [ "$$TOTAL_FAIL" -eq 0 ]; then \
|
||
echo "=== Matrice complète : SUCCÈS ==="; \
|
||
else \
|
||
echo "=== Matrice : $$TOTAL_FAIL combinaisons échouées ==="; \
|
||
exit 1; \
|
||
fi
|
||
done; \
|
||
echo ""; \
|
||
if [ "$$TOTAL_FAIL" -eq 0 ]; then \
|
||
echo "=== Matrice complète : SUCCÈS ==="; \
|
||
else \
|
||
echo "=== Matrice : $$TOTAL_FAIL combinaisons échouées ==="; \
|
||
exit 1; \
|
||
fi
|
||
|
||
test-vm-all-distros: ## Tests unitaires Go sur les 3 VMs (centos8 + rocky9 + rocky10)
|
||
@echo "=== Tests unitaires multi-distro ==="
|
||
@for vm in $(VMS); do \
|
||
echo ""; \
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
|
||
echo " VM: $$vm"; \
|
||
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"; \
|
||
cd $(CURDIR)/$(VM_DIR) && vagrant rsync $$vm && vagrant ssh $$vm -- \
|
||
'export PATH=/usr/local/go/bin:$$PATH && \
|
||
cd /ja4-platform/services/ja4ebpf && \
|
||
GOWORK=off go generate ./internal/loader/ 2>&1 | tail -2 && \
|
||
GOWORK=off CGO_ENABLED=0 go test ./... 2>&1 | tail -20'; \
|
||
echo ""; \
|
||
done
|
||
@echo "=== Tous les tests multi-distro terminés ==="
|
||
|
||
vm-reprovision: ## Re-provisionner les 3 VMs (installer nouveaux paquets)
|
||
@for vm in $(VMS); do \
|
||
echo "Re-provision $$vm..."; \
|
||
cd $(CURDIR)/$(VM_DIR) && vagrant rsync $$vm && vagrant provision $$vm; \
|
||
done
|
||
|
||
# ── Matrice multi-distro ─────────────────────────────────────────────────────
|
||
|
||
test-matrix: ## Toutes stacks × el8 + el9 + el10
|
||
cd tests/integration && bash run-distro-matrix.sh \
|
||
$${MATRIX_STACKS:+--stacks=$${MATRIX_STACKS}} \
|
||
$${MATRIX_DISTROS:+--distros=$${MATRIX_DISTROS}}
|
||
|
||
# ── Base de données ───────────────────────────────────────────────────────────
|
||
|
||
reload-prod-logs:
|
||
./scripts/reload-prod-logs.sh
|
||
|
||
init-stack:
|
||
./scripts/init-stack.sh
|
||
|
||
import-prod-data:
|
||
./scripts/import-prod-data.sh
|
||
|
||
init-and-import:
|
||
./scripts/init-stack.sh --import-prod
|
||
|
||
purge-db:
|
||
./scripts/init-stack.sh --reset
|