code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(ja4ebpf): split bpf2go generate into Ja4Tc + Ja4Ssl, fix RPM systemd-rpm-macros

Browse Source 
- Use two separate //go:generate directives (Ja4Tc for tc_capture.c, Ja4Ssl
  for uprobe_ssl.c) to avoid duplicate LICENSE symbol and multi-file clang issue
- Update loader.go to hold tcObjs/sslObjs separately with correct field names:
  UprobeSslSetFd, UprobeSslReadEntry, UretprobeSslReadExit,
  KprobeAccept4Entry, KretprobeAccept4Exit
- Add systemd-rpm-macros to all three RPM build stages (el8/el9/el10)
  so that %{_unitdir} macro resolves correctly
- RPMs now build successfully for el8, el9, el10

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-11 23:21:11 +02:00
parent a1e4c1dad5
commit 3b047b680a
155 changed files with 197011 additions and 599 deletions
Download Patch File Download Diff File Expand all files Collapse all files

182
Makefile
View File

@ -1,17 +1,19 @@
# =============================================================================
# ja4-platform — Monorepo Makefile
# All targets use new service names:
# sentinel, correlator, bot-detector, dashboard, mod-reqin-log
# Service de capture : ja4ebpf (eBPF CO-RE)
# =============================================================================
.PHONY: help build-all test-all rpm-all dist \
build-sentinel test-sentinel rpm-sentinel \
test-integration test-integration-keep test-integration-down \
test-mod-reqin-log rpm-mod-reqin-log \
build-correlator test-correlator rpm-correlator \
VERSION ?= $(shell git describe --tags --always 2>/dev/null || echo dev)
.PHONY: help \
build-all test-all rpm-all dist \
build-ja4ebpf test-ja4ebpf rpm-ja4ebpf \
build-bot-detector test-bot-detector \
build-dashboard test-dashboard \
test-ja4common-python \
test-all-stacks test-nginx test-nginx-varnish test-hitch-varnish test-apache \
test-matrix \
test-integration test-integration-keep test-integration-down \
reload-prod-logs init-stack import-prod-data init-and-import \
purge-db
@ -24,30 +26,31 @@ help: ## Affiche cette aide
@echo ""
@echo " Build"
@echo " make build-all Construit toutes les images Docker"
@echo " make build-sentinel Image sentinel (capture TLS/TCP)"
@echo " make build-correlator Image correlator (corrélation logs)"
@echo " make build-ja4ebpf Image ja4ebpf (agent eBPF CO-RE)"
@echo " make build-bot-detector Image bot-detector (détection ML)"
@echo " make build-dashboard Image dashboard (SOC UI)"
@echo ""
@echo " Tests"
@echo " Tests unitaires"
@echo " make test-all Lance tous les tests unitaires"
@echo " make test-sentinel Tests Go sentinel (NET_RAW)"
@echo " make test-correlator Tests Go correlator (80% coverage)"
@echo " make test-bot-detector Tests Python bot-detector (36 tests)"
@echo " make test-ja4ebpf Tests Go ja4ebpf"
@echo " make test-bot-detector Tests Python bot-detector"
@echo " make test-dashboard Tests Python dashboard"
@echo " make test-ja4common-python Tests Python ja4_common"
@echo " make test-mod-reqin-log Tests C mod-reqin-log (cmocka)"
@echo ""
@echo " Intégration"
@echo " make test-integration Tests full-stack (Docker Compose)"
@echo " make test-integration-keep Idem, stack reste active après"
@echo " make test-integration-down Arrête la stack d'intégration"
@echo " Tests d'intégration (par stack)"
@echo " make test-all-stacks Toutes les stacks sur Rocky Linux 9"
@echo " make test-apache Stack Apache + ja4ebpf"
@echo " make test-nginx Stack nginx + ja4ebpf"
@echo " make test-nginx-varnish Stack nginx + Varnish + ja4ebpf"
@echo " make test-hitch-varnish Stack hitch + Varnish + ja4ebpf"
@echo ""
@echo " Matrice multi-distro"
@echo " make test-matrix Toutes stacks × el8/el9/el10"
@echo " make test-matrix MATRIX_STACKS=nginx,apache MATRIX_DISTROS=el9,el10"
@echo ""
@echo " RPM"
@echo " make rpm-all Construit tous les RPMs (el8/el9/el10)"
@echo " make rpm-sentinel RPM sentinel"
@echo " make rpm-correlator RPM correlator"
@echo " make rpm-mod-reqin-log RPM mod-reqin-log"
@echo " make rpm-all Construit tous les RPMs ja4ebpf (el8/el9/el10)"
@echo " make rpm-ja4ebpf RPMs ja4ebpf (el8, el9, el10)"
@echo " make dist Alias de rpm-all"
@echo ""
@echo " Base de données"
@ -58,84 +61,50 @@ help: ## Affiche cette aide
@echo " make purge-db Supprime et recrée les bases ja4_*"
@echo ""
# --- Root -------------------------------------------------------------------
# ── Cibles agrégées ──────────────────────────────────────────────────────────
build-all: build-sentinel build-correlator build-bot-detector build-dashboard
build-all: build-ja4ebpf build-bot-detector build-dashboard
@echo "All services built."
test-all: test-sentinel test-correlator test-bot-detector test-dashboard test-ja4common-python
@echo "All tests completed."
test-all: test-ja4ebpf test-bot-detector test-dashboard test-ja4common-python
@echo "All unit tests completed."
rpm-all: rpm-sentinel rpm-correlator rpm-mod-reqin-log
rpm-all: rpm-ja4ebpf
@echo "All RPMs built."
dist: rpm-all
@echo "Distribution packages ready in services/*/dist/"
@echo "RPMs disponibles dans services/ja4ebpf/dist/"
# --- sentinel (was ja4sentinel) ---------------------------------------------
# ── ja4ebpf (agent eBPF CO-RE) ───────────────────────────────────────────────
build-sentinel:
build-ja4ebpf:
docker build \
--build-arg VERSION=$$(git -C services/sentinel describe --tags --always 2>/dev/null || echo dev) \
--build-arg GIT_COMMIT=$$(git rev-parse --short HEAD 2>/dev/null || echo unknown) \
--build-arg BUILD_TIME=$$(date -u +%Y-%m-%dT%H:%M:%SZ) \
-f services/sentinel/Dockerfile \
-t ja4-platform/sentinel:latest \
-f services/ja4ebpf/Dockerfile \
--build-arg BUILD_VERSION=$(VERSION) \
-t ja4-platform/ja4ebpf:latest \
.
test-sentinel:
# Tests run inside Docker — no native Go required on the host
docker build -f services/sentinel/Dockerfile.dev -t ja4-platform/sentinel-tests:latest .
docker run --rm --cap-add=NET_RAW --cap-add=NET_ADMIN ja4-platform/sentinel-tests:latest
rpm-sentinel:
# Méthode: Dockerfile.package → builder Go → rpm-builder (rpmbuild ×3) → output alpine
test-ja4ebpf:
docker build \
-f services/sentinel/Dockerfile.package \
-f services/ja4ebpf/Dockerfile.tests \
-t ja4-platform/ja4ebpf-tests:latest \
.
docker run --rm ja4-platform/ja4ebpf-tests:latest
rpm-ja4ebpf:
# Build multi-distro : el8 (AlmaLinux 8) + el9 (Rocky 9) + el10 (AlmaLinux 10)
# Sortie : services/ja4ebpf/dist/el{8,9,10}/ja4ebpf-*.rpm
docker build \
-f services/ja4ebpf/Dockerfile.package \
--target output \
--output type=local,dest=services/sentinel/dist \
--build-arg VERSION=$(shell git -C services/sentinel describe --tags --always 2>/dev/null || echo dev) \
--output type=local,dest=services/ja4ebpf/dist \
--build-arg BUILD_VERSION=$(VERSION) \
.
@echo "📦 RPMs sentinel dans services/sentinel/dist/"
@echo ""
@echo "RPMs produits :"
@find services/ja4ebpf/dist -name '*.rpm' | sort | sed 's/^/ /'
# --- mod-reqin-log (was mod_reqin_log) --------------------------------------
test-mod-reqin-log:
docker build -f services/mod-reqin-log/Dockerfile.tests -t ja4-platform/mod-reqin-log-tests:latest .
docker run --rm ja4-platform/mod-reqin-log-tests:latest
rpm-mod-reqin-log:
# Méthode: Dockerfile.package → builder C (×3 distros) → rpm-builder (rpmbuild ×3) → output alpine
docker build \
-f services/mod-reqin-log/Dockerfile.package \
--target output \
--output type=local,dest=services/mod-reqin-log/dist \
.
@echo "📦 RPMs mod-reqin-log dans services/mod-reqin-log/dist/"
# --- correlator (was logcorrelator) -----------------------------------------
build-correlator:
docker build \
-f services/correlator/Dockerfile \
-t ja4-platform/correlator:latest \
.
test-correlator:
# Tests run inside the Dockerfile builder stage (80% coverage gate enforced)
docker build --target builder -f services/correlator/Dockerfile -t ja4-platform/correlator-tests:latest .
rpm-correlator:
# Méthode: Dockerfile.package → builder Go → rpm-builder (rpmbuild ×3) → output alpine
docker build \
-f services/correlator/Dockerfile.package \
--target output \
--output type=local,dest=services/correlator/dist \
--build-arg VERSION=$(shell git -C services/correlator describe --tags --always 2>/dev/null || echo dev) \
.
@echo "📦 RPMs correlator dans services/correlator/dist/"
# --- bot-detector (was bot_detector) ----------------------------------------
# ── bot-detector ─────────────────────────────────────────────────────────────
build-bot-detector:
docker build \
@ -150,7 +119,7 @@ test-bot-detector:
.
docker run --rm ja4-platform/bot-detector-tests:latest
# --- dashboard --------------------------------------------------------------
# ── dashboard ────────────────────────────────────────────────────────────────
build-dashboard:
docker build \
@ -165,7 +134,7 @@ test-dashboard:
.
docker run --rm ja4-platform/dashboard-tests:latest
# --- shared/python/ja4_common -----------------------------------------------
# ── shared/python/ja4_common ─────────────────────────────────────────────────
test-ja4common-python:
docker build \
@ -174,18 +143,47 @@ test-ja4common-python:
shared/python/ja4_common/
docker run --rm ja4-platform/ja4common-python-tests:latest
# --- integration (full-stack) -----------------------------------------------
# ── Tests d'intégration par stack ────────────────────────────────────────────
test-integration:
cd tests/integration && ./run-tests.sh
test-all-stacks: ## Toutes les stacks sur la distro par défaut (Rocky Linux 9)
cd tests/integration && bash run-all-stacks.sh
test-apache:
cd tests/integration && bash apache/run-tests.sh
test-nginx:
cd tests/integration && bash nginx/run-tests.sh
test-nginx-varnish:
cd tests/integration && bash nginx-varnish/run-tests.sh
test-hitch-varnish:
cd tests/integration && bash hitch-varnish/run-tests.sh
# ── Matrice multi-distro ─────────────────────────────────────────────────────
test-matrix: ## Toutes stacks × el8 + el9 + el10
cd tests/integration && bash run-distro-matrix.sh \
$${MATRIX_STACKS:+--stacks=$${MATRIX_STACKS}} \
$${MATRIX_DISTROS:+--distros=$${MATRIX_DISTROS}}
# ── Compat : anciens targets d'intégration ───────────────────────────────────
test-integration: ## Ancien target — alias vers test-all-stacks
$(MAKE) test-all-stacks
test-integration-keep:
cd tests/integration && ./run-tests.sh --no-down
cd tests/integration && bash run-all-stacks.sh --no-down
test-integration-down:
cd tests/integration && docker compose down -v --remove-orphans
cd tests/integration && \
for stack in apache nginx nginx-varnish hitch-varnish; do \
[ -f "$$stack/docker-compose.yml" ] && \
docker compose -f "$$stack/docker-compose.yml" down -v --remove-orphans 2>/dev/null || true; \
done
# ── Base de données ───────────────────────────────────────────────────────────
# ── Dev data ─────────────────────────────────────────────────────────────────
reload-prod-logs:
./scripts/reload-prod-logs.sh

219
README.md
View File

@ -1,169 +1,166 @@
# ja4-platform
**ja4-platform** is a monorepo security pipeline for TLS fingerprinting (JA4/JA3) and bot detection. It captures live network traffic, correlates TLS handshakes with HTTP requests, applies triple-voice ML anomaly detection (Extended Isolation Forest + Autoencoder + XGBoost), and surfaces results through a SOC analyst dashboard — all backed by ClickHouse with a dual-database architecture.
**ja4-platform** est un pipeline de sécurité monorepo pour le fingerprinting TLS (JA4/JA3) et la détection de bots HTTP. Un agent unique basé sur eBPF observe passivement le trafic réseau de manière non-intrusive, reconstruit les sessions TCP/TLS/HTTP en mémoire, et alimente une base ClickHouse pour la détection d'anomalies par apprentissage automatique et la présentation dans un tableau de bord SOC.
## Pipeline Overview
```
┌──────────────────────────────────────────────────────────────────────────────┐
│ Linux Server (Apache) │
│ │
│ ┌─────────────────┐ UNIX socket (DGRAM) ┌──────────────────┐ │
│ │ mod-reqin-log │──── http.socket ────────────────▶│ │ │
│ │ (Apache C11) │ (source A) │ correlator │ │
│ └─────────────────┘ │ (Go · hex. │ │
│ │ architecture) │ │
│ ┌─────────────────┐ UNIX socket (DGRAM) │ │ │
│ │ sentinel │──── network.socket ─────────────▶│ Joins by │ │
│ │ (Go · libpcap) │ (source B) │ src_ip:src_port│ │
│ │ JA4/JA3 gen. │ └────────┬─────────┘ │
│ └─────────────────┘ │ │
└──────────────────────────────────────────────────────────────────┼────────────┘
│ INSERT
▼
┌──────────────────────────────────────┐
│ ClickHouse 24.8 │
│ │
│ ja4_logs ja4_processing │
│ ┌──────────┐ ┌──────────────┐ │
│ │_raw → MV │────▶│ agg_* (×6) │ │
│ │→ http_logs│ │ ml_* (×2) │ │
│ └──────────┘ │ views, dicts │ │
│ └──────────────┘ │
└─────────┬───────────────┬────────────┘
│ │
┌────────────────┘ └───────────────┐
▼ ▼
┌────────────────────┐ ┌────────────────────┐
│ bot-detector │ │ dashboard │
│ Python 3.11 │ │ FastAPI + Jinja2 │
│ EIF + AE + XGBoost │ │ htmx + Chart.js │
│ HDBSCAN · SHAP │ │ 55 routes · 14 pp │
└────────────────────┘ └────────────────────┘
+-----------------------------------------------------------------------+
| Linux Server (Apache / Nginx / Varnish / Hitch) |
| |
| +-------------------------------------------------------------+ |
| | ja4ebpf (agent eBPF GO) | |
| | | |
| | Hook TC ingress (L3/L4/L5 - passif) : | |
| reseau| SYN -> TTL, DF, IP-ID, MSS, Window, Scale | |
| ----->| TLS -> ClientHello : JA4, ALPN, SNI, extensions | |
| XDP/TC| HTTP -> payload port 80/8080 (magic bytes router) | |
| | | |
| uprobe| Hook SSL_read (L7 - trafic dechiffre) : | |
| ----->| HTTP/1.1 -> methode, path, headers (ordre exact) | |
| | HTTP/2 -> SETTINGS, WINDOW_UPDATE, pseudo-headers | |
| | | |
| | Correlation in-memory src_ip:src_port | |
| | 256 shards . timeout 500ms . GC 100ms | |
| +----------------------------+---------------------------------+ |
+-----------------------------------------------------------------------+
| INSERT batch (Native TCP :9000)
v
+------------------------------------------+
| ClickHouse 24.8 |
| |
| ja4_logs ja4_processing |
| +-----------+ +--------------+ |
| |_raw -> MV |------->| agg_* (x6) | |
| |-> http_logs | ml_* (x2) | |
| +-----------+ | views, dicts | |
+------+---------------------------+----+
| |
+--------+ +----------+
v v
+--------------------+ +--------------------+
| bot-detector | | dashboard |
| Python 3.11 | | FastAPI + Jinja2 |
| EIF + AE + XGBoost| | htmx + Chart.js |
| HDBSCAN . SHAP | | SOC analyst UI |
+--------------------+ +--------------------+
```
## Services
| Service | Language | Description | Interface |
|---------|----------|-------------|-----------|
| [sentinel](docs/services/sentinel.md) | Go 1.24.6 | TLS/TCP packet capture via libpcap, JA4/JA3 fingerprint generation | UNIX socket → `network.socket` |
| [mod-reqin-log](docs/services/mod-reqin-log.md) | C11 | Apache HTTPD module, HTTP request JSON logging | UNIX socket → `http.socket` |
| [correlator](docs/services/correlator.md) | Go 1.24.6 | Hexagonal architecture, correlates HTTP+TLS events by `src_ip:src_port` | ClickHouse INSERT (Native TCP) |
| [bot-detector](docs/services/bot-detector.md) | Python 3.11 | Triple-voice ML ensemble (EIF+AE+XGB), HDBSCAN campaigns, SHAP explainability | ClickHouse read/write, HTTP `:8080` |
| [dashboard](docs/services/dashboard.md) | Python 3.11 | SOC analyst dashboard: 55 routes, 15 templates, 14 pages | HTTP `:8000` |
| Service | Langage | Description | Interface |
|---------|---------|-------------|-----------|
| [ja4ebpf](docs/services/ja4ebpf.md) | Go 1.24 + C eBPF (CO-RE) | Agent eBPF passif : TC ingress (L3/L4/L5), uprobe SSL_read (L7 HTTPS), TC port 80/8080 (HTTP clair), corrélation in-memory, insert ClickHouse | INSERT batch ClickHouse |
| [bot-detector](docs/services/bot-detector.md) | Python 3.11 | Ensemble ML triple-voix (EIF+AE+XGB), clustering HDBSCAN, explicabilité SHAP | ClickHouse read/write, HTTP `:8080` |
| [dashboard](docs/services/dashboard.md) | Python 3.11 | Tableau de bord SOC : 9 endpoints JSON, 8 pages HTML | HTTP `:8000` |
## Shared Libraries
## Bibliothèques partagées
| Library | Language | Description |
|---------|----------|-------------|
| Bibliothèque | Langage | Description |
|--------------|---------|-------------|
| [go/ja4common](docs/shared/go-ja4common.md) | Go | Logger, config loader, graceful shutdown handler, IP filter |
| [python/ja4_common](docs/shared/python-ja4common.md) | Python | `ClickHouseClient` singleton, `ClickHouseSettings` (pydantic-settings) |
## Quickstart
### Prerequisites
### Prérequis
- Docker (with BuildKit) and Docker Compose
- Docker (avec BuildKit) et Docker Compose
- `make`
- No native Go, Python, or C toolchains required — all builds run inside Docker
- Aucune toolchain Go, Python, C ou eBPF n'est requise sur la machine hôte — tous les builds s'exécutent dans Docker Rocky Linux.
### Build All Services
### Build de tous les services
```bash
make build-all
```
### Run All Tests
### Exécution de tous les tests
```bash
make test-all
```
### Build RPM Packages
### Build des paquets RPM
```bash
make rpm-all
# RPMs written to services/<service>/dist/rpm/el{8,9,10}/
make rpm-ja4ebpf
# RPMs écrits dans services/ja4ebpf/dist/rpm/el{8,9,10}/
```
## Tests d'intégration
Tests full-stack avec Docker Compose et une vraie instance ClickHouse :
```bash
make test-integration # stack Apache référence (8 phases)
make test-nginx # stack nginx + ja4ebpf
make test-nginx-varnish # nginx + Varnish + ja4ebpf
make test-hitch-varnish # hitch (TLS) + Varnish + ja4ebpf
make test-all-stacks # les 3 stacks serveur en séquence
make test-integration-keep # laisse la stack en fonctionnement
make test-integration-down # démontage de la stack
```
La suite de tests se trouve dans `tests/integration/` et réinitialise la base entre chaque exécution.
## Scripts
Helper scripts are located in `scripts/`:
| Script | Description |
|--------|-------------|
| `init-stack.sh` | Full ClickHouse stack initialization — deploys schema, loads CSV data, verifies all components |
| `import-prod-data.sh` | Imports pre-exported production data into the dev database with dynamic date shifting |
| `reload-prod-logs.sh` | Exports `http_logs` from production and re-imports into the dev database |
| `update-csv-data.sh` | Downloads and generates all CSV reference data (bot IPs, JA4 signatures, ASN reputation) |
| `generate_bot_ip.py` | Generates `bot_ip.csv` from known scanner/bot sources + Tor exit nodes |
| `generate_bot_ja4.py` | Generates `bot_ja4.csv` from known bot TLS fingerprints |
| `generate_asn_data.py` | Generates `asn_reputation.csv` (ASN→label mapping) |
| `generate_browser_ja4.py` | Generates browser JA4 reference data for legitimate browser detection |
| `init-stack.sh` | Initialisation complète du schéma ClickHouse + chargement CSV |
| `import-prod-data.sh` | Import de données de production avec décalage temporel |
| `reload-prod-logs.sh` | Export prod -> réimport dev avec décalage |
| `update-csv-data.sh` | Génération des CSV de référence (bot IPs, signatures JA4, ASN) |
Corresponding Makefile targets:
Cibles Makefile correspondantes :
```bash
make init-stack # runs scripts/init-stack.sh
make import-prod-data # runs scripts/import-prod-data.sh
make init-stack # schéma + CSV
make import-prod-data # données prod
make init-and-import # init-stack + import-prod-data
make reload-prod-logs # runs scripts/reload-prod-logs.sh
make reload-prod-logs # rechargement depuis la prod
```
## Integration Tests
Full-stack integration tests run against Docker Compose with a real ClickHouse instance:
```bash
make test-integration # 8 phases: build → start → schema → traffic → pipeline → dashboard → bot-detector → sentinel
make test-integration-keep # same but leaves stack running after
make test-integration-down # tear down integration stack
```
The integration test suite is located in `tests/integration/` and resets the database between runs.
## Documentation
| Document | Description |
|----------|-------------|
| [Architecture](docs/architecture.md) | System architecture, data flow, component interactions |
| [Deployment](docs/deployment.md) | Step-by-step production deployment guide |
| [Development](docs/development.md) | Build, test, package, and extend the platform |
| [Database Schema](docs/database/schema.md) | Every ClickHouse table, view, dictionary, and materialized view |
| [Database Migrations](docs/database/migrations.md) | Migration order, application, verification, and rollback |
| [Commenting Standard](docs/commenting-standard.md) | Code commenting conventions (French comments, English identifiers) |
| [Thesis Reference](docs/THESIS_HTTP_Traffic_Detection.md) | Academic reference: HTTP traffic detection techniques |
| [Audit vs Thesis](docs/AUDIT_Detection_vs_Thesis.md) | Comparison between platform implementation and thesis techniques |
### Service Documentation
- [Sentinel](docs/services/sentinel.md) — TLS/TCP capture daemon (Go + libpcap)
- [mod-reqin-log](docs/services/mod-reqin-log.md) — Apache HTTP logging module (C11)
- [Correlator](docs/services/correlator.md) — HTTP/TLS event correlation engine (Go)
- [Bot Detector](docs/services/bot-detector.md) — Triple-voice ML anomaly detection (Python)
- [Dashboard](docs/services/dashboard.md) — SOC analyst dashboard and API (FastAPI)
### Shared Library Documentation
- [go-ja4common](docs/shared/go-ja4common.md) — Go shared library (logger, config, shutdown, ipfilter)
- [python-ja4common](docs/shared/python-ja4common.md) — Python shared library (ClickHouse client, settings)
## Go Workspace
The repository uses a Go workspace (`go.work`) to link the Go modules:
Le workspace `go.work` lie les modules Go du dépôt :
```
go 1.24.6
use (
./services/sentinel
./services/correlator
./shared/go/ja4common
./services/ja4ebpf
)
```
Both Go services have a `replace` directive in their `go.mod` pointing to `../../shared/go/ja4common`. The workspace takes precedence for local development; the `replace` is needed for Docker builds where `go.work` is not available.
`ja4ebpf` utilise une directive `replace` dans son `go.mod` vers `../../shared/go/ja4common`. Le workspace prend priorité en développement local ; la directive `replace` est nécessaire pour les builds Docker.
## Documentation
| Document | Description |
|----------|-------------|
| [Architecture](docs/architecture.md) | Architecture système, flux de données, interactions entre composants |
| [Deployment](docs/deployment.md) | Guide de déploiement en production |
| [Development](docs/development.md) | Build, test, packaging et extension de la plateforme |
| [Database Schema](docs/database/schema.md) | Tables, vues, dictionnaires et vues matérialisées ClickHouse |
| [Database Migrations](docs/database/migrations.md) | Ordre de migration, application, vérification et rollback |
| [Commenting Standard](docs/commenting-standard.md) | Conventions de commentaires (commentaires français, identifiants anglais) |
| [Thesis Reference](docs/THESIS_HTTP_Traffic_Detection.md) | Référence académique : techniques de détection du trafic HTTP |
| [Audit vs Thesis](docs/AUDIT_Detection_vs_Thesis.md) | Comparaison entre l'implémentation et les techniques de la thèse |
### Documentation des services
- [ja4ebpf](docs/services/ja4ebpf.md) — Agent eBPF CO-RE (Go + C), capture réseau passive multi-couches
- [Bot Detector](docs/services/bot-detector.md) — Détection ML d'anomalies triple-voix (Python)
- [Dashboard](docs/services/dashboard.md) — Tableau de bord SOC et API (FastAPI)
### Documentation des bibliothèques partagées
- [go-ja4common](docs/shared/go-ja4common.md) — Bibliothèque Go partagée (logger, config, shutdown, ipfilter)
- [python-ja4common](docs/shared/python-ja4common.md) — Bibliothèque Python partagée (client ClickHouse, settings)
## License
See individual service directories for license information.
Voir les répertoires des services individuels pour les informations de licence.

16
docs/AUDIT_Detection_vs_Thesis.md
View File

@ -23,9 +23,9 @@
| Composant thèse | Statut | Détail |
|-----------------|--------|--------|
| Pipeline L3-L5 (ja4sentinel) | ✅ | TTL, IP-ID, DF, TCP win/mss/scale, JA4/JA3, ALPN, SNI |
| Pipeline L7 (mod_reqin_log) | ✅ | Headers, méthode, path, query, timestamps ns |
| Corrélation (logcorrelator) | ✅ | Clé `src_ip:src_port`, Keep-Alive, orphelins |
| Pipeline L3-L5 (ja4ebpf TC ingress) | ✅ | TTL, IP-ID, DF, TCP win/mss/scale, JA4/JA3, ALPN, SNI via hooks TC + kprobe accept4 |
| Pipeline L7 (ja4ebpf uprobe SSL_read) | ✅ | Headers, méthode, path, query, timestamps ns — HTTP/1.1 et HTTP/2 via uprobe SSL_read |
| Corrélation (ja4ebpf in-memory) | ✅ | Clé `src_ip:src_port`, 256 shards, Keep-Alive, orphelins, Slowloris 10s |
| Enrichissement ASN | ✅ | `dict_iplocate_asn` (714K CIDRs, 4 colonnes) |
| Enrichissement Anubis | ✅ | Simplifié à `COALESCE(IP, ASN)` — 2 dictionnaires (`dict_anubis_ip` IP_TRIE, `dict_anubis_asn`) |
| Agrégation 1h | ✅ | 6 tables : `agg_host_ip_ja4_1h`, `agg_header_fingerprint_1h`, `agg_ip_behavior_1h`, `agg_request_timing_1h`, `agg_path_sequences_1h`, `agg_resource_cascade_1h` |
@ -132,7 +132,7 @@
| §5.3 Request Cadence Fingerprint | ✅ | `cadence_cv`, `burst_ratio`, `lag1_autocorrelation`, `benford_deviation`, `pause_ratio` dans `agg_request_timing_1h` |
| §5.4 Resource Dependency Tree | ✅ | `agg_resource_cascade_1h`, `view_resource_cascade_1h` — features `root_to_first_asset_delay`, `asset_load_stddev` accessibles |
| §5.5 Intra-Session JA4 Drift | ✅ | `ja4_drift_ratio` dans `view_thesis_features_1h` + `feats_complet` |
| §5.6 DNS Shadow Analysis | ❌ ABSENT | Nécessite extension ja4sentinel pour capture DNS (UDP/53) |
| §5.6 DNS Shadow Analysis | ❌ ABSENT | Nécessite extension ja4ebpf pour capture DNS (UDP/53) |
| §5.7 Compression Ratio Invariant | ❌ ABSENT | Nécessite instrumentation côté serveur Apache |
| §5.8 Cross-Domain Session Linking | ✅ | `host_diversity`, `host_sweep_speed`, `host_coverage_uniformity` + `cross_domain_path_similarity` (Jaccard) dans `view_thesis_features_1h` |
@ -293,7 +293,7 @@ Le module `api.py` expose **37 endpoints JSON** couvrant l'ensemble des besoins
| Priorité | Gap | Impact | Remarque |
|----------|-----|--------|----------|
| P2 🟡 | §5.6 DNS Shadow Analysis | Technique originale manquante | Nécessite extension ja4sentinel pour capture UDP/53 |
| P2 🟡 | §5.6 DNS Shadow Analysis | Technique originale manquante | Nécessite extension ja4ebpf pour capture UDP/53 |
| P2 🟡 | §5.7 Compression Ratio Invariant | Technique originale manquante | Nécessite instrumentation côté serveur Apache |
| P3 ⚪ | Authentification dashboard | Sécurité opérationnelle | Non exigé par la thèse — environnement SOC intranet |
| P3 ⚪ | CSRF sur `/api/classify` | Sécurité opérationnelle | Mitigé en déploiement restreint |
@ -319,11 +319,11 @@ Le module `api.py` expose **37 endpoints JSON** couvrant l'ensemble des besoins
- **`deploy_schema.sh`** : déploiement automatisé avec substitution des noms de bases depuis les variables d'environnement (`CLICKHOUSE_DB_LOGS`, `CLICKHOUSE_DB_PROCESSING`)
- **Dual database** : `ja4_logs` (logs bruts, enrichis, MV) + `ja4_processing` (agrégations, ML, vues, dictionnaires, audit)
- **7 dictionnaires** : `dict_iplocate_asn`, `dict_bot_ip`, `dict_bot_ja4`, `dict_browser_ja4`, `dict_asn_reputation`, `dict_anubis_ip`, `dict_anubis_asn`
- **Migrations post-déploiement** : `services/correlator/sql/migrations/` (ALTER TABLE pour déploiements existants)
- **Migrations post-déploiement** : `shared/clickhouse/` (ALTER TABLE pour déploiements existants)
### E3. Pipeline de build et tests
- **Docker-first** : chaque service dispose de `Dockerfile` (prod), `Dockerfile.dev` ou `Dockerfile.tests` (tests), et `Dockerfile.package` (RPM) pour les services Go/C
- **Tests d'intégration** : suite complète en 8 phases (build → start → schema → traffic → pipeline → dashboard → bot-detector → sentinel) via `make test-integration`
- **Couverture** : tests Go (80% gate pour le correlator), tests Python (pytest pour bot-detector, dashboard, ja4_common), tests C (cmocka pour mod_reqin_log)
- **Tests d'intégration** : suite complète en 8 phases (build → start → schema → traffic → pipeline → dashboard → bot-detector → ja4ebpf) via `make test-integration`
- **Couverture** : tests Go (ja4ebpf), tests Python (pytest pour bot-detector, dashboard, ja4_common)
- **RPM packaging** : 3 distributions (el8/el9/el10) via Rocky Linux / AlmaLinux

68
docs/THESIS_HTTP_Traffic_Detection.md
View File

@ -53,7 +53,7 @@ Ce document présente une architecture complète de détection et classification
- 3.3 Couche L4 — TCP
- 3.4 Couche L5 — TLS
- 3.5 Couche L7 — HTTP
- 3.6 Corrélation inter-couches (logcorrelator)
- 3.6 Corrélation inter-couches (ja4ebpf)
- 3.7 Agrégation temporelle et features dérivées
- 3.8 Détection ML semi-supervisée (full pipeline)
@ -112,7 +112,7 @@ La quatrième génération en émergence combine : analyse corrélée en temps r
Ce document décrit une architecture opérationnelle positionnée à la frontière Gen3/Gen4, déployée en production sur un serveur Apache. Les contributions techniques originales incluent :
1. **Corrélation TCP/TLS/HTTP** en temps réel via le module logcorrelator (clé : `src_ip:src_port`, timeout orphelin 500 ms)
1. **Corrélation TCP/TLS/HTTP** en temps réel via ja4ebpf (clé : `src_ip:src_port`, 256 shards, timeout orphelin 500 ms)
2. **Fingerprinting HTTP/2 passif** : extraction des trames SETTINGS, WINDOW_UPDATE, PRIORITY et de l'ordre des pseudo-headers directement depuis le stream TCP
3. **Architecture EIF bifurquée** : modèle complet (≈ 45 features L3→L7) et modèle applicatif (≈ 35 features L7 uniquement), évitant le biais de zérotage sur le trafic non corrélé
4. **Ensemble triple-voix + MetaLearner** : fusion EIF + AE + XGBoost avec régression logistique apprise sur étiquettes accumulées
@ -461,10 +461,10 @@ Résultat : les coupes ne sont plus liées aux axes, éliminant les artéfacts g
Deux modèles EIF s'exécutent en parallèle à chaque cycle de 300 secondes :
- **Modèle Complet** (≈ 45 features, L3→L7) : appliqué sur les sessions corrélées (correlated=1) pour lesquelles ja4sentinel a pu corréler les métadonnées TCP/TLS avec la requête HTTP. Inclut toutes les features des familles F1–F7.
- **Modèle Complet** (≈ 45 features, L3→L7) : appliqué sur les sessions corrélées (correlated=1) pour lesquelles ja4ebpf a pu corréler les métadonnées TCP/TLS avec la requête HTTP. Inclut toutes les features des familles F1–F7.
- **Modèle Applicatif** (≈ 35 features, L7 uniquement) : appliqué sur les sessions non corrélées (correlated=0) — trafic passant par un CDN ou proxy qui ne permet pas la corrélation TCP/TLS. Zeroed les features TCP/TLS pour ce modèle éviterait d'introduire un biais systématique de zéro-imputation.
La bifurcation est justifiée par le fait que les features TCP/TLS ne sont disponibles que lorsque ja4sentinel a corrélé la connexion réseau avec la requête HTTP. Imputer ces features à zéro pour le trafic non corrélé créerait un signal artificiel (zéro n'est pas neutre pour un EIF — il est interprété comme une valeur réelle).
La bifurcation est justifiée par le fait que les features TCP/TLS ne sont disponibles que lorsque ja4ebpf a corrélé la connexion réseau avec la requête HTTP. Imputer ces features à zéro pour le trafic non corrélé créerait un signal artificiel (zéro n'est pas neutre pour un EIF — il est interprété comme une valeur réelle).
#### 2.4.2b Autoencoders (AE) et détection d'anomalies
@ -799,18 +799,18 @@ httpcloak est un outil d'évasion qui tente d'imiter l'empreinte TLS de Chrome.
┌──────────────────────────────────────────────────────────────────┐
│ SOURCES DE DONNÉES │
├───────────────────────────┬──────────────────────────────────────┤
│ ja4sentinel │ mod_reqin_log │
│ (Source B) │ (Source A) │
│ TC ingress (XDP/TC) │ uprobe SSL_read │
│ Couches L3/L4/L5 │ Couche L7 HTTP déchiffré │
│ │ │
│ libpcap, CAP_NET_RAW │ Module Apache personnalisé │
│ Couches L3/L4/L5 │ Couches L7 HTTP + L5 HTTP/2 │
│ - IP header (TTL, DF) │ - src_ip, src_port │
│ - TCP SYN (MSS, Window, │ - timestamp_ns (nanoseconde) │
│ Options, Scale) │ - méthode, chemin, query │
│ - TLS ClientHello │ - version HTTP │
│ (JA4, JA4T, ALPN, SNI) │ - en-têtes (bruts + ordre) │
│ │ - statut, taille, durée_ms │
│ │ - HTTP/2 preface passif : │
│ │ │
│ réseau XDP/TC → │ Go Magic Bytes dispatcher → │
│ - SYN : TTL, IP-ID, DF, │ HTTP/1.1 : method, path, query, │
│ MSS, Window, Scale │ headers (bruts + ordre), │
│ - TLS ClientHello : │ status, taille, durée_ms, │
│ JA4, ALPN, SNI │ timestamp_ns (nanoseconde) │
│ - HTTP port 80/8080 │ HTTP/2 (depuis preface client) : │
│ │ ordre pseudo-headers │
│ │ WINDOW_UPDATE, PRIORITY flag, │
│ │ SETTINGS (7 params individuels), │
│ │ WINDOW_UPDATE, PRIORITY flag, │
│ │ ordre pseudo-headers │
@ -819,8 +819,8 @@ httpcloak est un outil d'évasion qui tente d'imiter l'empreinte TLS de Chrome.
└─────────────┬─────────────┘
│
┌─────────▼─────────┐
│ logcorrelator │
│ │
│ Corrélation in-memory │
│ (ja4ebpf) │
│ Clé: src_ip:port │
│ Keep-Alive multi-│
│ request tracking │
@ -899,7 +899,7 @@ Features : `avg_ttl` (TTL moyen sur la fenêtre), `ttl_std` (écart-type du TTL
### 3.3 Couche L4 — TCP
**TCP Keep-Alive et multiplexage HTTP** : à ne pas confondre avec HTTP Keep-Alive (`Connection: keep-alive`). Le TCP Keep-Alive est un mécanisme de détection de connexions mortes au niveau du noyau (envoi de paquets ACK vides après inactivité). L'HTTP Keep-Alive, en revanche, maintient la connexion TCP ouverte pour réutilisation par plusieurs requêtes HTTP successives. Le module mod_reqin_log trace le nombre de requêtes HTTP dans chaque connexion TCP via `max_keepalives`.
**TCP Keep-Alive et multiplexage HTTP** : à ne pas confondre avec HTTP Keep-Alive (`Connection: keep-alive`). Le TCP Keep-Alive est un mécanisme de détection de connexions mortes au niveau du noyau (envoi de paquets ACK vides après inactivité). L'HTTP Keep-Alive, en revanche, maintient la connexion TCP ouverte pour réutilisation par plusieurs requêtes HTTP successives. ja4ebpf trace le nombre de requêtes HTTP dans chaque connexion TCP via `max_keepalives` dans le gestionnaire de corrélation in-memory.
**Coefficient de variation (CV)** : mesure adimensionnelle de variabilité, CV = σ/μ. Un CV ≈ 0 indique une régularité élevée (automatisation à timer fixe) ; un CV ≈ 1–3 indique une variabilité naturelle (humain). Applicable à `syn_timing_cv` (variabilité du délai SYN→ClientHello) et à `cadence_cv` (variabilité des intervalles inter-requêtes).
@ -932,18 +932,18 @@ Features : `avg_ttl` (TTL moyen sur la fenêtre), `ttl_std` (écart-type du TTL
### 3.5 Couche L7 — HTTP
La couche L7 constitue la couche la plus riche en features comportementales. Le module **mod_reqin_log** est un module Apache personnalisé qui enregistre chaque requête HTTP avec une précision nanoseconde via `clock_gettime(CLOCK_REALTIME)`.
La couche L7 constitue la couche la plus riche en features comportementales. L'agent **ja4ebpf** capture le flux HTTP déchiffré via un uprobe sur `SSL_read` (OpenSSL/BoringSSL), avec une précision nanoseconde via `bpf_ktime_get_ns()`.
Données capturées par mod_reqin_log : `src_ip`, `src_port`, `timestamp_ns` (nanoseconde absolu), `method`, `path`, `query_string`, `http_version`, `headers_raw` (en-têtes bruts dans leur ordre d'émission), `header_order_signature` (hash de l'ordre), `status_code`, `response_size`, `duration_ms`. L'horodatage nanoseconde est critique pour le calcul des features temporelles F8 (cadence_cv, lag1_autocorrelation, benford_deviation, root_to_first_asset_delay).
Données capturées par ja4ebpf : `src_ip`, `src_port`, `timestamp_ns` (nanoseconde absolu), `method`, `path`, `query_string`, `http_version`, `headers_raw` (en-têtes bruts dans leur ordre d'émission), `header_order_signature` (hash de l'ordre), `status_code`, `response_size`, `duration_ms`. L'horodatage nanoseconde est critique pour le calcul des features temporelles F8 (cadence_cv, lag1_autocorrelation, benford_deviation, root_to_first_asset_delay).
**Fingerprinting HTTP/2 passif intégré** : pour les connexions HTTP/2, mod_reqin_log capture passivement le preface client via un hook `ap_hook_process_connection` enregistré en priorité `APR_HOOK_FIRST`, exécuté avant `mod_http2`. Ce hook effectue une lecture spéculative (`AP_MODE_SPECULATIVE`) de 512 octets sur les filtres d'entrée de la connexion, qui déclenche transparentement le handshake TLS (`mod_ssl`) et retourne les données déchiffrées sans les consommer. Le preface H2 est ensuite parsé pour en extraire :
**Fingerprinting HTTP/2 passif intégré** : pour les connexions HTTP/2, ja4ebpf identifie le protocole via le Go Magic Bytes dispatcher qui reconnaît le preface `PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n` dans le flux SSL_read déchiffré. Le parser HTTP/2 extrait ensuite les frames depuis ce flux déchiffré sans aucune instrumentation côté serveur web :
- Les 7 paramètres SETTINGS individuels (IDs 1–6 et 8), chacun stocké dans une colonne ClickHouse dédiée (`h2_header_table_size`, `h2_enable_push`, `h2_max_concurrent_streams`, `h2_initial_window_size`, `h2_max_frame_size`, `h2_max_header_list_size`, `h2_enable_connect_protocol`), avec la valeur -1 pour les paramètres absents du preface client
- L'incrément `h2_window_update` de la frame WINDOW_UPDATE sur la connexion (stream ID 0)
- Le flag `h2_has_priority` indiquant la présence d'un champ PRIORITY dans la frame HEADERS
- L'ordre des pseudo-headers `h2_pseudo_order` (ex. `m,a,s,p`) extrait par décodage HPACK partiel de la première frame HEADERS
- Le fingerprint composite `h2_fingerprint` au format Akamai et la chaîne brute `h2_settings_fp`
Les données H2 sont stockées dans les notes de la connexion primaire (`c->notes`). Pour les connexions HTTP/2, `mod_http2` crée des connexions secondaires (c2) par stream ; le hook `log_request` accède aux notes H2 via `r->connection->master` (connexion primaire c1). Le hook retourne `DECLINED` après stockage, laissant `mod_http2` gérer la suite normalement.
Les données H2 sont associées à la session courante via la clé `src_ip:src_port` dans le gestionnaire de corrélation in-memory de ja4ebpf, puis transmises en batch à ClickHouse avec les données TCP/TLS correspondantes.
**Colonnes HTTP/2 dans `ja4_logs.http_logs`** :
@ -966,11 +966,11 @@ La convention `-1` pour les paramètres SETTINGS absents est essentielle : elle
Toutes les features des familles F1–F6 et F8 proviennent de cette couche, agrégées sur des fenêtres temporelles de 300 secondes (5 minutes) par session (src_ip).
### 3.6 Corrélation inter-couches (logcorrelator)
### 3.6 Corrélation inter-couches (ja4ebpf)
**Clé de corrélation** : `(src_ip, src_port)` — le tuple source identifie de manière unique une connexion TCP à un instant donné (une connexion TCP est identifiée par le 4-tuple src_ip:src_port:dst_ip:dst_port, mais dst_ip:dst_port étant fixes pour un serveur, le 2-tuple src suffit).
**Gestion du HTTP Keep-Alive** : une connexion TCP peut transporter plusieurs requêtes HTTP successives. logcorrelator maintient une table de connexions indexée par `(src_ip, src_port)`. Chaque requête HTTP reçue de mod_reqin_log est associée à l'enregistrement TCP/TLS ouvert correspondant, et `max_keepalives` est incrémenté. L'enregistrement reste ouvert jusqu'à la déconnexion TCP ou l'expiration.
**Gestion du HTTP Keep-Alive** : une connexion TCP peut transporter plusieurs requêtes HTTP successives. ja4ebpf maintient un gestionnaire de corrélation in-memory organisé en 256 shards (partitionnement par hash de src_ip pour éviter la contention). Chaque requête HTTP capturée via l'uprobe SSL_read est associée à l'enregistrement TCP/TLS ouvert correspondant, et `max_keepalives` est incrémenté. Un GC toutes les 100 ms libère les sessions expirées.
**Timeout orphelin** : si aucun enregistrement réseau ne correspond à une requête HTTP dans les 500 ms, la requête est enregistrée comme orpheline (`correlated=0`). Cela se produit quand le trafic arrive via un CDN ou un proxy inverse qui établit une nouvelle connexion TCP entre le proxy et le serveur, rendant l'adresse source TCP celle du proxy plutôt que celle du client original.
@ -1099,9 +1099,7 @@ Le système `browser_confidence` à 6 axes (§3.8) fournit un score agrégé uti
- **Compression d'en-têtes HPACK** : [RFC 7541](https://www.rfc-editor.org/rfc/rfc7541) définit HPACK, une compression d'en-têtes HTTP par table d'indexation. L'ordre des entrées dans la table statique HPACK est normalisé, mais l'ordre de sérialisation des pseudo-headers est laissé à l'implémentation.
- **Contrôle de flux** : mécanisme de fenêtres (`WINDOW_UPDATE`) limitant le débit pour éviter la saturation du récepteur.
Après terminaison TLS par Apache (`mod_ssl`), le flux HTTP/2 est déchiffré et disponible en clair. Le fingerprinting passif est réalisé directement par **mod_reqin_log** via un hook `process_connection` enregistré en priorité `APR_HOOK_FIRST`. Ce hook s'exécute **avant** `mod_http2` et effectue une lecture spéculative (`AP_MODE_SPECULATIVE`, 512 octets) sur `c->input_filters`, déclenchant transparentement le handshake TLS. Si les données commencent par le magic HTTP/2 (`PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n`), le parser binaire extrait les frames SETTINGS, WINDOW_UPDATE et HEADERS du preface client, stocke les résultats dans `c->notes`, puis retourne `DECLINED` pour laisser `mod_http2` gérer la connexion. Cette approche ne consomme pas les données et n'interfère pas avec le traitement HTTP/2 normal.
**Note architecturale** : l'approche par filtre de connexion (`AP_FTYPE_CONNECTION`) a été abandonnée car `mod_http2` prend le contrôle complet de la connexion via son propre hook `process_connection` — les filtres d'entrée de connexion ne sont jamais invoqués pour les connexions HTTP/2. Le hook `process_connection` à priorité haute est la seule technique fiable pour intercepter le preface H2 avant `mod_http2`.
Après terminaison TLS par le serveur web, le flux HTTP/2 est déchiffré et disponible en clair dans le contexte d'OpenSSL/BoringSSL. Le fingerprinting passif est réalisé par **ja4ebpf** via un uprobe sur `SSL_read`. Lorsque le Go Magic Bytes dispatcher détecte le magic HTTP/2 (`PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n`) dans le flux déchiffré, le parser H2 extrait les frames SETTINGS, WINDOW_UPDATE et HEADERS du preface client, sans modifier ni interrompre le flux. Cette approche est agnostique au serveur web (Apache, Nginx, Varnish, HAProxy) et ne nécessite aucun module natif installé côté serveur.
#### Frame SETTINGS
@ -1397,7 +1395,7 @@ Le module `browser_matcher` génère les features suivantes dans le vecteur feat
| `h2_pseudo_order` | String | Ordre observé (ex. `m,a,s,p`) | `[impl.]` (colonne `h2_pseudo_order` dans `http_logs`) |
| `tls_h2_family_mismatch` | UInt8 | 1 si JA4 dit Chrome mais H2 SETTINGS dit Firefox/outil | `[impl.]` (feature F4) |
Les features `h2_window_update_value`, `h2_has_priority_frames` et `h2_pseudo_order` sont désormais capturées par mod_reqin_log et stockées dans des colonnes individuelles de `ja4_logs.http_logs`. De plus, chaque paramètre SETTINGS HTTP/2 dispose de sa propre colonne (`h2_header_table_size`, `h2_enable_push`, `h2_max_concurrent_streams`, `h2_initial_window_size`, `h2_max_frame_size`, `h2_max_header_list_size`, `h2_enable_connect_protocol`) avec la valeur -1 pour les paramètres absents du preface client. La feature `tls_h2_family_mismatch` est implémentée dans le vecteur feature global (famille F4 — TLS features) et se calcule à partir des données JA4 existantes et des colonnes H2 individuelles disponibles dans `ja4_logs.http_logs`. Les features `browser_match_*` requièrent l'exécution complète du module `browser_matcher.py`.
Les features `h2_window_update_value`, `h2_has_priority_frames` et `h2_pseudo_order` sont désormais capturées par ja4ebpf via le parser HTTP/2 du flux SSL_read déchiffré et stockées dans des colonnes individuelles de `ja4_logs.http_logs`. De plus, chaque paramètre SETTINGS HTTP/2 dispose de sa propre colonne (`h2_header_table_size`, `h2_enable_push`, `h2_max_concurrent_streams`, `h2_initial_window_size`, `h2_max_frame_size`, `h2_max_header_list_size`, `h2_enable_connect_protocol`) avec la valeur -1 pour les paramètres absents du preface client. La feature `tls_h2_family_mismatch` est implémentée dans le vecteur feature global (famille F4 — TLS features) et se calcule à partir des données JA4 existantes et des colonnes H2 individuelles disponibles dans `ja4_logs.http_logs`. Les features `browser_match_*` requièrent l'exécution complète du module `browser_matcher.py`.
### 3.9.5 Maintenance des signatures
@ -1508,7 +1506,7 @@ La famille 1 capture les signaux de volume et de cadence bruts. Ces features son
| Scraper simple (connexion par req) | 1 |
| Bot pipeline agressif | > 100 |
**Implémentation** : mod_reqin_log incrémente un compteur par connexion TCP (`src_ip:src_port`) à chaque requête HTTP reçue. Le maximum sur la session est agrégé dans `view_ai_features_1h`.
**Implémentation** : ja4ebpf incrémente un compteur via le gestionnaire de corrélation in-memory par connexion TCP (`src_ip:src_port`) à chaque requête HTTP reçue. Le maximum sur la session est agrégé dans `view_ai_features_1h`.
#### count_login_post `[impl.]`
@ -2637,7 +2635,7 @@ Cette asymétrie constitue un signal exploitable : un flux HTTP sans résolution
#### Technique : ratio DNS shadow
Capture DNS passive via `ja4sentinel` étendu au port UDP/53, puis corrélation avec les flux HTTP :
Capture DNS passive via `ja4ebpf` étendu au port UDP/53, puis corrélation avec les flux HTTP :
```
dns_shadow_ratio = requêtes HTTP vers hôte X / résolutions DNS de l'hôte X observées
@ -2658,7 +2656,7 @@ Les bots utilisant DoH contournent l'analyse DNS shadow. Cependant, les connexio
Le fingerprinting **JA4D/JA4D6** des requêtes DHCP/DHCPv6 pourrait compléter l'analyse en identifiant les dispositifs derrière NAT, ajoutant une couche d'identification au-delà de l'adresse IP. Cette extension est envisagée comme travail futur conjoint à DNS Shadow Analysis.
**`[todo]` Non implémenté** : nécessite l'extension de `ja4sentinel` pour la capture UDP/53. Travail futur priorité 1 (voir §6.6).
**`[todo]` Non implémenté** : nécessite l'extension de `ja4ebpf` pour la capture UDP/53. Travail futur priorité 1 (voir §6.6).
---
@ -2946,13 +2944,13 @@ Un seul utilisateur réel alterne quelques connexions (2–6 ports source actifs
**browser_matcher complet (§3.9)** :
- État actuel : modules `[partiel]` (logique de score partielle, base de signatures incomplète)
- Données H2 brutes : `[impl.]` — capture des 7 paramètres SETTINGS individuels, WINDOW_UPDATE, flag PRIORITY et ordre pseudo-headers par mod_reqin_log via hook `process_connection` (APR_HOOK_FIRST, AP_MODE_SPECULATIVE). Colonnes individuelles dans `ja4_logs.http_logs` (`h2_header_table_size`, `h2_enable_push`, `h2_max_concurrent_streams`, `h2_initial_window_size`, `h2_max_frame_size`, `h2_max_header_list_size`, `h2_enable_connect_protocol`, `h2_window_update`, `h2_has_priority`, `h2_pseudo_order`)
- Données H2 brutes : `[impl.]` — capture des 7 paramètres SETTINGS individuels, WINDOW_UPDATE, flag PRIORITY et ordre pseudo-headers par ja4ebpf via le parser HTTP/2 du flux SSL_read déchiffré. Colonnes individuelles dans `ja4_logs.http_logs` (`h2_header_table_size`, `h2_enable_push`, `h2_max_concurrent_streams`, `h2_initial_window_size`, `h2_max_frame_size`, `h2_max_header_list_size`, `h2_enable_connect_protocol`, `h2_window_update`, `h2_has_priority`, `h2_pseudo_order`)
- Travail restant : compléter `browser_signatures.py` avec signatures Firefox et Safari complètes, implémenter le rechargement ClickHouse toutes les 24h, intégrer les 5 features `browser_match_*` dérivées dans le vecteur feature global
- Dépendances : ~~capture des frames H2 SETTINGS~~ (réalisé) ; reste le module `browser_matcher.py`
**DNS Shadow Analysis (§5.6)** :
- État actuel : `[todo]` non implémenté
- Travail requis : extension de `ja4sentinel` pour capture UDP/53 + TCP/53, table ClickHouse `dns_resolutions`, corrélation avec table `sessions`
- Travail requis : extension de `ja4ebpf` pour capture UDP/53 + TCP/53, table ClickHouse `dns_resolutions`, corrélation avec table `sessions`
- Contraintes techniques : nécessite accès privilégié au trafic réseau niveau interface (pcap ou eBPF)
- Estimation : 6–8 semaines de développement + tests de charge
@ -3016,7 +3014,7 @@ L'**explainabilité** est assurée par ExIFFI ([Frizzo et al., 2024](https://arx
Extension du système `browser_confidence` à 6 axes vers une correspondance structurée par famille de navigateur, fondée sur l'analyse passive de 7 dimensions H2 : frame SETTINGS (7 paramètres), WINDOW_UPDATE, ordre des pseudo-headers, frames PRIORITY, cohérence des en-têtes HTTP, structure TLS, et lookup JA4 par dictionnaire.
La capture passive est réalisée par mod_reqin_log via un hook `ap_hook_process_connection` (APR_HOOK_FIRST) qui intercepte le preface HTTP/2 avant `mod_http2` par lecture spéculative. Chaque paramètre SETTINGS est stocké dans une colonne ClickHouse individuelle (`h2_header_table_size`, `h2_enable_push`, `h2_max_concurrent_streams`, `h2_initial_window_size`, `h2_max_frame_size`, `h2_max_header_list_size`, `h2_enable_connect_protocol`) avec -1 pour les paramètres absents, complétés par `h2_window_update`, `h2_has_priority` et `h2_pseudo_order`.
La capture passive est réalisée par ja4ebpf via un uprobe sur `SSL_read` (OpenSSL/BoringSSL). Le Go Magic Bytes dispatcher identifie le preface HTTP/2 dans le flux déchiffré, et le parser H2 extrait les frames SETTINGS, WINDOW_UPDATE et HEADERS sans instrumentation serveur. Chaque paramètre SETTINGS est stocké dans une colonne ClickHouse individuelle (`h2_header_table_size`, `h2_enable_push`, `h2_max_concurrent_streams`, `h2_initial_window_size`, `h2_max_frame_size`, `h2_max_header_list_size`, `h2_enable_connect_protocol`) avec -1 pour les paramètres absents, complétés par `h2_window_update`, `h2_has_priority` et `h2_pseudo_order`.
Cette technique permet de détecter des outils d'évasion qui reproduisent correctement la couche TLS (curl_cffi, httpcloak) mais échouent à reproduire les subtilités H2 — notamment l'ordre des pseudo-headers et la valeur WINDOW_UPDATE.
@ -3058,7 +3056,7 @@ Architecture de données fondée sur ClickHouse avec **AggregatingMergeTree view
| F11 — Futures (non impl.) | dns_shadow_ratio, compression_ratio_invariant (2 features) | 0 | 0 | 2 |
| **Total** | **85** | **70** | **5** | **2** + meta |
**Résumé quantitatif** : ~82 % des features entièrement implémentées (`[impl.]`), ~6 % partiellement implémentées (`[partiel]`, les 5 features `browser_match_*` de browser_matcher), ~2 % non implémentées (`[todo]`, §5.6 et §5.7). Les 3 features H2 brutes (`h2_window_update_value`, `h2_has_priority_frames`, `h2_pseudo_order`) sont passées de `[partiel]` à `[impl.]` suite à l'intégration de la capture HTTP/2 passive dans mod_reqin_log.
**Résumé quantitatif** : ~82 % des features entièrement implémentées (`[impl.]`), ~6 % partiellement implémentées (`[partiel]`, les 5 features `browser_match_*` de browser_matcher), ~2 % non implémentées (`[todo]`, §5.6 et §5.7). Les 3 features H2 brutes (`h2_window_update_value`, `h2_has_priority_frames`, `h2_pseudo_order`) sont passées de `[partiel]` à `[impl.]` suite à l'intégration de la capture HTTP/2 passive dans ja4ebpf via le parser HTTP/2 du flux SSL_read.
### Perspective

101
docs/architecture.md
View File

@ -5,30 +5,25 @@ ja4-platform est un pipeline de sécurité qui capture le trafic réseau en temp
## Architecture système
```
┌─────────────────────────────────────────────────────────────────────────────────────┐
│ Serveur Linux cible (Apache) │
│ │
│ ┌─────────────────┐ HTTP req ┌───────────────────────┐ │
│ │ Client │────────────▶│ Apache HTTPD │ │
│ │ (navigateur / │ │ + mod-reqin-log (C) │ │
│ │ bot) │ └───────────┬───────────┘ │
│ │ │ │ JSON / UNIX DGRAM │
│ │ │ │ /var/run/logcorrelator/http.socket │
│ │ │ ▼ │
│ │ │ TLS CH ┌──────────────────────────────────────┐ │
│ │ │────────────▶│ sentinel (Go · libpcap) │ │
│ └─────────────────┘ (pcap) │ → JA4/JA3 fingerprints │ │
│ └───────────┬──────────────────────────┘ │
│ │ JSON / UNIX DGRAM │
│ │ /var/run/logcorrelator/network.socket│
│ ▼ │
│ ┌───────────────────────────────┐ │
│ │ correlator (Go · hex. arch) │ │
│ │ join src_ip:src_port + TTL │ │
│ └───────────┬───────────────────┘ │
│ │ │
└───────────────────────────────────────────────┼─────────────────────────────────────┘
│ INSERT (Native TCP :9000)
+---------------------------------------------------+
| Serveur Linux cible (Apache/Nginx/...) |
| |
| Client --HTTPS (443)--> [Serveur web] |
| Client --HTTP (80)---> [Serveur web] |
| |
| +-----------------+ |
| | ja4ebpf | (s'exécute sur le meme host)|
| | (eBPF CO-RE) | |
| | TC ingress |<-- L3/L4/L5 (SYN, TLS CH) |
| | uprobe SSL_read|<-- L7 HTTPS (déchiffré) |
| | kprobe tcp_recv|<-- L7 HTTP port 80/8080 |
| | 256-shard mgr | corrélation src_ip:src_port |
| +--------+--------+ |
| | |
+-----------|-----------------------------------+ |
| INSERT batch | |
v v
INSERT (Native TCP :9000)
▼
┌───────────────────────────────────────────────────────┐
│ ClickHouse 24.8 │
@ -71,15 +66,17 @@ ja4-platform est un pipeline de sécurité qui capture le trafic réseau en temp
### Phase 1 — Capture
1. **mod-reqin-log** (module Apache C11) intercepte chaque requête HTTP dans le hook `post_read_request`. Il sérialise method, path, headers, client IP/port en JSON et envoie le datagramme vers `/var/run/logcorrelator/http.socket`.
1. **ja4ebpf TC ingress hook** capture les paquets réseau bruts. Pour chaque TCP SYN : `src_ip`, `src_port`, `ttl`, `window_size`, `mss`, `window_scale`, `df_bit`. Pour chaque TLS ClientHello : décodage du payload pour extraire `ciphers`, `extensions`, `elliptic_curves`, `alpn`, `sni` (calcul du hash JA4/JA3 en espace utilisateur Go).
2. **sentinel** (démon Go) capture les paquets TLS ClientHello via libpcap sur les ports configurés (défaut : 443, 8443). Il extrait les métadonnées IP/TCP, génère les empreintes JA4 et JA3, et envoie le résultat en JSON vers `/var/run/logcorrelator/network.socket`.
2. **ja4ebpf uprobe SSL_read** accroche `SSL_read` dans la bibliothèque OpenSSL/BoringSSL du serveur web. Les données déchiffrées sont écrites dans un RingBuffer eBPF. Un kprobe sur `accept4` fournit la correspondance `fd → src_ip:src_port` pour annoter chaque buffer L7.
### Phase 2 — Corrélation
3. **ja4ebpf kprobe tcp_recvmsg** (HTTP port 80/8080) intercepte le payload TCP avant consommation par le serveur pour les connexions non chiffrées.
3. **correlator** (démon Go, architecture hexagonale) écoute les deux sockets Unix. Il met en tampon les événements entrants et les corrèle par `src_ip:src_port` dans une fenêtre temporelle configurable (défaut : 10 s). Le mode `one_to_many` (Keep-Alive) permet de réutiliser un seul handshake TLS (source B) pour plusieurs requêtes HTTP (source A). Les événements corrélés fusionnent les champs HTTP + TLS en un objet `CorrelatedLog` JSON, inséré dans **`ja4_logs.http_logs_raw`**.
### Phase 2 — Corrélation en mémoire
### Phase 3 — Enrichissement (ClickHouse)
4. **ja4ebpf 256-shard manager** (espace utilisateur Go) consomme les trois RingBuffers eBPF via des goroutines dédiées. Les événements L3/L4/L5 et L7 sont corrélés par `src_ip:src_port` dans une table de sessions shardée (256 shards, mutex par shard). Timeout orphelin : 500 ms (émission avec `correlated=0`). Détection Slowloris : émission partielle après 10 s. GC des sessions fantômes : toutes les 100 ms. Le dispatcher magic bytes route vers le parser HTTP/1.1 ou HTTP/2. Pour HTTP/2, la première frame SETTINGS + WINDOW_UPDATE est décodée pour le fingerprinting passif. L’objet corrélé est inséré dans **`ja4_logs.http_logs_raw`** par batch.
### Phase 3 — Enrichissement### Phase 3 — Enrichissement (ClickHouse)
4. **mv_http_logs** (vue matérialisée) transforme le JSON de `http_logs_raw` en la table structurée `ja4_logs.http_logs`, enrichissant chaque ligne avec :
- Données ASN via `dict_iplocate_asn` (IP_TRIE)
@ -123,14 +120,12 @@ ja4-platform est un pipeline de sécurité qui capture le trafic réseau en temp
## Matrice d'interaction des composants
| De ↓ \ Vers → | mod-reqin-log | sentinel | correlator | ClickHouse | bot-detector | dashboard |
|----------------|:---:|:---:|:---:|:---:|:---:|:---:|
| **mod-reqin-log** | — | — | UNIX DGRAM (source A) | — | — | — |
| **sentinel** | — | — | UNIX DGRAM (source B) | — | — | — |
| **correlator** | — | — | — | Native TCP :9000 (INSERT) | — | — |
| **ClickHouse** | — | — | — | MVs internes | — | — |
| **bot-detector** | — | — | — | HTTP :8123 (SELECT/INSERT) | — | — |
| **dashboard** | — | — | — | HTTP :8123 (SELECT/INSERT) | — | — |
| De ↓ \ Vers → | ja4ebpf | ClickHouse | bot-detector | dashboard |
|----------------|:---:|:---:|:---:|:---:|
| **ja4ebpf** | — | Native TCP :9000 (INSERT batch) | — | — |
| **ClickHouse** | — | MVs internes | — | — |
| **bot-detector** | — | HTTP :8123 (SELECT/INSERT) | — | — |
| **dashboard** | — | HTTP :8123 (SELECT/INSERT) | — | — |
## Propriété des tables ClickHouse
@ -138,7 +133,7 @@ ja4-platform est un pipeline de sécurité qui capture le trafic réseau en temp
| Table / Vue | Écrit par | Lu par |
|-------------|-----------|--------|
| `http_logs_raw` | correlator | mv_http_logs (MV) |
| `http_logs_raw` | ja4ebpf | mv_http_logs (MV) |
| `http_logs` | mv_http_logs (MV) | mv_agg_* (6 MVs), dashboard |
| `mv_http_logs` | — (MV automatique) | — |
@ -187,17 +182,23 @@ ja4-platform est un pipeline de sécurité qui capture le trafic réseau en temp
## Algorithme de corrélation
Le correlator joint les événements HTTP (source A) avec les événements TLS/réseau (source B) via une corrélation à deux clés :
**ja4ebpf** corrèle les événements L3/L4/L5 (RingBuffer TC ingress) avec les événements L7 (RingBuffer uprobe) via une table de sessions en mémoire :
1. **Clé** : `src_ip + src_port` — l'IP source et le port éphémère du client identifient une connexion TCP de manière unique.
2. **Fenêtre temporelle** : Les événements doivent arriver dans la fenêtre configurée (défaut 10 secondes).
3. **Mode de correspondance** :
- `one_to_one` : Chaque événement B correspond à un seul événement A (consommé après correspondance).
- `one_to_many` (défaut, Keep-Alive) : Un seul B (handshake TLS) peut correspondre à plusieurs A (requêtes HTTP) sur la même connexion. Le B possède un TTL configurable (défaut 120 s) réinitialisé à chaque correspondance.
4. **Gestion des orphelins** : Les événements A sans correspondance sont émis après un délai configurable (défaut 500 ms) avec `correlated=false` et `orphan_side=A`.
1. **Clé** : `src_ip + src_port` — l’IP source et le port éphémère identifient une connexion TCP de manière unique.
2. **Sharding** : 256 shards (`src_port % 256`), chacun protégé par un `sync.Mutex`. Réduit la contention sous fort trafic.
3. **Gestion Keep-Alive** : Un seul état TLS/L4 est conservé par connexion TCP et partagé entre toutes les requêtes HTTP successives. Compteur `maxkeepalives` incrémenté à chaque requête.
4. **Magic Bytes dispatcher** : Les premiers octets du buffer L7 déterminent le protocole :
- `PRI * HTTP/2.0\r\n` → parser HTTP/2 (SETTINGS + WINDOW_UPDATE + pseudo-headers)
- `GET `, `POST `, `HEAD `, etc. → parser HTTP/1.1
- Autre → `ProtoUnknown` (bruit TLS ou protocole non supporté)
5. **Orphelins** : Événements L7 sans correspondance L3/L4 dans les 500 ms → `correlated=0`.
6. **Slowloris** : Connexion TCP sans requête HTTP complète après 10 s → export partiel.
7. **GC** : Goroutine dédiée, itération sur tous les shards toutes les 100 ms.
La sortie est insérée dans **`ja4_logs.http_logs_raw`** (base `ja4_logs`), pas dans `ja4_processing`.
## Pipeline ML
## Pipeline ML — bot-detector (détail)
```
@ -253,15 +254,15 @@ Format original de fingerprinting TLS :
Le `ja3_hash` est le hash MD5 de la chaîne JA3.
Les deux empreintes sont générées par sentinel à partir du payload TLS ClientHello.
Les deux empreintes sont générées par **ja4ebpf** (espace utilisateur Go) à partir du payload TLS ClientHello capturé par le TC ingress hook.
## Stack technologique
| Composant | Technologie |
|-----------|-------------|
| Capture de paquets | Go 1.24.6 + libpcap (gopacket) |
| Logging HTTP | Module Apache C11 (APR, `apxs`) |
| Corrélation d'événements | Go 1.24.6 (architecture hexagonale) |
| Capture réseau (L3/L4/L5) | Go 1.24.6 + eBPF CO-RE (TC ingress, cilium/ebpf) |
| Capture applicative (L7) | eBPF uprobe SSL_read + kprobe tcp_recvmsg |
| Corrélation en mémoire | Go 1.24.6 (256-shard manager, goroutines) |
| Détection ML — EIF | Python 3.11 + isotree |
| Détection ML — Autoencoder | Python 3.11 + PyTorch |
| Détection ML — Supervisé | Python 3.11 + XGBoost |
@ -272,7 +273,7 @@ Les deux empreintes sont générées par sentinel à partir du payload TLS Clien
| Frontend dashboard | htmx + Chart.js + ECharts + Tailwind CSS (CDN) |
| Magasin de données | ClickHouse 24.8 (dual-database) |
| Déploiement | systemd, Docker, RPM (Rocky 8/9/10) |
| IPC | Sockets UNIX datagramme |
| IPC | RingBuffers eBPF (kernel → userspace Go) |
| Workspace Go | `go.work` (Go 1.24.6) |
## Fichiers de schéma SQL (13)

170
docs/deployment.md
View File

@ -13,11 +13,9 @@
| Service | Runtime | Port par défaut |
|---------|---------|-----------------|
| sentinel | Go binary + libpcap (RPM) | — (capture réseau passive) |
| correlator | Go binary (RPM) | 8080 (metrics, optionnel) |
| mod-reqin-log | Apache module .so (RPM) | — (intégré à httpd) |
| ja4ebpf | Go binary + eBPF CO-RE (RPM) | — (capture réseau passive) |
| bot-detector | Python 3.11 (Docker) | 8080 (health check) |
| dashboard | Python 3.11 / FastAPI (Docker) | 8000 (API) + 3000 (frontend) |
| dashboard | Python 3.11 / FastAPI (Docker) | 8000 (API) |
---
@ -206,103 +204,64 @@ sudo chown -R clickhouse:clickhouse /var/lib/clickhouse/user_files/
---
## Étape 5 — Installation des services Go (RPM)
## Étape 5 — Installation de ja4ebpf (RPM)
### Build des RPMs
```bash
# Tous les RPMs (sentinel + correlator + mod-reqin-log) × 3 distros
make rpm-all
# Ou individuellement
make rpm-sentinel # → services/sentinel/dist/
make rpm-correlator # → services/correlator/dist/
make rpm-mod-reqin-log # → services/mod-reqin-log/dist/
# RPM ja4ebpf × 3 distros (el8, el9, el10)
make rpm-ja4ebpf
# → services/ja4ebpf/dist/rpm/el{8,9,10}/
```
Les RPMs sont générés dans `services/<nom>/dist/` avec un sous-dossier par distro (el8, el9, el10).
Le build utilise un pipeline Docker multi-étapes Rocky Linux :
1. L’étape `go-builder` compile le bytecode eBPF (clang/llvm) puis le binaire Go statique
2. Les étapes `rpm-el8`, `rpm-el9`, `rpm-el10` exécutent `rpmbuild` pour chaque distro cible
3. L’étape `alpine` collecte les RPMs via `--output type=local`
### Installation des RPMs
### Installation du RPM
```bash
# Sur le serveur cible (Rocky 9 par exemple)
sudo yum install -y ./ja4sentinel-*.el9.x86_64.rpm
sudo yum install -y ./logcorrelator-*.el9.x86_64.rpm
sudo yum install -y ./mod_reqin_log-*.el9.x86_64.rpm
sudo yum install -y ./ja4ebpf-*.el9.x86_64.rpm
```
### Configuration du sentinel
### Configuration de ja4ebpf
```bash
# Fichier de configuration principal
sudo cp /etc/ja4sentinel/config.yml.default /etc/ja4sentinel/config.yml
sudo vi /etc/ja4sentinel/config.yml
sudo cp /etc/ja4ebpf/config.yml.example /etc/ja4ebpf/config.yml
sudo vi /etc/ja4ebpf/config.yml
```
Variables d'environnement (dans `/etc/sysconfig/ja4sentinel` ou `.env`) :
Variables d’environnement clés (dans `/etc/sysconfig/ja4ebpf`) :
| Variable | Défaut | Description |
|----------|--------|-------------|
| `JA4EBPF_INTERFACE` | `eth0` | Interface réseau à observer |
| `JA4EBPF_CLICKHOUSE_DSN` | — | DSN ClickHouse (ex: `clickhouse://data_writer:pwd@host:9000/ja4_logs`) |
| `JA4EBPF_TARGET_BINARY` | `/usr/sbin/httpd` | Binaire OpenSSL à hooker (uprobe SSL_read) |
| `JA4EBPF_BATCH_SIZE` | `500` | Taille des batchs d’insertion ClickHouse |
| `JA4EBPF_FLUSH_INTERVAL_MS` | `200` | Intervalle de flush (ms) |
| `JA4EBPF_SESSION_TIMEOUT_MS` | `500` | Timeout orphelin (ms) |
```bash
JA4SENTINEL_INTERFACE=eth0 # Interface réseau à capturer
JA4SENTINEL_PORTS=443,8443 # Ports TLS à surveiller
sudo systemctl enable --now ja4ebpf
sudo systemctl status ja4ebpf
journalctl -u ja4ebpf -f
```
Le sentinel écrit vers le socket Unix du correlator : `/var/run/logcorrelator/network.socket`
### Capabilities Linux requises (SELinux Enforcing)
```bash
sudo systemctl enable --now ja4sentinel
sudo systemctl status ja4sentinel
journalctl -u ja4sentinel -f
```
`ja4ebpf` ne tourne **pas** en root absolu. Le fichier systemd utilise les capabilities minimales :
### Configuration du correlator
| Capability | Raison |
|------------|--------|
| `CAP_BPF` | Chargement des programmes eBPF (kernel 5.8+) |
| `CAP_SYS_ADMIN` | Requis pour les uprobes et RHEL 8 (kernel 4.18 <5.8) |
| `CAP_NET_ADMIN` | Attachement des hooks TC ingress |
| `CAP_PERFMON` | Accès aux perf events pour les uprobes |
```bash
sudo cp /etc/logcorrelator/config.yml.default /etc/logcorrelator/config.yml
sudo vi /etc/logcorrelator/config.yml
```
Configuration minimale pour activer ClickHouse (`config.yml`) :
```yaml
outputs:
clickhouse:
enabled: true
dsn: clickhouse://data_writer:VotreMotDePasse@localhost:9000/ja4_logs
table: http_logs_raw
batch_size: 500
flush_interval_ms: 200
file:
enabled: true
path: /var/log/logcorrelator/correlated.log
```
Variable d'environnement alternative (dans `/etc/sysconfig/logcorrelator`) :
```bash
LOGCORRELATOR_CLICKHOUSE_DSN=clickhouse://data_writer:VotreMotDePasse@localhost:9000/ja4_logs
```
```bash
sudo systemctl enable --now logcorrelator
sudo systemctl status logcorrelator
```
### Configuration de mod-reqin-log
Le module Apache écrit les requêtes HTTP en JSON vers le socket Unix du correlator.
```bash
# Le RPM installe automatiquement le module dans Apache
# Vérifier le chargement
httpd -M | grep reqin
# La configuration est dans /etc/httpd/conf.d/mod_reqin_log.conf
# Le socket par défaut : /var/run/logcorrelator/http.socket
sudo systemctl restart httpd
```
---
## Étape 6 — Installation des services Python (Docker)
@ -374,7 +333,7 @@ docker compose up -d
```bash
# Services systemd (Go)
sudo systemctl status ja4sentinel logcorrelator httpd
sudo systemctl status ja4ebpf
# Services Docker (Python)
docker compose -f services/bot-detector/docker-compose.yml ps
@ -384,7 +343,7 @@ docker compose -f services/dashboard/docker-compose.yaml ps
### 2. Vérifier l'ingestion des logs
```bash
# Logs bruts ingérés par le correlator
# Logs bruts ingérés par ja4ebpf
clickhouse-client --query "SELECT count() FROM ja4_logs.http_logs_raw"
# Logs parsés par la vue matérialisée
@ -480,27 +439,34 @@ Télécharge et génère tous les fichiers CSV de référence (bot IPs, JA4, ASN
## Schéma réseau récapitulatif
```
┌──────────────┐ Unix socket ┌──────────────┐ ClickHouse ┌──────────────────┐
│ mod-reqin-log│──── http.socket ─────→│ │ INSERT INTO │ │
│ (Apache C11) │ (source A) │ correlator │───→ ja4_logs. │ ClickHouse │
└──────────────┘ │ │ http_logs_raw │ │
│ │ │ ┌──────────────┐ │
┌──────────────┐ Unix socket │ │ MV parse JSON │ │ ja4_logs │ │
│ sentinel │──── network.socket ──→│ │ ↓ │ │ _raw → _logs│ │
│ (Go+libpcap) │ (source B) └──────────────┘ ja4_logs. │ └──────────────┘ │
└──────────────┘ http_logs │ │
│ │ ┌──────────────┐ │
6 MVs agrégation│ │ ja4_processing│ │
↓ │ │ agg_* (×6) │ │
┌──────────────┐ SELECT features ┌──────────────┐ ja4_processing. │ │ ml_* (×2) │ │
│ bot-detector │←─── view_ai_features ─│ │ agg_*, view_* │ │ views, dicts│ │
│ (Python 3.11)│ view_thesis_feat │ ClickHouse │ │ │ audit_logs │ │
│ EIF+AE+XGB │ │ │ ml_all_scores │ └──────────────┘ │
│ │───→ INSERT scores ────→│ │ ml_detected_* │ │
└──────────────┘ └──────────────┘ └──────────────────┘
↑
┌──────────────┐ SELECT * │
│ dashboard │←─── ja4_processing.ml_*, agg_*, views ────────────────────────────┘
│ (FastAPI) │←─── ja4_logs.http_logs ───────────────────────────────────────────┘
└──────────────┘
Trafic HTTPS/HTTP (port 80/443)
|
v
+-----------------+
| ja4ebpf | TC ingress hook -- L3/L4/L5 (SYN, TLS ClientHello)
| (eBPF CO-RE) | uprobe SSL_read -- L7 HTTPS (flux déchiffré)
| | kprobe tcp_recvmsg - L7 HTTP (port 80/8080)
+-----------------+
|
| INSERT batch (HTTP bulk)
v
+-----------------+ +------------------+
| ClickHouse |-- MV parse JSON -> | ja4_logs. |
| | | http_logs_raw |
| | | http_logs |
| |-- 6 MVs agg -----> | ja4_processing. |
| | | agg_*(x6) |
| | | ml_*(x2) |
+-----------------+ +------------------+
^
|
+-----------------+ SELECT view_ai_features
| bot-detector |<-- view_thesis_features
| (EIF+AE+XGB) |
| |--> INSERT ml_all_scores, ml_detected_anomalies
+-----------------+
+-----------------+
| dashboard |<-- ja4_processing.ml_*, agg_*, views
| (FastAPI) |<-- ja4_logs.http_logs
+-----------------+
```

198
docs/development.md
View File

@ -1,6 +1,6 @@
# Guide de développement
Ce guide couvre la compilation, les tests, le packaging et l'extension du monorepo ja4-platform. Toutes les opérations de build et de test s'exécutent dans Docker — aucune toolchain Go, Python ou C n'est requise sur la machine hôte.
Ce guide couvre la compilation, les tests, le packaging et l'extension du monorepo ja4-platform. Toutes les opérations de build et de test s'exécutent dans Docker — aucune toolchain Go, Python, C ou eBPF n'est requise sur la machine hôte.
## Prérequis
@ -11,8 +11,6 @@ Ce guide couvre la compilation, les tests, le packaging et l'extension du monore
| make | 3.81+ | GNU Make |
| git | 2.x | Pour le versioning |
Aucun compilateur Go, Python ou C n'est nécessaire sur la machine hôte.
## Compilation de tous les services
```bash
@ -20,18 +18,14 @@ make build-all
```
Cela construit les images Docker pour :
- `ja4-platform/sentinel:latest`
- `ja4-platform/correlator:latest`
- `ja4-platform/ja4ebpf:latest`
- `ja4-platform/bot-detector:latest`
- `ja4-platform/dashboard:latest`
mod-reqin-log est un module Apache et n'est construit que dans le cadre du processus RPM.
### Compilation individuelle
```bash
make build-sentinel # Binaire Go dans Docker
make build-correlator # Binaire Go dans Docker
make build-ja4ebpf # Bytecode eBPF (clang) + binaire Go dans Docker Rocky Linux
make build-bot-detector # Image Python
make build-dashboard # Image FastAPI + Jinja2
```
@ -46,9 +40,7 @@ make test-all
| Service | Commande | Détails |
|---------|----------|---------|
| sentinel | `make test-sentinel` | Tests Go avec `-race`, nécessite `NET_RAW`/`NET_ADMIN` |
| correlator | `make test-correlator` | Tests Go avec seuil de couverture 80% |
| mod-reqin-log | `make test-mod-reqin-log` | Tests unitaires C (sérialisation JSON, config, headers) |
| ja4ebpf | `make test-ja4ebpf` | Tests Go avec `-race`, nécessite `NET_RAW`/`NET_ADMIN`/`BPF` |
| bot-detector | `make test-bot-detector` | Suite pytest Python |
| dashboard | `make test-dashboard` | pytest pour les routes FastAPI |
| ja4_common (Python) | `make test-ja4common-python` | Tests de la librairie Python partagée |
@ -58,42 +50,32 @@ make test-all
Les tests d'intégration full-stack s'exécutent contre Docker Compose avec une instance ClickHouse réelle :
```bash
make test-integration # 8 phases : build → start → schema → traffic → pipeline → dashboard → bot-detector → sentinel
make test-integration # 8 phases : build -> start -> schema -> traffic -> pipeline -> dashboard -> bot-detector -> ja4ebpf
make test-integration-keep # idem mais laisse la stack en fonctionnement
make test-integration-down # démontage de la stack d'intégration
make test-nginx # stack nginx + ja4ebpf
make test-nginx-varnish # nginx + Varnish + ja4ebpf
make test-hitch-varnish # hitch (TLS) + Varnish + ja4ebpf
make test-all-stacks # les 3 stacks serveur en séquence
```
La suite de tests se trouve dans `tests/integration/` et réinitialise la base de données entre chaque exécution.
La suite de tests se trouve dans `tests/integration/` et réinitialise la base entre chaque exécution.
## Construction des paquets RPM
```bash
make rpm-all
make rpm-ja4ebpf
# RPMs dans services/ja4ebpf/dist/rpm/el{8,9,10}/
```
Construit les RPMs pour sentinel, correlator et mod-reqin-log ciblant Rocky Linux 8/9/10 :
```bash
make rpm-sentinel # → services/sentinel/dist/rpm/
make rpm-correlator # → services/correlator/dist/rpm/
make rpm-mod-reqin-log # → services/mod-reqin-log/dist/rpm/
```
Chaque build RPM utilise un pipeline Docker multi-étapes :
1. L'étape builder compile le binaire (Go) ou l'objet partagé (C)
2. L'étape rpmbuild exécute `rpmbuild` pour chaque distro cible (el8, el9, el10)
3. L'étape de sortie copie les RPMs sur l'hôte via `--output type=local`
### Paquets de distribution
```bash
make dist # Alias for rpm-all
# RPMs in services/<service>/dist/rpm/el{8,9,10}/
```
Le build RPM utilise un pipeline Docker multi-étapes Rocky Linux :
1. L'étape `go-builder` compile le bytecode eBPF (clang/llvm) puis le binaire Go statique
2. Les étapes `rpm-el8`, `rpm-el9`, `rpm-el10` exécutent `rpmbuild` pour chaque distro cible
3. L'étape de sortie `alpine` collecte tous les RPMs via `--output type=local`
## Développement local
### Services Go (sentinel, correlator)
### Service Go/eBPF (ja4ebpf)
Le fichier `go.work` relie les modules Go :
@ -101,23 +83,23 @@ Le fichier `go.work` relie les modules Go :
go 1.24.6
use (
./services/sentinel
./services/correlator
./shared/go/ja4common
./services/ja4ebpf
)
```
Si Go 1.24+ est installé localement, le développement sans Docker est possible :
La compilation locale du bytecode eBPF nécessite `clang`, `llvm` et `bpftool`. En pratique, on passe toujours par Docker :
```bash
# Tests sentinel en local
cd services/sentinel && go test ./... -race -v
# Rebuild du bytecode eBPF + binaire Go (Docker Rocky Linux)
make build-ja4ebpf
# Tests correlator en local
cd services/correlator && go test ./... -race -cover -v
# Tests unitaires ja4ebpf (Docker avec cap NET_RAW + BPF)
make test-ja4ebpf
# Compilation du binaire sentinel (nécessite libpcap-dev)
cd services/sentinel && go build -o ja4sentinel ./cmd/ja4sentinel/
# Test single dans Docker
docker run --rm --cap-add=NET_RAW --cap-add=NET_ADMIN --cap-add=BPF \
ja4ebpf:tests go test -v -run TestDispatcher ./internal/dispatcher/
```
### Services Python (bot-detector, dashboard)
@ -137,12 +119,10 @@ uvicorn backend.main:app --reload --host 0.0.0.0 --port 8000
#### Dépendances Python pour le développement local du bot-detector
Les dépendances principales sont listées dans `services/bot-detector/bot_detector/requirements.txt`. Les librairies ML clés sont :
| Librairie | Usage |
|-----------|-------|
| `isotree` | Extended Isolation Forest (scoreur principal non supervisé) |
| `torch` | Autoencoder (PyTorch, architecture n→64→32→16→32→64→n) |
| `torch` | Autoencoder (PyTorch, architecture n->64->32->16->32->64->n) |
| `xgboost` | Modèle supervisé (entraîné sur les labels SOC) |
| `hdbscan` | Clustering de campagnes d'attaque |
| `shap` | Explicabilité des scores d'anomalie |
@ -167,117 +147,48 @@ services/bot-detector/bot_detector/
└── tests/ # Tests pytest (self-contained, pas d'import lourd)
```
### Module C (mod-reqin-log)
Nécessite `apxs` (outil d'extension Apache) et les headers de développement :
```bash
cd services/mod-reqin-log
make build # Compile mod_reqin_log.so
make test # Exécute les tests unitaires
make rpm # Construit les paquets RPM
```
## Scripts utilitaires
Les scripts dans `scripts/` facilitent l'initialisation et la gestion de l'environnement de développement :
| Script | Commande Make | Description |
|--------|--------------|-------------|
| `init-stack.sh` | `make init-stack` | Initialisation complète de la stack ClickHouse (schéma + données CSV) |
| `import-prod-data.sh` | `make import-prod-data` | Import de données de prod avec décalage temporel |
| `reload-prod-logs.sh` | `make reload-prod-logs` | Export prod → réimport dev avec décalage |
| `update-csv-data.sh` | — | Téléchargement et génération de toutes les données CSV de référence |
| `reload-prod-logs.sh` | `make reload-prod-logs` | Export prod -> réimport dev avec décalage |
| `update-csv-data.sh` | -- | Téléchargement et génération de toutes les données CSV de référence |
```bash
# Initialisation complète (schéma + données CSV + import prod)
make init-and-import
# Initialisation seule (schéma + CSV)
make init-stack
# Import des données prod pré-exportées
make import-prod-data
# Rechargement depuis la prod
make reload-prod-logs
make init-and-import # Initialisation complète (schéma + CSV + import prod)
make init-stack # Initialisation seule (schéma + CSV)
make import-prod-data # Import des données prod pré-exportées
make reload-prod-logs # Rechargement depuis la prod
```
## Ajout d'un nouveau service
### Service Go
1. Créer le répertoire du service :
1. Créer le répertoire et initialiser le module :
```bash
mkdir -p services/my-service/cmd/my-service
mkdir -p services/my-service/internal
mkdir -p services/my-service/cmd/my-service services/my-service/internal
cd services/my-service && go mod init github.com/antitbone/ja4/my-service
```
2. Initialiser le module Go :
```bash
cd services/my-service
go mod init github.com/antitbone/ja4/my-service
```
3. Ajouter au `go.work` :
2. Ajouter au `go.work` :
```
use (
./services/sentinel
./services/correlator
./services/my-service # ← add this
./shared/go/ja4common
./services/ja4ebpf
./services/my-service # <- add this
)
```
4. Importer la librairie partagée :
```go
import (
"github.com/antitbone/ja4/ja4common/logger"
"github.com/antitbone/ja4/ja4common/config"
"github.com/antitbone/ja4/ja4common/shutdown"
)
```
5. Ajouter les cibles Makefile :
```makefile
build-my-service:
docker build -f services/my-service/Dockerfile -t ja4-platform/my-service:latest .
test-my-service:
docker build -f services/my-service/Dockerfile.dev -t ja4-platform/my-service-tests:latest .
docker run --rm ja4-platform/my-service-tests:latest
```
6. Mettre à jour les dépendances `build-all` et `test-all`.
3. Ajouter les cibles Makefile.
### Service Python
1. Créer le répertoire du service avec un `requirements.txt` ou `pyproject.toml`.
2. Ajouter `ja4-common` comme dépendance (installée depuis `shared/python/ja4_common`).
1. Créer le répertoire avec `requirements.txt`.
2. Ajouter `ja4-common` comme dépendance locale.
3. Utiliser `from ja4_common.clickhouse import get_client` pour l'accès ClickHouse.
4. Ajouter les cibles Makefile en suivant le modèle bot-detector/dashboard.
## Workspace go.work
Le fichier `go.work` à la racine du dépôt relie tous les modules Go, permettant le développement cross-module sans publication :
```
go 1.24.6
use (
./services/sentinel
./services/correlator
./shared/go/ja4common
)
```
Lors de l'ajout d'un nouveau module Go :
1. `go mod init` dans le répertoire du service
2. Ajouter le chemin dans `go.work`
3. Référencer les packages partagés via leur chemin de module : `github.com/antitbone/ja4/ja4common/...`
4. Exécuter `go work sync` pour mettre à jour le workspace
Les deux services Go ont une directive `replace` dans leur `go.mod` pointant vers `../../shared/go/ja4common`. Le workspace prend priorité en développement local ; la directive `replace` est nécessaire pour les builds Docker où `go.work` n'est pas disponible.
## Package Python ja4_common
@ -289,31 +200,10 @@ Le package Python partagé (`shared/python/ja4_common`) fournit :
> **Note :** le dashboard n'utilise PAS ja4_common — il possède son propre client léger `clickhouse-connect` dans `backend/database.py`.
### Extension de ja4_common
1. Ajouter de nouveaux modules sous `shared/python/ja4_common/ja4_common/`
2. Les exporter dans `__init__.py`
3. Ajouter les dépendances dans `pyproject.toml`
4. Lancer les tests : `make test-ja4common-python`
### Utilisation dans un nouveau service
Ajouter dans `requirements.txt` :
```
ja4-common @ file:///app/shared/python/ja4_common
```
Or in Docker, copy the shared library and install:
```dockerfile
COPY shared/python/ja4_common /app/shared/python/ja4_common
RUN pip install /app/shared/python/ja4_common
```
## Variables d'environnement
Chaque service lit sa configuration depuis des variables d'environnement et/ou des fichiers YAML. Consultez la documentation de chaque service pour la référence complète :
Consultez la documentation de chaque service pour la référence complète :
- [Configuration du Sentinel](services/sentinel.md#configuration-reference)
- [Configuration du Correlator](services/correlator.md#configuration-reference)
- [Configuration de ja4ebpf](services/ja4ebpf.md#configuration)
- [Configuration du Bot Detector](services/bot-detector.md#environment-variables)
- [Configuration du Dashboard](services/dashboard.md#configuration)

292
docs/services/ja4ebpf.md Normal file
View File

@ -0,0 +1,292 @@
# ja4ebpf — Agent eBPF CO-RE
## Description
`ja4ebpf` est l'agent de collecte de données de la plateforme ja4-platform. C'est un binaire Go unique qui utilise eBPF CO-RE (Compile Once — Run Everywhere) pour observer passivement le trafic réseau d'un serveur Linux, sans modifier ni interrompre le serveur web cible (Apache, Nginx, Varnish, HAProxy, ou tout processus utilisant OpenSSL/BoringSSL).
Il capture simultanément les métadonnées réseau L3/L4 (TCP SYN), les paramètres TLS L5 (ClientHello), et le contenu applicatif L7 (requêtes HTTP déchiffrées), corr le tout en mémoire par la clé `src_ip:src_port`, et envoie les données vers ClickHouse en batch.
## Architecture interne
```
+-----------------+ TC ingress (XDP/TC) +--------------------+
| réseau entrant |-------------------------->| bpf/tc_capture.c |
| | | |
| SYN packet | --> rb_tcp_syn (16 MB) | Programme eBPF |
| TLS ClientHello| --> rb_tls_hello (16 MB) | CO-RE |
| HTTP port 80 | --> rb_http_plain (32 MB)| |
+-----------------+ +--------------------+
|
+-----------------+ uprobe SSL_read +--------------------+
| serveur web |--------------------------> | bpf/uprobe_ssl.c |
| (OpenSSL) | | |
| flux déchiffré | --> rb_ssl_data (64 MB) | Programme eBPF |
+-----------------+ | CO-RE |
+--------------------+
|
+-----------v-----------+
| Go userspace |
| |
| internal/loader/ |
| - RingBuffer readers |
| - 5 goroutines |
| |
| internal/parser/ |
| - JA4 calculator |
| - H2 preface parser |
| - HTTP/1.1 parser |
| |
| internal/dispatcher/ |
| - Magic Bytes router |
| |
| internal/correlation/ |
| - 256-shard manager |
| - GC 100ms |
| - timeout 500ms |
| |
| internal/writer/ |
| - ClickHouse batch |
+----------+-------------+
|
INSERT batch TCP :9000
|
v
+----------------------+
| ja4_logs. |
| http_logs_raw |
+----------------------+
```
## Hooks eBPF
### TC ingress — Couches L3/L4/L5
Le programme `bpf/tc_capture.c` est attaché à l'interface réseau via **TC (Traffic Control)** en ingress. Il s'exécute pour chaque paquet entrant :
**Paquets TCP SYN** : extraction des options TCP et métadonnées IP depuis les headers du paquet.
- `bpf_skb_load_bytes()` pour lire les options TCP depuis le skb
- Envoyé dans le RingBuffer `rb_tcp_syn` (16 MB)
**ClientHello TLS** : détection du type 0x16 (Handshake) et sous-type 0x01 (ClientHello).
- `bpf_skb_load_bytes()` pour capturer 512 octets du payload
- Envoyé dans le RingBuffer `rb_tls_hello` (16 MB)
**HTTP en clair (port 80/8080)** : pour les connexions non chiffrées.
- SYN/FIN/RST exclus (uniquement les segments porteurs de données)
- Jusqu'à 4096 octets via `bpf_skb_load_bytes()`
- Envoyé dans le RingBuffer `rb_http_plain` (32 MB)
### Uprobe SSL_read — Couche L7
Les **uprobes** s'attachent dynamiquement à `SSL_read` (ou `SSL_read_ex`) dans le processus du serveur web. Elles s'exécutent *après* le déchiffrement TLS, capturant le flux HTTP en clair dans `rb_ssl_data` (64 MB).
La clé de corrélation `src_ip:src_port` est extraite depuis la structure `SSL*` → file descriptor → socket noyau via `bpf_probe_read_kernel()`.
### Kprobe accept4
Un kprobe sur `accept4` peuple `rb_accept` (4 MB) avec le tuple `(src_ip, src_port, fd, pid_tgid)`, permettant d'associer chaque fd SSL à une connexion TCP.
## Corrélation in-memory
Le gestionnaire `internal/correlation` maintient un état par connexion :
| Mécanisme | Valeur |
|-----------|--------|
| Sharding | 256 buckets (hash src_ip:src_port) |
| Timeout orphelin | 500 ms (→ `correlated=0`) |
| Détection Slowloris | 10 s (export partiel) |
| GC interval | 100 ms |
| Keep-Alive | max_keepalives incrémenté par requête |
### Routeur Magic Bytes (dispatcher)
```
Buffer reçu (SSL data ou HTTP plain)
|
+-- starts with "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" --> ProtoHTTP2
|
+-- starts with "GET " / "POST " / "PUT " / ... --> ProtoHTTP1
|
+-- partial H2 preface prefix --> ProtoHTTP2 (buffer)
|
+-- autre --> ProtoUnknown (ignoré)
```
**Fingerprinting HTTP/2 passif** : après détection du preface `PRI * HTTP/2.0...`, le parser itère les frames :
- Frame `SETTINGS` (type `0x04`) : extraction des 7 paramètres, valeur `-1` si absent
- Frame `WINDOW_UPDATE` (type `0x08`, stream 0) : incrément de fenêtre connexion
- Frame `HEADERS` (type `0x01`) : ordre des pseudo-headers (`m,a,s,p`)
## Données collectées
### L3/L4 (TCP SYN)
| Champ | Description |
|-------|-------------|
| `src_ip`, `src_port` | Clé de corrélation |
| `ttl` | Time To Live initial |
| `df_bit` | Don't Fragment bit |
| `ip_id` | IP Identification (0 = Linux/VPN/spoofé) |
| `window_size` | Taille fenêtre TCP SYN |
| `window_scale` | Option Window Scale (RFC 1323) |
| `mss` | Maximum Segment Size |
| `tcp_options` | Options TCP brutes (40 octets max) |
| `tcp_jitter_variance` | Variance jitter inter-SYN |
| `syn_timing_cv` | Délai SYN→ClientHello (ns) |
### L5 (TLS ClientHello)
| Champ | Description |
|-------|-------------|
| `tls_version` | Version TLS |
| `ciphers` | Liste suites cryptographiques |
| `extensions` | Liste extensions TLS |
| `elliptic_curves` | Courbes elliptiques supportées |
| `point_formats` | Formats de points EC |
| `alpn` | ALPN list (h2, http/1.1, ...) |
| `sni` | Server Name Indication |
| `ja4` | Empreinte JA4 calculée Go-side |
| `ja4t` | Empreinte JA4T (TCP) calculée Go-side |
### L7 HTTP/1.1
| Champ | Description |
|-------|-------------|
| `method` | Méthode HTTP |
| `path` | Chemin |
| `query_string` | Paramètres query |
| `http_version` | HTTP/1.0 ou HTTP/1.1 |
| `headers_raw` | En-têtes dans leur ordre d'émission |
| `header_order_signature` | Hash de l'ordre |
| `status_code` | Code de statut |
| `response_size` | Taille réponse (octets) |
| `duration_ms` | Durée requête |
| `timestamp_ns` | Horodatage ns absolu |
### L7 HTTP/2 (preface client)
| Champ | Description |
|-------|-------------|
| `h2_header_table_size` | SETTINGS ID 1 (-1 si absent) |
| `h2_enable_push` | SETTINGS ID 2 |
| `h2_max_concurrent_streams` | SETTINGS ID 3 |
| `h2_initial_window_size` | SETTINGS ID 4 |
| `h2_max_frame_size` | SETTINGS ID 5 |
| `h2_max_header_list_size` | SETTINGS ID 6 |
| `h2_enable_connect_protocol` | SETTINGS ID 8 (RFC 8441) |
| `h2_window_update` | Incrément WINDOW_UPDATE connexion |
| `h2_has_priority` | Flag PRIORITY dans HEADERS frame |
| `h2_pseudo_order` | Ordre pseudo-headers (ex: `m,a,s,p`) |
## eBPF CO-RE
| Aspect | Détail |
|--------|--------|
| Compilateur | `clang` + `llvm` |
| Target | `bpf` (architecture BPF 64 bits) |
| BTF source | `/sys/kernel/btf/vmlinux` (disponible RHEL 8+) |
| Relocations | `cilium/ebpf` résout automatiquement les offsets struct |
| Embed | `go:generate` génère `bpf_bpfel.go` avec bytecode embarqué |
| Compatibilité | Rocky/RHEL Linux 8, 9, 10 (kernel 4.18+) |
## Configuration
```yaml
# /etc/ja4ebpf/config.yml
interface: eth0
target_binary: /usr/sbin/httpd
clickhouse:
dsn: clickhouse://data_writer:pwd@localhost:9000/ja4_logs
table: http_logs_raw
batch_size: 500
flush_interval_ms: 200
correlation:
session_timeout_ms: 500
slowloris_threshold_s: 10
gc_interval_ms: 100
```
### Variables d'environnement
| Variable | Défaut | Description |
|----------|--------|-------------|
| `JA4EBPF_INTERFACE` | `eth0` | Interface réseau |
| `JA4EBPF_TARGET_BINARY` | `/usr/sbin/httpd` | Binaire à hooker (uprobe SSL_read) |
| `JA4EBPF_CLICKHOUSE_DSN` | — | DSN ClickHouse |
| `JA4EBPF_BATCH_SIZE` | `500` | Taille des batchs d'insertion |
| `JA4EBPF_FLUSH_INTERVAL_MS` | `200` | Intervalle de flush (ms) |
| `JA4EBPF_SESSION_TIMEOUT_MS` | `500` | Timeout session orpheline |
| `JA4EBPF_SLOWLORIS_THRESHOLD_S` | `10` | Seuil détection Slowloris (s) |
## Build
```bash
# Build complet (bytecode eBPF + binaire Go) — Docker Rocky Linux
make build-ja4ebpf
# Tests unitaires (nécessite NET_RAW/NET_ADMIN/BPF capabilities)
make test-ja4ebpf
# Build RPMs el8/el9/el10
make rpm-ja4ebpf
# → services/ja4ebpf/dist/rpm/el{8,9,10}/
```
## Structure du code
```
services/ja4ebpf/
├── bpf/
│ ├── bpf_types.h # Structs C partagées + déclarations maps eBPF
│ ├── tc_capture.c # Programme TC ingress (L3/L4/L5 + HTTP plain)
│ └── uprobe_ssl.c # Programme uprobe SSL_read (L7 déchiffré)
├── cmd/ja4ebpf/
│ └── main.go # Point d'entrée : 5 goroutines consumer
├── internal/
│ ├── loader/
│ │ └── loader.go # Chargement eBPF + RingBuffer readers + désérialisation
│ ├── parser/
│ │ ├── ja4.go # Calcul empreintes JA4 / JA4T
│ │ ├── http2.go # Parser HTTP/2 preface (SETTINGS, WINDOW_UPDATE, HEADERS)
│ │ └── http1.go # Parser HTTP/1.1
│ ├── dispatcher/
│ │ └── dispatcher.go # Routeur Magic Bytes (ProtoHTTP1/2/Unknown)
│ ├── correlation/
│ │ └── manager.go # Gestionnaire sessions 256-shard
│ └── writer/
│ └── writer.go # Writer ClickHouse (batch + retry)
├── packaging/
│ ├── rpm/ja4ebpf.spec # Spec RPM (el8/el9/el10)
│ └── systemd/ja4ebpf.service # Unit systemd
├── Dockerfile # Image de production
├── Dockerfile.tests # Image de tests
├── Dockerfile.package # Build RPM multi-distro (5 stages)
└── Makefile
```
## Capabilities Linux requises (SELinux Enforcing)
L'agent tourne sous l'utilisateur `ja4ebpf` (UID/GID 490 fixe). Les capabilities Linux accordées via `AmbientCapabilities` :
| Capability | Raison |
|------------|--------|
| `CAP_BPF` | Chargement des programmes eBPF (kernel 5.8+) |
| `CAP_SYS_ADMIN` | Uprobes + RHEL 8 (kernel 4.18, pré-CAP_BPF) |
| `CAP_NET_ADMIN` | Attachement hooks TC ingress |
| `CAP_PERFMON` | Accès perf events pour les uprobes |
`LimitMEMLOCK=infinity` est requis pour le `mlock()` des maps eBPF.
## Tests d'intégration
Stacks Docker Compose testant l'agent contre différents serveurs web :
```bash
make test-integration # Apache httpd (référence)
make test-nginx # Nginx
make test-nginx-varnish # Nginx + Varnish (reverse proxy)
make test-hitch-varnish # Hitch (TLS) + Varnish
make test-all-stacks # Les 4 stacks en séquence
```

2
go.work
View File

@ -1,8 +1,6 @@
go 1.24.6
use (
./services/sentinel
./services/correlator
./shared/go/ja4common
./services/ja4ebpf
)

19
old/services/correlator/.dockerignore Normal file
View File

@ -0,0 +1,19 @@
# Build outputs
dist/
# Dependency directories
vendor/
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
# OS
.DS_Store
Thumbs.db
# Aider cache
.aider*

2
old/services/correlator/.env.example Normal file
View File

@ -0,0 +1,2 @@
# correlator configuration — DO NOT COMMIT real values
LOGCORRELATOR_CLICKHOUSE_DSN=clickhouse://data_writer:ChangeMe@clickhouse:9000/ja4_logs

73
old/services/correlator/.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,73 @@
name: Build and Test
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
- name: Download dependencies
run: go mod download
- name: Run tests with coverage
run: |
go test -race -coverprofile=coverage.txt -covermode=atomic ./...
TOTAL=$(go tool cover -func=coverage.txt | grep total | awk '{gsub(/%/, "", $3); print $3}')
echo "Coverage: ${TOTAL}%"
if (( $(echo "$TOTAL < 80" | bc -l) )); then
echo "Coverage ${TOTAL}% is below 80% threshold"
exit 1
fi
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:
file: ./coverage.txt
build:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v5
with:
go-version: '1.21'
- name: Build binary
run: |
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-ldflags="-w -s" \
-o logcorrelator \
./cmd/logcorrelator
- name: Upload binary artifact
uses: actions/upload-artifact@v4
with:
name: logcorrelator-linux-amd64
path: logcorrelator
docker:
runs-on: ubuntu-latest
needs: test
steps:
- uses: actions/checkout@v4
- name: Build Docker image
run: docker build -t logcorrelator:latest .
- name: Run tests in Docker
run: |
docker run --rm logcorrelator:latest --help || true

32
old/services/correlator/.gitignore vendored Normal file
View File

@ -0,0 +1,32 @@
# Build directory
/build/
/dist/
# Binaries
*.exe
*.exe~
*.dll
*.so
*.dylib
/logcorrelator
# Test binary
*.test
# Output of the go coverage tool
*.out
# Dependency directories
vendor/
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
# OS
.DS_Store
Thumbs.db
.aider*

43
old/services/correlator/Dockerfile Normal file
View File

@ -0,0 +1,43 @@
# syntax=docker/dockerfile:1
FROM golang:1.24 AS builder
WORKDIR /build
RUN apt-get update && apt-get install -y --no-install-recommends git bc && rm -rf /var/lib/apt/lists/*
COPY go.work go.work.sum* ./
COPY shared/go/ja4common/ ./shared/go/ja4common/
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
WORKDIR /build/services/correlator
RUN --mount=type=cache,target=/go/pkg/mod go mod download
COPY services/correlator/ /build/services/correlator/
ARG SKIP_TESTS=false
RUN --mount=type=cache,target=/go/pkg/mod \
if [ "$SKIP_TESTS" = "false" ]; then \
go test -race -coverprofile=coverage.txt -covermode=atomic ./... && \
echo "=== Coverage Report ===" && \
go tool cover -func=coverage.txt | grep total && \
TOTAL=$(go tool cover -func=coverage.txt | grep total | awk '{gsub(/%/, "", $3); print $3}') && \
echo "Total coverage: ${TOTAL}%" && \
if (( $(echo "$TOTAL < 60" | bc -l) )); then \
echo "ERROR: Coverage ${TOTAL}% is below 60% threshold"; \
exit 1; \
fi && \
echo "Coverage check passed!"; \
else \
echo "Skipping tests (SKIP_TESTS=true)"; \
fi
RUN --mount=type=cache,target=/go/pkg/mod \
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
-ldflags="-w -s" -o /usr/bin/correlator ./cmd/logcorrelator
FROM scratch AS runtime
COPY --from=builder /usr/bin/correlator /usr/bin/correlator
COPY --from=builder /build/services/correlator/config.example.yml /etc/correlator/correlator.yml
ENTRYPOINT ["/usr/bin/correlator"]
CMD ["-config", "/etc/correlator/correlator.yml"]

108
old/services/correlator/Dockerfile.package Normal file
View File

@ -0,0 +1,108 @@
# syntax=docker/dockerfile:1
# =============================================================================
# correlator — Dockerfile de packaging RPM (Rocky Linux 8/9, AlmaLinux 10)
# Build context: monorepo root (ja4-platform/)
# Méthode: 1 builder Rocky → 1 rpm-builder (rpmbuild, 3 × dist) → 1 output alpine
# =============================================================================
# =============================================================================
# Stage 1: Builder — compilation du binaire Go sur Rocky Linux 9
# CGO_ENABLED=0 → binaire statique, mais compilé sur la même distro cible
# =============================================================================
FROM rockylinux:9 AS builder
WORKDIR /build
RUN dnf install -y golang git && dnf clean all
# Copie du workspace Go et du module partagé en premier (meilleur cache)
COPY go.work go.work.sum* ./
COPY shared/go/ja4common/ ./shared/go/ja4common/
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
WORKDIR /build/services/correlator
RUN go mod download
COPY services/correlator/ /build/services/correlator/
ARG VERSION=dev
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
go build -ldflags="-w -s -X main.Version=${VERSION}" \
-o /tmp/correlator \
./cmd/logcorrelator
# =============================================================================
# Stage 2: rpm-builder — construction des RPMs avec rpmbuild
# Un seul stage, trois appels rpmbuild successifs (el8, el9, el10).
# Le spec lit les fichiers depuis %{_builddir} (répertoire BUILD de rpmbuild).
# =============================================================================
FROM rockylinux:9 AS rpm-builder
WORKDIR /package
ARG VERSION=dev
RUN dnf install -y rpm-build rpmdevtools systemd-rpm-macros && dnf clean all
RUN mkdir -p /root/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} && \
mkdir -p /packages/rpm/{el8,el9,el10}
# Disposition des fichiers dans BUILD/ (attendue par le spec correlator)
RUN mkdir -p /root/rpmbuild/BUILD/usr/bin \
/root/rpmbuild/BUILD/etc/logcorrelator \
/root/rpmbuild/BUILD/etc/systemd/system \
/root/rpmbuild/BUILD/etc/logrotate.d
COPY --from=builder /tmp/correlator /root/rpmbuild/BUILD/usr/bin/logcorrelator
COPY services/correlator/config.example.yml /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml
COPY services/correlator/config.example.yml /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml.example
COPY services/correlator/logcorrelator.service /root/rpmbuild/BUILD/etc/systemd/system/logcorrelator.service
COPY services/correlator/packaging/rpm/logrotate /root/rpmbuild/BUILD/etc/logrotate.d/logcorrelator
RUN chmod 755 /root/rpmbuild/BUILD/usr/bin/logcorrelator && \
chmod 640 /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml && \
chmod 640 /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml.example && \
chmod 644 /root/rpmbuild/BUILD/etc/systemd/system/logcorrelator.service && \
chmod 644 /root/rpmbuild/BUILD/etc/logrotate.d/logcorrelator
COPY services/correlator/packaging/rpm/logcorrelator.spec /root/rpmbuild/SPECS/logcorrelator.spec
# el8
RUN rpmbuild --define "_topdir /root/rpmbuild" \
--define "dist .el8" \
--define "build_version ${VERSION}" \
--target x86_64 \
-bb /root/rpmbuild/SPECS/logcorrelator.spec && \
cp /root/rpmbuild/RPMS/x86_64/*.el8.x86_64.rpm /packages/rpm/el8/
# el9
RUN rpmbuild --define "_topdir /root/rpmbuild" \
--define "dist .el9" \
--define "build_version ${VERSION}" \
--target x86_64 \
-bb /root/rpmbuild/SPECS/logcorrelator.spec && \
cp /root/rpmbuild/RPMS/x86_64/*.el9.x86_64.rpm /packages/rpm/el9/
# el10
RUN rpmbuild --define "_topdir /root/rpmbuild" \
--define "dist .el10" \
--define "build_version ${VERSION}" \
--target x86_64 \
-bb /root/rpmbuild/SPECS/logcorrelator.spec && \
cp /root/rpmbuild/RPMS/x86_64/*.el10.x86_64.rpm /packages/rpm/el10/
# =============================================================================
# Stage 3: output — image finale contenant uniquement les RPMs
# =============================================================================
FROM alpine:latest AS output
WORKDIR /packages
COPY --from=rpm-builder /packages/rpm/el8/*.rpm /packages/rpm/el8/
COPY --from=rpm-builder /packages/rpm/el9/*.rpm /packages/rpm/el9/
COPY --from=rpm-builder /packages/rpm/el10/*.rpm /packages/rpm/el10/
CMD ["sh", "-c", \
"echo '=== RPM el8 ===' && ls -la /packages/rpm/el8/ && \
echo '' && echo '=== RPM el9 ===' && ls -la /packages/rpm/el9/ && \
echo '' && echo '=== RPM el10 ===' && ls -la /packages/rpm/el10/"]

148
old/services/correlator/Makefile Normal file
View File

@ -0,0 +1,148 @@
.PHONY: build build-docker test test-docker lint clean help docker-build-dev docker-build-runtime package package-rpm
# Docker parameters
DOCKER=docker
# Use buildx for better cache management and parallel builds
DOCKER_BUILD=$(DOCKER) build
DOCKER_BUILDX=$(DOCKER) buildx
DOCKER_RUN=$(DOCKER) run
# Image names
DEV_IMAGE=logcorrelator-dev:latest
RUNTIME_IMAGE=logcorrelator:latest
PACKAGER_IMAGE=logcorrelator-packager:latest
PACKAGER_IMAGE_EL8=logcorrelator-packager-el8:latest
PACKAGER_IMAGE_EL9=logcorrelator-packager-el9:latest
PACKAGER_IMAGE_EL10=logcorrelator-packager-el10:latest
# Binary name
BINARY_NAME=logcorrelator
DIST_DIR=dist
# Package version
PKG_VERSION ?= 1.1.22
# Enable BuildKit for better performance
export DOCKER_BUILDKIT=1
## build: Build the logcorrelator binary locally
build:
mkdir -p $(DIST_DIR)
go build -ldflags="-w -s" -o $(DIST_DIR)/$(BINARY_NAME) ./cmd/$(BINARY_NAME)
## docker-build-dev: Build the development Docker image (with tests and coverage)
docker-build-dev:
$(DOCKER_BUILD) --target builder -t $(DEV_IMAGE) -f Dockerfile .
## docker-build-dev-no-test: Build the development Docker image WITHOUT tests (faster)
docker-build-dev-no-test:
$(DOCKER_BUILD) --target builder --no-cache --build-arg SKIP_TESTS=true -t $(DEV_IMAGE) -f Dockerfile .
## docker-build-runtime: Build the runtime Docker image (fast, no tests)
docker-build-runtime:
$(DOCKER_BUILD) --target runtime -t $(RUNTIME_IMAGE) -f Dockerfile .
## test: Run unit tests locally
test:
go test -race -coverprofile=coverage.out ./...
## test-docker: Run unit tests inside Docker container
test-docker: docker-build-dev
@echo "Tests already run in builder stage"
## lint: Run linters
lint:
go vet ./...
gofmt -l .
## fmt: Format all Go files
fmt:
gofmt -w .
## package: Build RPM packages for all target distributions
package: package-rpm
## package-rpm: Build RPM packages for Rocky Linux 8/9, AlmaLinux 10 (requires Docker)
## Uses buildx for parallel builds (el8, el9, el10 built simultaneously)
package-rpm:
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
@echo "Starting parallel RPM builds for el8, el9, el10..."
# Build all three distributions in parallel using buildx
$(DOCKER_BUILDX) build --target output -t $(PACKAGER_IMAGE) \
--build-arg VERSION=$(PKG_VERSION) \
-f Dockerfile.package . \
--load
@echo "Extracting RPM packages from Docker image..."
$(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm $(PACKAGER_IMAGE) sh -c \
"cp -r /packages/rpm/el8 /output/rpm/ && \
cp -r /packages/rpm/el9 /output/rpm/ && \
cp -r /packages/rpm/el10 /output/rpm/"
@echo "RPM packages created:"
@echo " Enterprise Linux 8 (el8):"
ls -la $(DIST_DIR)/rpm/el8/ 2>/dev/null || echo " (no packages)"
@echo " Enterprise Linux 9 (el9):"
ls -la $(DIST_DIR)/rpm/el9/ 2>/dev/null || echo " (no packages)"
@echo " Enterprise Linux 10 (el10):"
ls -la $(DIST_DIR)/rpm/el10/ 2>/dev/null || echo " (no packages)"
## package-rpm-sequential: Build RPM packages sequentially (fallback if parallel fails)
package-rpm-sequential:
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
@echo "Building RPM for el8..."
$(DOCKER_BUILD) --target rpm-el8-builder -t $(PACKAGER_IMAGE_EL8) \
--build-arg VERSION=$(PKG_VERSION) \
-f Dockerfile.package .
@echo "Building RPM for el9..."
$(DOCKER_BUILD) --target rpm-el9-builder -t $(PACKAGER_IMAGE_EL9) \
--build-arg VERSION=$(PKG_VERSION) \
-f Dockerfile.package .
@echo "Building RPM for el10..."
$(DOCKER_BUILD) --target rpm-el10-builder -t $(PACKAGER_IMAGE_EL10) \
--build-arg VERSION=$(PKG_VERSION) \
-f Dockerfile.package .
@echo "Extracting RPM packages..."
$(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm \
-v $(PACKAGER_IMAGE_EL8):/el8:ro \
-v $(PACKAGER_IMAGE_EL9):/el9:ro \
-v $(PACKAGER_IMAGE_EL10):/el10:ro \
alpine:latest sh -c \
"cp -r /el8/packages/rpm/el8 /output/rpm/ && \
cp -r /el9/packages/rpm/el9 /output/rpm/ && \
cp -r /el10/packages/rpm/el10 /output/rpm/"
## test-package-rpm: Test RPM package installation in Docker
test-package-rpm: package-rpm
./packaging/test/test-rpm.sh
## test-package: Test RPM package installation
test-package: test-package-rpm
## ci: Full CI pipeline (tests, build, packages, package tests)
ci: ci-test ci-build ci-package ci-package-test
## ci-test: Run all tests for CI
ci-test: test lint
## ci-build: Build for CI (production binary)
ci-build: build
## ci-package: Build all packages for CI
ci-package: package
## ci-package-test: Test all packages for CI
ci-package-test: test-package
## clean: Clean build artifacts and Docker images
clean:
rm -rf $(DIST_DIR)/
rm -f coverage.out
$(DOCKER) rmi $(DEV_IMAGE) 2>/dev/null || true
$(DOCKER) rmi $(RUNTIME_IMAGE) 2>/dev/null || true
$(DOCKER) rmi $(PACKAGER_IMAGE) 2>/dev/null || true
## help: Show this help message
help:
@echo "Usage: make [target]"
@echo ""
@echo "Targets:"
@sed -n 's/^##//p' $(MAKEFILE_LIST) | column -t -s ':' | sed -e 's/^/ /'

426
old/services/correlator/README.md Normal file
View File

@ -0,0 +1,426 @@
# logcorrelator
Service de corrélation de logs HTTP et réseau écrit en Go.
## Description
**logcorrelator** reçoit deux flux de logs JSON via des sockets Unix datagrammes (SOCK_DGRAM) :
- **Source A** : logs HTTP applicatifs (Apache, reverse proxy)
- **Source B** : logs réseau (métadonnées IP/TCP, JA3/JA4, etc.)
Il corrèle les événements sur la base de `src_ip + src_port` dans une fenêtre temporelle configurable, et produit des logs corrélés vers :
- Un fichier local (JSON lines)
- ClickHouse (pour analyse et archivage)
Les logs opérationnels du service (démarrage, erreurs, métriques) sont écrits sur **stderr** et collectés par journald. Aucune donnée corrélée n'apparaît sur stdout.
## Architecture
```
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
│ Source A │────▶│ │────▶│ File Sink │
│ HTTP/Apache │ │ Correlation │ │ (JSON lines) │
│ (Unix DGRAM) │ │ Service │ └─────────────────┘
└─────────────────┘ │ │
│ - Buffers │ ┌─────────────────┐
┌─────────────────┐ │ - Time Window │────▶│ ClickHouse │
│ Source B │────▶│ - Orphan Policy │ │ Sink │
│ Réseau/JA4 │ │ - Keep-Alive │ └─────────────────┘
│ (Unix DGRAM) │ └──────────────────┘
└─────────────────┘
```
Architecture hexagonale : domaine pur (`internal/domain`), ports abstraits (`internal/ports`), adaptateurs (`internal/adapters`), orchestration (`internal/app`).
## Build (100% Docker)
Tout le build, les tests et le packaging RPM s'exécutent dans des conteneurs :
```bash
# Build complet avec tests (builder stage)
make docker-build-dev
# Packaging RPM (el8, el9, el10)
make package-rpm
# Build rapide sans tests
make docker-build-dev-no-test
# Tests en local (nécessite Go 1.21+)
make test
```
### Prérequis
- Docker 20.10+
## Installation
### Packages RPM
```bash
# Générer les packages
make package-rpm
# Installer (Rocky Linux / AlmaLinux)
sudo dnf install -y dist/rpm/el8/logcorrelator-1.1.12-1.el8.x86_64.rpm
sudo dnf install -y dist/rpm/el9/logcorrelator-1.1.12-1.el9.x86_64.rpm
sudo dnf install -y dist/rpm/el10/logcorrelator-1.1.12-1.el10.x86_64.rpm
# Démarrer
sudo systemctl enable --now logcorrelator
sudo systemctl status logcorrelator
```
### Build manuel
```bash
# Binaire local (nécessite Go 1.21+)
go build -o logcorrelator ./cmd/logcorrelator
./logcorrelator -config config.example.yml
```
## Configuration
Fichier YAML. Voir `config.example.yml` pour un exemple complet.
```yaml
log:
level: INFO # DEBUG, INFO, WARN, ERROR
inputs:
unix_sockets:
- name: http
source_type: A # Source HTTP
path: /var/run/logcorrelator/http.socket
format: json
socket_permissions: "0666"
- name: network
source_type: B # Source réseau
path: /var/run/logcorrelator/network.socket
format: json
socket_permissions: "0666"
outputs:
file:
path: /var/log/logcorrelator/correlated.log
clickhouse:
enabled: false
dsn: clickhouse://user:pass@localhost:9000/db
table: http_logs_raw
batch_size: 500
flush_interval_ms: 200
max_buffer_size: 5000
drop_on_overflow: true
timeout_ms: 1000
stdout:
enabled: false # no-op pour les données ; logs opérationnels toujours sur stderr
correlation:
time_window:
value: 10
unit: s
orphan_policy:
apache_always_emit: true
apache_emit_delay_ms: 500 # délai avant émission orphelin A (ms)
network_emit: false
matching:
mode: one_to_many # Keep-Alive : un B peut corréler plusieurs A successifs
buffers:
max_http_items: 10000
max_network_items: 20000
ttl:
network_ttl_s: 120 # TTL remis à zéro à chaque corrélation (Keep-Alive)
# Exclure des IPs source (IPs uniques ou plages CIDR)
exclude_source_ips:
- 10.0.0.1
- 172.16.0.0/12
# Restreindre la corrélation à certains ports de destination (optionnel)
# Si la liste est vide, tous les ports sont corrélés
include_dest_ports:
- 80
- 443
metrics:
enabled: false
addr: ":8080"
```
### Format du DSN ClickHouse
```
clickhouse://username:password@host:port/database
```
Ports : `9000` (natif, recommandé) ou `8123` (HTTP).
## Format des logs
### Source A (HTTP)
```json
{
"src_ip": "192.168.1.1", "src_port": 8080,
"dst_ip": "10.0.0.1", "dst_port": 443,
"timestamp": 1704110400000000000,
"method": "GET", "path": "/api/test"
}
```
### Source B (Réseau)
```json
{
"src_ip": "192.168.1.1", "src_port": 8080,
"dst_ip": "10.0.0.1", "dst_port": 443,
"ja3": "abc123", "ja4": "xyz789"
}
```
### Log corrélé (sortie)
Structure JSON plate — tous les champs A et B sont fusionnés à la racine :
```json
{
"timestamp": "2024-01-01T12:00:00Z",
"src_ip": "192.168.1.1", "src_port": 8080,
"dst_ip": "10.0.0.1", "dst_port": 443,
"correlated": true,
"method": "GET", "path": "/api/test",
"ja3": "abc123", "ja4": "xyz789"
}
```
En cas de collision de champ entre A et B, les deux valeurs sont conservées avec préfixes `a_` et `b_`.
Les orphelins A (sans B correspondant) sont émis avec `"correlated": false, "orphan_side": "A"`.
## Schema ClickHouse
Le fichier `sql/init.sql` contient le schéma complet prêt à l'emploi.
```bash
clickhouse-client --multiquery < sql/init.sql
```
### Architecture des tables
```
http_logs_raw ← inserts du service (raw_json String)
│
└─ mv_http_logs ← vue matérialisée (parse JSON → colonnes typées)
│
▼
http_logs ← table requêtable par les analystes
```
### Table `http_logs` — colonnes
| Groupe | Colonnes |
|---|---|
| Temporel | `time` DateTime, `log_date` Date |
| Réseau | `src_ip` IPv4, `src_port` UInt16, `dst_ip` IPv4, `dst_port` UInt16 |
| HTTP | `method`, `scheme`, `host`, `path`, `query`, `http_version` (LowCardinality) |
| Corrélation | `orphan_side`, `correlated` UInt8, `keepalives` UInt16, `a_timestamp`/`b_timestamp` UInt64, `conn_id` |
| IP meta | `ip_meta_df` UInt8, `ip_meta_id` UInt16, `ip_meta_total_length` UInt16, `ip_meta_ttl` UInt8 |
| TCP meta | `tcp_meta_options`, `tcp_meta_window_size` UInt32, `tcp_meta_mss` UInt16, `tcp_meta_window_scale` UInt8, `syn_to_clienthello_ms` Int32 |
| TLS / fingerprint | `tls_version`, `tls_sni`, `tls_alpn` (LowCardinality), `ja3`, `ja3_hash`, `ja4` |
| En-têtes HTTP | `header_user_agent`, `header_accept`, `header_accept_encoding`, `header_accept_language`, `header_x_request_id`, `header_x_trace_id`, `header_x_forwarded_for`, `header_sec_ch_ua*`, `header_sec_fetch_*` |
### Utilisateurs et permissions
```sql
-- data_writer : INSERT sur http_logs_raw uniquement (compte du service)
GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
-- analyst : lecture sur la table parsée
GRANT SELECT ON ja4_logs.http_logs TO analyst;
```
### Vérification de l'ingestion
```sql
-- Données brutes reçues
SELECT count(*), min(ingest_time), max(ingest_time) FROM ja4_logs.http_logs_raw;
-- Données parsées par la vue matérialisée
SELECT count(*), min(time), max(time) FROM ja4_logs.http_logs;
-- Derniers logs corrélés
SELECT time, src_ip, dst_ip, method, host, path, ja4
FROM ja4_logs.http_logs
WHERE correlated = 1
ORDER BY time DESC LIMIT 10;
```
## Signaux
| Signal | Comportement |
|--------|--------------|
| `SIGINT` / `SIGTERM` | Arrêt gracieux (drain buffers, flush sinks) |
| `SIGHUP` | Réouverture des fichiers de sortie (log rotation) |
## Logs internes
Les logs opérationnels vont sur **stderr** :
```bash
# Systemd
journalctl -u logcorrelator -f
# Docker
docker logs -f logcorrelator
```
## Structure du projet
```
cmd/logcorrelator/ # Point d'entrée
internal/
adapters/
inbound/unixsocket/ # Lecture SOCK_DGRAM → NormalizedEvent
outbound/
clickhouse/ # Sink ClickHouse (batch, retry, logging complet)
file/ # Sink fichier (JSON lines, SIGHUP reopen)
multi/ # Fan-out vers plusieurs sinks
stdout/ # No-op pour les données (logs opérationnels sur stderr)
app/ # Orchestrator (sources → corrélation → sinks)
config/ # Chargement/validation YAML
domain/ # CorrelationService, NormalizedEvent, CorrelatedLog
observability/ # Logger, métriques, serveur HTTP /metrics /health
ports/ # Interfaces EventSource, CorrelatedLogSink, CorrelationProcessor
config.example.yml # Exemple de configuration
Dockerfile # Build multi-stage (builder, runtime, dev)
Dockerfile.package # Packaging RPM multi-distros (el8, el9, el10)
Makefile # Cibles de build
architecture.yml # Spécification architecture
logcorrelator.service # Unité systemd
```
## Débogage
### Logs DEBUG
```yaml
log:
level: DEBUG
```
Exemples de logs produits :
```
[unixsocket:http] DEBUG event received: source=A src_ip=192.168.1.1 src_port=8080
[correlation] DEBUG processing A event: key=192.168.1.1:8080
[correlation] DEBUG correlation found: A(src_ip=... src_port=... ts=...) + B(...)
[correlation] DEBUG A event has no matching B key in buffer: key=...
[correlation] DEBUG event excluded by IP filter: source=A src_ip=10.0.0.1 src_port=8080
[correlation] DEBUG event excluded by dest port filter: source=A dst_port=22
[correlation] DEBUG TTL reset for B event (Keep-Alive): key=... new_ttl=120s
[clickhouse] DEBUG batch sent: rows=42 table=http_logs_raw
```
### Serveur de métriques
```yaml
metrics:
enabled: true
addr: ":8080"
```
`GET /health` → `{"status":"healthy"}`
`GET /metrics` :
```json
{
"events_received_a": 1542, "events_received_b": 1498,
"correlations_success": 1450, "correlations_failed": 92,
"failed_no_match_key": 45, "failed_time_window": 23,
"failed_buffer_eviction": 5, "failed_ttl_expired": 12,
"failed_ip_excluded": 7, "failed_dest_port_filtered": 3,
"buffer_a_size": 23, "buffer_b_size": 18,
"orphans_emitted_a": 92, "orphans_pending_a": 4,
"keepalive_resets": 892
}
```
### Diagnostic par métriques
| Métrique élevée | Cause | Solution |
|---|---|---|
| `failed_no_match_key` | A et B n'ont pas le même `src_ip:src_port` | Vérifier les deux sources |
| `failed_time_window` | Timestamps trop éloignés | Augmenter `time_window.value` ou vérifier NTP |
| `failed_ttl_expired` | B expire avant corrélation | Augmenter `ttl.network_ttl_s` |
| `failed_buffer_eviction` | Buffers trop petits | Augmenter `buffers.max_http_items` / `max_network_items` |
| `failed_ip_excluded` | Traffic depuis IPs exclues | Normal si attendu |
| `failed_dest_port_filtered` | Traffic sur ports non listés | Vérifier `include_dest_ports` |
| `orphans_emitted_a` élevé | Beaucoup de A sans B | Vérifier que la source B envoie des événements |
### Filtrage par IP source
```yaml
correlation:
exclude_source_ips:
- 10.0.0.1 # IP unique (health checks)
- 172.16.0.0/12 # Plage CIDR
```
Les événements depuis ces IPs sont silencieusement ignorés (non corrélés, non émis en orphelin). La métrique `failed_ip_excluded` comptabilise les exclusions.
### Filtrage par port de destination
```yaml
correlation:
include_dest_ports:
- 80 # HTTP
- 443 # HTTPS
- 8080
- 8443
```
Si la liste est non vide, seuls les événements dont le `dst_port` est dans la liste participent à la corrélation. Les autres sont silencieusement ignorés. Liste vide = tous les ports corrélés (comportement par défaut). La métrique `failed_dest_port_filtered` comptabilise les exclusions.
### Scripts de test
```bash
# Script Bash (simple)
./scripts/test-correlation.sh -c 10 -v
# Script Python (scénarios complets : basic, time window, keepalive, différentes IPs)
pip install requests
python3 scripts/test-correlation-advanced.py --all
```
## Troubleshooting
### ClickHouse : erreurs d'insertion
- **`No such column`** : vérifier que la table `http_logs_raw` utilise la colonne unique `raw_json` (pas de colonnes séparées)
- **`ACCESS_DENIED`** : `GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;`
- Les erreurs de flush sont loggées en ERROR dans les logs du service
### Vue matérialisée vide
Si `http_logs_raw` a des données mais `http_logs` est vide :
```sql
-- Vérifier la vue
SHOW CREATE TABLE ja4_logs.mv_http_logs;
-- Vérifier les permissions (la MV s'exécute sous le compte du service)
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
```
### Sockets Unix : permission denied
Vérifier que `socket_permissions: "0666"` est configuré et que le répertoire `/var/run/logcorrelator` appartient à l'utilisateur `logcorrelator`.
### Service systemd ne démarre pas
```bash
journalctl -u logcorrelator -n 50 --no-pager
/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml
```
## License
MIT

974
old/services/correlator/architecture.yml Normal file
View File

@ -0,0 +1,974 @@
service:
name: logcorrelator
context: http-network-correlation
language: go
pattern: hexagonal
description: >
logcorrelator est un service système (lancé par systemd) écrit en Go, chargé
de recevoir deux flux de logs JSON via des sockets Unix, de corréler les
événements HTTP applicatifs (source A, typiquement Apache ou reverse proxy)
avec des événements réseau (source B, métadonnées IP/TCP, JA3/JA4, etc.)
sur la base de la combinaison strictement définie src_ip + src_port, avec
une fenêtre temporelle configurable. Le service supporte les connexions
HTTP Keep-Alive : un log réseau peut être corrélé à plusieurs logs HTTP
successifs (stratégie 1‑à‑N). La rétention en mémoire est bornée par des
tailles de caches configurables et un TTL dynamique pour la source B. Le
service émet toujours les événements A même lorsqu'aucun événement B n'est
disponible, n'émet jamais de logs B seuls, et pousse les résultats vers
ClickHouse et/ou un fichier local.
Fonctionnalités de débogage incluses :
- Serveur de métriques HTTP (/metrics, /health)
- Logs DEBUG détaillés avec raisons des échecs de corrélation
- Filtrage des IPs source (exclude_source_ips)
- Scripts de test (Bash et Python)
- Métriques : événements reçus, corrélations, échecs par raison, buffers, orphelins
runtime:
deployment:
unit_type: systemd
description: >
logcorrelator est livré sous forme de binaire autonome, exécuté comme un
service systemd. L'unité systemd assure le démarrage automatique au boot,
le redémarrage en cas de crash, et une intégration standard dans l'écosystème
Linux.
binary_path: /usr/bin/logcorrelator
config_path: /etc/logcorrelator/logcorrelator.yml
user: logcorrelator
group: logcorrelator
restart: on-failure
systemd_unit:
path: /etc/systemd/system/logcorrelator.service
content_example: |
[Unit]
Description=logcorrelator service
After=network.target
[Service]
Type=simple
User=logcorrelator
Group=logcorrelator
ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=5
# Security hardening
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/logcorrelator /var/run/logcorrelator /etc/logcorrelator
# Resource limits
LimitNOFILE=65536
# Systemd timeouts
TimeoutStartSec=10
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target
os:
supported:
- rocky-linux-8
- rocky-linux-9
- almalinux-10
- autres-linux-recentes
logs:
stdout_stderr: journald
structured: true
description: >
Les logs internes du service (erreurs, messages d'information) sont envoyés
vers stdout/stderr et collectés par journald. Ils sont structurés et ne
contiennent pas de données personnelles.
signals:
graceful_shutdown:
- SIGINT
- SIGTERM
reload:
- SIGHUP
description: >
SIGINT/SIGTERM : arrêt propre (arrêt des sockets, vidage des buffers, fermeture
des sinks). SIGHUP : réouverture des fichiers de sortie (utile pour la
rotation des logs via logrotate) sans arrêter le service.
filesystem:
description: >
Permissions et propriété des fichiers et répertoires utilisés par logcorrelator.
directories:
- path: /var/run/logcorrelator
owner: logcorrelator:logcorrelator
permissions: "0755"
purpose: >
Contient les sockets Unix (http.socket, network.socket).
Les sockets sont créés avec des permissions 0666 (world read/write).
- path: /var/log/logcorrelator
owner: logcorrelator:logcorrelator
permissions: "0750"
purpose: >
Contient les logs corrélés (correlated.log).
- path: /var/lib/logcorrelator
owner: logcorrelator:logcorrelator
permissions: "0750"
purpose: >
Répertoire home du service (données internes).
- path: /etc/logcorrelator
owner: logcorrelator:logcorrelator
permissions: "0750"
purpose: >
Contient la configuration (logcorrelator.yml, logcorrelator.yml.example).
files:
- path: /etc/logcorrelator/logcorrelator.yml
owner: logcorrelator:logcorrelator
permissions: "0640"
rpm_directive: "%config(noreplace)"
- path: /etc/logcorrelator/logcorrelator.yml.example
owner: logcorrelator:logcorrelator
permissions: "0640"
- path: /etc/systemd/system/logcorrelator.service
owner: root:root
permissions: "0644"
- path: /etc/logrotate.d/logcorrelator
owner: root:root
permissions: "0644"
rpm_directive: "%config(noreplace)"
sockets:
- path: /var/run/logcorrelator/http.socket
owner: logcorrelator:logcorrelator
permissions: "0666"
type: unix_datagram
purpose: "Source A - logs HTTP applicatifs"
- path: /var/run/logcorrelator/network.socket
owner: logcorrelator:logcorrelator
permissions: "0666"
type: unix_datagram
purpose: "Source B - logs réseau"
packaging:
description: >
logcorrelator est distribué sous forme de packages .rpm (Rocky Linux, AlmaLinux,
RHEL), construits intégralement dans des conteneurs. Le changelog RPM est mis
à jour à chaque changement de version.
Tous les numéros de version doivent être cohérents entre le spec RPM, le Makefile
(PKG_VERSION), le CHANGELOG.md et les tags git.
Politique de mise à jour de la configuration :
- Le fichier logcorrelator.yml est marqué %config(noreplace) : il n'est JAMAIS
écrasé lors d'une mise à jour. La configuration existante est préservée.
- Le fichier logcorrelator.yml.example est TOUJOURS mis à jour pour refléter
les nouvelles options de configuration disponibles.
- Lors de la première installation, si logcorrelator.yml n'existe pas, il est
créé à partir de logcorrelator.yml.example.
formats:
- rpm
target_distros:
- rocky-linux-8
- rocky-linux-9
- almalinux-10
- rhel-8
- rhel-9
- rhel-10
rpm:
tool: fpm
changelog:
source: git # ou CHANGELOG.md
description: >
À chaque build, un script génère un fichier de changelog RPM à partir de
l'historique (tags/commits) et le passe à fpm (option --rpm-changelog).
contents:
- path: /usr/bin/logcorrelator
type: binary
- path: /etc/logcorrelator/logcorrelator.yml
type: config
directives: "%config(noreplace)"
behavior: >
Jamais écrasé lors des mises à jour. Préservé automatiquement par RPM.
Créé uniquement lors de la première installation s'il n'existe pas.
- path: /etc/logcorrelator/logcorrelator.yml.example
type: doc
behavior: >
TOUJOURS mis à jour lors des mises à jour. Sert de référence pour les
nouvelles options de configuration disponibles.
- path: /etc/systemd/system/logcorrelator.service
type: systemd_unit
- path: /etc/logrotate.d/logcorrelator
type: logrotate_script
directives: "%config(noreplace)"
logrotate_example: |
/var/log/logcorrelator/correlated.log {
daily
rotate 7
compress
delaycompress
missingok
notifempty
create 0640 logcorrelator logcorrelator
sharedscripts
postrotate
/bin/systemctl reload logcorrelator > /dev/null 2>&1 || true
endscript
}
config:
format: yaml
location: /etc/logcorrelator/logcorrelator.yml
reload_strategy: signal_sighup_for_files
description: >
Toute la configuration est centralisée dans un fichier YAML lisible. Le RPM
fournit aussi un fichier d'exemple mis à jour à chaque version.
example: |
# /etc/logcorrelator/logcorrelator.yml
log:
level: INFO # DEBUG, INFO, WARN, ERROR
inputs:
unix_sockets:
# Source HTTP (A) : logs applicatifs en JSON, 1 datagramme = 1 log.
- name: http
source_type: A
path: /var/run/logcorrelator/http.socket
format: json
socket_permissions: "0666"
# Source réseau (B) : logs IP/TCP/JA3... en JSON, 1 datagramme = 1 log.
- name: network
source_type: B
path: /var/run/logcorrelator/network.socket
format: json
socket_permissions: "0666"
outputs:
file:
enabled: true
path: /var/log/logcorrelator/correlated.log
clickhouse:
enabled: false
dsn: clickhouse://user:pass@localhost:9000/db
table: correlated_logs_http_network
batch_size: 500
flush_interval_ms: 200
max_buffer_size: 5000
drop_on_overflow: true
async_insert: true
timeout_ms: 1000
stdout:
enabled: false
level: INFO # DEBUG: tous les logs (y compris orphelins), INFO: seulement corrélés, WARN: corrélés seulement, ERROR: aucun
correlation:
# Fenêtre de corrélation : si le log HTTP arrive avant le réseau, il attend
# au plus cette durée (sauf éviction du cache HTTP).
# Augmentée à 10s pour supporter le Keep-Alive HTTP.
time_window:
value: 10
unit: s
orphan_policy:
apache_always_emit: true # Toujours émettre les événements A, même sans correspondance B
network_emit: false # Ne jamais émettre les événements B seuls
matching:
mode: one_to_many # Keep‑Alive : un B peut corréler plusieurs A.
buffers:
# Tailles max des caches en mémoire (en nombre de logs).
max_http_items: 10000
max_network_items: 20000
ttl:
# Durée de vie standard d'un log réseau (B) en mémoire. Chaque corrélation
# réussie avec un A réinitialise ce TTL.
# Augmenté à 120s pour supporter les sessions HTTP Keep-Alive longues.
network_ttl_s: 120
# Filtrage des IPs source à exclure (optionnel)
exclude_source_ips:
- 10.0.0.1 # IP unique
- 172.16.0.0/12 # Plage CIDR
# Les événements depuis ces IPs sont silencieusement ignorés
# Serveur de métriques HTTP (optionnel, pour débogage et monitoring)
metrics:
enabled: false
addr: ":8080" # Adresse d'écoute du serveur HTTP
# Endpoints:
# GET /metrics - Retourne les métriques de corrélation en JSON
# GET /health - Health check
inputs:
description: >
Deux flux de logs JSON via sockets Unix datagram (SOCK_DGRAM). Chaque datagramme
contient un JSON complet. Le champ source_type ("A" ou "B") doit être spécifié
pour chaque socket. À défaut, la source est déduite automatiquement (présence de
headers = source A, sinon source B).
unix_sockets:
- name: http
id: A
description: >
Source A, logs HTTP applicatifs (Apache, reverse proxy, etc.). Schéma JSON
variable, champ timestamp (int64, nanosecondes) obligatoire, headers dynamiques (header_*).
path: /var/run/logcorrelator/http.socket
source_type: A
permissions: "0666"
protocol: unix
socket_type: dgram
mode: datagram
format: json
framing: message
max_datagram_bytes: 65535
retry_on_error: true
- name: network
id: B
description: >
Source B, logs réseau (métadonnées IP/TCP, JA3/JA4, etc.). Seuls src_ip
et src_port sont requis pour la corrélation. Le champ timestamp est optionnel ;
s'il est absent, l'heure de réception est utilisée.
path: /var/run/logcorrelator/network.socket
source_type: B
permissions: "0666"
protocol: unix
socket_type: dgram
mode: datagram
format: json
framing: message
max_datagram_bytes: 65535
retry_on_error: true
outputs:
description: >
Les logs corrélés sont envoyés vers un ou plusieurs sinks (MultiSink).
sinks:
file:
enabled: true
description: >
Sink fichier local. Un JSON par ligne. Rotation gérée par logrotate,
réouverture du fichier sur SIGHUP. Le champ `enabled: false` coupe
completement l'ecriture du fichier (le sink n'est pas cree).
path: /var/log/logcorrelator/correlated.log
format: json_lines
rotate_managed_by: external_logrotate
clickhouse:
enabled: false
description: >
Sink principal pour l'archivage et l'analyse quasi temps réel. Inserts
batch asynchrones, drop en cas de saturation. Le service insère uniquement
dans une table RAW (raw_json String, ingest_time DateTime DEFAULT now()).
La table parsée et la vue matérialisée sont gérées en externe (DDL séparés).
Toutes les erreurs de connexion, de flush et de retry sont loggées :
INFO à la connexion, ERROR sur échec de flush, WARN sur drop/retry, DEBUG sur envoi réussi.
dsn: clickhouse://user:pass@host:9000/db
table: correlated_logs_http_network
batch_size: 500
flush_interval_ms: 200
max_buffer_size: 5000
drop_on_overflow: true
async_insert: true
timeout_ms: 1000
stdout:
enabled: false
description: >
Sink no-op pour les données. Aucune donnée corrélée ou orpheline n'est
jamais écrite sur stdout. Ce sink existe uniquement pour satisfaire
l'interface CorrelatedLogSink. Les logs opérationnels du service
(démarrage, erreurs, métriques de débogage) sont toujours sur stderr
via observability.Logger, indépendamment de ce sink.
correlation:
description: >
Corrélation stricte basée sur src_ip + src_port et une fenêtre temporelle
configurable. Aucun autre champ n'est utilisé pour la décision de corrélation.
key:
- src_ip
- src_port
time_window:
value: 10
unit: s
description: >
Fenêtre de temps appliquée aux timestamps de A et B. Si B n'arrive pas dans
ce délai, A est émis comme orphelin. Augmentée à 10s pour le Keep-Alive.
retention_limits:
max_http_items: 10000
max_network_items: 20000
description: >
Limites des caches. Si max_http_items est atteint, le plus ancien A est
évincé et émis orphelin. Si max_network_items est atteint, le plus ancien B
est supprimé silencieusement.
ttl_management:
network_ttl_s: 120
description: >
TTL des logs réseau. Chaque fois qu'un B est corrélé à un A (Keep-Alive),
son TTL est remis à cette valeur. Augmenté à 120s pour les sessions longues.
timestamp_source:
apache: timestamp (champ int64, nanosecondes)
network: timestamp (champ int64, nanosecondes) si présent, sinon time (RFC3339),
sinon reception_time (time.Now())
orphan_policy:
apache_always_emit: true
network_emit: false
matching:
mode: one_to_many
description: >
Stratégie 1‑à‑N : un log réseau peut être utilisé pour plusieurs logs HTTP
successifs tant qu'il n'a pas expiré ni été évincé.
ip_filtering:
directive: exclude_source_ips
description: >
Liste d'IPs source (exactes ou plages CIDR) à ignorer silencieusement.
Événements non corrélés, non émis en orphelin. Métrique : failed_ip_excluded.
dest_port_filtering:
directive: include_dest_ports
description: >
Liste blanche de ports de destination. Si non vide, seuls les événements
dont le dst_port est dans la liste participent à la corrélation. Les autres
sont silencieusement ignorés (non corrélés, non émis en orphelin).
Liste vide = tous les ports autorisés (comportement par défaut).
Métrique : failed_dest_port_filtered.
example:
include_dest_ports: [80, 443, 8080, 8443]
schema:
description: >
Schémas variables pour A et B. Quelques champs seulement sont obligatoires
pour la corrélation, les autres sont acceptés sans modification de code.
source_A:
description: >
Logs HTTP applicatifs au format JSON.
required_fields:
- name: src_ip
type: string
- name: src_port
type: int
- name: timestamp
type: int64
unit: ns
optional_fields:
- name: dst_ip
type: string
- name: dst_port
type: int
- name: method
type: string
- name: path
type: string
- name: host
type: string
- name: http_version
type: string
dynamic_fields:
- pattern: header_*
target_map: headers
- pattern: "*"
target_map: extra
source_B:
description: Logs réseau JSON (IP/TCP, JA3/JA4...).
required_fields:
- name: src_ip
type: string
- name: src_port
type: int
optional_fields:
- name: dst_ip
type: string
- name: dst_port
type: int
- name: timestamp
type: int64
unit: ns
- name: time
type: string
format: RFC3339 ou RFC3339Nano
dynamic_fields:
- pattern: "*"
target_map: extra
normalized_event:
description: >
Représentation interne unifiée des événements A/B.
fields:
- name: source
type: enum("A","B")
- name: timestamp
type: time.Time
- name: src_ip
type: string
- name: src_port
type: int
- name: dst_ip
type: string
optional: true
- name: dst_port
type: int
optional: true
- name: headers
type: map[string]string
optional: true
- name: extra
type: map[string]any
correlated_log:
description: >
Structure du log corrélé émis vers les sinks.
fields:
- name: timestamp
type: time.Time
- name: src_ip
type: string
- name: src_port
type: int
- name: dst_ip
type: string
optional: true
- name: dst_port
type: int
optional: true
- name: correlated
type: bool
- name: orphan_side
type: string
- name: "*"
type: map[string]any
clickhouse_schema:
strategy: external_ddls
database: ja4_processing
description: >
La table ClickHouse est gérée en dehors du service. Le service insère dans une
table RAW avec une seule colonne raw_json contenant le log corrélé complet
sérialisé en JSON. La colonne ingest_time utilise DEFAULT now().
Toute extraction de champs (table parsée, vue matérialisée) est gérée en externe
via des DDL séparés, non implémentés dans le service.
tables:
- name: http_logs_raw
description: >
Table d'ingestion brute. Une seule colonne raw_json contient le log corrélé
complet sérialisé en JSON. La colonne ingest_time est auto-générée avec
DEFAULT now(). Partitionnée par jour pour optimiser le TTL.
engine: MergeTree
partition_by: toDate(ingest_time)
order_by: ingest_time
columns:
- name: raw_json
type: String
- name: ingest_time
type: DateTime
default: now()
insert_format: |
INSERT INTO ja4_processing.http_logs_raw (raw_json) VALUES
('{...log corrélé sérialisé en JSON...}')
notes: >
Le service utilise l'API native clickhouse-go/v2 (PrepareBatch + Append + Send).
La colonne ingest_time n'est PAS explicitement insérée (DEFAULT now() est utilisé).
- name: http_logs
description: >
Table parsée (optionnelle, gérée en externe). Le service n'implémente PAS
l'extraction des champs suivants. Si cette table est utilisée, elle doit être
alimentée par une vue matérialisée ou un traitement ETL externe.
engine: MergeTree
partition_by: log_date
order_by: (time, src_ip, dst_ip, ja4)
columns:
- name: time
type: DateTime
- name: log_date
type: Date
default: toDate(time)
- name: src_ip
type: IPv4
- name: src_port
type: UInt16
- name: dst_ip
type: IPv4
- name: dst_port
type: UInt16
- name: method
type: LowCardinality(String)
- name: scheme
type: LowCardinality(String)
- name: host
type: LowCardinality(String)
- name: path
type: String
- name: query
type: String
- name: http_version
type: LowCardinality(String)
- name: orphan_side
type: LowCardinality(String)
- name: correlated
type: UInt8
- name: keepalives
type: UInt16
status: non_implémenté
- name: a_timestamp
type: UInt64
status: non_implémenté
- name: b_timestamp
type: UInt64
status: non_implémenté
- name: conn_id
type: String
status: non_implémenté
- name: ip_meta_df
type: UInt8
status: non_implémenté
- name: ip_meta_id
type: UInt32
status: non_implémenté
- name: ip_meta_total_length
type: UInt32
status: non_implémenté
- name: ip_meta_ttl
type: UInt8
status: non_implémenté
- name: tcp_meta_options
type: LowCardinality(String)
status: non_implémenté
- name: tcp_meta_window_size
type: UInt32
status: non_implémenté
- name: syn_to_clienthello_ms
type: Int32
status: non_implémenté
- name: tls_version
type: LowCardinality(String)
status: non_implémenté
- name: tls_sni
type: LowCardinality(String)
status: non_implémenté
- name: ja3
type: String
status: non_implémenté
- name: ja3_hash
type: String
status: non_implémenté
- name: ja4
type: String
status: non_implémenté
- name: header_user_agent
type: String
status: non_implémenté
- name: header_accept
type: String
status: non_implémenté
- name: header_accept_encoding
type: String
status: non_implémenté
- name: header_accept_language
type: String
status: non_implémenté
- name: header_x_request_id
type: String
status: non_implémenté
- name: header_x_trace_id
type: String
status: non_implémenté
- name: header_x_forwarded_for
type: String
status: non_implémenté
- name: header_sec_ch_ua
type: String
status: non_implémenté
- name: header_sec_ch_ua_mobile
type: String
status: non_implémenté
- name: header_sec_ch_ua_platform
type: String
status: non_implémenté
- name: header_sec_fetch_dest
type: String
status: non_implémenté
- name: header_sec_fetch_mode
type: String
status: non_implémenté
- name: header_sec_fetch_site
type: String
status: non_implémenté
notes: >
Cette table et la vue matérialisée associée sont gérées en externe (DDL séparés).
Le service se contente d'insérer le JSON brut dans http_logs_raw.
Les champs marqués "non_implémenté" ne sont PAS extraits par le service.
users:
description: >
La gestion des utilisateurs ClickHouse est externe au service. Le DSN est
configuré dans le fichier de configuration YAML.
notes: >
Cette section est fournie à titre indicatif pour l'administration ClickHouse.
migration:
description: >
Aucune migration n'est implémentée dans le service. La gestion des schémas
(tables, vues matérialisées) est entièrement externe (DDL séparés).
architecture:
description: >
Architecture hexagonale : domaine de corrélation indépendant, ports abstraits
pour les sources/sinks, adaptateurs pour sockets Unix, fichier, ClickHouse et
stdout, couche application d'orchestration, et modules infra (config, observabilité).
modules:
- name: cmd/logcorrelator
type: entrypoint
responsibilities:
- Chargement de la configuration YAML.
- Initialisation des adaptateurs d'entrée/sortie.
- Création du CorrelationService.
- Démarrage de l'orchestrateur.
- Gestion des signaux (SIGINT, SIGTERM, SIGHUP).
- Versioning via -ldflags (main.Version).
- name: internal/domain
type: domain
responsibilities:
- Modèles NormalizedEvent et CorrelatedLog.
- CorrelationService (fenêtre, TTL, buffers bornés, one-to-many/Keep-Alive, orphelins).
- Filtrage par IP source (exclude_source_ips, CIDR).
- Filtrage par port destination (include_dest_ports, liste blanche).
- Custom JSON marshaling pour CorrelatedLog (structure plate).
- name: internal/ports
type: ports
responsibilities:
- Interfaces EventSource, CorrelatedLogSink, CorrelationProcessor.
- name: internal/app
type: application
responsibilities:
- Orchestrator : EventSource → CorrelationService → MultiSink.
- Gestion du contexte de shutdown et drain des événements.
- name: internal/adapters/inbound/unixsocket
type: adapter_inbound
responsibilities:
- Lecture Unix datagram (SOCK_DGRAM) et parsing JSON → NormalizedEvent.
- Détection automatique de la source (A/B) via source_type ou headers.
- Gestion des permissions de socket (défaut 0666).
- Cleanup du fichier socket à l'arrêt.
- name: internal/adapters/outbound/file
type: adapter_outbound
responsibilities:
- Écriture JSON lines.
- Réouverture du fichier sur SIGHUP (log rotation).
- Validation des chemins (répertoire autorisé).
- name: internal/adapters/outbound/clickhouse
type: adapter_outbound
responsibilities:
- Bufferisation + inserts batch asynchrones.
- Gestion du drop_on_overflow.
- Retry avec backoff exponentiel (MaxRetries=3).
- API native clickhouse-go/v2 (PrepareBatch + Append + Send).
- Logging complet via observability.Logger (SetLogger) : INFO à la connexion,
DEBUG sur envoi réussi (rows/table), WARN sur drop buffer et retries,
ERROR sur échec de flush (périodique, batch, fermeture).
- name: internal/adapters/outbound/stdout
type: adapter_outbound
responsibilities:
- Sink no-op pour les données corrélées.
- Write/Flush/Close ne font rien : les données ne passent jamais par stdout.
- Les logs opérationnels sont sur stderr via observability.Logger (indépendant de ce sink).
- name: internal/adapters/outbound/multi
type: adapter_outbound
responsibilities:
- Fan-out vers plusieurs sinks.
- Implémentation de Reopen() pour la rotation des logs.
- name: internal/config
type: infrastructure
responsibilities:
- Chargement/validation de la configuration YAML.
- Valeurs par défaut et fallback pour champs dépréciés.
- name: internal/observability
type: infrastructure
responsibilities:
- Logger structuré avec niveaux (DEBUG, INFO, WARN, ERROR).
- CorrelationMetrics : suivi des statistiques de corrélation.
- MetricsServer : serveur HTTP pour exposition des métriques (/metrics, /health).
- Traçage des événements exclus (exclude_source_ips).
- Logs pour : événements reçus, corrélations, orphelins, buffer plein.
testing:
unit:
description: >
Tests unitaires table‑driven, couverture cible ≥ 80 %. La couverture actuelle
est d'environ 74-80% selon les versions. Les tests se concentrent sur la logique
de corrélation, les caches, les sinks et le parsing des datagrammes.
coverage_minimum: 0.8
coverage_actual: ~0.74-0.80
focus:
- CorrelationService (fenêtre, TTL, évictions, one-to-many/Keep-Alive)
- Parsing A/B → NormalizedEvent (datagrammes JSON)
- ClickHouseSink (batching, retry, overflow, logging erreurs/succès)
- FileSink (réouverture sur SIGHUP)
- MultiSink (fan-out)
- StdoutSink (no-op data, test stdout reste vide)
- Config (validation, valeurs par défaut, exclude_source_ips)
- UnixSocketSource (lecture, permissions, cleanup)
- CorrelationMetrics (suivi des statistiques)
- MetricsServer (endpoints /metrics et /health)
integration:
description: >
Tests d'intégration limités. Le flux complet A+B → corrélation → sinks est
testé via des tests unitaires avec mocks. ClickHouse est mocké (pas de tests
avec vrai ClickHouse). Scénarios Keep-Alive testés dans correlation_service_test.go.
Scripts de test fournis : scripts/test-correlation.sh et scripts/test-correlation-advanced.py.
docker:
description: >
Build, tests et packaging RPM sont exécutés intégralement dans des conteneurs
via un multi‑stage build. Deux Dockerfiles : Dockerfile (build + runtime + dev)
et Dockerfile.package (RPM multi-distros : el8, el9, el10).
build_pipeline:
multi_stage: true
stages:
- name: builder
base: golang:1.21
description: >
go test -race -coverprofile=coverage.txt ./... avec vérification de couverture
(échec si < 80 %). Compilation d'un binaire statique (CGO_ENABLED=0,
GOOS=linux, GOARCH=amd64).
- name: runtime
base: scratch
description: >
Image minimale contenant uniquement le binaire et la config exemple.
- name: rpm_builder_el8
base: rockylinux:8
description: >
Installation de fpm (via Ruby), construction RPM pour Enterprise Linux 8.
- name: rpm_builder_el9
base: rockylinux:9
description: >
Installation de fpm (via Ruby), construction RPM pour Enterprise Linux 9.
- name: rpm_builder_el10
base: almalinux:10
description: >
Installation de fpm (via Ruby), construction RPM pour Enterprise Linux 10.
- name: output_export
base: alpine:latest
description: >
Export des paquets RPM produits pour les 3 distributions (el8, el9, el10).
files:
- path: Dockerfile
description: Build principal (builder, runtime, dev) et packaging RPM mono-distro.
- path: Dockerfile.package
description: Packaging RPM multi-distros (el8, el9, el10) avec scripts post/preun/postun.
observability:
description: >
Le service inclut des fonctionnalités complètes de débogage et de monitoring
pour diagnostiquer les problèmes de corrélation et surveiller les performances.
logging:
levels:
- DEBUG: Tous les événements reçus, tentatives de corrélation, raisons des échecs
- INFO: Événements corrélés, démarrage/arrêt du service
- WARN: Orphelins émis, buffer plein, TTL expiré
- ERROR: Erreurs de parsing, échecs de sink, erreurs critiques
debug_logs:
- "event received: source=A src_ip=192.168.1.1 src_port=8080 timestamp=..."
- "processing A event: key=192.168.1.1:8080 timestamp=..."
- "correlation found: A(src_ip=... src_port=... ts=...) + B(src_ip=... src_port=... ts=...)"
- "A event has no matching B key in buffer: key=..."
- "A event has same key as B but outside time window: key=... time_diff=5s window=10s"
- "event excluded by IP filter: source=A src_ip=10.0.0.1 src_port=8080"
- "event excluded by dest port filter: source=A dst_port=22"
- "TTL reset for B event (Keep-Alive): key=... new_ttl=120s"
- "[clickhouse] DEBUG batch sent: rows=42 table=correlated_logs_http_network"
info_logs:
- "[clickhouse] INFO connected to ClickHouse: table=... batch_size=500 flush_interval_ms=200"
warn_logs:
- "[clickhouse] WARN buffer full, dropping log: table=... buffer_size=5000"
- "[clickhouse] WARN retrying batch insert: attempt=2/3 delay=100ms rows=42 err=connection refused"
error_logs:
- "[clickhouse] ERROR periodic flush failed: ..."
- "[clickhouse] ERROR batch flush failed: ..."
- "[clickhouse] ERROR final flush on close failed: ..."
metrics_server:
enabled: true
endpoints:
- path: /metrics
method: GET
description: Retourne les métriques de corrélation au format JSON
response_example: |
{
"events_received_a": 1542,
"events_received_b": 1498,
"correlations_success": 1450,
"correlations_failed": 92,
"failed_no_match_key": 45,
"failed_time_window": 23,
"failed_buffer_eviction": 5,
"failed_ttl_expired": 12,
"failed_ip_excluded": 7,
"failed_dest_port_filtered": 3,
"buffer_a_size": 23,
"buffer_b_size": 18,
"orphans_emitted_a": 92,
"keepalive_resets": 892
}
- path: /health
method: GET
description: Health check
response_example: |
{"status":"healthy"}
metrics_tracked:
events_received:
- events_received_a: Nombre d'événements HTTP (source A) reçus
- events_received_b: Nombre d'événements réseau (source B) reçus
correlations:
- correlations_success: Corrélations réussies
- correlations_failed: Échecs de corrélation
failure_reasons:
- failed_no_match_key: Clé src_ip:src_port non trouvée dans le buffer
- failed_time_window: Événements hors fenêtre temporelle
- failed_buffer_eviction: Buffer plein, événement évincé
- failed_ttl_expired: TTL du événement B expiré
- failed_ip_excluded: Événement exclu par filtre IP (exclude_source_ips)
- failed_dest_port_filtered: Événement exclu par filtre port destination (include_dest_ports)
buffers:
- buffer_a_size: Taille actuelle du buffer HTTP
- buffer_b_size: Taille actuelle du buffer réseau
orphans:
- orphans_emitted_a: Orphelins A émis (sans correspondance B)
- orphans_emitted_b: Orphelins B émis (toujours 0, policy: network_emit=false)
- orphans_pending_a: Orphelins A en attente (délai avant émission)
- pending_orphan_match: B a corrélé avec un orphelin A en attente
keepalive:
- keepalive_resets: Resets TTL pour mode Keep-Alive (one-to-many)
troubleshooting:
description: >
Guide de diagnostic basé sur les métriques et logs
common_issues:
- symptom: failed_no_match_key élevé
cause: Les logs A et B n'ont pas le même src_ip + src_port
solution: Vérifier que les deux sources utilisent la même combinaison IP/port
- symptom: failed_time_window élevé
cause: Timestamps trop éloignés (> time_window.value)
solution: Augmenter correlation.time_window.value ou synchroniser les horloges (NTP)
- symptom: failed_ttl_expired élevé
cause: Les événements B expirent avant corrélation
solution: Augmenter correlation.ttl.network_ttl_s
- symptom: failed_buffer_eviction élevé
cause: Buffers trop petits pour le volume de logs
solution: Augmenter correlation.buffers.max_http_items et max_network_items
- symptom: failed_ip_excluded élevé
cause: Traffic depuis des IPs configurées dans exclude_source_ips
solution: Vérifier la configuration, c'est normal si attendu
- symptom: failed_dest_port_filtered élevé
cause: Traffic sur des ports non listés dans include_dest_ports
solution: Vérifier la configuration include_dest_ports, ou vider la liste pour tout accepter
- symptom: orphans_emitted_a élevé
cause: Beaucoup de logs A sans correspondance B
solution: Vérifier que la source B envoie bien les événements attendus
test_scripts:
- name: scripts/test-correlation.sh
description: Script Bash pour tester la corrélation avec des événements synthétiques
features:
- Envoi de paires A+B avec mêmes src_ip:src_port
- Vérification des métriques avant/après
- Options: -c (count), -d (delay), -v (verbose), -m (metrics-url)
- name: scripts/test-correlation-advanced.py
description: Script Python avancé avec multiples scénarios de test
features:
- Basic test: corrélations simples
- Time window test: vérifie l'expiration de la fenêtre temporelle
- Different IP test: vérifie non-corrélation avec IPs différentes
- Keep-Alive test: vérifie le mode one-to-many
- Métriques en temps réel

204
old/services/correlator/cmd/logcorrelator/main.go Normal file
View File

@ -0,0 +1,204 @@
// Package main initialise et démarre le service logcorrelator.
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"syscall"
"time"
"github.com/antitbone/ja4/correlator/internal/adapters/inbound/unixsocket"
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/clickhouse"
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/file"
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/multi"
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/stdout"
"github.com/antitbone/ja4/correlator/internal/app"
"github.com/antitbone/ja4/correlator/internal/config"
"github.com/antitbone/ja4/correlator/internal/domain"
"github.com/antitbone/ja4/correlator/internal/observability"
"github.com/antitbone/ja4/correlator/internal/ports"
)
var Version = "dev"
// main configure les sources, les puits et le service de corrélation, puis démarre l'orchestrateur.
func main() {
configPath := flag.String("config", "config.yml", "path to configuration file")
version := flag.Bool("version", false, "print version and exit")
flag.Parse()
if *version {
fmt.Println(Version)
os.Exit(0)
}
// Load configuration
cfg, err := config.Load(*configPath)
if err != nil {
fmt.Fprintf(os.Stderr, "Error loading configuration: %v\n", err)
os.Exit(1)
}
// Initialize logger with configured level
logger := observability.NewLoggerWithLevel("logcorrelator", cfg.Log.GetLevel())
logger.Info(fmt.Sprintf("Starting logcorrelator version %s (log_level=%s)", Version, cfg.Log.GetLevel()))
// Create sources
sources := make([]ports.EventSource, 0, len(cfg.Inputs.UnixSockets))
for _, inputCfg := range cfg.Inputs.UnixSockets {
source := unixsocket.NewUnixSocketSource(unixsocket.Config{
Name: inputCfg.Name,
Path: inputCfg.Path,
SourceType: inputCfg.SourceType,
SocketPermissions: inputCfg.GetSocketPermissions(),
})
// Set logger for debug logging
source.SetLogger(logger)
sources = append(sources, source)
logger.Info(fmt.Sprintf("Configured input source: name=%s, path=%s, permissions=%o", inputCfg.Name, inputCfg.Path, inputCfg.GetSocketPermissions()))
}
// Create sinks
sinks := make([]ports.CorrelatedLogSink, 0)
if cfg.Outputs.File.Enabled && cfg.Outputs.File.Path != "" {
fileSink, err := file.NewFileSink(file.Config{
Path: cfg.Outputs.File.Path,
})
if err != nil {
logger.Error("Failed to create file sink", err)
os.Exit(1)
}
sinks = append(sinks, fileSink)
logger.Info(fmt.Sprintf("Configured file sink: path=%s", cfg.Outputs.File.Path))
}
if cfg.Outputs.ClickHouse.Enabled {
clickHouseSink, err := clickhouse.NewClickHouseSink(clickhouse.Config{
DSN: cfg.Outputs.ClickHouse.DSN,
Table: cfg.Outputs.ClickHouse.Table,
BatchSize: cfg.Outputs.ClickHouse.BatchSize,
FlushIntervalMs: cfg.Outputs.ClickHouse.FlushIntervalMs,
MaxBufferSize: cfg.Outputs.ClickHouse.MaxBufferSize,
DropOnOverflow: cfg.Outputs.ClickHouse.DropOnOverflow,
AsyncInsert: cfg.Outputs.ClickHouse.AsyncInsert,
TimeoutMs: cfg.Outputs.ClickHouse.TimeoutMs,
})
if err != nil {
logger.Error("Failed to create ClickHouse sink", err)
os.Exit(1)
}
clickHouseSink.SetLogger(logger)
sinks = append(sinks, clickHouseSink)
logger.Info(fmt.Sprintf("Configured ClickHouse sink: table=%s", cfg.Outputs.ClickHouse.Table))
}
if cfg.Outputs.Stdout.Enabled {
stdoutSink := stdout.NewStdoutSink(stdout.Config{Enabled: true})
sinks = append(sinks, stdoutSink)
logger.Info("Configured stdout sink (operational logs on stderr)")
}
// Create multi-sink wrapper
multiSink := multi.NewMultiSink(sinks...)
// Create correlation service
correlationSvc := domain.NewCorrelationService(domain.CorrelationConfig{
TimeWindow: cfg.Correlation.GetTimeWindow(),
ApacheAlwaysEmit: cfg.Correlation.GetApacheAlwaysEmit(),
ApacheEmitDelayMs: cfg.Correlation.GetApacheEmitDelayMs(),
NetworkEmit: false,
MaxHTTPBufferSize: cfg.Correlation.GetMaxHTTPBufferSize(),
MaxNetworkBufferSize: cfg.Correlation.GetMaxNetworkBufferSize(),
NetworkTTLS: cfg.Correlation.GetNetworkTTLS(),
MatchingMode: cfg.Correlation.GetMatchingMode(),
ExcludeSourceIPs: cfg.Correlation.GetExcludeSourceIPs(),
IncludeDestPorts: cfg.Correlation.GetIncludeDestPorts(),
}, &domain.RealTimeProvider{})
// Set logger for correlation service
correlationSvc.SetLogger(logger.WithFields(map[string]any{"component": "correlation"}))
logger.Info(fmt.Sprintf("Correlation service initialized: time_window=%s, emit_orphans=%v, emit_delay_ms=%d",
cfg.Correlation.GetTimeWindow().String(),
cfg.Correlation.GetApacheAlwaysEmit(),
cfg.Correlation.GetApacheEmitDelayMs()))
// Start metrics server if enabled
var metricsServer *observability.MetricsServer
if cfg.Metrics.Enabled {
addr := cfg.Metrics.Addr
if addr == "" {
addr = ":8080" // Default address
}
var err error
metricsServer, err = observability.NewMetricsServer(addr, correlationSvc.GetMetricsSnapshot)
if err != nil {
logger.Error("Failed to create metrics server", err)
os.Exit(1)
}
if err := metricsServer.Start(); err != nil {
logger.Error("Failed to start metrics server", err)
os.Exit(1)
}
logger.Info(fmt.Sprintf("Metrics server started: addr=%s", metricsServer.Addr()))
logger.Info("Metrics endpoints: /metrics (JSON), /health")
}
// Create orchestrator
orchestrator := app.NewOrchestrator(app.OrchestratorConfig{
Sources: sources,
Sink: multiSink,
}, correlationSvc)
// Start the application
if err := orchestrator.Start(); err != nil {
logger.Error("Failed to start orchestrator", err)
os.Exit(1)
}
logger.Info("logcorrelator started successfully")
// Wait for shutdown signal
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
for {
sig := <-sigChan
if sig == syscall.SIGHUP {
// Reopen file sinks for log rotation
logger.Info("SIGHUP received, reopening file sinks...")
if err := multiSink.Reopen(); err != nil {
logger.Error("Error reopening file sinks", err)
} else {
logger.Info("File sinks reopened successfully")
}
continue
}
// Shutdown signal received
logger.Info(fmt.Sprintf("Shutdown signal received: %v", sig))
break
}
// Graceful shutdown
if err := orchestrator.Stop(); err != nil {
logger.Error("Error during shutdown", err)
}
// Stop metrics server
if metricsServer != nil {
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := metricsServer.Stop(shutdownCtx); err != nil {
logger.Error("Error stopping metrics server", err)
}
}
logger.Info("logcorrelator stopped")
}

92
old/services/correlator/config.example.yml Normal file
View File

@ -0,0 +1,92 @@
# logcorrelator configuration file
# Format: YAML
# Logging configuration
log:
level: INFO # DEBUG, INFO, WARN, ERROR
inputs:
unix_sockets:
- name: http
source_type: A
path: /var/run/logcorrelator/http.socket
format: json
socket_permissions: "0666" # world read/write
- name: network
source_type: B
path: /var/run/logcorrelator/network.socket
format: json
socket_permissions: "0666"
outputs:
file:
enabled: true
path: /var/log/logcorrelator/correlated.log
clickhouse:
enabled: false
dsn: clickhouse://user:pass@localhost:9000/ja4_logs
table: http_logs_raw
batch_size: 500
flush_interval_ms: 200
max_buffer_size: 5000
drop_on_overflow: true
async_insert: true
timeout_ms: 1000
stdout:
enabled: false
correlation:
# Time window for correlation (A and B must be within this window)
# Increased to 10s to support HTTP Keep-Alive scenarios
time_window:
value: 10
unit: s
# Orphan policy: what to do when no match is found
orphan_policy:
apache_always_emit: true # Always emit A events, even without B match
apache_emit_delay_ms: 500 # Wait 500ms before emitting as orphan (allows B to arrive)
network_emit: false # Never emit B events alone
# Matching mode: one_to_one or one_to_many (Keep-Alive)
matching:
mode: one_to_many
# Buffer limits (max events in memory)
buffers:
max_http_items: 10000
max_network_items: 20000
# TTL for network events (source B)
# Increased to 120s to support long-lived HTTP Keep-Alive sessions
ttl:
network_ttl_s: 120
# Exclude specific source IPs or CIDR ranges from correlation
# Events from these IPs will be silently dropped (not correlated, not emitted)
# Useful for excluding health checks, internal traffic, or known bad actors
exclude_source_ips:
- 10.0.0.1 # Single IP
- 192.168.1.100 # Another single IP
- 172.16.0.0/12 # CIDR range (private network)
- 10.10.10.0/24 # Another CIDR range
# Restrict correlation to specific destination ports (optional)
# If non-empty, only events whose dst_port matches one of these values will be correlated
# Events on other ports are silently ignored (not correlated, not emitted as orphans)
# Useful to focus on HTTP/HTTPS traffic only and ignore unrelated connections
# include_dest_ports:
# - 80 # HTTP
# - 443 # HTTPS
# - 8080 # HTTP alt
# - 8443 # HTTPS alt
# Metrics server configuration (optional, for debugging/monitoring)
metrics:
enabled: false
addr: ":8080" # Address to listen on (e.g., ":8080", "localhost:8080")
# Endpoints:
# GET /metrics - Returns correlation metrics as JSON
# GET /health - Health check endpoint

224
old/services/correlator/docs/detection.md Normal file
View File

@ -0,0 +1,224 @@
# Architecture de détection — logcorrelator
## Vue d'ensemble
Le système de détection est composé de **trois couches** qui s'enchaînent en pipeline :
```
Trafic HTTP/TLS capturé
│
▼
┌───────────────────┐
│ ClickHouse │ Stockage, agrégation, vues heuristiques
│ (SQL pipeline) │
└────────┬──────────┘
│
▼
┌───────────────────┐
│ bot_detector.py │ Modèle IA (Isolation Forest, cycle 5 min)
│ (Python / ML) │
└────────┬──────────┘
│
▼
┌───────────────────┐
│ ml_detected_ │ Table de résultats (ReplacingMergeTree)
│ anomalies │
└───────────────────┘
```
---
## 1. Ingestion des logs (`http_logs_raw` → `http_logs`)
Les logs bruts arrivent en JSON dans la table `http_logs_raw`. Une **vue matérialisée** (`mv_http_logs`) les parse en temps réel et alimente la table `http_logs`, qui contient les champs structurés suivants :
| Catégorie | Champs clés |
|---|---|
| Réseau | `src_ip`, `src_port`, `dst_ip`, `dst_port` |
| Enrichissement | `src_asn`, `src_country_code`, `src_as_name` (via dictionnaire IPLocate) |
| HTTP | `method`, `host`, `path`, `query`, `http_version` |
| Corrélation | `correlated`, `orphan_side`, `conn_id`, `keepalives` |
| Métadonnées IP | `ip_meta_ttl`, `ip_meta_id`, `ip_meta_df`, `ip_meta_total_length` |
| Métadonnées TCP | `tcp_meta_window_size`, `tcp_meta_mss`, `tcp_meta_window_scale`, `tcp_meta_options` |
| TLS / Fingerprint | `tls_version`, `tls_sni`, `tls_alpn`, `ja3`, `ja3_hash`, `ja4` |
| En-têtes HTTP | `header_user_agent`, `header_sec_ch_ua*`, `header_sec_fetch_*`, … |
L'enrichissement IP est réalisé via le dictionnaire `dict_iplocate_asn` (fichier CSV chargé en mémoire, rechargé toutes les 1-2 heures).
---
## 2. Agrégation comportementale (fenêtre horaire)
Deux tables d'agrégation `AggregatingMergeTree` sont alimentées en continu par des vues matérialisées.
### 2.1 `agg_host_ip_ja4_1h` — Comportement réseau & applicatif
Agrège par triplet **(window_start, src_ip, ja4, host)** toutes les heures :
| Métrique agrégée | Signification |
|---|---|
| `hits` | Nombre total de requêtes |
| `count_post` | Requêtes POST |
| `uniq_paths` | Chemins distincts visités |
| `uniq_query_params` | Paramètres de query distincts |
| `unique_src_ports` | Ports sources distincts |
| `unique_conn_id` | Connexions TCP distinctes |
| `max_keepalives` | Réutilisation maximale d'une connexion |
| `orphan_count` | Requêtes sans corrélation TCP complète |
| `ip_id_zero_count` | Paquets avec IP ID = 0 (spoofing potentiel) |
| `tcp_fp_raw` | Hash de l'empreinte TCP (window, MSS, scale, options) |
| `tcp_jitter_variance` | Variance du délai SYN→ClientHello (jitter TLS) |
| `total_ip_length_var` | Variance de la taille des paquets IP |
| `mss_1460_count` | Requêtes avec MSS = 1460 (signature Ethernet/desktop) |
### 2.2 `agg_header_fingerprint_1h` — Empreinte des en-têtes HTTP
Agrège par **(window_start, src_ip)** :
| Métrique | Signification |
|---|---|
| `header_order_hash` | Hash de l'ordre des en-têtes (fingerprint JA4H) |
| `header_count` | Nombre d'en-têtes distincts |
| `has_accept_language` | Présence de `Accept-Language` |
| `has_cookie` | Présence de `Cookie` |
| `has_referer` | Présence de `Referer` |
| `modern_browser_score` | Score 0/50/100 selon présence UA et `Sec-CH-UA` |
| `ua_ch_mismatch` | Incohérence entre `User-Agent` et `Sec-CH-UA-Platform` |
| `sec_fetch_mode/dest` | Contexte de navigation déclaré |
---
## 3. Exclusions (listes blanches)
Avant toute analyse, deux tables permettent d'**exclure les robots légitimes** connus :
- `bot_ip` (fichier `bot_ip.csv`) — IPs à ignorer (crawlers, monitoring…)
- `bot_ja4` (fichier `bot_ja4.csv`) — Fingerprints JA4 à ignorer
- `ref_bot_networks` — Réseaux CIDR IPv4/IPv6 catégorisés (légitimes ou malveillants)
Ces exclusions sont appliquées dans la vue `view_ai_features_1h`.
---
## 4. Vue IA : `view_ai_features_1h`
Cette vue consolidée **sur 24 heures glissantes** calcule les **28 features** passées au modèle ML. Elle joint les deux tables d'agrégation et dérive les métriques suivantes :
| Feature | Calcul | Signal détecté |
|---|---|---|
| `hit_velocity` | `hits / durée_en_secondes` | Volume de requêtes anormalement élevé |
| `fuzzing_index` | `uniq_query_params / uniq_paths` | Exploration paramétrique (fuzzing) |
| `post_ratio` | `count_post / hits` | Soumission de formulaires en masse |
| `port_exhaustion_ratio` | `unique_src_ports / hits` | Rotation de ports (scan) |
| `orphan_ratio` | `orphan_count / hits` | Requêtes sans handshake complet |
| `ip_id_zero_ratio` | `ip_id_zero_count / hits` | Spoofing d'adresse IP |
| `multiplexing_efficiency` | `hits / unique_conn_id` | Réutilisation des connexions (H2/H3) |
| `true_window_size` | `tcp_win * 2^tcp_scale` | Taille réelle de la fenêtre TCP |
| `window_mss_ratio` | `tcp_win / tcp_mss` | Cohérence TCP stack |
| `tcp_jitter_variance` | Variance SYN→ClientHello | Irrégularité du timing TLS |
| `alpn_http_mismatch` | ALPN=h2 mais HTTP/1.1 | Négociation TLS mensongère |
| `is_alpn_missing` | ALPN absent ou `00` | Client non-standard |
| `sni_host_mismatch` | SNI ≠ Host header | Proxy transparent / bot |
| `mss_mobile_mismatch` | MSS=1460 + score navigateur élevé | Client mobile simulé depuis desktop |
| `is_fake_navigation` | `sec_fetch_mode=navigate` mais `sec_fetch_dest≠document` | Navigation simulée |
| `tcp_shared_count` | Nb d'IPs partageant la même empreinte TCP | Infrastructure partagée / botnet |
| `header_order_shared_count` | Nb d'IPs partageant le même ordre d'en-têtes | Outil automatisé commun |
---
## 5. Modèle IA : Isolation Forest (`bot_detector.py`)
### Cycle d'exécution
Le service tourne en boucle avec un **cycle de 5 minutes** :
```
fetch_and_analyze()
│
├─ Requête SELECT * FROM view_ai_features_1h
│
├─ Nettoyage des données (fillna)
│
├─ Dual-Model routing :
│ ├─ [Complet] correlated=1 → 23 features (réseau + TLS + headers)
│ └─ [Applicatif] correlated=0 → 19 features (headers + comportement)
│
└─ INSERT INTO ml_detected_anomalies
```
### Paramétrage du modèle
| Paramètre | Valeur | Signification |
|---|---|---|
| `n_estimators` | 200 | Nombre d'arbres d'isolation |
| `contamination` | 0.2% | Proportion de bots attendue dans le trafic |
| `seuil de score` | < -0.05 | Score en dessous duquel une session est marquée anomalie |
| `volume minimum` | 500 sessions | En dessous, le modèle est ignoré (trop peu de données) |
### Dual-Model routing
Le trafic est **séparé en deux populations** selon le champ `correlated` :
- **Modèle Complet** (`correlated=1`) : la corrélation TCP↔HTTP est disponible → les features réseau (TTL, jitter TLS, ALPN, SNI) sont fiables et ajoutées à l'analyse.
- **Modèle Applicatif** (`correlated=0`) : seule la couche HTTP est disponible → l'analyse se concentre sur le comportement applicatif (headers, paths, POST ratio…).
---
## 6. Vues heuristiques statiques
En parallèle du modèle IA, cinq vues SQL fournissent des **détections déterministes** sans ML, sur fenêtre 24h :
| Vue | Règle de détection |
|---|---|
| `view_host_ip_ja4_rotation` | IP avec ≥ 5 fingerprints JA4 distincts et > 100 requêtes → rotation d'identité |
| `view_host_ja4_anomalies` | Fingerprint JA4 vu depuis ≥ 20 IPs sur ≥ 3 hôtes → outil de scan distribué |
| `view_form_bruteforce_detected` | ≥ 10 query params distincts et ≥ 20 hits → brute-force de formulaire |
| `view_alpn_mismatch_detected` | HTTP/1.1 avec ALPN h2 ou h3 et ≥ 10 hits → négociation TLS frauduleuse |
| `view_tcp_spoofing_detected` | TTL ≤ 64 avec User-Agent Windows ou iPhone → empreinte OS incohérente |
---
## 7. Résultats : `ml_detected_anomalies`
Les anomalies détectées sont stockées dans une table `ReplacingMergeTree(detected_at)` avec **TTL 30 jours**. La clé d'ordre `(src_ip, ja4, host)` garantit que chaque triplet ne conserve que la **détection la plus récente** (dédoublonnage automatique).
Chaque enregistrement contient :
- Les scores et features ayant conduit à la détection
- Le champ `reason` : texte lisible avec score, vélocité, et indice de fuzzing
- Le champ `is_headless` : déduit de l'incohérence `sec_fetch_mode`
---
## 8. Schéma de flux complet
```
┌─────────────────────────────────────┐
│ http_logs_raw (JSON) │
└──────────────┬──────────────────────┘
│ mv_http_logs (MV)
▼
┌─────────────────────────────────────┐
│ http_logs (parsée) │
└────────┬──────────────┬─────────────┘
│ │
mv_agg_host_ip_ja4 │ │ mv_agg_header_fingerprint
▼ ▼
┌──────────────────┐ ┌──────────────────────────┐
│ agg_host_ip_ja4 │ │ agg_header_fingerprint │
│ _1h │ │ _1h │
└────────┬─────────┘ └──────────┬──────────────┘
│ │
└──────────┬─────────────┘
│ view_ai_features_1h (JOIN + calculs)
▼
┌─────────────────────────────────────┐
│ bot_detector.py (Isolation Forest) │
│ Cycle : 5 min | Fenêtre : 24h │
└──────────────┬──────────────────────┘
│
▼
┌─────────────────────────────────────┐
│ ml_detected_anomalies │
│ (ReplacingMergeTree, TTL 30j) │
└─────────────────────────────────────┘
```

29
old/services/correlator/go.mod Normal file
View File

@ -0,0 +1,29 @@
module github.com/antitbone/ja4/correlator
go 1.21
require (
github.com/ClickHouse/clickhouse-go/v2 v2.23.0
gopkg.in/yaml.v3 v3.0.1
)
require (
github.com/ClickHouse/ch-go v0.61.5 // indirect
github.com/andybalholm/brotli v1.1.0 // indirect
github.com/go-faster/city v1.0.1 // indirect
github.com/go-faster/errors v0.7.1 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/paulmach/orb v0.11.1 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/segmentio/asm v1.2.0 // indirect
github.com/shopspring/decimal v1.3.1 // indirect
go.opentelemetry.io/otel v1.24.0 // indirect
go.opentelemetry.io/otel/trace v1.24.0 // indirect
golang.org/x/sys v0.18.0 // indirect
)
require github.com/antitbone/ja4/ja4common v0.1.0
replace github.com/antitbone/ja4/ja4common => ../../shared/go/ja4common

110
old/services/correlator/go.sum Normal file
View File

@ -0,0 +1,110 @@
github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
github.com/ClickHouse/clickhouse-go/v2 v2.23.0 h1:srmRrkS0BR8gEut87u8jpcZ7geOob6nGj9ifrb+aKmg=
github.com/ClickHouse/clickhouse-go/v2 v2.23.0/go.mod h1:tBhdF3f3RdP7sS59+oBAtTyhWpy0024ZxDMhgxra0QE=
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

111
old/services/correlator/idees/champs.md Normal file
View File

@ -0,0 +1,111 @@
time
log_date
src_ip
- ip source de la connexion
src_port
- port source de la connexion
dst_ip
- ip de destination de la connexion
dst_port
- port de destination de la connexion
src_asn
- Numero d'AS de l'ip source
src_country_code
- Code Pays de l'ip source
src_as_name
- Nom de l'AS de l ip source
src_org
- Organisation de l AS source
src_domain
- domaine de l'AS de l ip source
method
- Methode HTTP [GET, POST, ... ]
scheme
- Type de connexion http [http, https]
host
- Hostname demandé dans l'url
path
- Path demandé dans l'url
query
- Query demandé dans l'url
http_version
- Version du protocol http utilisé
orphan_side
- Indique si le log HTTP a pu etre enrichi avec les informations ip_, tcp, ja3_ et ja4_
- "A" indique que seul le log HTTP est present, sans enrichissement
correlated
- l'algorithm de correlation log http + parametres tcp a il réussi (tcp + ja4/3)
keepalives
- Numero de desquance dans une connexion http avec keepalive.
a_timestamp
b_timestamp
conn_id
ip_meta_df
- Flag dont fragement
ip_meta_id
- id du packet ip
ip_meta_total_length
- Taille des metadata dans pe packet ip
ip_meta_ttl
- TTL du packet ip vu par le serveur destinataire du packet
tcp_meta_options
- options du packet TCP vu par le serveur destinataire du packet
tcp_meta_window_size
- TCP window size vu par le serveur destinataire du packet
tcp_meta_mss
- TCP mss vu par le serveur destinataire du packet
tcp_meta_window_scale
- TCP windows scale vu par le serveur destinataire du packet
syn_to_clienthello_ms
- durée en ms entre le 1er packet SYN et le ClienHello du TLS
tls_version
- Version de TLS negocié avec le serveur destinataire du packet
tls_sni
- SNI, nom de domaine demandé pour le cerificat TLS
tls_alpn
- ALPN annoncé lors du TLS
ja3
- liste des agos utiliés pour la signature ja3
ja3_hash
- hash ja3
ja4
- hash ja4
client_headers
- liste des headers envoyés par le client http sous forme de liste Header,Header2,Header3,...
header_user_agent
- Header HTTP User-Agent
header_accept
- Header HTTP Accept
header_accept_encoding
- Header HTTP Accept-Encoding
header_accept_language
- Header HTTP Accept-Language
header_content_type
- Header Content-Type
header_x_request_id
- Header X-Request-ID
header_x_trace_id
- Header X-Trace-ID
header_x_forwarded_for
- Header X-Forwarded-For
header_sec_ch_ua
- Header Sec-Ch-UA
header_sec_ch_ua_mobile
- Header -Sec-Ch-UA-Mobile
header_sec_ch_ua_platform
- Header Sec-Ch-UA-Plateform
header_sec_fetch_dest
- Header -Sec-Fetch-Dest
header_sec_fetch_mode
- Header Sec-Fetch-Mode
header_sec_fetch_site
- Header Sec-Fetch-Site

30
old/services/correlator/idees/idees.txt Normal file
View File

@ -0,0 +1,30 @@
1. Incohérences de Signatures (Spoofing)
User-Agent vs TLS : Le header_user_agent prétend être un navigateur (Chrome/Safari) mais le ja3/ja4 correspond à un outil de script.
User-Agent vs Headers modernes : Le header_user_agent indique un navigateur récent, mais les headers header_sec_ch_ua_* sont vides ou absents de client_headers.
User-Agent vs ALPN : Le navigateur déclaré ne correspond pas au protocole négocié dans tls_alpn (ex: Chrome sans h2).
OS vs TTL TCP : L'OS déclaré dans le header_user_agent (ex: Windows) contredit la valeur de ip_meta_ttl (ex: 64, typique de Linux).
Host vs SNI : Le nom de domaine dans le header host ne correspond pas au tls_sni demandé lors du handshake TLS.
2. Anomalies de Headers (HTTP Fingerprinting)
Empreinte d'ordre (Fingerprint) : Apparition soudaine d'une disposition de client_headers (ordre exact) très rare, générant beaucoup de trafic.
Pauvreté des headers : Le nombre total de headers dans client_headers est anormalement bas (ex: < 5), typique des scripts basiques.
Absence de headers vitaux : Le trafic prétend être humain mais n'envoie pas header_accept_language ou header_accept_encoding.
Combinaison fatale : Le croisement d'un ja4 spécifique avec un ordre de client_headers inédit (détection de bots modifiant leur TLS mais trahis par l'applicatif).
3. Anomalies Réseau et TCP (Couche 3 & 4)
Mécanique TCP de masse : Une même combinaison (tcp_meta_window_size, tcp_meta_window_scale, tcp_meta_mss) vue sur des milliers d'IP différentes.
Handshake robotique : Un délai syn_to_clienthello_ms anormalement constant (variance quasi nulle) sur un grand nombre de connexions, typique d'un bot en datacenter.
Options TCP atypiques : Des paramètres tcp_meta_options inhabituels pour le trafic web classique de tes vrais utilisateurs.
4. Anomalies Comportementales et Volumétriques (Côté Requête)
Rafale de requêtes (Spike) : Volume d'appels (count) par src_ip ou par ja4 dépassant drastiquement le 99ème percentile historique sur 5 minutes.
Scraping furtif distribué : Un même ja4 (non standard) utilisé par des centaines de src_ip différentes, chacune faisant très peu de requêtes.
Balayage aveugle (Scanner) : Un volume anormal de path uniques (ou path + query) visités par une même IP ou un même ja4 en quelques minutes (remplace la détection des erreurs 404).
Acharnement sur cible (Brute force aveugle) : Une concentration extrême de requêtes ciblant uniquement les path sensibles (login, API, password-reset) sans navigation normale sur le reste du site (remplace la détection des 401/403).
Méthodes suspectes : Utilisation massive ou inhabituelle de method non standards (PUT, DELETE, OPTIONS, TRACE) par rapport à la baseline.
Payloads suspects : Présence de patterns d'injection ou de caractères très inhabituels dans query ou path (longueur extrême, encodages multiples).
Bot "Low and Slow" : IP ou ja4 qui passe sous les radars sur 5 minutes, mais dont le volume cumulé sur 24h ou 7 jours est mathématiquement improbable pour un humain.

521
old/services/correlator/idees/views.md Normal file
View File

@ -0,0 +1,521 @@
# 🛡️ Manuel de Référence Technique : Moteur de Détection Antispam & Bot
Ce document détaille les algorithmes de détection implémentés dans les vues ClickHouse pour la plateforme.
---
## 1. Analyse de la Couche Transport (L4) : La "Trace Physique"
Avant même d'analyser l'URL, le moteur inspecte la manière dont la connexion a été établie. C'est la couche la plus difficile à falsifier pour un attaquant.
### A. Fingerprint de la Pile TCP (`tcp_fingerprint`)
* **Fonctionnement :** Nous utilisons `cityHash64` pour créer un identifiant unique basé sur trois paramètres immuables du handshake : le **MSS** (Maximum Segment Size), la **Window Size** et le **Window Scale**.
* **Ce que ça détecte :** L'unicité logicielle. Un bot tournant sur une image Alpine Linux aura une signature TCP différente d'un utilisateur sur iOS 17 ou Windows 11.
* **Détection de botnet :** Si 500 IPs différentes partagent exactement le même `tcp_fingerprint` ET le même `ja4`, il y a une probabilité de 99% qu'il s'agisse d'un cluster de bots clonés.
### B. Analyse de la gigue (Jitter) et Handshake
* **Fonctionnement :** On calcule la variance (`varPop`) du délai entre le `SYN` et le `ClientHello` TLS.
* **Ce que ça détecte :** La stabilité robotique.
* **Humain :** Latence variable (4G, Wi-Fi, mouvements). La variance est élevée.
* **Bot Datacenter :** Latence ultra-stable (fibre optique dédiée). Une variance proche de 0 indique une connexion automatisée depuis une infrastructure cloud.
---
## 2. Analyse de la Session (L5) : Le "Passeport TLS"
Le handshake TLS est une mine d'or pour identifier la bibliothèque logicielle (OpenSSL, Go-TLS, etc.).
### A. Incohérence UA vs JA4
* **Fonctionnement :** Le moteur croise le `header_user_agent` (déclaratif) avec le `ja4` (structurel).
* **Ce que ça détecte :** Le **Spoofing de Browser**. Un script Python peut facilement écrire `User-Agent: Mozilla/5.0...Chrome/120`, mais il ne peut pas simuler l'ordre exact des extensions TLS et des algorithmes de chiffrement d'un vrai Chrome sans une ingénierie complexe (comme `utls`).
* **Logique de score :** Si UA = Chrome mais JA4 != Signature_Chrome -> **+50 points de risque**.
### B. Discordance Host vs SNI
* **Fonctionnement :** Comparaison entre le champ `tls_sni` (négocié en clair lors du handshake) et le header `Host` (envoyé plus tard dans la requête chiffrée).
* **Ce que ça détecte :** Le **Domain Fronting** ou les attaques par tunnel. Un bot peut demander un certificat pour `domaine-innocent.com` (SNI) mais tenter d'attaquer `api-critique.com` (Host).
---
## 3. Analyse Applicative (L7) : Le "Comportement HTTP"
Une fois le tunnel établi, on analyse la structure de la requête HTTP.
### A. Empreinte d'ordre des Headers (`http_fp`)
* **Fonctionnement :** Nous hashons la liste ordonnée des clés de headers (`Accept`, `User-Agent`, `Connection`, etc.).
* **Ce que ça détecte :** La signature du moteur de rendu. Chaque navigateur (Firefox, Safari, Chromium) a un ordre immuable pour envoyer ses headers.
* **Détection :** Si un client envoie les headers dans un ordre inhabituel ou minimaliste (pauvreté des headers < 6), il est marqué comme suspect.
### B. Analyse des Payloads et Entropie
* **Fonctionnement :** Recherche de patterns via regex dans `query` et `path` (détection SQLi, XSS, Path Traversal).
* **Complexité :** Nous détectons les encodages multiples (ex: `%2520`) qui tentent de tromper les pare-feux simples.
---
## 4. Corrélation Temporelle & Baseline : Le "Voisinage Statistique"
Le score final dépend du passé de la signature TLS.
### A. Le Malus de Nouveauté (`agg_novelty`)
* **Logique :** Une signature (JA4 + FP) vue pour la première fois aujourd'hui est "froide".
* **Traitement :** On applique un malus si `first_seen` date de moins de 2 heures. Un botnet qui vient de lancer une campagne de rotation de signatures sera immédiatement pénalisé par son manque d'historique.
### B. Le Dépassement de Baseline (`tbl_baseline_ja4_7d`)
* **Fonctionnement :** On compare les `hits` actuels au 99ème percentile (`p99`) historique de cette signature précise.
* **Exemple :** Si le JA4 de "Chrome 122" fait habituellement 10 requêtes/min/IP sur votre site, et qu'une IP en fait soudainement 300, le score explose même si la requête est techniquement parfaite.
---
## 5. Synthèse du Scoring (Le Verdict)
| Algorithme | Signal | Impact Score |
| :--- | :--- | :--- |
| **Fingerprint Mismatch** | UA vs TLS (Spoofing) | **Haut (50)** |
| **L4 Anomaly** | Variance latence < 0.5ms | **Moyen (30)** |
| **Path Sensitivity** | Hit sur `/admin` ou `/config` | **Haut (40)** |
| **Payload Security** | Caractères d'injection (SQL/XSS) | **Critique (60)** |
| **Mass Distribution** | 1 JA4 sur > 50 IPs différentes | **Moyen (30)** |
---
## 6. Identification des Hosts par IP et JA4 (sql/hosts.sql)
Cette section détaille les vues d'agrégation et de détection pour identifier quels hosts sont associés à quelles signatures (IP + JA4).
### A. Agrégats de Base
| Table | Granularité | Description |
|-------|-------------|-------------|
| `agg_host_ip_ja4_1h` | heure | Hits, paths uniques, query params, méthodes par (IP, JA4, host) |
| `agg_host_ip_ja4_24h` | jour | Rollup quotidien pour historique long terme |
### B. Vues d'Identification
**`view_host_identification`** - Top hosts par signature
```sql
-- Quel host est associé à cette IP/JA4 ?
SELECT src_ip, ja4, host, total_hits, unique_paths, user_agent
FROM ja4_processing.view_host_identification
WHERE src_ip = '1.2.3.4'
ORDER BY total_hits DESC;
```
**`view_host_ja4_anomalies`** - JA4 partagé par plusieurs hosts (botnet)
```sql
-- Ce JA4 est-il utilisé par plusieurs hosts différents ?
SELECT ja4, hosts, unique_hosts, unique_ips
FROM ja4_processing.view_host_ja4_anomalies
HAVING unique_hosts >= 3;
-- Interprétation : 1 JA4 sur 3+ hosts = botnet cloné probable
```
**`view_host_ip_ja4_rotation`** - IP avec rotation de fingerprints
```sql
-- Cette IP change-t-elle de JA4 fréquemment ?
SELECT src_ip, ja4s, unique_ja4s
FROM ja4_processing.view_host_ip_ja4_rotation
HAVING unique_ja4s >= 5;
-- Interprétation : 1 IP avec 5+ JA4 différents = fingerprint spoofing
```
---
## 7. Détection de Brute Force (sql/hosts.sql)
### A. Brute Force sur POST (endpoints sensibles)
**Table :** `agg_bruteforce_post_5m` - Fenêtres de 5 minutes
**Vue :** `view_bruteforce_post_detected`
```sql
-- Détecter les tentatives de brute force sur les login
SELECT window, src_ip, ja4, host, path, attempts, attempts_per_minute
FROM ja4_processing.view_bruteforce_post_detected
WHERE host = 'api.example.com'
ORDER BY attempts DESC;
-- Threshold : ≥10 POST en 5 minutes sur endpoints sensibles
-- Endpoints ciblés : login, auth, signin, password, admin, wp-login, etc.
```
### B. Brute Force sur Formulaire (Query params variables)
**Table :** `agg_form_bruteforce_5m`
**Vue :** `view_form_bruteforce_detected`
```sql
-- Détecter les requêtes avec query params hautement variables
SELECT window, src_ip, ja4, host, path, requests, unique_query_patterns
FROM ja4_processing.view_form_bruteforce_detected
HAVING requests >= 20 AND unique_query_patterns >= 10;
-- Interprétation : 20+ requêtes avec 10+ patterns query différents
-- = tentative de fuzzing ou brute force sur paramètres
```
---
## 8. Header Fingerprinting (sql/hosts.sql)
Le champ `client_headers` contient la liste comma-separated des headers présents.
Exemple : `"Accept,Accept-Encoding,Sec-CH-UA,Sec-Fetch-Dest,User-Agent"`
### A. Signature par Ordre de Headers
**Table :** `agg_header_fingerprint_1h`
| Champ | Description |
|-------|-------------|
| `header_count` | Nombre total de headers (virgules + 1) |
| `has_*` | Flags pour chaque header moderne (Sec-CH-UA, Sec-Fetch-*, etc.) |
| `header_order_hash` | MD5(client_headers) = signature unique de l'ordre |
| `modern_browser_score` | Score 0-100 basé sur les headers modernes présents |
### B. Vues de Détection
**`view_header_missing_modern_headers`** - Headers modernes manquants
```sql
-- Navigateurs "modernes" avec headers manquants
SELECT src_ip, ja4, header_user_agent, modern_browser_score, header_count
FROM ja4_processing.view_header_missing_modern_headers
WHERE header_user_agent ILIKE '%Chrome%';
-- Threshold : score < 70 pour Chrome/Firefox = suspect
-- Un vrai Chrome envoie automatiquement Sec-CH-UA, Sec-Fetch-*, etc.
```
**`view_header_ua_order_mismatch`** - Spoofing détecté
```sql
-- Même User-Agent avec ordre de headers différent
SELECT header_user_agent, ja4, unique_hashes, unique_ips
FROM ja4_processing.view_header_ua_order_mismatch
HAVING unique_hashes > 1;
-- Interprétation : 1 UA avec 2+ ordres de headers = spoofing ou outil custom
```
**`view_header_minimalist_count`** - Bot minimaliste
```sql
-- Clients avec trop peu de headers
SELECT src_ip, ja4, header_count, header_user_agent
FROM ja4_processing.view_header_minimalist_count
WHERE header_count < 6;
-- Threshold : < 6 headers = bot scripté (curl, Python requests, etc.)
```
**`view_header_sec_ch_missing`** - Incohérence Chrome
```sql
-- Chrome sans Sec-CH-UA (impossible pour un vrai Chrome)
SELECT src_ip, ja4, header_user_agent
FROM ja4_processing.view_header_sec_ch_missing
WHERE header_user_agent ILIKE '%Chrome/%';
```
**`view_header_known_bot_signature`** - Signature botnet
```sql
-- Même ordre de headers sur 10+ IPs différentes
SELECT header_order_hash, header_user_agent, unique_ips, total_hits
FROM ja4_processing.view_header_known_bot_signature
HAVING unique_ips >= 10;
-- Interprétation : 1 signature sur 10+ IPs = cluster de bots clonés
```
---
## 9. ALPN Mismatch Detection (sql/hosts.sql)
### Principe
ALPN (Application-Layer Protocol Negotiation) est une extension TLS qui négocie le protocole HTTP **avant** la requête.
| ALPN déclaré | HTTP réel | Interprétation |
|--------------|-----------|----------------|
| `h2` | `HTTP/2` | ✅ Normal |
| `h2` | `HTTP/1.1` | ❌ Bot mal configuré |
| `http/1.1` | `HTTP/1.1` | ✅ Normal |
### Vue de Détection
**`view_alpn_mismatch_detected`**
```sql
-- Clients déclarant h2 mais parlant HTTP/1.1
SELECT src_ip, ja4, declared_alpn, actual_http_version, mismatches, mismatch_pct
FROM ja4_processing.view_alpn_mismatch_detected
HAVING mismatch_pct >= 80;
-- Threshold : ≥5 requêtes avec ≥80% d'incohérence
-- Cause : curl mal configuré, Python requests, bots spoofant ALPN
```
---
## 10. Rate Limiting & Burst Detection (sql/hosts.sql)
### A. Rate Limiting (1 minute)
**Table :** `agg_rate_limit_1m`
**Vue :** `view_rate_limit_exceeded`
```sql
-- IPs dépassant 50 requêtes/minute
SELECT minute, src_ip, ja4, requests_per_min, unique_paths
FROM ja4_processing.view_rate_limit_exceeded
ORDER BY requests_per_min DESC;
-- Threshold : > 50 req/min = trafic automatisé
-- Un humain ne peut pas soutenir 50+ req/min de manière cohérente
```
### B. Burst Detection (10 secondes)
**Table :** `agg_burst_10s`
**Vue :** `view_burst_detected`
```sql
-- Pics soudains de trafic
SELECT window, src_ip, ja4, burst_count
FROM ja4_processing.view_burst_detected
HAVING burst_count > 20;
-- Threshold : > 20 requêtes en 10 secondes = burst suspect
-- Utile pour détecter les attaques par vagues
```
---
## 11. Path Enumeration / Scanning (sql/hosts.sql)
### Vue de Détection
**`view_path_scan_detected`**
```sql
-- Détection de scanning de paths sensibles
SELECT window, src_ip, ja4, host, sensitive_hits, sensitive_ratio
FROM ja4_processing.view_path_scan_detected
HAVING sensitive_hits >= 5;
-- Paths surveillés : admin, backup, config, .env, .git, wp-admin,
-- phpinfo, test, debug, log, sql, dump, passwd, shadow, htaccess, etc.
-- Threshold : ≥5 paths sensibles en 5 minutes = scanning
```
### Exemple de Résultat
| src_ip | ja4 | host | sensitive_hits | sensitive_ratio |
|--------|-----|------|----------------|-----------------|
| 1.2.3.4 | t13d... | api.example.com | 47 | 94.00 |
| 5.6.7.8 | t13d... | www.example.com | 12 | 80.00 |
**Interprétation :** Ces IPs testent systématiquement les paths sensibles = outils comme Nikto, Dirb, Gobuster.
---
## 12. Payload Attack Detection (sql/hosts.sql)
### A. Types d'Attaques Détectées
| Type | Patterns Détectés |
|------|-------------------|
| **SQL Injection** | `UNION SELECT`, `OR 1=1`, `DROP TABLE`, `; --`, `/* */`, `WAITFOR DELAY`, `SLEEP()` |
| **XSS** | `<script>`, `javascript:`, `onerror=`, `onload=`, `<img src=data:`, `<svg onload>` |
| **Path Traversal** | `../`, `..\\`, `%2e%2e%2f`, `%252e%252e`, `%%32%65%%32%65` |
### Vue de Détection
**`view_payload_attacks_detected`**
```sql
-- Toutes les tentatives d'injection
SELECT window, src_ip, ja4, host, path,
sqli_attempts, xss_attempts, traversal_attempts
FROM ja4_processing.view_payload_attacks_detected
ORDER BY sqli_attempts DESC, xss_attempts DESC, traversal_attempts DESC;
-- Threshold : ≥1 tentative = alerte (zero tolerance)
```
---
## 13. JA4 Botnet Detection (sql/hosts.sql)
### Principe
Un vrai navigateur a un fingerprint TLS unique. Un bot déployé sur 100 machines aura le **même JA4**.
### Vue de Détection
**`view_ja4_botnet_suspected`**
```sql
-- JA4 partagé par 20+ IPs différentes
SELECT ja4, ja3_hash, unique_ips, unique_asns, unique_countries, total_hits
FROM ja4_processing.view_ja4_botnet_suspected
HAVING unique_ips >= 20;
-- Threshold : ≥20 IPs avec le même JA4 = botnet cloné
```
### Exemple de Résultat
| ja4 | ja3_hash | unique_ips | unique_asns | unique_countries |
|-----|----------|------------|-------------|------------------|
| t13d1512... | a3b5c7... | 147 | 12 | 8 |
| t13d0918... | f1e2d3... | 52 | 3 | 2 |
**Interprétation :** 147 IPs différentes avec le même fingerprint = cluster de bots clonés.
---
## 14. Correlation Quality (sql/hosts.sql)
### Principe
Mesure le ratio d'événements non-corrélés (orphelins). Un trafic légitime a une bonne corrélation HTTP/TCP.
### Vue de Détection
**`view_high_orphan_ratio`**
```sql
-- Trafic avec >80% d'événements non-corrélés
SELECT hour, src_ip, ja4, host, correlated, orphans, orphan_pct
FROM ja4_processing.view_high_orphan_ratio
ORDER BY orphan_pct DESC;
-- Threshold : orphan_pct > 80% = trafic suspect
-- Peut indiquer du trafic généré artificiellement
```
---
## 15. Maintenance et Faux Positifs
### Exceptions Connues
| Source | Faux Positif | Solution |
|--------|--------------|----------|
| **Googlebot/Bingbot** | Scan agressif mais légitime | Filtrer par ASN + Reverse DNS |
| **Monitoring interne** | Rate limit élevé | Whitelist par IP/ASN |
| **CDN/Proxy** | JA4 partagé (clients derrière proxy) | Vérifier ASN (Cloudflare, Akamai) |
| **Navigateurs anciens** | Headers modernes manquants | Vérifier UA version |
### Reset des Scores
Les agrégats sont automatiquement purgés par TTL :
- `agg_*_1h` : TTL 7 jours
- `agg_*_5m` : TTL 1 jour
- `agg_*_1m` : TTL 1 jour
Un IP bloquée par erreur retrouvera un score normal après expiration du TTL.
---
## 16. Synthèse des Vues de Détection
| Vue | Détection | Threshold | Impact |
|-----|-----------|-----------|--------|
| `view_bruteforce_post_detected` | POST endpoints sensibles | ≥10 en 5min | 🔴 Haut |
| `view_form_bruteforce_detected` | Query params variables | ≥20 req, ≥10 patterns | 🔴 Haut |
| `view_header_missing_modern_headers` | Headers modernes manquants | score < 70 | 🔴 Haut |
| `view_header_ua_order_mismatch` | UA spoofing (ordre) | >1 hash | 🔴 Haut |
| `view_header_minimalist_count` | Bot minimaliste | < 6 headers | 🔴 Haut |
| `view_header_sec_ch_missing` | Chrome sans Sec-CH | absent | 🟡 Moyen |
| `view_header_known_bot_signature` | Signature connue (botnet) | 10+ IPs | 🔴 Haut |
| `view_alpn_mismatch_detected` | h2 déclaré, HTTP/1.1 parlé | ≥80% mismatch | 🔴 Haut |
| `view_rate_limit_exceeded` | Rate limit dépassé | >50 req/min | 🔴 Haut |
| `view_burst_detected` | Burst soudain | >20 req/10s | 🟡 Moyen |
| `view_path_scan_detected` | Scanning de paths | ≥5 sensibles | 🔴 Haut |
| `view_payload_attacks_detected` | Injections SQLi/XSS | ≥1 tentative | 🔴 Critique |
| `view_ja4_botnet_suspected` | JA4 partagé (botnet) | ≥20 IPs | 🔴 Haut |
| `view_high_orphan_ratio` | Trafic non-corrélé | >80% orphans | 🟡 Moyen |
| `view_host_ja4_anomalies` | JA4 sur plusieurs hosts | ≥3 hosts | 🟡 Moyen |
| `view_host_ip_ja4_rotation` | IP rotate JA4 | ≥5 JA4 | 🟡 Moyen |
---
## 17. Exemples de Requêtes d'Investigation
### Top 10 des IPs les plus suspectes (score cumulé)
```sql
WITH threats AS (
SELECT src_ip, ja4, 'bruteforce' AS type, sum(attempts) AS score
FROM ja4_processing.view_bruteforce_post_detected GROUP BY src_ip, ja4
UNION ALL
SELECT src_ip, ja4, 'path_scan', sum(sensitive_hits)
FROM ja4_processing.view_path_scan_detected GROUP BY src_ip, ja4
UNION ALL
SELECT src_ip, ja4, 'payload', sum(sqli_attempts + xss_attempts)
FROM ja4_processing.view_payload_attacks_detected GROUP BY src_ip, ja4
)
SELECT src_ip, ja4, sum(score) AS total_score, groupArray(type) AS threat_types
FROM threats
GROUP BY src_ip, ja4
ORDER BY total_score DESC
LIMIT 10;
```
### Historique d'une IP suspecte
```sql
SELECT
hour,
host,
countMerge(hits) AS requests,
uniqMerge(uniq_paths) AS unique_paths
FROM ja4_processing.agg_host_ip_ja4_1h
WHERE src_ip = '1.2.3.4'
AND hour >= now() - INTERVAL 24 HOUR
GROUP BY hour, host
ORDER BY hour DESC;
```
### Corrélation JA4 → User-Agent → Hosts
```sql
SELECT
ja4,
any(first_ua) AS user_agent,
groupArray(DISTINCT host) AS hosts,
sum(countMerge(hits)) AS total_requests
FROM ja4_processing.agg_host_ip_ja4_1h
WHERE hour >= now() - INTERVAL 1 HOUR
GROUP BY ja4
ORDER BY total_requests DESC
LIMIT 20;
```
---
## 18. Installation et Maintenance
### Installation
```bash
# Exécuter après init.sql
clickhouse-client --multiquery < sql/hosts.sql
```
### Vérification
```sql
-- Compter les enregistrements
SELECT count(*) FROM ja4_processing.agg_host_ip_ja4_1h;
SELECT count(*) FROM ja4_processing.agg_header_fingerprint_1h;
-- Tester les vues
SELECT * FROM ja4_processing.view_host_identification LIMIT 10;
SELECT * FROM ja4_processing.view_bruteforce_post_detected LIMIT 10;
SELECT * FROM ja4_processing.view_payload_attacks_detected LIMIT 10;
```
### Monitoring
```sql
-- Vues les plus actives (dernière heure)
SELECT
'bruteforce_post' AS view_name, count() AS alerts
FROM ja4_processing.view_bruteforce_post_detected
UNION ALL
SELECT 'path_scan', count() FROM ja4_processing.view_path_scan_detected
UNION ALL
SELECT 'payload_attacks', count() FROM ja4_processing.view_payload_attacks_detected
UNION ALL
SELECT 'ja4_botnet', count() FROM ja4_processing.view_ja4_botnet_suspected
ORDER BY alerts DESC;
```

386
old/services/correlator/internal/adapters/inbound/unixsocket/source.go Normal file
View File

@ -0,0 +1,386 @@
package unixsocket
import (
"context"
"encoding/json"
"fmt"
"math"
"net"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/antitbone/ja4/correlator/internal/domain"
"github.com/antitbone/ja4/correlator/internal/observability"
)
const (
// Maximum datagram size for JSON logs (64KB - Unix datagram limit)
MaxDatagramSize = 65535
// Rate limit: max events per second
MaxEventsPerSecond = 10000
)
// Config holds the Unix socket source configuration.
type Config struct {
Name string
Path string
SourceType string // "A" for Apache/HTTP, "B" for Network, "" for auto-detect
SocketPermissions os.FileMode
}
// UnixSocketSource reads JSON events from a Unix datagram socket.
type UnixSocketSource struct {
config Config
mu sync.Mutex
conn *net.UnixConn
done chan struct{}
wg sync.WaitGroup
stopOnce sync.Once
logger *observability.Logger
}
// NewUnixSocketSource creates a new Unix socket source.
func NewUnixSocketSource(config Config) *UnixSocketSource {
return &UnixSocketSource{
config: config,
done: make(chan struct{}),
logger: observability.NewLogger("unixsocket:" + config.Name),
}
}
// SetLogger sets the logger for the source (for debug mode).
func (s *UnixSocketSource) SetLogger(logger *observability.Logger) {
s.logger = logger.WithFields(map[string]any{"source": s.config.Name})
}
// Name returns the source name.
func (s *UnixSocketSource) Name() string {
return s.config.Name
}
// Start begins listening on the Unix datagram socket.
func (s *UnixSocketSource) Start(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) error {
if strings.TrimSpace(s.config.Path) == "" {
return fmt.Errorf("socket path cannot be empty")
}
// Create parent directory if it doesn't exist
socketDir := filepath.Dir(s.config.Path)
if err := os.MkdirAll(socketDir, 0755); err != nil {
return fmt.Errorf("failed to create socket directory %s: %w", socketDir, err)
}
// Remove existing socket file if present
if info, err := os.Stat(s.config.Path); err == nil {
if info.Mode()&os.ModeSocket != 0 {
if err := os.Remove(s.config.Path); err != nil {
return fmt.Errorf("failed to remove existing socket: %w", err)
}
} else {
return fmt.Errorf("path exists but is not a socket: %s", s.config.Path)
}
}
// Create Unix datagram socket
addr, err := net.ResolveUnixAddr("unixgram", s.config.Path)
if err != nil {
return fmt.Errorf("failed to resolve unix socket address: %w", err)
}
conn, err := net.ListenUnixgram("unixgram", addr)
if err != nil {
return fmt.Errorf("failed to create unix datagram socket: %w", err)
}
s.conn = conn
// Set permissions - fail if we can't
permissions := s.config.SocketPermissions
if permissions == 0 {
permissions = 0666 // default
}
if err := os.Chmod(s.config.Path, permissions); err != nil {
_ = conn.Close()
_ = os.Remove(s.config.Path)
return fmt.Errorf("failed to set socket permissions: %w", err)
}
s.wg.Add(1)
go func() {
defer s.wg.Done()
s.readDatagrams(ctx, eventChan)
}()
return nil
}
// readDatagrams lit en continu les datagrammes sur la socket Unix et envoie les événements normalisés sur le canal.
func (s *UnixSocketSource) readDatagrams(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) {
buf := make([]byte, MaxDatagramSize)
for {
select {
case <-s.done:
return
case <-ctx.Done():
return
default:
}
// Set read deadline to allow periodic context checks
_ = s.conn.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
n, _, err := s.conn.ReadFromUnix(buf)
if err != nil {
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
// Read timeout, continue to check context
continue
}
// Other errors (e.g., closed socket)
select {
case <-s.done:
return
case <-ctx.Done():
return
default:
s.logger.Warnf("read error: %v", err)
continue
}
}
if n == 0 {
continue
}
data := make([]byte, n)
copy(data, buf[:n])
event, err := parseJSONEvent(data, s.config.SourceType)
if err != nil {
// Log parse errors with the raw data for debugging
s.logger.Warnf("parse error: %v | raw: %s", err, string(data))
continue
}
// Debug: log raw events with all key details
s.logger.Debugf("event received: source=%s src_ip=%s src_port=%d timestamp=%v raw_timestamp=%v",
event.Source, event.SrcIP, event.SrcPort, event.Timestamp, event.Raw["timestamp"])
select {
case eventChan <- event:
case <-ctx.Done():
return
}
}
}
// resolveSource détermine la source d'un événement à partir du type déclaré ou de la présence d'en-têtes HTTP.
func resolveSource(sourceType string, headers map[string]string) domain.EventSource {
switch strings.ToLower(strings.TrimSpace(sourceType)) {
case "a", "apache", "http":
return domain.SourceA
case "b", "network", "net":
return domain.SourceB
default:
// fallback compat
if len(headers) > 0 {
return domain.SourceA
}
return domain.SourceB
}
}
// parseJSONEvent désérialise un datagramme JSON et construit un NormalizedEvent validé avec ses champs obligatoires.
func parseJSONEvent(data []byte, sourceType string) (*domain.NormalizedEvent, error) {
var raw map[string]any
if err := json.Unmarshal(data, &raw); err != nil {
return nil, fmt.Errorf("invalid JSON: %w", err)
}
event := &domain.NormalizedEvent{
Raw: raw,
Extra: make(map[string]any),
Headers: make(map[string]string),
}
// Extract headers (header_* fields) first
for k, v := range raw {
if strings.HasPrefix(k, "header_") {
if sv, ok := v.(string); ok {
event.Headers[k[7:]] = sv
}
}
}
// Resolve source first (strict timestamp logic depends on source)
event.Source = resolveSource(sourceType, event.Headers)
// Extract and validate src_ip
if v, ok := getString(raw, "src_ip"); ok {
v = strings.TrimSpace(v)
if v == "" {
return nil, fmt.Errorf("src_ip cannot be empty")
}
event.SrcIP = v
} else {
return nil, fmt.Errorf("missing required field: src_ip")
}
// Extract and validate src_port
if v, ok := getInt(raw, "src_port"); ok {
if v < 1 || v > 65535 {
return nil, fmt.Errorf("src_port must be between 1 and 65535, got %d", v)
}
event.SrcPort = v
} else {
return nil, fmt.Errorf("missing required field: src_port")
}
// Extract dst_ip (optional)
if v, ok := getString(raw, "dst_ip"); ok {
event.DstIP = strings.TrimSpace(v)
}
// Extract dst_port (optional)
if v, ok := getInt(raw, "dst_port"); ok {
if v < 0 || v > 65535 {
return nil, fmt.Errorf("dst_port must be between 0 and 65535, got %d", v)
}
event.DstPort = v
}
// Extract timestamp based on source contract
switch event.Source {
case domain.SourceA:
ts, ok := getInt64(raw, "timestamp_ns")
if !ok {
// Fallback to legacy "timestamp" field name
ts, ok = getInt64(raw, "timestamp")
}
if !ok {
return nil, fmt.Errorf("missing required numeric field: timestamp/timestamp_ns for source A")
}
// Assume nanoseconds
event.Timestamp = time.Unix(0, ts)
case domain.SourceB:
// For network source, try to use event timestamp if available,
// fallback to reception time. This improves correlation accuracy
// when network logs include their own timestamp (e.g., from packet capture).
if ts, ok := getInt64(raw, "timestamp"); ok {
event.Timestamp = time.Unix(0, ts)
} else if timeStr, ok := getString(raw, "time"); ok {
// Try RFC3339 format
if t, err := time.Parse(time.RFC3339, timeStr); err == nil {
event.Timestamp = t
} else if t, err := time.Parse(time.RFC3339Nano, timeStr); err == nil {
event.Timestamp = t
} else {
event.Timestamp = time.Now()
}
} else {
event.Timestamp = time.Now()
}
default:
return nil, fmt.Errorf("unsupported source type: %s", event.Source)
}
// Extra fields
knownFields := map[string]bool{
"src_ip": true, "src_port": true, "dst_ip": true, "dst_port": true,
"timestamp": true, "time": true,
}
for k, v := range raw {
if knownFields[k] {
continue
}
if strings.HasPrefix(k, "header_") {
continue
}
event.Extra[k] = v
}
return event, nil
}
// getString extrait la valeur d'une clé sous forme de chaîne depuis une map JSON désérialisée.
func getString(m map[string]any, key string) (string, bool) {
if v, ok := m[key]; ok {
if s, ok := v.(string); ok {
return s, true
}
}
return "", false
}
// getInt extrait la valeur d'une clé sous forme d'entier depuis une map JSON en gérant les conversions de types courants.
func getInt(m map[string]any, key string) (int, bool) {
if v, ok := m[key]; ok {
switch val := v.(type) {
case float64:
if math.Trunc(val) != val {
return 0, false
}
return int(val), true
case int:
return val, true
case int64:
return int(val), true
case string:
if i, err := strconv.Atoi(val); err == nil {
return i, true
}
}
}
return 0, false
}
// getInt64 extrait la valeur d'une clé sous forme d'entier 64 bits depuis une map JSON en gérant les conversions de types courants.
func getInt64(m map[string]any, key string) (int64, bool) {
if v, ok := m[key]; ok {
switch val := v.(type) {
case float64:
if math.Trunc(val) != val {
return 0, false
}
return int64(val), true
case int:
return int64(val), true
case int64:
return val, true
case string:
if i, err := strconv.ParseInt(val, 10, 64); err == nil {
return i, true
}
}
}
return 0, false
}
// Stop gracefully stops the source.
func (s *UnixSocketSource) Stop() error {
var stopErr error
s.stopOnce.Do(func() {
s.mu.Lock()
defer s.mu.Unlock()
close(s.done)
if s.conn != nil {
_ = s.conn.Close()
}
s.wg.Wait()
// Clean up socket file
if err := os.Remove(s.config.Path); err != nil && !os.IsNotExist(err) {
stopErr = fmt.Errorf("failed to remove socket file: %w", err)
return
}
})
return stopErr
}

596
old/services/correlator/internal/adapters/inbound/unixsocket/source_test.go Normal file
View File

@ -0,0 +1,596 @@
package unixsocket
import (
"context"
"fmt"
"net"
"os"
"testing"
"time"
"github.com/antitbone/ja4/correlator/internal/domain"
)
func TestParseJSONEvent_Apache(t *testing.T) {
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080,
"dst_ip": "10.0.0.1",
"dst_port": 80,
"timestamp": 1704110400000000000,
"method": "GET",
"path": "/api/test",
"header_host": "example.com",
"header_user_agent": "Mozilla/5.0"
}`)
event, err := parseJSONEvent(data, "A")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if event.SrcIP != "192.168.1.1" {
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
}
if event.SrcPort != 8080 {
t.Errorf("expected src_port 8080, got %d", event.SrcPort)
}
if event.Headers["host"] != "example.com" {
t.Errorf("expected header host example.com, got %s", event.Headers["host"])
}
if event.Headers["user_agent"] != "Mozilla/5.0" {
t.Errorf("expected header_user_agent Mozilla/5.0, got %s", event.Headers["user_agent"])
}
if event.Source != domain.SourceA {
t.Errorf("expected source A, got %s", event.Source)
}
expectedTs := time.Unix(0, 1704110400000000000)
if !event.Timestamp.Equal(expectedTs) {
t.Errorf("expected timestamp %v, got %v", expectedTs, event.Timestamp)
}
}
func TestParseJSONEvent_Network(t *testing.T) {
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080,
"dst_ip": "10.0.0.1",
"dst_port": 443,
"timestamp": 1704110400000000000,
"ja3": "abc123def456",
"ja4": "xyz789",
"tcp_meta_flags": "SYN"
}`)
event, err := parseJSONEvent(data, "B")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if event.SrcIP != "192.168.1.1" {
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
}
if event.Extra["ja3"] != "abc123def456" {
t.Errorf("expected ja3 abc123def456, got %v", event.Extra["ja3"])
}
if event.Source != domain.SourceB {
t.Errorf("expected source B, got %s", event.Source)
}
// Network source now uses payload timestamp if available
expectedTs := time.Unix(0, 1704110400000000000)
if !event.Timestamp.Equal(expectedTs) {
t.Errorf("expected network timestamp %v, got %v", expectedTs, event.Timestamp)
}
}
func TestParseJSONEvent_InvalidJSON(t *testing.T) {
data := []byte(`{invalid json}`)
_, err := parseJSONEvent(data, "")
if err == nil {
t.Error("expected error for invalid JSON")
}
}
func TestParseJSONEvent_MissingFields(t *testing.T) {
data := []byte(`{"other_field": "value"}`)
_, err := parseJSONEvent(data, "")
if err == nil {
t.Error("expected error for missing src_ip/src_port")
}
}
func TestParseJSONEvent_SourceARequiresNumericTimestamp(t *testing.T) {
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080,
"time": "2024-01-01T12:00:00Z"
}`)
_, err := parseJSONEvent(data, "A")
if err == nil {
t.Fatal("expected error for source A without numeric timestamp")
}
}
func TestParseJSONEvent_SourceBUsesPayloadTimestamp(t *testing.T) {
expectedTs := int64(1704110400000000000)
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080,
"timestamp": 1704110400000000000
}`)
event, err := parseJSONEvent(data, "B")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectedTime := time.Unix(0, expectedTs)
if !event.Timestamp.Equal(expectedTime) {
t.Errorf("expected source B to use payload timestamp %v, got %v", expectedTime, event.Timestamp)
}
}
func TestParseJSONEvent_SourceBUsesTimeField(t *testing.T) {
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080,
"time": "2024-01-01T12:00:00Z"
}`)
event, err := parseJSONEvent(data, "B")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expectedTime := time.Unix(0, 1704110400000000000)
if !event.Timestamp.Equal(expectedTime) {
t.Errorf("expected source B to use time field %v, got %v", expectedTime, event.Timestamp)
}
}
func TestParseJSONEvent_SourceBFallbackToNow(t *testing.T) {
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080
}`)
before := time.Now()
event, err := parseJSONEvent(data, "B")
after := time.Now()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if event.Timestamp.Before(before.Add(-2*time.Second)) || event.Timestamp.After(after.Add(2*time.Second)) {
t.Errorf("expected source B timestamp near now, got %v", event.Timestamp)
}
}
func TestParseJSONEvent_ExplicitSourceType(t *testing.T) {
tests := []struct {
name string
data string
sourceType string
expected domain.EventSource
}{
{
name: "explicit A",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000}`,
sourceType: "A",
expected: domain.SourceA,
},
{
name: "explicit B",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
sourceType: "B",
expected: domain.SourceB,
},
{
name: "explicit apache",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000}`,
sourceType: "apache",
expected: domain.SourceA,
},
{
name: "explicit network",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
sourceType: "network",
expected: domain.SourceB,
},
{
name: "auto-detect A with headers",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000, "header_host": "example.com"}`,
sourceType: "",
expected: domain.SourceA,
},
{
name: "auto-detect B without headers",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "ja3": "abc"}`,
sourceType: "",
expected: domain.SourceB,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
event, err := parseJSONEvent([]byte(tt.data), tt.sourceType)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if event.Source != tt.expected {
t.Errorf("expected source %s, got %s", tt.expected, event.Source)
}
})
}
}
func TestUnixSocketSource_Name(t *testing.T) {
source := NewUnixSocketSource(Config{
Name: "test_source",
Path: "/tmp/test.sock",
})
if source.Name() != "test_source" {
t.Errorf("expected name 'test_source', got %s", source.Name())
}
}
func TestUnixSocketSource_StopWithoutStart(t *testing.T) {
source := NewUnixSocketSource(Config{
Name: "test_source",
Path: "/tmp/test.sock",
})
// Should not panic
err := source.Stop()
if err != nil {
t.Errorf("expected no error on stop without start, got %v", err)
}
}
func TestUnixSocketSource_EmptyPath(t *testing.T) {
source := NewUnixSocketSource(Config{
Name: "test_source",
Path: "",
})
ctx := context.Background()
eventChan := make(chan *domain.NormalizedEvent, 10)
err := source.Start(ctx, eventChan)
if err == nil {
t.Error("expected error for empty path")
}
}
func TestGetString(t *testing.T) {
m := map[string]any{
"string": "hello",
"int": 42,
"nil": nil,
}
v, ok := getString(m, "string")
if !ok || v != "hello" {
t.Errorf("expected 'hello', got %v, %v", v, ok)
}
_, ok = getString(m, "int")
if ok {
t.Error("expected false for int")
}
_, ok = getString(m, "missing")
if ok {
t.Error("expected false for missing key")
}
}
func TestGetInt(t *testing.T) {
m := map[string]any{
"float": 42.5,
"int": 42,
"int64": int64(42),
"string": "42",
"bad": "not a number",
"nil": nil,
}
tests := []struct {
key string
expected int
ok bool
}{
{"float", 0, false},
{"int", 42, true},
{"int64", 42, true},
{"string", 42, true},
{"bad", 0, false},
{"nil", 0, false},
{"missing", 0, false},
}
for _, tt := range tests {
t.Run(tt.key, func(t *testing.T) {
v, ok := getInt(m, tt.key)
if ok != tt.ok {
t.Errorf("getInt(%q) ok = %v, want %v", tt.key, ok, tt.ok)
}
if v != tt.expected {
t.Errorf("getInt(%q) = %v, want %v", tt.key, v, tt.expected)
}
})
}
}
func TestGetInt64(t *testing.T) {
m := map[string]any{
"float": 42.5,
"int": 42,
"int64": int64(42),
"string": "42",
"bad": "not a number",
"nil": nil,
}
tests := []struct {
key string
expected int64
ok bool
}{
{"float", 0, false},
{"int", 42, true},
{"int64", 42, true},
{"string", 42, true},
{"bad", 0, false},
{"nil", 0, false},
{"missing", 0, false},
}
for _, tt := range tests {
t.Run(tt.key, func(t *testing.T) {
v, ok := getInt64(m, tt.key)
if ok != tt.ok {
t.Errorf("getInt64(%q) ok = %v, want %v", tt.key, ok, tt.ok)
}
if v != tt.expected {
t.Errorf("getInt64(%q) = %v, want %v", tt.key, v, tt.expected)
}
})
}
}
func TestParseJSONEvent_PortValidation(t *testing.T) {
tests := []struct {
name string
data string
sourceType string
wantErr bool
}{
{
name: "valid src_port",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
sourceType: "B",
wantErr: false,
},
{
name: "src_port zero",
data: `{"src_ip": "192.168.1.1", "src_port": 0}`,
sourceType: "B",
wantErr: true,
},
{
name: "src_port negative",
data: `{"src_ip": "192.168.1.1", "src_port": -1}`,
sourceType: "B",
wantErr: true,
},
{
name: "src_port too high",
data: `{"src_ip": "192.168.1.1", "src_port": 70000}`,
sourceType: "B",
wantErr: true,
},
{
name: "valid dst_port zero",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 0}`,
sourceType: "B",
wantErr: false,
},
{
name: "dst_port too high",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 70000}`,
sourceType: "B",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := parseJSONEvent([]byte(tt.data), tt.sourceType)
if (err != nil) != tt.wantErr {
t.Errorf("parseJSONEvent() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestParseJSONEvent_TimestampFallback(t *testing.T) {
data := []byte(`{"src_ip": "192.168.1.1", "src_port": 8080}`)
event, err := parseJSONEvent(data, "B")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// For source B, timestamp is reception time
if event.Timestamp.IsZero() {
t.Error("expected non-zero timestamp")
}
}
func TestUnixSocketSource_StartStopDatagram(t *testing.T) {
tmpPath := "/tmp/test_logcorrelator_datagram.sock"
// Clean up any existing socket
os.Remove(tmpPath)
source := NewUnixSocketSource(Config{
Name: "test_datagram",
Path: tmpPath,
SourceType: "B",
SocketPermissions: 0666,
})
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
eventChan := make(chan *domain.NormalizedEvent, 10)
err := source.Start(ctx, eventChan)
if err != nil {
t.Fatalf("failed to start source: %v", err)
}
// Give socket time to start
time.Sleep(100 * time.Millisecond)
// Verify socket file exists
if _, err := os.Stat(tmpPath); os.IsNotExist(err) {
t.Error("socket file should exist")
}
// Stop the source
err = source.Stop()
if err != nil {
t.Errorf("failed to stop source: %v", err)
}
// Socket file should be cleaned up
time.Sleep(100 * time.Millisecond)
if _, err := os.Stat(tmpPath); !os.IsNotExist(err) {
t.Error("socket file should be removed after stop")
}
}
func TestUnixSocketSource_SendDatagram(t *testing.T) {
tmpPath := "/tmp/test_logcorrelator_send.sock"
os.Remove(tmpPath)
source := NewUnixSocketSource(Config{
Name: "test_send",
Path: tmpPath,
SourceType: "B",
SocketPermissions: 0666,
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
eventChan := make(chan *domain.NormalizedEvent, 10)
err := source.Start(ctx, eventChan)
if err != nil {
t.Fatalf("failed to start source: %v", err)
}
// Give socket time to start
time.Sleep(100 * time.Millisecond)
// Connect and send a datagram
conn, err := net.Dial("unixgram", tmpPath)
if err != nil {
t.Fatalf("failed to dial socket: %v", err)
}
defer conn.Close()
data := []byte(`{"src_ip": "192.168.1.1", "src_port": 8080, "ja3": "test"}`)
_, err = conn.Write(data)
if err != nil {
t.Fatalf("failed to write: %v", err)
}
// Wait for event
select {
case event := <-eventChan:
if event.SrcIP != "192.168.1.1" {
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
}
if event.SrcPort != 8080 {
t.Errorf("expected src_port 8080, got %d", event.SrcPort)
}
case <-time.After(2 * time.Second):
t.Error("timeout waiting for event")
case <-ctx.Done():
t.Error("context cancelled")
}
err = source.Stop()
if err != nil {
t.Errorf("failed to stop source: %v", err)
}
}
func TestUnixSocketSource_MultipleDatagrams(t *testing.T) {
tmpPath := "/tmp/test_logcorrelator_multi.sock"
os.Remove(tmpPath)
source := NewUnixSocketSource(Config{
Name: "test_multi",
Path: tmpPath,
SourceType: "B",
SocketPermissions: 0666,
})
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
eventChan := make(chan *domain.NormalizedEvent, 100)
err := source.Start(ctx, eventChan)
if err != nil {
t.Fatalf("failed to start source: %v", err)
}
// Give socket time to start
time.Sleep(100 * time.Millisecond)
// Connect and send multiple datagrams
conn, err := net.Dial("unixgram", tmpPath)
if err != nil {
t.Fatalf("failed to dial socket: %v", err)
}
defer conn.Close()
for i := 0; i < 5; i++ {
data := []byte(fmt.Sprintf(`{"src_ip": "192.168.1.%d", "src_port": %d, "ja3": "test%d"}`, i+1, 8080+i, i))
_, err = conn.Write(data)
if err != nil {
t.Fatalf("failed to write datagram %d: %v", i, err)
}
}
// Wait for all events
received := 0
timeout := time.After(3 * time.Second)
for received < 5 {
select {
case event := <-eventChan:
received++
t.Logf("received event %d: src_ip=%s", received, event.SrcIP)
case <-timeout:
t.Errorf("timeout waiting for events, received %d/5", received)
goto done
case <-ctx.Done():
t.Error("context cancelled")
goto done
}
}
done:
err = source.Stop()
if err != nil {
t.Errorf("failed to stop source: %v", err)
}
}

391
old/services/correlator/internal/adapters/outbound/clickhouse/sink.go Normal file
View File

@ -0,0 +1,391 @@
package clickhouse
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"strings"
"sync"
"time"
"github.com/ClickHouse/clickhouse-go/v2"
"github.com/antitbone/ja4/correlator/internal/domain"
"github.com/antitbone/ja4/correlator/internal/observability"
)
const (
// DefaultBatchSize is the default number of records per batch
DefaultBatchSize = 500
// DefaultFlushIntervalMs is the default flush interval in milliseconds
DefaultFlushIntervalMs = 200
// DefaultMaxBufferSize is the default maximum buffer size
DefaultMaxBufferSize = 5000
// DefaultTimeoutMs is the default timeout for operations in milliseconds
DefaultTimeoutMs = 1000
// DefaultPingTimeoutMs is the timeout for initial connection ping
DefaultPingTimeoutMs = 5000
// MaxRetries is the maximum number of retry attempts for failed inserts
MaxRetries = 3
// RetryBaseDelay is the base delay between retries
RetryBaseDelay = 100 * time.Millisecond
)
// Config holds the ClickHouse sink configuration.
type Config struct {
DSN string
Table string
BatchSize int
FlushIntervalMs int
MaxBufferSize int
DropOnOverflow bool
AsyncInsert bool
TimeoutMs int
}
// ClickHouseSink writes correlated logs to ClickHouse.
type ClickHouseSink struct {
config Config
conn clickhouse.Conn
mu sync.Mutex
buffer []domain.CorrelatedLog
flushChan chan struct{}
done chan struct{}
wg sync.WaitGroup
closeOnce sync.Once
logger *observability.Logger
}
// SetLogger sets the logger used by the sink.
func (s *ClickHouseSink) SetLogger(logger *observability.Logger) {
s.logger = logger.WithFields(map[string]any{"sink": "clickhouse"})
}
// NewClickHouseSink creates a new ClickHouse sink.
func NewClickHouseSink(config Config) (*ClickHouseSink, error) {
if strings.TrimSpace(config.DSN) == "" {
return nil, fmt.Errorf("clickhouse DSN is required")
}
if strings.TrimSpace(config.Table) == "" {
return nil, fmt.Errorf("clickhouse table is required")
}
// Apply defaults
if config.BatchSize <= 0 {
config.BatchSize = DefaultBatchSize
}
if config.FlushIntervalMs <= 0 {
config.FlushIntervalMs = DefaultFlushIntervalMs
}
if config.MaxBufferSize <= 0 {
config.MaxBufferSize = DefaultMaxBufferSize
}
if config.TimeoutMs <= 0 {
config.TimeoutMs = DefaultTimeoutMs
}
s := &ClickHouseSink{
config: config,
buffer: make([]domain.CorrelatedLog, 0, config.BatchSize),
flushChan: make(chan struct{}, 1),
done: make(chan struct{}),
logger: observability.NewLogger("clickhouse"),
}
// Parse DSN and create options
options, err := clickhouse.ParseDSN(config.DSN)
if err != nil {
return nil, fmt.Errorf("failed to parse ClickHouse DSN: %w", err)
}
// Connect to ClickHouse using native API
conn, err := clickhouse.Open(options)
if err != nil {
return nil, fmt.Errorf("failed to connect to ClickHouse: %w", err)
}
// Ping with timeout to verify connection
pingCtx, pingCancel := context.WithTimeout(context.Background(), time.Duration(DefaultPingTimeoutMs)*time.Millisecond)
defer pingCancel()
if err := conn.Ping(pingCtx); err != nil {
_ = conn.Close()
return nil, fmt.Errorf("failed to ping ClickHouse: %w", err)
}
s.conn = conn
s.log().Infof("connected to ClickHouse: table=%s batch_size=%d flush_interval_ms=%d",
config.Table, config.BatchSize, config.FlushIntervalMs)
// Start flush goroutine
s.wg.Add(1)
go s.flushLoop()
return s, nil
}
// Name returns the sink name.
func (s *ClickHouseSink) Name() string {
return "clickhouse"
}
// log returns the logger, initializing a default one if not set (e.g. in tests).
func (s *ClickHouseSink) log() *observability.Logger {
if s.logger == nil {
s.logger = observability.NewLogger("clickhouse")
}
return s.logger
}
// Reopen is a no-op for ClickHouse (connection is managed internally).
func (s *ClickHouseSink) Reopen() error {
return nil
}
// Write adds a log to the buffer.
func (s *ClickHouseSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
deadline := time.Now().Add(time.Duration(s.config.TimeoutMs) * time.Millisecond)
for {
s.mu.Lock()
if len(s.buffer) < s.config.MaxBufferSize {
s.buffer = append(s.buffer, log)
if len(s.buffer) >= s.config.BatchSize {
select {
case s.flushChan <- struct{}{}:
default:
}
}
s.mu.Unlock()
return nil
}
drop := s.config.DropOnOverflow
s.mu.Unlock()
if drop {
s.log().Warnf("buffer full, dropping log: table=%s buffer_size=%d", s.config.Table, s.config.MaxBufferSize)
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("buffer full, timeout exceeded")
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(10 * time.Millisecond):
}
}
}
// Flush flushes the buffer to ClickHouse.
func (s *ClickHouseSink) Flush(ctx context.Context) error {
return s.doFlush(ctx)
}
// Close closes the sink.
func (s *ClickHouseSink) Close() error {
var closeErr error
s.closeOnce.Do(func() {
if s.done != nil {
close(s.done)
}
s.wg.Wait()
flushCtx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
defer cancel()
if err := s.doFlush(flushCtx); err != nil {
closeErr = err
}
if s.conn != nil {
if err := s.conn.Close(); err != nil && closeErr == nil {
closeErr = err
}
}
})
return closeErr
}
func (s *ClickHouseSink) flushLoop() {
defer s.wg.Done()
ticker := time.NewTicker(time.Duration(s.config.FlushIntervalMs) * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-s.done:
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
if err := s.doFlush(ctx); err != nil {
s.log().Error("final flush on close failed", err)
}
cancel()
return
case <-ticker.C:
s.mu.Lock()
needsFlush := len(s.buffer) > 0
s.mu.Unlock()
if needsFlush {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
if err := s.doFlush(ctx); err != nil {
s.log().Error("periodic flush failed", err)
}
cancel()
}
case <-s.flushChan:
s.mu.Lock()
needsFlush := len(s.buffer) >= s.config.BatchSize
s.mu.Unlock()
if needsFlush {
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
if err := s.doFlush(ctx); err != nil {
s.log().Error("batch flush failed", err)
}
cancel()
}
}
}
}
func (s *ClickHouseSink) doFlush(ctx context.Context) error {
s.mu.Lock()
if len(s.buffer) == 0 {
s.mu.Unlock()
return nil
}
// Copy buffer to flush
buffer := make([]domain.CorrelatedLog, len(s.buffer))
copy(buffer, s.buffer)
s.buffer = make([]domain.CorrelatedLog, 0, s.config.BatchSize)
s.mu.Unlock()
if s.conn == nil {
return fmt.Errorf("clickhouse connection is not initialized")
}
batchSize := len(buffer)
// Retry logic with exponential backoff
var lastErr error
for attempt := 0; attempt < MaxRetries; attempt++ {
if attempt > 0 {
delay := RetryBaseDelay * time.Duration(1<<uint(attempt-1))
s.log().Warnf("retrying batch insert: attempt=%d/%d delay=%s rows=%d err=%v",
attempt+1, MaxRetries, delay, batchSize, lastErr)
select {
case <-time.After(delay):
case <-ctx.Done():
return ctx.Err()
}
}
lastErr = s.executeBatch(ctx, buffer)
if lastErr == nil {
s.log().Debugf("batch sent: rows=%d table=%s", batchSize, s.config.Table)
return nil
}
if !isRetryableError(lastErr) {
return fmt.Errorf("non-retryable error: %w", lastErr)
}
}
return fmt.Errorf("failed after %d retries (batch size: %d): %w", MaxRetries, batchSize, lastErr)
}
func (s *ClickHouseSink) executeBatch(ctx context.Context, buffer []domain.CorrelatedLog) error {
if s.conn == nil {
return fmt.Errorf("clickhouse connection is not initialized")
}
// Table schema: http_logs_raw (raw_json String)
// Single column insert - the entire log is serialized as JSON string
query := fmt.Sprintf(`INSERT INTO %s (raw_json)`, s.config.Table)
// Prepare batch using native clickhouse-go/v2 API
batch, err := s.conn.PrepareBatch(ctx, query)
if err != nil {
return fmt.Errorf("failed to prepare batch: %w", err)
}
for i, log := range buffer {
// Marshal the entire CorrelatedLog to JSON
logJSON, marshalErr := json.Marshal(log)
if marshalErr != nil {
return fmt.Errorf("failed to marshal log %d to JSON: %w", i, marshalErr)
}
// Append the JSON string as the raw_json column value
appendErr := batch.Append(string(logJSON))
if appendErr != nil {
return fmt.Errorf("failed to append log %d to batch: %w", i, appendErr)
}
}
// Send the batch - DO NOT FORGET this step
sendErr := batch.Send()
if sendErr != nil {
return fmt.Errorf("failed to send batch (%d rows): %w", len(buffer), sendErr)
}
return nil
}
// isRetryableError checks if an error is retryable.
func isRetryableError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
if errors.Is(err, context.Canceled) {
return false
}
var netErr net.Error
if errors.As(err, &netErr) {
if netErr.Timeout() {
return true
}
}
errStr := strings.ToLower(err.Error())
// Explicit non-retryable SQL/schema errors
if strings.Contains(errStr, "syntax error") ||
strings.Contains(errStr, "unknown table") ||
strings.Contains(errStr, "unknown column") ||
(strings.Contains(errStr, "table") && strings.Contains(errStr, "not found")) {
return false
}
// Fallback network/transient errors
retryableErrors := []string{
"connection refused",
"connection reset",
"timeout",
"temporary failure",
"network is unreachable",
"broken pipe",
"no route to host",
}
for _, re := range retryableErrors {
if strings.Contains(errStr, re) {
return true
}
}
return false
}

538
old/services/correlator/internal/adapters/outbound/clickhouse/sink_test.go Normal file
View File

@ -0,0 +1,538 @@
package clickhouse
import (
"context"
"testing"
"time"
"github.com/antitbone/ja4/correlator/internal/domain"
"github.com/antitbone/ja4/correlator/internal/observability"
)
func TestClickHouseSink_Name(t *testing.T) {
sink := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
},
}
if sink.Name() != "clickhouse" {
t.Errorf("expected name 'clickhouse', got %s", sink.Name())
}
}
func TestClickHouseSink_ConfigDefaults(t *testing.T) {
// Test that defaults are applied correctly
config := Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
// Other fields are zero, should get defaults
}
// Verify defaults would be applied (we can't actually connect in tests)
if config.BatchSize <= 0 {
config.BatchSize = DefaultBatchSize
}
if config.FlushIntervalMs <= 0 {
config.FlushIntervalMs = DefaultFlushIntervalMs
}
if config.MaxBufferSize <= 0 {
config.MaxBufferSize = DefaultMaxBufferSize
}
if config.TimeoutMs <= 0 {
config.TimeoutMs = DefaultTimeoutMs
}
if config.BatchSize != DefaultBatchSize {
t.Errorf("expected BatchSize %d, got %d", DefaultBatchSize, config.BatchSize)
}
if config.FlushIntervalMs != DefaultFlushIntervalMs {
t.Errorf("expected FlushIntervalMs %d, got %d", DefaultFlushIntervalMs, config.FlushIntervalMs)
}
if config.MaxBufferSize != DefaultMaxBufferSize {
t.Errorf("expected MaxBufferSize %d, got %d", DefaultMaxBufferSize, config.MaxBufferSize)
}
if config.TimeoutMs != DefaultTimeoutMs {
t.Errorf("expected TimeoutMs %d, got %d", DefaultTimeoutMs, config.TimeoutMs)
}
}
func TestClickHouseSink_Write_BufferOverflow(t *testing.T) {
// This test verifies the buffer overflow logic without actually connecting
config := Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
BatchSize: 10,
MaxBufferSize: 10,
DropOnOverflow: true,
TimeoutMs: 100,
FlushIntervalMs: 1000,
}
// We can't test actual writes without a ClickHouse instance,
// but we can verify the config is valid
if config.BatchSize > config.MaxBufferSize {
t.Error("BatchSize should not exceed MaxBufferSize")
}
}
func TestClickHouseSink_IsRetryableError(t *testing.T) {
tests := []struct {
name string
err error
expected bool
}{
{"nil error", nil, false},
{"connection refused", &mockError{"connection refused"}, true},
{"connection reset", &mockError{"connection reset by peer"}, true},
{"timeout", &mockError{"timeout waiting for response"}, true},
{"network unreachable", &mockError{"network is unreachable"}, true},
{"broken pipe", &mockError{"broken pipe"}, true},
{"syntax error", &mockError{"syntax error in SQL"}, false},
{"table not found", &mockError{"table test not found"}, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := isRetryableError(tt.err)
if result != tt.expected {
t.Errorf("expected %v, got %v", tt.expected, result)
}
})
}
}
func TestClickHouseSink_FlushEmpty(t *testing.T) {
// Test that flushing an empty buffer doesn't cause issues
// (We can't test actual ClickHouse operations without a real instance)
s := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
},
buffer: make([]domain.CorrelatedLog, 0),
}
// Should not panic or error on empty flush
ctx := context.Background()
err := s.Flush(ctx)
if err != nil {
t.Errorf("expected no error on empty flush, got %v", err)
}
}
func TestClickHouseSink_CloseWithoutConnect(t *testing.T) {
// Test that closing without connecting doesn't panic
s := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
},
buffer: make([]domain.CorrelatedLog, 0),
done: make(chan struct{}),
}
err := s.Close()
if err != nil {
t.Errorf("expected no error on close without connect, got %v", err)
}
}
func TestClickHouseSink_Constants(t *testing.T) {
// Verify constants have reasonable values
if DefaultBatchSize <= 0 {
t.Error("DefaultBatchSize should be positive")
}
if DefaultFlushIntervalMs <= 0 {
t.Error("DefaultFlushIntervalMs should be positive")
}
if DefaultMaxBufferSize <= 0 {
t.Error("DefaultMaxBufferSize should be positive")
}
if DefaultTimeoutMs <= 0 {
t.Error("DefaultTimeoutMs should be positive")
}
if DefaultPingTimeoutMs <= 0 {
t.Error("DefaultPingTimeoutMs should be positive")
}
if MaxRetries <= 0 {
t.Error("MaxRetries should be positive")
}
if RetryBaseDelay <= 0 {
t.Error("RetryBaseDelay should be positive")
}
}
// mockError implements error for testing
type mockError struct {
msg string
}
func (e *mockError) Error() string {
return e.msg
}
// Test the doFlush function with empty buffer (no actual DB connection)
func TestClickHouseSink_DoFlushEmpty(t *testing.T) {
s := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
},
buffer: make([]domain.CorrelatedLog, 0),
}
ctx := context.Background()
err := s.doFlush(ctx)
if err != nil {
t.Errorf("expected no error when flushing empty buffer, got %v", err)
}
}
// Test that buffer is properly managed (without actual DB operations)
func TestClickHouseSink_BufferManagement(t *testing.T) {
log := domain.CorrelatedLog{
SrcIP: "192.168.1.1",
SrcPort: 8080,
Correlated: 1,
}
s := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
MaxBufferSize: 100, // Allow more than 1 element
DropOnOverflow: false,
TimeoutMs: 1000,
},
buffer: []domain.CorrelatedLog{log},
}
// Verify buffer has data
if len(s.buffer) != 1 {
t.Fatalf("expected buffer length 1, got %d", len(s.buffer))
}
// Test that Write properly adds to buffer
ctx := context.Background()
err := s.Write(ctx, log)
if err != nil {
t.Errorf("unexpected error on Write: %v", err)
}
if len(s.buffer) != 2 {
t.Errorf("expected buffer length 2 after Write, got %d", len(s.buffer))
}
}
// Test Write with context cancellation
func TestClickHouseSink_Write_ContextCancel(t *testing.T) {
s := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
MaxBufferSize: 1,
DropOnOverflow: false,
TimeoutMs: 10,
},
buffer: make([]domain.CorrelatedLog, 0, 1),
}
// Fill the buffer
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
s.buffer = append(s.buffer, log)
// Try to write with cancelled context
ctx, cancel := context.WithCancel(context.Background())
cancel() // Cancel immediately
err := s.Write(ctx, log)
if err == nil {
t.Error("expected error when writing with cancelled context")
}
}
// Test DropOnOverflow behavior
func TestClickHouseSink_Write_DropOnOverflow(t *testing.T) {
s := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
MaxBufferSize: 1,
DropOnOverflow: true,
TimeoutMs: 10,
},
buffer: make([]domain.CorrelatedLog, 0, 1),
}
// Fill the buffer
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
s.buffer = append(s.buffer, log)
// Try to write when buffer is full - should drop silently
ctx := context.Background()
err := s.Write(ctx, log)
if err != nil {
t.Errorf("expected no error when DropOnOverflow is true, got %v", err)
}
}
// TestIsRetryableError_ContextDeadlineExceeded tests context.DeadlineExceeded is retryable.
func TestIsRetryableError_ContextDeadlineExceeded(t *testing.T) {
if !isRetryableError(context.DeadlineExceeded) {
t.Error("context.DeadlineExceeded should be retryable")
}
}
// TestIsRetryableError_ContextCanceled tests context.Canceled is NOT retryable.
func TestIsRetryableError_ContextCanceled(t *testing.T) {
if isRetryableError(context.Canceled) {
t.Error("context.Canceled should not be retryable")
}
}
// TestIsRetryableError_NetTimeout tests net.Error with Timeout() = true is retryable.
func TestIsRetryableError_NetTimeout(t *testing.T) {
err := &mockNetError{timeout: true, temporary: false}
if !isRetryableError(err) {
t.Error("net.Error with Timeout()=true should be retryable")
}
}
// TestIsRetryableError_NetNoTimeout tests net.Error with Timeout() = false is NOT retryable.
func TestIsRetryableError_NetNoTimeout(t *testing.T) {
err := &mockNetError{timeout: false, temporary: false}
if isRetryableError(err) {
t.Error("net.Error with Timeout()=false should not be retryable (unless msg matches)")
}
}
// TestIsRetryableError_UnknownTable tests "unknown table" is NOT retryable.
func TestIsRetryableError_UnknownTable(t *testing.T) {
if isRetryableError(&mockError{"unknown table users"}) {
t.Error("unknown table error should not be retryable")
}
}
// TestIsRetryableError_UnknownColumn tests "unknown column" is NOT retryable.
func TestIsRetryableError_UnknownColumn(t *testing.T) {
if isRetryableError(&mockError{"unknown column foo"}) {
t.Error("unknown column error should not be retryable")
}
}
// TestIsRetryableError_RandomError tests a random error is NOT retryable.
func TestIsRetryableError_RandomError(t *testing.T) {
if isRetryableError(&mockError{"some random unrecognized error"}) {
t.Error("random error should not be retryable")
}
}
// TestIsRetryableError_NoRouteToHost tests "no route to host" is retryable.
func TestIsRetryableError_NoRouteToHost(t *testing.T) {
if !isRetryableError(&mockError{"no route to host"}) {
t.Error("'no route to host' should be retryable")
}
}
// TestIsRetryableError_TemporaryFailure tests "temporary failure" is retryable.
func TestIsRetryableError_TemporaryFailure(t *testing.T) {
if !isRetryableError(&mockError{"temporary failure in name resolution"}) {
t.Error("'temporary failure' should be retryable")
}
}
// mockNetError implements net.Error for testing.
type mockNetError struct {
timeout bool
temporary bool
msg string
}
func (e *mockNetError) Error() string { return e.msg }
func (e *mockNetError) Timeout() bool { return e.timeout }
func (e *mockNetError) Temporary() bool { return e.temporary }
// TestNewClickHouseSink_EmptyDSN tests that empty DSN returns error.
func TestNewClickHouseSink_EmptyDSN(t *testing.T) {
_, err := NewClickHouseSink(Config{
DSN: "",
Table: "test_table",
})
if err == nil {
t.Error("expected error for empty DSN")
}
}
// TestNewClickHouseSink_WhitespaceDSN tests that whitespace DSN returns error.
func TestNewClickHouseSink_WhitespaceDSN(t *testing.T) {
_, err := NewClickHouseSink(Config{
DSN: " ",
Table: "test_table",
})
if err == nil {
t.Error("expected error for whitespace-only DSN")
}
}
// TestNewClickHouseSink_EmptyTable tests that empty Table returns error.
func TestNewClickHouseSink_EmptyTable(t *testing.T) {
_, err := NewClickHouseSink(Config{
DSN: "clickhouse://localhost:9000/test",
Table: "",
})
if err == nil {
t.Error("expected error for empty Table")
}
}
// TestNewClickHouseSink_WhitespaceTable tests that whitespace Table returns error.
func TestNewClickHouseSink_WhitespaceTable(t *testing.T) {
_, err := NewClickHouseSink(Config{
DSN: "clickhouse://localhost:9000/test",
Table: " ",
})
if err == nil {
t.Error("expected error for whitespace-only Table")
}
}
// TestNewClickHouseSink_InvalidDSN tests that an invalid DSN (no real connection) returns error.
func TestNewClickHouseSink_InvalidDSN(t *testing.T) {
_, err := NewClickHouseSink(Config{
DSN: "not-a-valid-dsn",
Table: "test_table",
})
if err == nil {
t.Error("expected error for invalid DSN")
}
}
// TestClickHouseSink_SetLogger tests that SetLogger sets a logger.
func TestClickHouseSink_SetLogger(t *testing.T) {
s := &ClickHouseSink{
config: Config{Table: "test_table"},
buffer: make([]domain.CorrelatedLog, 0),
}
testLogger := observability.NewLogger("test")
s.SetLogger(testLogger)
if s.logger == nil {
t.Error("expected logger to be set")
}
}
// TestClickHouseSink_LogNilLogger tests that log() returns a logger even when s.logger is nil.
func TestClickHouseSink_LogNilLogger(t *testing.T) {
s := &ClickHouseSink{
config: Config{Table: "test_table"},
buffer: make([]domain.CorrelatedLog, 0),
}
s.logger = nil
// log() should auto-initialize
logger := s.log()
if logger == nil {
t.Error("expected non-nil logger from log()")
}
}
// TestClickHouseSink_Reopen tests that Reopen is a no-op and returns nil.
func TestClickHouseSink_Reopen(t *testing.T) {
s := &ClickHouseSink{
config: Config{Table: "test_table"},
buffer: make([]domain.CorrelatedLog, 0),
}
if err := s.Reopen(); err != nil {
t.Errorf("Reopen() should return nil, got: %v", err)
}
}
// TestClickHouseSink_DoFlushNilConn tests doFlush returns error when conn is nil and buffer non-empty.
func TestClickHouseSink_DoFlushNilConn(t *testing.T) {
log := domain.CorrelatedLog{SrcIP: "1.2.3.4", SrcPort: 1234}
s := &ClickHouseSink{
config: Config{
Table: "test_table",
BatchSize: DefaultBatchSize,
},
buffer: []domain.CorrelatedLog{log},
conn: nil,
}
err := s.doFlush(context.Background())
if err == nil {
t.Error("expected error from doFlush when conn is nil")
}
}
// TestClickHouseSink_CloseTwice tests that calling Close() twice does not panic or error.
func TestClickHouseSink_CloseTwice(t *testing.T) {
s := &ClickHouseSink{
config: Config{
Table: "test_table",
TimeoutMs: DefaultTimeoutMs,
},
buffer: make([]domain.CorrelatedLog, 0),
done: make(chan struct{}),
}
if err := s.Close(); err != nil {
t.Errorf("first Close() should not error, got: %v", err)
}
if err := s.Close(); err != nil {
t.Errorf("second Close() should not error (closeOnce), got: %v", err)
}
}
// TestClickHouseSink_WriteTimeout tests that Write returns error when buffer is full and timeout exceeded.
func TestClickHouseSink_Write_Timeout(t *testing.T) {
s := &ClickHouseSink{
config: Config{
Table: "test_table",
MaxBufferSize: 1,
DropOnOverflow: false,
TimeoutMs: 1, // 1ms timeout
},
buffer: make([]domain.CorrelatedLog, 0, 1),
}
log := domain.CorrelatedLog{SrcIP: "1.2.3.4", SrcPort: 1234}
// Fill the buffer
s.buffer = append(s.buffer, log)
ctx := context.Background()
err := s.Write(ctx, log)
if err == nil {
t.Error("expected error when buffer full and timeout exceeded")
}
}
// Benchmark Write operation (without actual DB)
func BenchmarkClickHouseSink_Write(b *testing.B) {
s := &ClickHouseSink{
config: Config{
DSN: "clickhouse://test:test@localhost:9000/test",
Table: "test_table",
MaxBufferSize: 10000,
DropOnOverflow: true,
},
buffer: make([]domain.CorrelatedLog, 0, 10000),
}
log := domain.CorrelatedLog{
Timestamp: time.Now(),
SrcIP: "192.168.1.1",
SrcPort: 8080,
Correlated: 1,
}
ctx := context.Background()
b.ResetTimer()
for i := 0; i < b.N; i++ {
s.Write(ctx, log)
}
}

191
old/services/correlator/internal/adapters/outbound/file/sink.go Normal file
View File

@ -0,0 +1,191 @@
package file
import (
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"github.com/antitbone/ja4/correlator/internal/domain"
)
const (
// DefaultFilePermissions for output files
DefaultFilePermissions os.FileMode = 0644
// DefaultDirPermissions for output directories
DefaultDirPermissions os.FileMode = 0750
)
// Config holds the file sink configuration.
type Config struct {
Path string
}
// FileSink writes correlated logs to a file as JSON lines.
type FileSink struct {
config Config
mu sync.Mutex
file *os.File
}
// NewFileSink creates a new file sink.
func NewFileSink(config Config) (*FileSink, error) {
// Validate path
if err := validateFilePath(config.Path); err != nil {
return nil, fmt.Errorf("invalid file path: %w", err)
}
s := &FileSink{
config: config,
}
// Open file on creation
if err := s.openFile(); err != nil {
return nil, err
}
return s, nil
}
// Name returns the sink name.
func (s *FileSink) Name() string {
return "file"
}
// Reopen closes and reopens the file (for log rotation on SIGHUP).
func (s *FileSink) Reopen() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.file != nil {
if err := s.file.Close(); err != nil {
return fmt.Errorf("failed to close file: %w", err)
}
}
return s.openFile()
}
// Write writes a correlated log to the file.
func (s *FileSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.file == nil {
if err := s.openFile(); err != nil {
return err
}
}
data, err := json.Marshal(log)
if err != nil {
return fmt.Errorf("failed to marshal log: %w", err)
}
line := append(data, '\n')
if _, err := s.file.Write(line); err != nil {
return fmt.Errorf("failed to write log line: %w", err)
}
if err := s.file.Sync(); err != nil {
return fmt.Errorf("failed to sync log line: %w", err)
}
return nil
}
// Flush flushes any buffered data.
func (s *FileSink) Flush(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.file != nil {
return s.file.Sync()
}
return nil
}
// Close closes the sink.
func (s *FileSink) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.file != nil {
err := s.file.Close()
s.file = nil
return err
}
return nil
}
func (s *FileSink) openFile() error {
// Validate path again before opening
if err := validateFilePath(s.config.Path); err != nil {
return fmt.Errorf("invalid file path: %w", err)
}
// Ensure directory exists
dir := filepath.Dir(s.config.Path)
if err := os.MkdirAll(dir, DefaultDirPermissions); err != nil {
return fmt.Errorf("failed to create directory: %w", err)
}
file, err := os.OpenFile(s.config.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, DefaultFilePermissions)
if err != nil {
return fmt.Errorf("failed to open file: %w", err)
}
s.file = file
return nil
}
// validateFilePath validates that the file path is safe and allowed.
func validateFilePath(path string) error {
if strings.TrimSpace(path) == "" {
return fmt.Errorf("path cannot be empty")
}
cleanPath := filepath.Clean(path)
// Allow relative paths for testing/dev
if !filepath.IsAbs(cleanPath) {
return nil
}
absPath, err := filepath.Abs(cleanPath)
if err != nil {
return fmt.Errorf("failed to resolve absolute path: %w", err)
}
allowedRoots := []string{
"/var/log/logcorrelator",
"/var/log",
"/tmp",
}
for _, root := range allowedRoots {
absRoot, err := filepath.Abs(filepath.Clean(root))
if err != nil {
continue
}
rel, err := filepath.Rel(absRoot, absPath)
if err != nil {
continue
}
if rel == "." {
return nil
}
if rel == ".." {
continue
}
if !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return nil
}
}
return fmt.Errorf("path must be under allowed directories: %v", allowedRoots)
}

524
old/services/correlator/internal/adapters/outbound/file/sink_test.go Normal file
View File

@ -0,0 +1,524 @@
package file
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/antitbone/ja4/correlator/internal/domain"
)
func TestFileSink_Write(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
log := domain.CorrelatedLog{
SrcIP: "192.168.1.1",
SrcPort: 8080,
Correlated: 1,
}
if err := sink.Write(context.Background(), log); err != nil {
t.Fatalf("failed to write: %v", err)
}
if err := sink.Flush(context.Background()); err != nil {
t.Fatalf("failed to flush: %v", err)
}
// Verify file exists and contains data
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
if len(data) == 0 {
t.Error("expected non-empty file")
}
}
func TestFileSink_WriteImmediatePersist_NoFlushNeeded(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
log := domain.CorrelatedLog{
SrcIP: "192.168.1.1",
SrcPort: 8080,
Correlated: 1,
}
if err := sink.Write(context.Background(), log); err != nil {
t.Fatalf("failed to write: %v", err)
}
// Must be visible immediately without Flush()
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
if len(data) == 0 {
t.Error("expected data to be present immediately after Write without Flush")
}
}
func TestFileSink_MultipleWrites(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
for i := 0; i < 5; i++ {
log := domain.CorrelatedLog{
SrcIP: "192.168.1.1",
SrcPort: 8080 + i,
}
if err := sink.Write(context.Background(), log); err != nil {
t.Fatalf("failed to write: %v", err)
}
}
sink.Close()
// Verify file has 5 lines
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
lines := 0
for _, b := range data {
if b == '\n' {
lines++
}
}
if lines != 5 {
t.Errorf("expected 5 lines, got %d", lines)
}
}
func TestFileSink_Name(t *testing.T) {
sink, err := NewFileSink(Config{Path: "/tmp/test.log"})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
if sink.Name() != "file" {
t.Errorf("expected name 'file', got %s", sink.Name())
}
}
func TestFileSink_ValidateFilePath(t *testing.T) {
tests := []struct {
name string
path string
wantErr bool
}{
{"empty path", "", true},
{"valid /var/log/logcorrelator", "/var/log/logcorrelator/test.log", false},
{"valid /var/log", "/var/log/test.log", false},
{"valid /tmp", "/tmp/test.log", false},
{"reject lookalike /var/logevil", "/var/logevil/test.log", true},
{"invalid directory", "/etc/logcorrelator/test.log", true},
{"relative path", "test.log", false}, // Allowed for testing
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
err := validateFilePath(tt.path)
if (err != nil) != tt.wantErr {
t.Errorf("validateFilePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
}
})
}
}
func TestFileSink_OpenFile(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "subdir", "test.log")
sink := &FileSink{
config: Config{Path: testPath},
}
err := sink.openFile()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
defer sink.Close()
if sink.file == nil {
t.Error("expected file to be opened")
}
}
func TestFileSink_WriteBeforeOpen(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
// Write should open file automatically
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
err = sink.Write(context.Background(), log)
if err != nil {
t.Fatalf("failed to write: %v", err)
}
// Verify file was created
if _, err := os.Stat(testPath); os.IsNotExist(err) {
t.Error("expected file to be created")
}
}
func TestFileSink_FlushBeforeOpen(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
// Flush before any write should not error
err = sink.Flush(context.Background())
if err != nil {
t.Errorf("expected no error on flush before open, got %v", err)
}
}
func TestFileSink_InvalidPath(t *testing.T) {
// Test with invalid path (outside allowed directories)
_, err := NewFileSink(Config{Path: "/etc/../passwd"})
if err == nil {
t.Error("expected error for invalid path")
}
}
func TestFileSink_Reopen(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
// Write initial data
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
if err := sink.Write(context.Background(), log); err != nil {
t.Fatalf("failed to write: %v", err)
}
// Reopen should close and reopen the file
err = sink.Reopen()
if err != nil {
t.Errorf("expected no error on Reopen, got %v", err)
}
// Write after reopen
log2 := domain.CorrelatedLog{SrcIP: "192.168.1.2", SrcPort: 8081}
if err := sink.Write(context.Background(), log2); err != nil {
t.Fatalf("failed to write after reopen: %v", err)
}
sink.Close()
// Verify both writes are present
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
lines := 0
for _, b := range data {
if b == '\n' {
lines++
}
}
if lines != 2 {
t.Errorf("expected 2 lines after reopen, got %d", lines)
}
}
func TestFileSink_Close(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
// Close should succeed
err = sink.Close()
if err != nil {
t.Errorf("expected no error on Close, got %v", err)
}
// Write after close should fail or reopen
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
err = sink.Write(context.Background(), log)
if err != nil {
// Expected - file was closed
t.Logf("write after close returned error (expected): %v", err)
}
}
func TestFileSink_EmptyPath(t *testing.T) {
_, err := NewFileSink(Config{Path: ""})
if err == nil {
t.Error("expected error for empty path")
}
}
func TestFileSink_WhitespacePath(t *testing.T) {
_, err := NewFileSink(Config{Path: " "})
if err == nil {
t.Error("expected error for whitespace-only path")
}
}
func TestFileSink_ValidateFilePath_AllowedRoots(t *testing.T) {
// Test paths under allowed roots
allowedPaths := []string{
"/var/log/logcorrelator/correlated.log",
"/var/log/test.log",
"/tmp/test.log",
"/tmp/subdir/test.log",
"relative/path/test.log",
"./test.log",
}
for _, path := range allowedPaths {
err := validateFilePath(path)
if err != nil {
t.Errorf("validateFilePath(%q) unexpected error: %v", path, err)
}
}
}
func TestFileSink_ValidateFilePath_RejectedPaths(t *testing.T) {
// Test paths that should be rejected
rejectedPaths := []string{
"",
" ",
"/etc/passwd",
"/etc/logcorrelator/test.log",
"/root/test.log",
"/home/user/test.log",
"/var/logevil/test.log",
}
for _, path := range rejectedPaths {
err := validateFilePath(path)
if err == nil {
t.Errorf("validateFilePath(%q) should have been rejected", path)
}
}
}
func TestFileSink_ConcurrentWrites(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
done := make(chan bool)
for i := 0; i < 10; i++ {
go func(n int) {
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080 + n}
sink.Write(context.Background(), log)
done <- true
}(i)
}
for i := 0; i < 10; i++ {
<-done
}
// Verify all writes completed
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
lines := 0
for _, b := range data {
if b == '\n' {
lines++
}
}
if lines != 10 {
t.Errorf("expected 10 lines from concurrent writes, got %d", lines)
}
}
func TestFileSink_Flush(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
if err := sink.Write(context.Background(), log); err != nil {
t.Fatalf("failed to write: %v", err)
}
// Flush should succeed
err = sink.Flush(context.Background())
if err != nil {
t.Errorf("expected no error on Flush, got %v", err)
}
}
func TestFileSink_MarshalError(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
// Create a log with unmarshalable data (channel)
log := domain.CorrelatedLog{
SrcIP: "192.168.1.1",
SrcPort: 8080,
Fields: map[string]any{"chan": make(chan int)},
}
err = sink.Write(context.Background(), log)
if err == nil {
t.Error("expected error when marshaling unmarshalable data")
}
}
// TestFileSink_CloseTwice tests that closing an already-closed sink does not error.
func TestFileSink_CloseTwice(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
if err := sink.Close(); err != nil {
t.Errorf("first Close() should not error, got: %v", err)
}
// After close, file is nil, so second close should return nil
if err := sink.Close(); err != nil {
t.Errorf("second Close() on already-closed sink should not error, got: %v", err)
}
}
// TestFileSink_WriteAfterClose tests that Write after Close re-opens the file.
func TestFileSink_WriteAfterCloseReopens(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
if err := sink.Close(); err != nil {
t.Fatalf("Close() failed: %v", err)
}
// Write after close: FileSink.Write reopens the file when file == nil
log := domain.CorrelatedLog{SrcIP: "1.2.3.4", SrcPort: 80}
if err := sink.Write(context.Background(), log); err != nil {
t.Errorf("Write after close should succeed (auto-reopen), got: %v", err)
}
// Verify data was written
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
if len(data) == 0 {
t.Error("expected data to be present after write on re-opened file")
}
}
// TestFileSink_ReopenAfterWrite tests Reopen then write produces correct output.
func TestFileSink_ReopenThenWrite(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
// Write before reopen
log1 := domain.CorrelatedLog{SrcIP: "1.1.1.1", SrcPort: 80}
if err := sink.Write(context.Background(), log1); err != nil {
t.Fatalf("first Write failed: %v", err)
}
// Simulate log rotation
if err := sink.Reopen(); err != nil {
t.Fatalf("Reopen failed: %v", err)
}
// Write after reopen
log2 := domain.CorrelatedLog{SrcIP: "2.2.2.2", SrcPort: 443}
if err := sink.Write(context.Background(), log2); err != nil {
t.Fatalf("second Write failed: %v", err)
}
sink.Close()
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
lines := 0
for _, b := range data {
if b == '\n' {
lines++
}
}
if lines != 2 {
t.Errorf("expected 2 lines after reopen+write, got %d", lines)
}
}

137
old/services/correlator/internal/adapters/outbound/multi/sink.go Normal file
View File

@ -0,0 +1,137 @@
package multi
import (
"context"
"sync"
"github.com/antitbone/ja4/correlator/internal/domain"
"github.com/antitbone/ja4/correlator/internal/ports"
)
// MultiSink fans out correlated logs to multiple sinks.
type MultiSink struct {
mu sync.RWMutex
sinks []ports.CorrelatedLogSink
}
// NewMultiSink creates a new multi-sink.
func NewMultiSink(sinks ...ports.CorrelatedLogSink) *MultiSink {
return &MultiSink{
sinks: sinks,
}
}
// Name returns the sink name.
func (s *MultiSink) Name() string {
return "multi"
}
// AddSink adds a sink to the fan-out.
func (s *MultiSink) AddSink(sink ports.CorrelatedLogSink) {
s.mu.Lock()
defer s.mu.Unlock()
s.sinks = append(s.sinks, sink)
}
// Write writes a correlated log to all sinks concurrently.
// Returns the first error encountered (but all sinks are attempted).
func (s *MultiSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
s.mu.RLock()
sinks := make([]ports.CorrelatedLogSink, len(s.sinks))
copy(sinks, s.sinks)
s.mu.RUnlock()
if len(sinks) == 0 {
return nil
}
var wg sync.WaitGroup
var firstErr error
var firstErrMu sync.Mutex
errChan := make(chan error, len(sinks))
for _, sink := range sinks {
wg.Add(1)
go func(sk ports.CorrelatedLogSink) {
defer wg.Done()
if err := sk.Write(ctx, log); err != nil {
// Non-blocking send to errChan
select {
case errChan <- err:
default:
// Channel full, error will be handled via firstErr
}
}
}(sink)
}
// Wait for all writes to complete in a separate goroutine
done := make(chan struct{})
go func() {
wg.Wait()
close(done)
}()
// Collect errors with timeout
select {
case <-done:
close(errChan)
// Collect first error
for err := range errChan {
if err != nil {
firstErrMu.Lock()
if firstErr == nil {
firstErr = err
}
firstErrMu.Unlock()
}
}
case <-ctx.Done():
return ctx.Err()
}
firstErrMu.Lock()
defer firstErrMu.Unlock()
return firstErr
}
// Flush flushes all sinks.
func (s *MultiSink) Flush(ctx context.Context) error {
s.mu.RLock()
defer s.mu.RUnlock()
for _, sink := range s.sinks {
if err := sink.Flush(ctx); err != nil {
return err
}
}
return nil
}
// Close closes all sinks.
func (s *MultiSink) Close() error {
s.mu.RLock()
defer s.mu.RUnlock()
var firstErr error
for _, sink := range s.sinks {
if err := sink.Close(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}
// Reopen reopens all sinks (for log rotation on SIGHUP).
func (s *MultiSink) Reopen() error {
s.mu.RLock()
defer s.mu.RUnlock()
var firstErr error
for _, sink := range s.sinks {
if err := sink.Reopen(); err != nil && firstErr == nil {
firstErr = err
}
}
return firstErr
}

233
old/services/correlator/internal/adapters/outbound/multi/sink_test.go Normal file
View File

@ -0,0 +1,233 @@
package multi
import (
"context"
"sync"
"testing"
"github.com/antitbone/ja4/correlator/internal/domain"
)
type mockSink struct {
name string
mu sync.Mutex
writeFunc func(domain.CorrelatedLog) error
flushFunc func() error
closeFunc func() error
reopenFunc func() error
}
func (m *mockSink) Name() string { return m.name }
func (m *mockSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
m.mu.Lock()
defer m.mu.Unlock()
return m.writeFunc(log)
}
func (m *mockSink) Flush(ctx context.Context) error { return m.flushFunc() }
func (m *mockSink) Close() error { return m.closeFunc() }
func (m *mockSink) Reopen() error {
if m.reopenFunc != nil {
return m.reopenFunc()
}
return nil
}
func TestMultiSink_Write(t *testing.T) {
var mu sync.Mutex
writeCount := 0
sink1 := &mockSink{
name: "sink1",
writeFunc: func(log domain.CorrelatedLog) error {
mu.Lock()
writeCount++
mu.Unlock()
return nil
},
flushFunc: func() error { return nil },
closeFunc: func() error { return nil },
}
sink2 := &mockSink{
name: "sink2",
writeFunc: func(log domain.CorrelatedLog) error {
mu.Lock()
writeCount++
mu.Unlock()
return nil
},
flushFunc: func() error { return nil },
closeFunc: func() error { return nil },
}
ms := NewMultiSink(sink1, sink2)
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
err := ms.Write(context.Background(), log)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if writeCount != 2 {
t.Errorf("expected 2 writes, got %d", writeCount)
}
}
func TestMultiSink_Write_OneFails(t *testing.T) {
sink1 := &mockSink{
name: "sink1",
writeFunc: func(log domain.CorrelatedLog) error {
return nil
},
flushFunc: func() error { return nil },
closeFunc: func() error { return nil },
}
sink2 := &mockSink{
name: "sink2",
writeFunc: func(log domain.CorrelatedLog) error {
return context.Canceled
},
flushFunc: func() error { return nil },
closeFunc: func() error { return nil },
}
ms := NewMultiSink(sink1, sink2)
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
err := ms.Write(context.Background(), log)
if err == nil {
t.Error("expected error when one sink fails")
}
}
func TestMultiSink_AddSink(t *testing.T) {
ms := NewMultiSink()
sink := &mockSink{
name: "dynamic",
writeFunc: func(log domain.CorrelatedLog) error { return nil },
flushFunc: func() error { return nil },
closeFunc: func() error { return nil },
}
ms.AddSink(sink)
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
err := ms.Write(context.Background(), log)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
}
func TestMultiSink_Name(t *testing.T) {
ms := NewMultiSink()
if ms.Name() != "multi" {
t.Errorf("expected name 'multi', got %s", ms.Name())
}
}
func TestMultiSink_Flush(t *testing.T) {
flushed := false
sink := &mockSink{
name: "test",
writeFunc: func(log domain.CorrelatedLog) error { return nil },
flushFunc: func() error {
flushed = true
return nil
},
closeFunc: func() error { return nil },
}
ms := NewMultiSink(sink)
err := ms.Flush(context.Background())
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !flushed {
t.Error("expected sink to be flushed")
}
}
func TestMultiSink_Flush_Error(t *testing.T) {
sink := &mockSink{
name: "test",
writeFunc: func(log domain.CorrelatedLog) error { return nil },
flushFunc: func() error { return context.Canceled },
closeFunc: func() error { return nil },
}
ms := NewMultiSink(sink)
err := ms.Flush(context.Background())
if err != context.Canceled {
t.Errorf("expected context.Canceled error, got %v", err)
}
}
func TestMultiSink_Close(t *testing.T) {
closed := false
sink := &mockSink{
name: "test",
writeFunc: func(log domain.CorrelatedLog) error { return nil },
flushFunc: func() error { return nil },
closeFunc: func() error {
closed = true
return nil
},
}
ms := NewMultiSink(sink)
err := ms.Close()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if !closed {
t.Error("expected sink to be closed")
}
}
func TestMultiSink_Close_Error(t *testing.T) {
sink := &mockSink{
name: "test",
writeFunc: func(log domain.CorrelatedLog) error { return nil },
flushFunc: func() error { return nil },
closeFunc: func() error { return context.Canceled },
}
ms := NewMultiSink(sink)
err := ms.Close()
if err != context.Canceled {
t.Errorf("expected context.Canceled error, got %v", err)
}
}
func TestMultiSink_Write_EmptySinks(t *testing.T) {
ms := NewMultiSink()
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
err := ms.Write(context.Background(), log)
if err != nil {
t.Fatalf("unexpected error with empty sinks: %v", err)
}
}
func TestMultiSink_Write_ContextCancelled(t *testing.T) {
sink := &mockSink{
name: "test",
writeFunc: func(log domain.CorrelatedLog) error {
<-context.Background().Done()
return nil
},
flushFunc: func() error { return nil },
closeFunc: func() error { return nil },
}
ms := NewMultiSink(sink)
ctx, cancel := context.WithCancel(context.Background())
cancel()
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
err := ms.Write(ctx, log)
if err != context.Canceled {
t.Errorf("expected context.Canceled error, got %v", err)
}
}

46
old/services/correlator/internal/adapters/outbound/stdout/sink.go Normal file
View File

@ -0,0 +1,46 @@
package stdout
import (
"context"
"github.com/antitbone/ja4/correlator/internal/domain"
)
// Config holds the stdout sink configuration.
type Config struct {
Enabled bool
}
// StdoutSink is a no-op data sink. Operational logs are written to stderr
// by the observability.Logger; correlated data must never appear on stdout.
type StdoutSink struct{}
// NewStdoutSink creates a new stdout sink.
func NewStdoutSink(config Config) *StdoutSink {
return &StdoutSink{}
}
// Name returns the sink name.
func (s *StdoutSink) Name() string {
return "stdout"
}
// Reopen is a no-op for stdout.
func (s *StdoutSink) Reopen() error {
return nil
}
// Write is a no-op: correlated data must never be written to stdout.
func (s *StdoutSink) Write(_ context.Context, _ domain.CorrelatedLog) error {
return nil
}
// Flush is a no-op for stdout.
func (s *StdoutSink) Flush(_ context.Context) error {
return nil
}
// Close is a no-op for stdout.
func (s *StdoutSink) Close() error {
return nil
}

81
old/services/correlator/internal/adapters/outbound/stdout/sink_test.go Normal file
View File

@ -0,0 +1,81 @@
package stdout
import (
"bytes"
"context"
"os"
"testing"
"time"
"github.com/antitbone/ja4/correlator/internal/domain"
)
func makeLog(correlated int) domain.CorrelatedLog {
return domain.CorrelatedLog{
Timestamp: time.Unix(1700000000, 0),
SrcIP: "1.2.3.4",
SrcPort: 12345,
Correlated: correlated,
}
}
// captureStdout replaces os.Stdout temporarily and returns what was written.
func captureStdout(t *testing.T, fn func()) string {
t.Helper()
r, w, err := os.Pipe()
if err != nil {
t.Fatalf("os.Pipe: %v", err)
}
old := os.Stdout
os.Stdout = w
fn()
w.Close()
os.Stdout = old
var buf bytes.Buffer
buf.ReadFrom(r)
r.Close()
return buf.String()
}
func TestStdoutSink_Name(t *testing.T) {
s := NewStdoutSink(Config{Enabled: true})
if s.Name() != "stdout" {
t.Errorf("expected name 'stdout', got %q", s.Name())
}
}
// TestStdoutSink_WriteDoesNotProduceOutput verifies that no JSON data
// (correlated or not) is ever written to stdout.
func TestStdoutSink_WriteDoesNotProduceOutput(t *testing.T) {
s := NewStdoutSink(Config{Enabled: true})
got := captureStdout(t, func() {
if err := s.Write(context.Background(), makeLog(1)); err != nil {
t.Fatalf("Write(correlated) returned error: %v", err)
}
if err := s.Write(context.Background(), makeLog(0)); err != nil {
t.Fatalf("Write(orphan) returned error: %v", err)
}
})
if got != "" {
t.Errorf("stdout must be empty but got: %q", got)
}
}
func TestStdoutSink_NoopMethods(t *testing.T) {
s := NewStdoutSink(Config{Enabled: true})
if err := s.Flush(context.Background()); err != nil {
t.Errorf("Flush returned error: %v", err)
}
if err := s.Close(); err != nil {
t.Errorf("Close returned error: %v", err)
}
if err := s.Reopen(); err != nil {
t.Errorf("Reopen returned error: %v", err)
}
}

161
old/services/correlator/internal/app/orchestrator.go Normal file
View File

@ -0,0 +1,161 @@
package app
import (
"context"
"sync"
"sync/atomic"
"time"
"github.com/antitbone/ja4/correlator/internal/domain"
"github.com/antitbone/ja4/correlator/internal/ports"
)
const (
// DefaultEventChannelBufferSize is the default size for event channels
DefaultEventChannelBufferSize = 1000
// OrphanTickInterval is how often the orchestrator drains pending orphans.
// Set to half the default emit delay (500ms/2) so orphans are emitted promptly
// even when no new events arrive.
OrphanTickInterval = 250 * time.Millisecond
)
// OrchestratorConfig holds the orchestrator configuration.
type OrchestratorConfig struct {
Sources []ports.EventSource
Sink ports.CorrelatedLogSink
}
// Orchestrator connects sources to the correlation service and sinks.
type Orchestrator struct {
config OrchestratorConfig
correlationSvc ports.CorrelationProcessor
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
running atomic.Bool
}
// NewOrchestrator creates a new orchestrator.
func NewOrchestrator(config OrchestratorConfig, correlationSvc ports.CorrelationProcessor) *Orchestrator {
ctx, cancel := context.WithCancel(context.Background())
return &Orchestrator{
config: config,
correlationSvc: correlationSvc,
ctx: ctx,
cancel: cancel,
}
}
// Start begins the orchestration.
func (o *Orchestrator) Start() error {
if !o.running.CompareAndSwap(false, true) {
return nil // Already running
}
// Start each source
for _, source := range o.config.Sources {
eventChan := make(chan *domain.NormalizedEvent, DefaultEventChannelBufferSize)
o.wg.Add(1)
go func(src ports.EventSource, evChan chan *domain.NormalizedEvent) {
defer o.wg.Done()
// Start the source in a separate goroutine
sourceErr := make(chan error, 1)
go func() {
if err := src.Start(o.ctx, evChan); err != nil {
sourceErr <- err
}
}()
// Process events in the current goroutine
o.processEvents(evChan)
// Check for source start errors
if err := <-sourceErr; err != nil {
// Source failed to start, log error and exit
return
}
}(source, eventChan)
}
// Start a periodic ticker to drain pending orphan A events independently of the
// event flow. Without this, orphans are only emitted when a new event arrives,
// causing them to accumulate silently when the source goes quiet.
o.wg.Add(1)
go func() {
defer o.wg.Done()
ticker := time.NewTicker(OrphanTickInterval)
defer ticker.Stop()
for {
select {
case <-o.ctx.Done():
return
case <-ticker.C:
logs := o.correlationSvc.EmitPendingOrphans()
for _, log := range logs {
o.config.Sink.Write(o.ctx, log) //nolint:errcheck
}
}
}
}()
return nil
}
// processEvents lit les événements du canal, les soumet au service de corrélation et écrit les résultats dans le puits.
func (o *Orchestrator) processEvents(eventChan <-chan *domain.NormalizedEvent) {
for {
select {
case <-o.ctx.Done():
// Drain remaining events before exiting
for {
select {
case event, ok := <-eventChan:
if !ok {
return
}
logs := o.correlationSvc.ProcessEvent(event)
for _, log := range logs {
o.config.Sink.Write(o.ctx, log)
}
default:
return
}
}
case event, ok := <-eventChan:
if !ok {
return
}
// Process through correlation service
logs := o.correlationSvc.ProcessEvent(event)
// Write correlated logs to sink
for _, log := range logs {
if err := o.config.Sink.Write(o.ctx, log); err != nil {
// Log error but continue processing
}
}
}
}
}
// Stop gracefully stops the orchestrator.
// It stops all sources and closes sinks immediately without waiting for queue drainage.
// systemd TimeoutStopSec handles forced termination if needed.
func (o *Orchestrator) Stop() error {
if !o.running.CompareAndSwap(true, false) {
return nil // Not running
}
// Cancel context to stop accepting new events immediately
o.cancel()
// Close sink (flush skipped - in-flight events are dropped)
if err := o.config.Sink.Close(); err != nil {
// Log error
}
return nil
}

300
old/services/correlator/internal/app/orchestrator_test.go Normal file
View File

@ -0,0 +1,300 @@
package app
import (
"context"
"sync"
"testing"
"time"
"github.com/antitbone/ja4/correlator/internal/domain"
"github.com/antitbone/ja4/correlator/internal/ports"
)
type mockEventSource struct {
name string
mu sync.RWMutex
eventChan chan<- *domain.NormalizedEvent
started bool
stopped bool
}
func (m *mockEventSource) Name() string { return m.name }
func (m *mockEventSource) Start(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) error {
m.mu.Lock()
m.started = true
m.eventChan = eventChan
m.mu.Unlock()
<-ctx.Done()
m.mu.Lock()
m.stopped = true
m.mu.Unlock()
return nil
}
func (m *mockEventSource) Stop() error { return nil }
func (m *mockEventSource) getEventChan() chan<- *domain.NormalizedEvent {
m.mu.RLock()
defer m.mu.RUnlock()
return m.eventChan
}
func (m *mockEventSource) isStarted() bool {
m.mu.RLock()
defer m.mu.RUnlock()
return m.started
}
type mockSink struct {
mu sync.Mutex
written []domain.CorrelatedLog
}
func (m *mockSink) Name() string { return "mock" }
func (m *mockSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
m.mu.Lock()
defer m.mu.Unlock()
m.written = append(m.written, log)
return nil
}
func (m *mockSink) Flush(ctx context.Context) error { return nil }
func (m *mockSink) Close() error { return nil }
func (m *mockSink) Reopen() error { return nil }
func (m *mockSink) getWritten() []domain.CorrelatedLog {
m.mu.Lock()
defer m.mu.Unlock()
result := make([]domain.CorrelatedLog, len(m.written))
copy(result, m.written)
return result
}
func TestOrchestrator_StartStop(t *testing.T) {
source := &mockEventSource{name: "test"}
sink := &mockSink{}
corrConfig := domain.CorrelationConfig{
TimeWindow: time.Second,
ApacheAlwaysEmit: true,
NetworkEmit: false,
}
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
orchestrator := NewOrchestrator(OrchestratorConfig{
Sources: []ports.EventSource{source},
Sink: sink,
}, correlationSvc)
if err := orchestrator.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
// Let it run briefly
time.Sleep(100 * time.Millisecond)
if err := orchestrator.Stop(); err != nil {
t.Fatalf("failed to stop: %v", err)
}
if !source.isStarted() {
t.Error("expected source to be started")
}
}
func TestOrchestrator_ProcessEvent(t *testing.T) {
source := &mockEventSource{name: "test"}
sink := &mockSink{}
corrConfig := domain.CorrelationConfig{
TimeWindow: time.Second,
ApacheAlwaysEmit: true,
NetworkEmit: false,
}
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
orchestrator := NewOrchestrator(OrchestratorConfig{
Sources: []ports.EventSource{source},
Sink: sink,
}, correlationSvc)
if err := orchestrator.Start(); err != nil {
t.Fatalf("failed to start: %v", err)
}
// Wait for source to start and get the channel
var eventChan chan<- *domain.NormalizedEvent
for i := 0; i < 50; i++ {
eventChan = source.getEventChan()
if eventChan != nil {
break
}
time.Sleep(10 * time.Millisecond)
}
if eventChan == nil {
t.Fatal("source did not start properly")
}
// Send an event through the source
event := &domain.NormalizedEvent{
Source: domain.SourceA,
Timestamp: time.Now(),
SrcIP: "192.168.1.1",
SrcPort: 8080,
Raw: map[string]any{"method": "GET"},
}
// Send event
eventChan <- event
// Give it time to process
time.Sleep(100 * time.Millisecond)
if err := orchestrator.Stop(); err != nil {
t.Fatalf("failed to stop: %v", err)
}
// Should have written at least one log (the orphan A)
written := sink.getWritten()
if len(written) == 0 {
t.Error("expected at least one log to be written")
}
}
// TestOrchestrator_StartTwice tests that calling Start() twice is a no-op (already running).
func TestOrchestrator_StartTwice(t *testing.T) {
source := &mockEventSource{name: "test"}
sink := &mockSink{}
corrConfig := domain.CorrelationConfig{
TimeWindow: time.Second,
ApacheAlwaysEmit: true,
}
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
o := NewOrchestrator(OrchestratorConfig{
Sources: []ports.EventSource{source},
Sink: sink,
}, correlationSvc)
if err := o.Start(); err != nil {
t.Fatalf("first Start() failed: %v", err)
}
if err := o.Start(); err != nil {
t.Errorf("second Start() should be no-op, got: %v", err)
}
o.Stop()
}
// TestOrchestrator_StopTwice tests that calling Stop() twice is a no-op.
func TestOrchestrator_StopTwice(t *testing.T) {
source := &mockEventSource{name: "test"}
sink := &mockSink{}
corrConfig := domain.CorrelationConfig{
TimeWindow: time.Second,
ApacheAlwaysEmit: true,
}
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
o := NewOrchestrator(OrchestratorConfig{
Sources: []ports.EventSource{source},
Sink: sink,
}, correlationSvc)
o.Start()
if err := o.Stop(); err != nil {
t.Errorf("first Stop() failed: %v", err)
}
if err := o.Stop(); err != nil {
t.Errorf("second Stop() should be no-op, got: %v", err)
}
}
// TestOrchestrator_NoSources tests that Orchestrator works with no sources.
func TestOrchestrator_NoSources(t *testing.T) {
sink := &mockSink{}
corrConfig := domain.CorrelationConfig{TimeWindow: time.Second}
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
o := NewOrchestrator(OrchestratorConfig{
Sources: []ports.EventSource{},
Sink: sink,
}, correlationSvc)
if err := o.Start(); err != nil {
t.Fatalf("Start() with no sources failed: %v", err)
}
time.Sleep(50 * time.Millisecond)
if err := o.Stop(); err != nil {
t.Errorf("Stop() failed: %v", err)
}
}
// TestOrchestrator_OrphanEmission tests that orphan A events are emitted via tick.
func TestOrchestrator_OrphanEmission(t *testing.T) {
source := &mockEventSource{name: "test"}
sink := &mockSink{}
corrConfig := domain.CorrelationConfig{
TimeWindow: 50 * time.Millisecond,
ApacheAlwaysEmit: true,
ApacheEmitDelayMs: 10, // Very short delay so orphans emit quickly
}
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
o := NewOrchestrator(OrchestratorConfig{
Sources: []ports.EventSource{source},
Sink: sink,
}, correlationSvc)
if err := o.Start(); err != nil {
t.Fatalf("Start() failed: %v", err)
}
// Wait for source to be ready
var eventChan chan<- *domain.NormalizedEvent
for i := 0; i < 50; i++ {
eventChan = source.getEventChan()
if eventChan != nil {
break
}
time.Sleep(5 * time.Millisecond)
}
if eventChan == nil {
t.Fatal("source did not start")
}
// Send a source A event (Apache/HTTP)
eventChan <- &domain.NormalizedEvent{
Source: domain.SourceA,
Timestamp: time.Now(),
SrcIP: "10.0.0.1",
SrcPort: 12345,
Raw: map[string]any{"method": "GET"},
}
// Allow time for orphan ticker to fire (OrphanTickInterval = 250ms, but emit delay is 10ms)
time.Sleep(600 * time.Millisecond)
o.Stop()
written := sink.getWritten()
if len(written) == 0 {
t.Error("expected at least one orphan log to be emitted")
}
}
// TestOrchestrator_Constants tests that constants have reasonable values.
func TestOrchestrator_Constants(t *testing.T) {
if DefaultEventChannelBufferSize <= 0 {
t.Error("DefaultEventChannelBufferSize should be positive")
}
if OrphanTickInterval <= 0 {
t.Error("OrphanTickInterval should be positive")
}
}

412
old/services/correlator/internal/config/config.go Normal file
View File

@ -0,0 +1,412 @@
package config
import (
"fmt"
"net"
"os"
"strconv"
"strings"
"time"
"github.com/antitbone/ja4/correlator/internal/domain"
ja4config "github.com/antitbone/ja4/ja4common/config"
"gopkg.in/yaml.v3"
)
// Config holds the complete application configuration.
type Config struct {
Log LogConfig `yaml:"log"`
Inputs InputsConfig `yaml:"inputs"`
Outputs OutputsConfig `yaml:"outputs"`
Correlation CorrelationConfig `yaml:"correlation"`
Metrics MetricsConfig `yaml:"metrics"`
}
// MetricsConfig holds metrics server configuration.
type MetricsConfig struct {
Enabled bool `yaml:"enabled"`
Addr string `yaml:"addr"` // e.g., ":8080", "localhost:8080"
}
// LogConfig holds logging configuration.
type LogConfig struct {
Level string `yaml:"level" env:"LOG_LEVEL"` // DEBUG, INFO, WARN, ERROR
}
// GetLogLevel returns the log level, defaulting to INFO if not set.
func (c *LogConfig) GetLevel() string {
if c.Level == "" {
return "INFO"
}
return strings.ToUpper(c.Level)
}
// ServiceConfig holds service-level configuration.
type ServiceConfig struct {
Name string `yaml:"name"`
Language string `yaml:"language"`
}
// InputsConfig holds input sources configuration.
type InputsConfig struct {
UnixSockets []UnixSocketConfig `yaml:"unix_sockets"`
}
// UnixSocketConfig holds a Unix socket source configuration.
type UnixSocketConfig struct {
Name string `yaml:"name"`
Path string `yaml:"path"`
Format string `yaml:"format"`
SourceType string `yaml:"source_type"` // "A" for Apache/HTTP, "B" for Network
SocketPermissions string `yaml:"socket_permissions"` // octal string, e.g., "0660", "0666"
}
// OutputsConfig holds output sinks configuration.
type OutputsConfig struct {
File FileOutputConfig `yaml:"file"`
ClickHouse ClickHouseOutputConfig `yaml:"clickhouse"`
Stdout StdoutOutputConfig `yaml:"stdout"`
}
// FileOutputConfig holds file sink configuration.
type FileOutputConfig struct {
Enabled bool `yaml:"enabled"`
Path string `yaml:"path"`
}
// ClickHouseOutputConfig holds ClickHouse sink configuration.
type ClickHouseOutputConfig struct {
Enabled bool `yaml:"enabled" env:"CLICKHOUSE_ENABLED"`
DSN string `yaml:"dsn" env:"CLICKHOUSE_DSN"`
Table string `yaml:"table" env:"CLICKHOUSE_TABLE"`
BatchSize int `yaml:"batch_size" env:"CLICKHOUSE_BATCH_SIZE"`
FlushIntervalMs int `yaml:"flush_interval_ms" env:"CLICKHOUSE_FLUSH_INTERVAL_MS"`
MaxBufferSize int `yaml:"max_buffer_size" env:"CLICKHOUSE_MAX_BUFFER_SIZE"`
DropOnOverflow bool `yaml:"drop_on_overflow"`
AsyncInsert bool `yaml:"async_insert"`
TimeoutMs int `yaml:"timeout_ms" env:"CLICKHOUSE_TIMEOUT_MS"`
}
// StdoutOutputConfig holds stdout sink configuration.
type StdoutOutputConfig struct {
Enabled bool `yaml:"enabled"`
Level string `yaml:"level"` // DEBUG, INFO, WARN, ERROR - filters output verbosity
}
// CorrelationConfig holds correlation configuration.
type CorrelationConfig struct {
TimeWindow TimeWindowConfig `yaml:"time_window"`
OrphanPolicy OrphanPolicyConfig `yaml:"orphan_policy"`
Matching MatchingConfig `yaml:"matching"`
Buffers BuffersConfig `yaml:"buffers"`
TTL TTLConfig `yaml:"ttl"`
ExcludeSourceIPs []string `yaml:"exclude_source_ips"` // List of source IPs or CIDR ranges to exclude
IncludeDestPorts []int `yaml:"include_dest_ports"` // If non-empty, only correlate events matching these destination ports
// Deprecated: Use TimeWindow.Value instead
TimeWindowS int `yaml:"time_window_s"`
// Deprecated: Use OrphanPolicy.ApacheAlwaysEmit instead
EmitOrphans bool `yaml:"emit_orphans"`
}
// TimeWindowConfig holds time window configuration.
type TimeWindowConfig struct {
Value int `yaml:"value"`
Unit string `yaml:"unit"` // s, ms, etc.
}
// GetDuration returns the time window as a duration.
func (c *TimeWindowConfig) GetDuration() time.Duration {
value := c.Value
if value <= 0 {
value = 1
}
switch c.Unit {
case "ms", "millisecond", "milliseconds":
return time.Duration(value) * time.Millisecond
case "s", "sec", "second", "seconds":
fallthrough
default:
return time.Duration(value) * time.Second
}
}
// OrphanPolicyConfig holds orphan event policy configuration.
type OrphanPolicyConfig struct {
ApacheAlwaysEmit bool `yaml:"apache_always_emit"`
ApacheEmitDelayMs int `yaml:"apache_emit_delay_ms"` // Delay in ms before emitting orphan A
NetworkEmit bool `yaml:"network_emit"`
}
// MatchingConfig holds matching mode configuration.
type MatchingConfig struct {
Mode string `yaml:"mode"` // one_to_one or one_to_many
}
// BuffersConfig holds buffer size configuration.
type BuffersConfig struct {
MaxHTTPItems int `yaml:"max_http_items"`
MaxNetworkItems int `yaml:"max_network_items"`
}
// TTLConfig holds TTL configuration.
type TTLConfig struct {
NetworkTTLS int `yaml:"network_ttl_s"`
}
// Load loads configuration from a YAML file.
func Load(path string) (*Config, error) {
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
cfg := defaultConfig()
if err := yaml.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("failed to parse config file: %w", err)
}
// Surcharge par variables d'environnement (préfixe LOGCORRELATOR_)
if err := ja4config.OverrideFromEnv(cfg, "LOGCORRELATOR"); err != nil {
return nil, fmt.Errorf("failed to apply env overrides: %w", err)
}
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
// defaultConfig returns a Config with default values.
func defaultConfig() *Config {
return &Config{
Log: LogConfig{
Level: "INFO",
},
Inputs: InputsConfig{
UnixSockets: make([]UnixSocketConfig, 0),
},
Outputs: OutputsConfig{
File: FileOutputConfig{
Enabled: true,
Path: "/var/log/logcorrelator/correlated.log",
},
ClickHouse: ClickHouseOutputConfig{
Enabled: false,
BatchSize: 500,
FlushIntervalMs: 200,
MaxBufferSize: 5000,
DropOnOverflow: true,
AsyncInsert: true,
TimeoutMs: 1000,
},
Stdout: StdoutOutputConfig{Enabled: false},
},
Correlation: CorrelationConfig{
TimeWindowS: 1,
EmitOrphans: true,
},
}
}
// Validate validates the configuration.
func (c *Config) Validate() error {
if len(c.Inputs.UnixSockets) < 2 {
return fmt.Errorf("at least two unix socket inputs are required")
}
seenNames := make(map[string]struct{}, len(c.Inputs.UnixSockets))
seenPaths := make(map[string]struct{}, len(c.Inputs.UnixSockets))
for i, input := range c.Inputs.UnixSockets {
if strings.TrimSpace(input.Name) == "" {
return fmt.Errorf("inputs.unix_sockets[%d].name is required", i)
}
if strings.TrimSpace(input.Path) == "" {
return fmt.Errorf("inputs.unix_sockets[%d].path is required", i)
}
if _, exists := seenNames[input.Name]; exists {
return fmt.Errorf("duplicate unix socket input name: %s", input.Name)
}
seenNames[input.Name] = struct{}{}
if _, exists := seenPaths[input.Path]; exists {
return fmt.Errorf("duplicate unix socket input path: %s", input.Path)
}
seenPaths[input.Path] = struct{}{}
}
// At least one output must be enabled
hasOutput := false
if c.Outputs.File.Enabled && c.Outputs.File.Path != "" {
hasOutput = true
}
if c.Outputs.ClickHouse.Enabled {
hasOutput = true
}
if c.Outputs.Stdout.Enabled {
hasOutput = true
}
if !hasOutput {
return fmt.Errorf("at least one output must be enabled (file, clickhouse, or stdout)")
}
if c.Outputs.ClickHouse.Enabled {
if strings.TrimSpace(c.Outputs.ClickHouse.DSN) == "" {
return fmt.Errorf("clickhouse DSN is required when enabled")
}
if strings.TrimSpace(c.Outputs.ClickHouse.Table) == "" {
return fmt.Errorf("clickhouse table is required when enabled")
}
if c.Outputs.ClickHouse.BatchSize <= 0 {
return fmt.Errorf("clickhouse batch_size must be > 0")
}
if c.Outputs.ClickHouse.MaxBufferSize <= 0 {
return fmt.Errorf("clickhouse max_buffer_size must be > 0")
}
if c.Outputs.ClickHouse.TimeoutMs <= 0 {
return fmt.Errorf("clickhouse timeout_ms must be > 0")
}
}
if c.Correlation.TimeWindowS <= 0 {
return fmt.Errorf("correlation.time_window_s must be > 0")
}
return nil
}
// GetTimeWindow returns the time window as a duration.
// Deprecated: Use TimeWindow.GetDuration() instead.
func (c *CorrelationConfig) GetTimeWindow() time.Duration {
// New config takes precedence
if c.TimeWindow.Value > 0 {
return c.TimeWindow.GetDuration()
}
// Fallback to deprecated field
value := c.TimeWindowS
if value <= 0 {
value = 1
}
return time.Duration(value) * time.Second
}
// GetApacheAlwaysEmit returns whether to always emit Apache events.
func (c *CorrelationConfig) GetApacheAlwaysEmit() bool {
if c.OrphanPolicy.ApacheAlwaysEmit {
return true
}
// Fallback to deprecated field
return c.EmitOrphans
}
// GetApacheEmitDelayMs returns the delay in milliseconds before emitting orphan A events.
func (c *CorrelationConfig) GetApacheEmitDelayMs() int {
if c.OrphanPolicy.ApacheEmitDelayMs > 0 {
return c.OrphanPolicy.ApacheEmitDelayMs
}
return domain.DefaultApacheEmitDelayMs // Default: 500ms
}
// GetMatchingMode returns the matching mode.
func (c *CorrelationConfig) GetMatchingMode() string {
if c.Matching.Mode != "" {
return c.Matching.Mode
}
return "one_to_many" // Default to Keep-Alive
}
// GetMaxHTTPBufferSize returns the max HTTP buffer size.
func (c *CorrelationConfig) GetMaxHTTPBufferSize() int {
if c.Buffers.MaxHTTPItems > 0 {
return c.Buffers.MaxHTTPItems
}
return domain.DefaultMaxHTTPBufferSize
}
// GetMaxNetworkBufferSize returns the max network buffer size.
func (c *CorrelationConfig) GetMaxNetworkBufferSize() int {
if c.Buffers.MaxNetworkItems > 0 {
return c.Buffers.MaxNetworkItems
}
return domain.DefaultMaxNetworkBufferSize
}
// GetNetworkTTLS returns the network TTL in seconds.
func (c *CorrelationConfig) GetNetworkTTLS() int {
if c.TTL.NetworkTTLS > 0 {
return c.TTL.NetworkTTLS
}
return domain.DefaultNetworkTTLS
}
// GetSocketPermissions returns the socket permissions as os.FileMode.
// Default is 0666 (world read/write).
func (c *UnixSocketConfig) GetSocketPermissions() os.FileMode {
trimmed := strings.TrimSpace(c.SocketPermissions)
if trimmed == "" {
return 0666
}
// Parse octal string (e.g., "0660", "660", "0666")
perms, err := strconv.ParseUint(trimmed, 8, 32)
if err != nil {
return 0666
}
return os.FileMode(perms)
}
// GetIncludeDestPorts returns the list of destination ports allowed for correlation.
// An empty list means all ports are allowed.
func (c *CorrelationConfig) GetIncludeDestPorts() []int {
return c.IncludeDestPorts
}
// GetExcludeSourceIPs returns the list of excluded source IPs or CIDR ranges.
func (c *CorrelationConfig) GetExcludeSourceIPs() []string {
return c.ExcludeSourceIPs
}
// IsSourceIPExcluded checks if a source IP should be excluded.
// Supports both exact IP matches and CIDR ranges.
func (c *CorrelationConfig) IsSourceIPExcluded(ip string) bool {
if len(c.ExcludeSourceIPs) == 0 {
return false
}
// Parse the IP once
parsedIP := net.ParseIP(ip)
if parsedIP == nil {
return false // Invalid IP
}
for _, exclude := range c.ExcludeSourceIPs {
// Try CIDR first
if strings.Contains(exclude, "/") {
_, cidr, err := net.ParseCIDR(exclude)
if err != nil {
continue // Invalid CIDR, skip
}
if cidr.Contains(parsedIP) {
return true
}
} else {
// Exact IP match
if exclude == ip {
return true
}
// Also try parsing as IP (handles different formats like 192.168.1.1 vs 192.168.001.001)
if excludeIP := net.ParseIP(exclude); excludeIP != nil {
if excludeIP.Equal(parsedIP) {
return true
}
}
}
}
return false
}

1253
old/services/correlator/internal/config/config_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

155
old/services/correlator/internal/domain/correlated_log.go Normal file
View File

@ -0,0 +1,155 @@
package domain
import (
"encoding/json"
"reflect"
"time"
)
// CorrelatedLog represents the output correlated log entry.
// All fields are flattened into a single-level structure.
type CorrelatedLog struct {
Timestamp time.Time `json:"timestamp"`
SrcIP string `json:"src_ip"`
SrcPort int `json:"src_port"`
DstIP string `json:"dst_ip,omitempty"`
DstPort int `json:"dst_port,omitempty"`
Correlated int `json:"correlated"` // 0 = orphelin, 1 = corrélé
OrphanSide string `json:"orphan_side,omitempty"`
Fields map[string]any `json:"-"` // Additional fields, merged at marshal time
}
// MarshalJSON implements custom JSON marshaling to flatten the structure.
func (c CorrelatedLog) MarshalJSON() ([]byte, error) {
// Create a flat map with all fields
flat := make(map[string]any)
// Add core fields
flat["timestamp"] = c.Timestamp
flat["src_ip"] = c.SrcIP
flat["src_port"] = c.SrcPort
if c.DstIP != "" {
flat["dst_ip"] = c.DstIP
}
if c.DstPort != 0 {
flat["dst_port"] = c.DstPort
}
flat["correlated"] = c.Correlated
if c.OrphanSide != "" {
flat["orphan_side"] = c.OrphanSide
}
// Merge additional fields while preserving reserved keys
reservedKeys := map[string]struct{}{
"timestamp": {},
"src_ip": {},
"src_port": {},
"dst_ip": {},
"dst_port": {},
"correlated": {},
"orphan_side": {},
}
for k, v := range c.Fields {
if _, reserved := reservedKeys[k]; reserved {
continue
}
flat[k] = v
}
return json.Marshal(flat)
}
// NewCorrelatedLogFromEvent creates a correlated log from a single event (orphan).
func NewCorrelatedLogFromEvent(event *NormalizedEvent, orphanSide string) CorrelatedLog {
fields := extractFields(event)
if event.KeepAliveSeq > 0 {
fields["keepalives"] = event.KeepAliveSeq
}
return CorrelatedLog{
Timestamp: event.Timestamp,
SrcIP: event.SrcIP,
SrcPort: event.SrcPort,
DstIP: event.DstIP,
DstPort: event.DstPort,
Correlated: 0,
OrphanSide: orphanSide,
Fields: fields,
}
}
// NewCorrelatedLog creates a correlated log from two matched events.
func NewCorrelatedLog(apacheEvent, networkEvent *NormalizedEvent) CorrelatedLog {
ts := apacheEvent.Timestamp
if networkEvent.Timestamp.After(ts) {
ts = networkEvent.Timestamp
}
fields := mergeFields(apacheEvent, networkEvent)
if apacheEvent.KeepAliveSeq > 0 {
fields["keepalives"] = apacheEvent.KeepAliveSeq
}
return CorrelatedLog{
Timestamp: ts,
SrcIP: apacheEvent.SrcIP,
SrcPort: apacheEvent.SrcPort,
DstIP: coalesceString(apacheEvent.DstIP, networkEvent.DstIP),
DstPort: coalesceInt(apacheEvent.DstPort, networkEvent.DstPort),
Correlated: 1,
OrphanSide: "",
Fields: fields,
}
}
// extractFields copie l'ensemble des champs bruts d'un événement dans une nouvelle map.
func extractFields(e *NormalizedEvent) map[string]any {
result := make(map[string]any)
for k, v := range e.Raw {
result[k] = v
}
return result
}
// mergeFields fusionne les champs bruts de deux événements en préfixant les clés en collision par "a_" et "b_".
func mergeFields(a, b *NormalizedEvent) map[string]any {
result := make(map[string]any)
// Start with A fields
for k, v := range a.Raw {
result[k] = v
}
// Merge B fields with collision handling
for k, v := range b.Raw {
if existing, exists := result[k]; exists {
if reflect.DeepEqual(existing, v) {
continue
}
// Collision with different values: keep both with prefixes
delete(result, k)
result["a_"+k] = existing
result["b_"+k] = v
continue
}
result[k] = v
}
return result
}
// coalesceString retourne la première chaîne non vide parmi les deux arguments.
func coalesceString(a, b string) string {
if a != "" {
return a
}
return b
}
// coalesceInt retourne le premier entier non nul parmi les deux arguments.
func coalesceInt(a, b int) int {
if a != 0 {
return a
}
return b
}

365
old/services/correlator/internal/domain/correlated_log_test.go Normal file
View File

@ -0,0 +1,365 @@
package domain
import (
"encoding/json"
"testing"
"time"
)
func TestNormalizedEvent_CorrelationKey(t *testing.T) {
tests := []struct {
name string
event *NormalizedEvent
expected string
}{
{
name: "basic key",
event: &NormalizedEvent{
SrcIP: "192.168.1.1",
SrcPort: 8080,
},
expected: "192.168.1.1:8080",
},
{
name: "different port",
event: &NormalizedEvent{
SrcIP: "10.0.0.1",
SrcPort: 443,
},
expected: "10.0.0.1:443",
},
{
name: "port zero",
event: &NormalizedEvent{
SrcIP: "127.0.0.1",
SrcPort: 0,
},
expected: "127.0.0.1:0",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
key := tt.event.CorrelationKey()
if key != tt.expected {
t.Errorf("expected %s, got %s", tt.expected, key)
}
})
}
}
func TestNewCorrelatedLogFromEvent(t *testing.T) {
event := &NormalizedEvent{
Source: SourceA,
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
SrcIP: "192.168.1.1",
SrcPort: 8080,
DstIP: "10.0.0.1",
DstPort: 80,
Raw: map[string]any{
"method": "GET",
"path": "/api/test",
},
}
log := NewCorrelatedLogFromEvent(event, "A")
if log.Correlated != 0 {
t.Error("expected correlated to be 0")
}
if log.OrphanSide != "A" {
t.Errorf("expected orphan_side A, got %s", log.OrphanSide)
}
if log.SrcIP != "192.168.1.1" {
t.Errorf("expected src_ip 192.168.1.1, got %s", log.SrcIP)
}
if log.Fields == nil {
t.Error("expected fields to be non-nil")
}
}
func TestNewCorrelatedLog(t *testing.T) {
apacheEvent := &NormalizedEvent{
Source: SourceA,
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
SrcIP: "192.168.1.1",
SrcPort: 8080,
DstIP: "10.0.0.1",
DstPort: 80,
Raw: map[string]any{"method": "GET"},
}
networkEvent := &NormalizedEvent{
Source: SourceB,
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 500000000, time.UTC),
SrcIP: "192.168.1.1",
SrcPort: 8080,
DstIP: "10.0.0.1",
DstPort: 80,
Raw: map[string]any{"ja3": "abc123"},
}
log := NewCorrelatedLog(apacheEvent, networkEvent)
if log.Correlated != 1 {
t.Error("expected correlated to be 1")
}
if log.OrphanSide != "" {
t.Errorf("expected orphan_side to be empty, got %s", log.OrphanSide)
}
if log.Fields == nil {
t.Error("expected fields to be non-nil")
}
}
// TestNewCorrelatedLog_TimestampSelectionAEarlier verifies that when A is earlier the later (B) timestamp is used.
func TestNewCorrelatedLog_TimestampSelectionAEarlier(t *testing.T) {
tsA := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
tsB := time.Date(2024, 1, 1, 12, 0, 1, 0, time.UTC) // B is later
a := &NormalizedEvent{Source: SourceA, Timestamp: tsA, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
b := &NormalizedEvent{Source: SourceB, Timestamp: tsB, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
log := NewCorrelatedLog(a, b)
if !log.Timestamp.Equal(tsB) {
t.Errorf("expected timestamp to be B's (later), got %v", log.Timestamp)
}
}
// TestNewCorrelatedLog_TimestampSelectionBEarlier verifies that when B is earlier, A's timestamp is used.
func TestNewCorrelatedLog_TimestampSelectionBEarlier(t *testing.T) {
tsA := time.Date(2024, 1, 1, 12, 0, 1, 0, time.UTC) // A is later
tsB := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
a := &NormalizedEvent{Source: SourceA, Timestamp: tsA, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
b := &NormalizedEvent{Source: SourceB, Timestamp: tsB, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
log := NewCorrelatedLog(a, b)
// The later timestamp wins. Since B is not After A, ts stays as A's timestamp.
if !log.Timestamp.Equal(tsA) {
t.Errorf("expected timestamp to be A's (later), got %v", log.Timestamp)
}
}
// TestNewCorrelatedLog_TimestampEqual verifies equal timestamps yield A's timestamp.
func TestNewCorrelatedLog_TimestampEqual(t *testing.T) {
ts := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
a := &NormalizedEvent{Source: SourceA, Timestamp: ts, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
b := &NormalizedEvent{Source: SourceB, Timestamp: ts, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
log := NewCorrelatedLog(a, b)
if !log.Timestamp.Equal(ts) {
t.Errorf("expected timestamp to be equal to both events' timestamp, got %v", log.Timestamp)
}
}
// TestNewCorrelatedLogFromEvent_WithKeepAlive verifies keepalives field is added when KeepAliveSeq > 0.
func TestNewCorrelatedLogFromEvent_WithKeepAlive(t *testing.T) {
event := &NormalizedEvent{
Source: SourceA,
Timestamp: time.Now(),
SrcIP: "1.1.1.1",
SrcPort: 9999,
KeepAliveSeq: 3,
Raw: map[string]any{"method": "GET"},
}
log := NewCorrelatedLogFromEvent(event, "A")
if log.Fields["keepalives"] != 3 {
t.Errorf("expected keepalives=3, got %v", log.Fields["keepalives"])
}
}
// TestNewCorrelatedLogFromEvent_NoKeepAlive verifies keepalives field is absent when KeepAliveSeq == 0.
func TestNewCorrelatedLogFromEvent_NoKeepAlive(t *testing.T) {
event := &NormalizedEvent{
Source: SourceA,
Timestamp: time.Now(),
SrcIP: "1.1.1.1",
SrcPort: 9999,
KeepAliveSeq: 0,
Raw: map[string]any{"method": "GET"},
}
log := NewCorrelatedLogFromEvent(event, "A")
if _, ok := log.Fields["keepalives"]; ok {
t.Error("keepalives field should not be present when KeepAliveSeq == 0")
}
}
// TestMergeFields_NoCollision verifies fields from A and B are merged without conflict.
func TestMergeFields_NoCollision(t *testing.T) {
a := &NormalizedEvent{Raw: map[string]any{"method": "GET", "path": "/foo"}}
b := &NormalizedEvent{Raw: map[string]any{"ja4": "abc123", "proto": "TLS"}}
fields := mergeFields(a, b)
if fields["method"] != "GET" {
t.Errorf("expected method=GET, got %v", fields["method"])
}
if fields["ja4"] != "abc123" {
t.Errorf("expected ja4=abc123, got %v", fields["ja4"])
}
}
// TestMergeFields_SameValueNoPrefix verifies same-value fields are not prefixed.
func TestMergeFields_SameValueNoPrefix(t *testing.T) {
a := &NormalizedEvent{Raw: map[string]any{"proto": "TCP"}}
b := &NormalizedEvent{Raw: map[string]any{"proto": "TCP"}}
fields := mergeFields(a, b)
if fields["proto"] != "TCP" {
t.Errorf("expected proto=TCP (no prefix), got %v", fields["proto"])
}
if _, ok := fields["a_proto"]; ok {
t.Error("a_proto should not exist for same-value collision")
}
if _, ok := fields["b_proto"]; ok {
t.Error("b_proto should not exist for same-value collision")
}
}
// TestMergeFields_DifferentValuePrefix verifies different-value fields get a_/b_ prefix.
func TestMergeFields_DifferentValuePrefix(t *testing.T) {
a := &NormalizedEvent{Raw: map[string]any{"port": 80}}
b := &NormalizedEvent{Raw: map[string]any{"port": 443}}
fields := mergeFields(a, b)
if fields["a_port"] != 80 {
t.Errorf("expected a_port=80, got %v", fields["a_port"])
}
if fields["b_port"] != 443 {
t.Errorf("expected b_port=443, got %v", fields["b_port"])
}
if _, ok := fields["port"]; ok {
t.Error("original 'port' key should be removed on collision")
}
}
// TestCoalesceString_EmptyA tests that when a is empty, b is returned.
func TestCoalesceString_EmptyA(t *testing.T) {
result := coalesceString("", "fallback")
if result != "fallback" {
t.Errorf("expected 'fallback', got %q", result)
}
}
// TestCoalesceString_NonEmptyA tests that when a is non-empty, a is returned.
func TestCoalesceString_NonEmptyA(t *testing.T) {
result := coalesceString("primary", "fallback")
if result != "primary" {
t.Errorf("expected 'primary', got %q", result)
}
}
// TestCoalesceInt_ZeroA tests that when a is zero, b is returned.
func TestCoalesceInt_ZeroA(t *testing.T) {
result := coalesceInt(0, 443)
if result != 443 {
t.Errorf("expected 443, got %d", result)
}
}
// TestCoalesceInt_NonZeroA tests that when a is non-zero, a is returned.
func TestCoalesceInt_NonZeroA(t *testing.T) {
result := coalesceInt(80, 443)
if result != 80 {
t.Errorf("expected 80, got %d", result)
}
}
// TestMarshalJSON_ReservedKeyProtection verifies reserved keys in Fields are not overwritten.
func TestMarshalJSON_ReservedKeyProtection(t *testing.T) {
log := CorrelatedLog{
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
SrcIP: "1.2.3.4",
SrcPort: 1234,
Correlated: 1,
Fields: map[string]any{
"src_ip": "EVIL_OVERRIDE", // should be ignored
"correlated": false, // should be ignored
"extra": "value",
},
}
data, err := json.Marshal(log)
if err != nil {
t.Fatalf("MarshalJSON failed: %v", err)
}
var flat map[string]any
if err := json.Unmarshal(data, &flat); err != nil {
t.Fatalf("Unmarshal failed: %v", err)
}
if flat["src_ip"] != "1.2.3.4" {
t.Errorf("reserved key src_ip should not be overwritten, got %v", flat["src_ip"])
}
if flat["correlated"] != float64(1) {
t.Errorf("reserved key correlated should not be overwritten, got %v", flat["correlated"])
}
if flat["extra"] != "value" {
t.Errorf("non-reserved key extra should be present, got %v", flat["extra"])
}
}
// TestMarshalJSON_OptionalFieldsOmittedWhenZero verifies DstIP/DstPort are omitted when zero.
func TestMarshalJSON_OptionalFieldsOmittedWhenZero(t *testing.T) {
log := CorrelatedLog{
Timestamp: time.Now(),
SrcIP: "1.2.3.4",
SrcPort: 1234,
Correlated: 0,
}
data, err := json.Marshal(log)
if err != nil {
t.Fatalf("MarshalJSON failed: %v", err)
}
var flat map[string]any
if err := json.Unmarshal(data, &flat); err != nil {
t.Fatalf("Unmarshal failed: %v", err)
}
if _, ok := flat["dst_ip"]; ok {
t.Error("dst_ip should be omitted when empty")
}
if _, ok := flat["dst_port"]; ok {
t.Error("dst_port should be omitted when zero")
}
if _, ok := flat["orphan_side"]; ok {
t.Error("orphan_side should be omitted when empty")
}
}
// TestExtractFields_Basic verifies extractFields copies Raw fields.
func TestExtractFields_Basic(t *testing.T) {
e := &NormalizedEvent{
Raw: map[string]any{"key1": "val1", "key2": 42},
}
fields := extractFields(e)
if fields["key1"] != "val1" {
t.Errorf("expected key1=val1, got %v", fields["key1"])
}
if fields["key2"] != 42 {
t.Errorf("expected key2=42, got %v", fields["key2"])
}
}
// TestNewCorrelatedLog_KeepAliveSeq verifies keepalives is set from apache event.
func TestNewCorrelatedLog_KeepAliveSeq(t *testing.T) {
a := &NormalizedEvent{
Source: SourceA, Timestamp: time.Now(), SrcIP: "1.1.1.1", SrcPort: 100,
KeepAliveSeq: 5,
Raw: map[string]any{},
}
b := &NormalizedEvent{
Source: SourceB, Timestamp: time.Now(), SrcIP: "1.1.1.1", SrcPort: 100,
Raw: map[string]any{},
}
log := NewCorrelatedLog(a, b)
if log.Fields["keepalives"] != 5 {
t.Errorf("expected keepalives=5, got %v", log.Fields["keepalives"])
}
}

1027
old/services/correlator/internal/domain/correlation_service.go Normal file
View File

File diff suppressed because it is too large Load Diff

1865
old/services/correlator/internal/domain/correlation_service_test.go Normal file
View File

File diff suppressed because it is too large Load Diff

33
old/services/correlator/internal/domain/event.go Normal file
View File

@ -0,0 +1,33 @@
package domain
import (
"strconv"
"time"
)
// EventSource identifies the source of an event.
type EventSource string
const (
SourceA EventSource = "A" // Apache/HTTP source
SourceB EventSource = "B" // Network source
)
// NormalizedEvent represents a unified internal event from either source.
type NormalizedEvent struct {
Source EventSource
Timestamp time.Time
SrcIP string
SrcPort int
DstIP string
DstPort int
Headers map[string]string
Extra map[string]any
Raw map[string]any // Original raw data
KeepAliveSeq int // Request sequence number within the Keep-Alive connection (1-based)
}
// CorrelationKey returns the key used for correlation (src_ip + src_port).
func (e *NormalizedEvent) CorrelationKey() string {
return e.SrcIP + ":" + strconv.Itoa(e.SrcPort)
}

27
old/services/correlator/internal/observability/logger.go Normal file
View File

@ -0,0 +1,27 @@
// Package observability provides structured logging for the correlator service.
// Implementation is delegated to shared/go/ja4common/logger to avoid duplication.
package observability
import jalogger "github.com/antitbone/ja4/ja4common/logger"
// Logger est un alias du type Logger de ja4common pour la journalisation structurée.
type Logger = jalogger.Logger
// LogLevel est un alias du type LogLevel de ja4common pour le niveau de journalisation.
type LogLevel = jalogger.LogLevel
const (
DEBUG LogLevel = jalogger.DEBUG
INFO LogLevel = jalogger.INFO
WARN LogLevel = jalogger.WARN
ERROR LogLevel = jalogger.ERROR
)
// NewLogger creates a new Logger with INFO level.
func NewLogger(prefix string) *Logger { return jalogger.New(prefix) }
// NewLoggerWithLevel creates a new Logger with the specified minimum level.
func NewLoggerWithLevel(prefix, level string) *Logger { return jalogger.NewWithLevel(prefix, level) }
// ParseLogLevel converts a string to LogLevel.
func ParseLogLevel(level string) LogLevel { return jalogger.ParseLogLevel(level) }

296
old/services/correlator/internal/observability/logger_test.go Normal file
View File

@ -0,0 +1,296 @@
// Package observability tests — behavioral tests for the Logger type alias.
// Since Logger = jalogger.Logger, we test the observable API only.
package observability_test
import (
"testing"
"github.com/antitbone/ja4/correlator/internal/observability"
)
func TestNewLogger_NonNil(t *testing.T) {
logger := observability.NewLogger("test")
if logger == nil {
t.Fatal("expected non-nil logger")
}
}
func TestLogger_DefaultLevel_IsInfo(t *testing.T) {
logger := observability.NewLogger("test")
if !logger.ShouldLog(observability.INFO) {
t.Error("INFO should be enabled by default")
}
if logger.ShouldLog(observability.DEBUG) {
t.Error("DEBUG should be disabled by default")
}
}
func TestLogger_Info_NoPanic(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "INFO")
if !logger.ShouldLog(observability.INFO) {
t.Error("INFO should be enabled")
}
logger.Info("test message")
}
func TestLogger_Error_NoPanic(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "ERROR")
if !logger.ShouldLog(observability.ERROR) {
t.Error("ERROR should be enabled")
}
logger.Error("error message", nil)
}
func TestLogger_Debug_NoPanic(t *testing.T) {
logger := observability.NewLogger("test")
logger.SetLevel("DEBUG")
if !logger.ShouldLog(observability.DEBUG) {
t.Error("DEBUG should be enabled after SetLevel(DEBUG)")
}
logger.Debug("test message")
}
func TestLogger_SetLevel(t *testing.T) {
logger := observability.NewLogger("test")
logger.SetLevel("DEBUG")
if !logger.ShouldLog(observability.DEBUG) {
t.Error("DEBUG should be enabled after SetLevel(DEBUG)")
}
logger.SetLevel("INFO")
if logger.ShouldLog(observability.DEBUG) {
t.Error("DEBUG should be disabled after SetLevel(INFO)")
}
logger.SetLevel("WARN")
if logger.ShouldLog(observability.INFO) {
t.Error("INFO should be disabled after SetLevel(WARN)")
}
if !logger.ShouldLog(observability.WARN) {
t.Error("WARN should be enabled after SetLevel(WARN)")
}
logger.SetLevel("ERROR")
if logger.ShouldLog(observability.WARN) {
t.Error("WARN should be disabled after SetLevel(ERROR)")
}
if !logger.ShouldLog(observability.ERROR) {
t.Error("ERROR should be enabled after SetLevel(ERROR)")
}
}
func TestParseLogLevel(t *testing.T) {
cases := []struct {
input string
expected observability.LogLevel
}{
{"DEBUG", observability.DEBUG},
{"debug", observability.DEBUG},
{"INFO", observability.INFO},
{"info", observability.INFO},
{"WARN", observability.WARN},
{"warn", observability.WARN},
{"WARNING", observability.WARN},
{"ERROR", observability.ERROR},
{"error", observability.ERROR},
{"", observability.INFO},
{"invalid", observability.INFO},
}
for _, tt := range cases {
t.Run(tt.input, func(t *testing.T) {
result := observability.ParseLogLevel(tt.input)
if result != tt.expected {
t.Errorf("ParseLogLevel(%q) = %v, want %v", tt.input, result, tt.expected)
}
})
}
}
func TestLogger_WithFields_NoPanic(t *testing.T) {
logger := observability.NewLogger("test")
child := logger.WithFields(map[string]any{"key1": "value1", "key2": 42})
if child == logger {
t.Error("expected different logger instance")
}
child.Info("message with fields")
}
func TestLogLevel_String(t *testing.T) {
cases := []struct {
level observability.LogLevel
expected string
}{
{observability.DEBUG, "DEBUG"},
{observability.INFO, "INFO"},
{observability.WARN, "WARN"},
{observability.ERROR, "ERROR"},
}
for _, tt := range cases {
t.Run(tt.expected, func(t *testing.T) {
if got := tt.level.String(); got != tt.expected {
t.Errorf("LogLevel(%d).String() = %q, want %q", tt.level, got, tt.expected)
}
})
}
}
func TestLogger_Warn_NoPanic(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "WARN")
if !logger.ShouldLog(observability.WARN) {
t.Error("WARN should be enabled")
}
logger.Warn("warning message")
}
func TestLogger_Formatted_NoPanic(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "DEBUG")
logger.Warnf("formatted %s %d", "message", 42)
logger.Infof("formatted %s %d", "message", 42)
logger.Debugf("formatted %s %d", "message", 42)
}
func TestLogger_Error_WithError(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "ERROR")
logger.Error("error occurred", &testErr{"test error"})
}
func TestLogger_ShouldLog_Concurrent(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "DEBUG")
done := make(chan bool)
for i := 0; i < 10; i++ {
go func() {
_ = logger.ShouldLog(observability.DEBUG)
done <- true
}()
}
for i := 0; i < 10; i++ {
<-done
}
}
func TestLogger_Log_Concurrent(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "DEBUG")
done := make(chan bool)
for i := 0; i < 10; i++ {
go func(n int) {
logger.Debugf("message %d", n)
done <- true
}(i)
}
for i := 0; i < 10; i++ {
<-done
}
}
func TestLogger_WithFields_Concurrent(t *testing.T) {
logger := observability.NewLogger("test")
done := make(chan bool)
for i := 0; i < 10; i++ {
go func(n int) {
_ = logger.WithFields(map[string]any{"key": n})
done <- true
}(i)
}
for i := 0; i < 10; i++ {
<-done
}
}
func TestLogger_SetLevel_Concurrent(t *testing.T) {
logger := observability.NewLogger("test")
done := make(chan bool)
for i := 0; i < 10; i++ {
go func() {
logger.SetLevel("DEBUG")
logger.SetLevel("INFO")
done <- true
}()
}
for i := 0; i < 10; i++ {
<-done
}
}
type testErr struct{ msg string }
func (e *testErr) Error() string { return e.msg }
func TestNewLoggerWithLevel_AllLevels(t *testing.T) {
levels := []string{"DEBUG", "INFO", "WARN", "WARNING", "ERROR", "invalid", ""}
for _, level := range levels {
t.Run(level, func(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", level)
if logger == nil {
t.Errorf("NewLoggerWithLevel(%q) returned nil", level)
}
})
}
}
func TestLogLevel_Constants(t *testing.T) {
if observability.DEBUG >= observability.INFO {
t.Error("DEBUG should be less than INFO")
}
if observability.INFO >= observability.WARN {
t.Error("INFO should be less than WARN")
}
if observability.WARN >= observability.ERROR {
t.Error("WARN should be less than ERROR")
}
}
func TestLogger_ShouldLog_AllLevels(t *testing.T) {
cases := []struct {
minLevel string
level observability.LogLevel
want bool
}{
{"DEBUG", observability.DEBUG, true},
{"DEBUG", observability.INFO, true},
{"DEBUG", observability.WARN, true},
{"DEBUG", observability.ERROR, true},
{"INFO", observability.DEBUG, false},
{"INFO", observability.INFO, true},
{"INFO", observability.WARN, true},
{"WARN", observability.INFO, false},
{"WARN", observability.WARN, true},
{"WARN", observability.ERROR, true},
{"ERROR", observability.WARN, false},
{"ERROR", observability.ERROR, true},
}
for _, tc := range cases {
t.Run(tc.minLevel+"_"+tc.level.String(), func(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", tc.minLevel)
got := logger.ShouldLog(tc.level)
if got != tc.want {
t.Errorf("ShouldLog(%v) with min=%s: expected %v, got %v",
tc.level, tc.minLevel, tc.want, got)
}
})
}
}
func TestParseLogLevel_WarningAlias(t *testing.T) {
got := observability.ParseLogLevel("WARNING")
if got != observability.WARN {
t.Errorf("ParseLogLevel(WARNING) = %v, want WARN", got)
}
}
func TestLogger_Errorf_NoPanic(t *testing.T) {
logger := observability.NewLoggerWithLevel("test", "DEBUG")
// Errorf is not defined in the interface, but Warnf/Infof/Debugf are tested
// Just ensure Error with a formatted message doesn't panic
logger.Error("formatted error", &testErr{"err detail"})
}
func TestNewLogger_PrefixIsUsed(t *testing.T) {
logger := observability.NewLogger("my-prefix")
if logger == nil {
t.Fatal("expected non-nil logger")
}
// The logger should be usable
logger.Infof("hello from %s", "my-prefix")
}

176
old/services/correlator/internal/observability/metrics.go Normal file
View File

@ -0,0 +1,176 @@
package observability
import (
"encoding/json"
"fmt"
"strings"
"sync"
"sync/atomic"
)
// CorrelationMetrics tracks correlation statistics for debugging and monitoring.
type CorrelationMetrics struct {
mu sync.RWMutex
// Events received
eventsReceivedA atomic.Int64
eventsReceivedB atomic.Int64
// Correlation results
correlationsSuccess atomic.Int64
correlationsFailed atomic.Int64
// Failure reasons
failedNoMatchKey atomic.Int64 // No event with same key in buffer
failedTimeWindow atomic.Int64 // Key found but outside time window
failedBufferEviction atomic.Int64 // Event evicted due to buffer full
failedTTLExpired atomic.Int64 // B event TTL expired before match
failedIPExcluded atomic.Int64 // Event excluded by IP filter
// Buffer stats
bufferASize atomic.Int64
bufferBSize atomic.Int64
// Orphan stats
orphansEmittedA atomic.Int64
orphansEmittedB atomic.Int64
orphansPendingA atomic.Int64
pendingOrphanMatch atomic.Int64 // B matched with pending orphan A
// Keep-Alive stats
keepAliveResets atomic.Int64 // Number of TTL resets (one-to-many mode)
}
// NewCorrelationMetrics creates a new metrics tracker.
func NewCorrelationMetrics() *CorrelationMetrics {
return &CorrelationMetrics{}
}
// RecordEventReceived records an event received from a source.
func (m *CorrelationMetrics) RecordEventReceived(source string) {
if source == "A" {
m.eventsReceivedA.Add(1)
} else if source == "B" {
m.eventsReceivedB.Add(1)
}
}
// RecordCorrelationSuccess records a successful correlation.
func (m *CorrelationMetrics) RecordCorrelationSuccess() {
m.correlationsSuccess.Add(1)
}
// RecordCorrelationFailed records a failed correlation attempt with the reason.
func (m *CorrelationMetrics) RecordCorrelationFailed(reason string) {
m.correlationsFailed.Add(1)
switch reason {
case "no_match_key":
m.failedNoMatchKey.Add(1)
case "time_window":
m.failedTimeWindow.Add(1)
case "buffer_eviction":
m.failedBufferEviction.Add(1)
case "ttl_expired":
m.failedTTLExpired.Add(1)
case "ip_excluded":
m.failedIPExcluded.Add(1)
}
}
// RecordBufferEviction records an event evicted from buffer.
func (m *CorrelationMetrics) RecordBufferEviction(source string) {
// Can be used for additional tracking if needed
}
// RecordOrphanEmitted records an orphan event emitted.
func (m *CorrelationMetrics) RecordOrphanEmitted(source string) {
if source == "A" {
m.orphansEmittedA.Add(1)
} else if source == "B" {
m.orphansEmittedB.Add(1)
}
}
// RecordPendingOrphan records an A event added to pending orphans.
func (m *CorrelationMetrics) RecordPendingOrphan() {
m.orphansPendingA.Add(1)
}
// RecordPendingOrphanMatch records a B event matching a pending orphan A.
func (m *CorrelationMetrics) RecordPendingOrphanMatch() {
m.pendingOrphanMatch.Add(1)
}
// RecordKeepAliveReset records a TTL reset for Keep-Alive.
func (m *CorrelationMetrics) RecordKeepAliveReset() {
m.keepAliveResets.Add(1)
}
// UpdateBufferSizes updates the current buffer sizes.
func (m *CorrelationMetrics) UpdateBufferSizes(sizeA, sizeB int64) {
m.bufferASize.Store(sizeA)
m.bufferBSize.Store(sizeB)
}
// Snapshot returns a point-in-time snapshot of all metrics.
func (m *CorrelationMetrics) Snapshot() MetricsSnapshot {
return MetricsSnapshot{
EventsReceivedA: m.eventsReceivedA.Load(),
EventsReceivedB: m.eventsReceivedB.Load(),
CorrelationsSuccess: m.correlationsSuccess.Load(),
CorrelationsFailed: m.correlationsFailed.Load(),
FailedNoMatchKey: m.failedNoMatchKey.Load(),
FailedTimeWindow: m.failedTimeWindow.Load(),
FailedBufferEviction: m.failedBufferEviction.Load(),
FailedTTLExpired: m.failedTTLExpired.Load(),
FailedIPExcluded: m.failedIPExcluded.Load(),
BufferASize: m.bufferASize.Load(),
BufferBSize: m.bufferBSize.Load(),
OrphansEmittedA: m.orphansEmittedA.Load(),
OrphansEmittedB: m.orphansEmittedB.Load(),
OrphansPendingA: m.orphansPendingA.Load(),
PendingOrphanMatch: m.pendingOrphanMatch.Load(),
KeepAliveResets: m.keepAliveResets.Load(),
}
}
// MetricsSnapshot is a point-in-time snapshot of metrics.
type MetricsSnapshot struct {
EventsReceivedA int64 `json:"events_received_a"`
EventsReceivedB int64 `json:"events_received_b"`
CorrelationsSuccess int64 `json:"correlations_success"`
CorrelationsFailed int64 `json:"correlations_failed"`
FailedNoMatchKey int64 `json:"failed_no_match_key"`
FailedTimeWindow int64 `json:"failed_time_window"`
FailedBufferEviction int64 `json:"failed_buffer_eviction"`
FailedTTLExpired int64 `json:"failed_ttl_expired"`
FailedIPExcluded int64 `json:"failed_ip_excluded"`
BufferASize int64 `json:"buffer_a_size"`
BufferBSize int64 `json:"buffer_b_size"`
OrphansEmittedA int64 `json:"orphans_emitted_a"`
OrphansEmittedB int64 `json:"orphans_emitted_b"`
OrphansPendingA int64 `json:"orphans_pending_a"`
PendingOrphanMatch int64 `json:"pending_orphan_match"`
KeepAliveResets int64 `json:"keepalive_resets"`
}
// MarshalJSON implements json.Marshaler.
func (m *CorrelationMetrics) MarshalJSON() ([]byte, error) {
return json.Marshal(m.Snapshot())
}
// String returns a human-readable string of metrics.
func (m *CorrelationMetrics) String() string {
s := m.Snapshot()
var b strings.Builder
b.WriteString("Correlation Metrics:\n")
fmt.Fprintf(&b, " Events Received: A=%d B=%d Total=%d\n", s.EventsReceivedA, s.EventsReceivedB, s.EventsReceivedA+s.EventsReceivedB)
fmt.Fprintf(&b, " Correlations: Success=%d Failed=%d\n", s.CorrelationsSuccess, s.CorrelationsFailed)
fmt.Fprintf(&b, " Failure Reasons: no_match_key=%d time_window=%d buffer_eviction=%d ttl_expired=%d ip_excluded=%d\n",
s.FailedNoMatchKey, s.FailedTimeWindow, s.FailedBufferEviction, s.FailedTTLExpired, s.FailedIPExcluded)
fmt.Fprintf(&b, " Buffer Sizes: A=%d B=%d\n", s.BufferASize, s.BufferBSize)
fmt.Fprintf(&b, " Orphans: Emitted A=%d B=%d Pending A=%d\n", s.OrphansEmittedA, s.OrphansEmittedB, s.OrphansPendingA)
fmt.Fprintf(&b, " Pending Orphan Match: %d\n", s.PendingOrphanMatch)
fmt.Fprintf(&b, " Keep-Alive Resets: %d\n", s.KeepAliveResets)
return b.String()
}

128
old/services/correlator/internal/observability/metrics_server.go Normal file
View File

@ -0,0 +1,128 @@
package observability
import (
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"sync"
"time"
)
// MetricsServer exposes correlation metrics via HTTP.
type MetricsServer struct {
mu sync.Mutex
server *http.Server
listener net.Listener
metricsFunc func() MetricsSnapshot
running bool
}
// NewMetricsServer creates a new metrics HTTP server.
func NewMetricsServer(addr string, metricsFunc func() MetricsSnapshot) (*MetricsServer, error) {
if metricsFunc == nil {
return nil, fmt.Errorf("metricsFunc cannot be nil")
}
ms := &MetricsServer{
metricsFunc: metricsFunc,
}
mux := http.NewServeMux()
mux.HandleFunc("/metrics", ms.handleMetrics)
mux.HandleFunc("/health", ms.handleHealth)
ms.server = &http.Server{
Addr: addr,
Handler: mux,
ReadTimeout: 5 * time.Second,
WriteTimeout: 10 * time.Second,
}
return ms, nil
}
// Start begins listening on the configured address.
func (ms *MetricsServer) Start() error {
ms.mu.Lock()
defer ms.mu.Unlock()
if ms.running {
return nil
}
listener, err := net.Listen("tcp", ms.server.Addr)
if err != nil {
return fmt.Errorf("failed to start metrics server: %w", err)
}
ms.listener = listener
ms.running = true
go func() {
if err := ms.server.Serve(listener); err != nil && err != http.ErrServerClosed {
// Server error or closed
}
}()
return nil
}
// Stop gracefully stops the metrics server.
func (ms *MetricsServer) Stop(ctx context.Context) error {
ms.mu.Lock()
defer ms.mu.Unlock()
if !ms.running {
return nil
}
ms.running = false
return ms.server.Shutdown(ctx)
}
// handleMetrics returns the correlation metrics as JSON.
func (ms *MetricsServer) handleMetrics(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
metrics := ms.metricsFunc()
w.Header().Set("Content-Type", "application/json")
if err := json.NewEncoder(w).Encode(metrics); err != nil {
http.Error(w, "Failed to encode metrics", http.StatusInternalServerError)
return
}
}
// handleHealth returns a simple health check response.
func (ms *MetricsServer) handleHealth(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, `{"status":"healthy"}`)
}
// IsRunning returns true if the server is running.
func (ms *MetricsServer) IsRunning() bool {
ms.mu.Lock()
defer ms.mu.Unlock()
return ms.running
}
// Addr returns the listening address.
func (ms *MetricsServer) Addr() string {
ms.mu.Lock()
defer ms.mu.Unlock()
if ms.listener == nil {
return ""
}
return ms.listener.Addr().String()
}

57
old/services/correlator/internal/ports/source.go Normal file
View File

@ -0,0 +1,57 @@
package ports
import (
"context"
"github.com/antitbone/ja4/correlator/internal/domain"
)
// EventSource defines the interface for log sources.
type EventSource interface {
// Start begins reading events and sending them to the channel.
// Returns an error if the source cannot be started.
Start(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) error
// Stop gracefully stops the source.
Stop() error
// Name returns the source name.
Name() string
}
// CorrelatedLogSink defines the interface for correlated log destinations.
type CorrelatedLogSink interface {
// Write sends a correlated log to the sink.
Write(ctx context.Context, log domain.CorrelatedLog) error
// Flush flushes any buffered logs.
Flush(ctx context.Context) error
// Close closes the sink.
Close() error
// Name returns the sink name.
Name() string
// Reopen closes and reopens the sink (for log rotation on SIGHUP).
// Optional: only FileSink implements this.
Reopen() error
}
// CorrelationProcessor defines the interface for the correlation service.
// This allows for easier testing and alternative implementations.
type CorrelationProcessor interface {
// ProcessEvent processes an incoming event and returns correlated logs.
ProcessEvent(event *domain.NormalizedEvent) []domain.CorrelatedLog
// Flush forces emission of remaining buffered events.
Flush() []domain.CorrelatedLog
// EmitPendingOrphans emits orphan A events whose delay has expired.
// Called periodically by the Orchestrator ticker so orphans are not blocked
// waiting for the next incoming event.
EmitPendingOrphans() []domain.CorrelatedLog
// GetBufferSizes returns the current buffer sizes for monitoring.
GetBufferSizes() (int, int)
}

34
old/services/correlator/logcorrelator.service Normal file
View File

@ -0,0 +1,34 @@
[Unit]
Description=logcorrelator service
After=network.target
[Service]
Type=simple
User=logcorrelator
Group=logcorrelator
ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml
ExecReload=/bin/kill -HUP $MAINPID
Restart=on-failure
RestartSec=5
# Runtime directory: systemd crée /run/logcorrelator (= /var/run/logcorrelator)
# avec le bon propriétaire (logcorrelator:logcorrelator) à chaque démarrage/restart,
# ce qui évite que les sockets se retrouvent en root:root après un reboot (tmpfs vidé).
RuntimeDirectory=logcorrelator
RuntimeDirectoryMode=0755
# Security hardening
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=true
ReadWritePaths=/var/log/logcorrelator /etc/logcorrelator
# Resource limits
LimitNOFILE=65536
# Systemd timeouts
TimeoutStartSec=10
TimeoutStopSec=30
[Install]
WantedBy=multi-user.target

366
old/services/correlator/packaging/rpm/logcorrelator.spec Normal file
View File

@ -0,0 +1,366 @@
# logcorrelator RPM spec file
# Compatible with CentOS 7, Rocky Linux 8, 9, 10
# Built with rpmbuild (not FPM)
Name: logcorrelator
Version: %{spec_version}
Release: 1%{?dist}
Summary: Log correlation service for HTTP and network events
License: MIT
URL: https://github.com/logcorrelator/logcorrelator
Vendor: logcorrelator <dev@example.com>
Packager: logcorrelator <dev@example.com>
BuildArch: x86_64
# Version macro with safe fallback
%if %{defined build_version}
%define spec_version %{build_version}
%else
%define spec_version 1.1.22
%endif
# Dependencies
Requires: systemd
Requires(pre): shadow-utils
Requires(post): systemd
Requires(preun): systemd
Requires(postun): systemd
BuildRequires: systemd-rpm-macros
%description
logcorrelator est un service système écrit en Go qui reçoit deux flux de logs JSON
via des sockets Unix, corrèle les événements HTTP applicatifs avec des événements
réseau, et produit des logs corrélés en temps réel vers ClickHouse et/ou fichier local.
Notes de sécurité :
- Le service s'exécute sous l'utilisateur logcorrelator (non-root)
- Les sockets Unix sont créés avec des permissions 0666 (world read/write)
- Les répertoires critiques sont protégés : /var/log (750), /var/lib (750), /etc (750)
- /var/run/logcorrelator est en 755 pour permettre la création de sockets
%prep
# Files are already in BUILD directory (copied by build-rpm.sh)
# No extraction needed
echo "Files available in BUILD directory:"
ls -la %{_builddir}/
%install
# Create directory structure in buildroot
mkdir -p %{buildroot}/usr/bin
mkdir -p %{buildroot}/etc/logcorrelator
mkdir -p %{buildroot}/var/log/logcorrelator
mkdir -p %{buildroot}/var/lib/logcorrelator
mkdir -p %{buildroot}%{_unitdir}
mkdir -p %{buildroot}/etc/logrotate.d
# Install binary (from BUILD directory)
install -m 0755 %{_builddir}/usr/bin/logcorrelator %{buildroot}/usr/bin/logcorrelator
# Install config files
install -m 0640 %{_builddir}/etc/logcorrelator/logcorrelator.yml %{buildroot}/etc/logcorrelator/logcorrelator.yml
install -m 0640 %{_builddir}/etc/logcorrelator/logcorrelator.yml.example %{buildroot}/etc/logcorrelator/logcorrelator.yml.example
# Install systemd service — destination is %{_unitdir} (/usr/lib/systemd/system), never /etc/
install -m 0644 %{_builddir}/etc/systemd/system/logcorrelator.service %{buildroot}%{_unitdir}/logcorrelator.service
# Install logrotate config
install -m 0644 %{_builddir}/etc/logrotate.d/logcorrelator %{buildroot}/etc/logrotate.d/logcorrelator
%pre
# Create service user and group before file installation so %attr() works correctly
if ! getent group logcorrelator >/dev/null 2>&1; then
groupadd --system logcorrelator
fi
if ! getent passwd logcorrelator >/dev/null 2>&1; then
useradd --system \
--gid logcorrelator \
--home-dir /var/lib/logcorrelator \
--no-create-home \
--shell /usr/sbin/nologin \
logcorrelator
fi
%post
# Use standard systemd RPM macros (daemon-reload, preset, no-op in containers)
%systemd_post logcorrelator.service
# Explicitly enable+start on fresh install
if [ $1 -eq 1 ] && [ -x /usr/bin/systemctl ] && [ -d /run/systemd/system ]; then
/usr/bin/systemctl enable logcorrelator.service 2>/dev/null || :
/usr/bin/systemctl start logcorrelator.service 2>/dev/null || :
fi
# Set ownership and permissions (idempotent — safe on both install and upgrade)
chown -R logcorrelator:logcorrelator /var/lib/logcorrelator 2>/dev/null || true
chown -R logcorrelator:logcorrelator /var/log/logcorrelator 2>/dev/null || true
chown -R logcorrelator:logcorrelator /etc/logcorrelator 2>/dev/null || true
chmod 750 /var/lib/logcorrelator 2>/dev/null || true
chmod 750 /var/log/logcorrelator 2>/dev/null || true
chmod 750 /etc/logcorrelator 2>/dev/null || true
# Copy default config if not exists
if [ ! -f /etc/logcorrelator/logcorrelator.yml ]; then
cp /etc/logcorrelator/logcorrelator.yml.example /etc/logcorrelator/logcorrelator.yml
chown logcorrelator:logcorrelator /etc/logcorrelator/logcorrelator.yml
chmod 640 /etc/logcorrelator/logcorrelator.yml
fi
%preun
%systemd_preun logcorrelator.service
%postun
%systemd_postun_with_restart logcorrelator.service
%files
/usr/bin/logcorrelator
%config(noreplace) /etc/logcorrelator/logcorrelator.yml
/etc/logcorrelator/logcorrelator.yml.example
%dir /var/log/logcorrelator
%dir /var/lib/logcorrelator
%{_unitdir}/logcorrelator.service
%config(noreplace) /etc/logrotate.d/logcorrelator
%changelog
* Wed Mar 11 2026 logcorrelator <dev@example.com> - 1.1.22-1
- Feat(outputs): file output enabled/disabled toggle
Ajout du champ enabled: true/false dans outputs.file de la configuration.
Le sink fichier n'est cree que si enabled: true ET path: defini.
Permet de desactiver completement la sortie fichier tout en gardant stdout/clickhouse.
Tests: TestValidate_FileOutputDisabled, TestLoadConfig_FileOutputDisabled
- Fix(systemd): arret immediat sans vidage de queue
orchestrator.Stop() ne vide plus les buffers (events en transit perdus).
Suppression de ShutdownTimeout et de la logique de flush/attente.
systemd TimeoutStopSec=30 gere l'arret force si besoin.
Simplification: cancel() + Close() uniquement.
- Feat(sql): TTL et compression ZSTD sur tables ClickHouse
http_logs_raw: TTL 1 jour, compression ZSTD sur raw_json
http_logs: TTL 7 jours, compression ZSTD sur champs texte volumineux
Parametre ttl_only_drop_parts = 1 pour optimiser les suppressions
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.21-1
- Update: vues ClickHouse et schema SQL
Ajout de bots.sql pour l'identification des bots (User-Agent parsing)
Ajout de tables.sql pour les tables de reference
Mise a jour de mv1.sql (vue materialisee) avec nouvelle structure de correlation
Documentation views.md enrichie avec exemples de requetes et schema complet
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.20-1
- Fix(rpm): suppression de systemd-tmpfiles.conf redondant
RuntimeDirectory=logcorrelator dans le service systemd gere deja /run/logcorrelator
automatiquement. La commande systemd-tmpfiles --create causait des erreurs sur
les systemes avec /var/lib/mysql existant (fichier au lieu de repertoire).
Suppression de /usr/lib/tmpfiles.d/logcorrelator.conf et de systemd-tmpfiles --create.
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.19-1
- Fix(systemd): stop/restart immediat sans attendre vidage queue
L'arret du service ne vide plus les buffers (events en transit perdus).
systemd TimeoutStopSec=30 gere deja l'arret force si besoin.
Simplification de orchestrator.Stop() : cancel() + Close() uniquement.
Suppression de ShutdownTimeout devenu inutile.
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.18-1
- Fix(outputs): file output enabled: false ne coupait pas l ecriture du fichier
Le champ Enabled manquait dans FileOutputConfig. Le sink fichier etait cree
meme avec enabled: false tant que path etait defini. Desormais, la condition
verifie explicitement enabled && path != "" dans main.go et Validate().
Test: TestValidate_FileOutputDisabled et TestLoadConfig_FileOutputDisabled ajoutes.
* Fri Mar 06 2026 logcorrelator <dev@example.com> - 1.1.17-1
- Fix(correlation): champ keepalives non peuple dans ClickHouse
Le champ KeepAliveSeq de NormalizedEvent n'etait pas transfere dans les Fields
de CorrelatedLog. La vue materialisee ClickHouse extrayait keepalives du JSON
mais trouvait toujours 0. Desormais, NewCorrelatedLog et NewCorrelatedLogFromEvent
ajoutent explicitement keepalives = KeepAliveSeq dans les Fields.
* Fri Mar 06 2026 logcorrelator <dev@example.com> - 1.1.16-1
- Feat(correlation): emettre les evenements A filtrés par include_dest_ports vers ClickHouse
Quand un evenement A (HTTP) etait exclu par le filtre include_dest_ports, il etait
silencieusement ignore. Desormais, si ApacheAlwaysEmit=true, l evenement est emis comme
non-correle (orphan_side=A) afin d apparaitre dans ClickHouse. Les evenements B restent
ignores. Test: TestCorrelationService_IncludeDestPorts_FilteredPort mis a jour +
TestCorrelationService_IncludeDestPorts_FilteredPort_NoAlwaysEmit ajoute.
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.15-1
- Fix(correlation/bug3): perte de donnees quand B expire avec des orphelins en attente
cleanNetworkBufferByTTL supprimait les pendingOrphans sans les emettre (perte silencieuse).
Desormais, les orphelins A sont retournes immediatement a l'appelant quand B expire,
et cleanExpired/ProcessEvent propagent ces resultats vers le sink.
Test: TestBTTLExpiry_PurgesPendingOrphans etendu pour verifier l'emission effective.
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.14-1
- Fix(correlation/bug1): Keep-Alive sessions au-dela de TimeWindow ne correlent plus en orphelins
Le matcher dans processSourceA utilisait eventsMatch (comparaison de timestamps) en mode
one_to_many. Apres ~10s, B.Timestamp_original depasse la TimeWindow et toutes les requetes
suivantes devenaient orphelines. Nouveau matcher bEventHasValidTTL : un B event est valide
tant que son TTL n'a pas expire (le TTL est reset a chaque correlation Keep-Alive).
- Fix(correlation/bug4): checkPendingOrphansForCorrelation utilisait eventsMatch (meme bug)
En mode one_to_many, un B arrivant avec un vieux timestamp ne matchait plus les pending orphans
pour la meme cle. Remplace par une verification de cle uniquement (meme cle = meme connexion).
- Fix(correlation/bug3): pendingOrphans non purges quand le B expire (cleanNetworkBufferByTTL)
Quand un B event expirait (TTL), les pending orphan A associes etaient bloques indefiniment.
Ils sont desormais emis immediatement lors de l'expiration du B correspondant.
- Fix(correlation/bug2): orphans emis uniquement sur reception d'evenement (pas de timer dedie)
EmitPendingOrphans() est maintenant une methode publique thread-safe. L'Orchestrateur
demarre un goroutine ticker (250ms) qui appelle EmitPendingOrphans() independamment du flux,
garantissant l'emission meme en l'absence de nouveaux evenements.
- Feat(ports): ajout de EmitPendingOrphans() dans l'interface CorrelationProcessor
- Test: 4 nouveaux tests de non-regression (Bug #1, #2, #3, #4)
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.13-1
- Fix: Unix sockets ne passent plus en root:root lors des restarts du service
- Fix: Ajout de RuntimeDirectory=logcorrelator dans le service systemd (systemd gère /run/logcorrelator avec le bon propriétaire à chaque démarrage/restart)
- Fix: Ajout de /usr/lib/tmpfiles.d/logcorrelator.conf pour recréer /run/logcorrelator au boot
- Chore: Retrait de /var/run/logcorrelator du RPM %files (géré par tmpfiles.d)
- Fix(correlation): emitPendingOrphans - corruption de slice lors de l expiration simultanée de plusieurs orphelins pour la même clé (slice aliasing bug, émissions en double)
- Fix(correlation): rotateOldestA - l événement rotaté était perdu silencieusement même avec ApacheAlwaysEmit=true (retourne désormais le CorrelatedLog)
- Fix(correlation): Keep-Alive cassé dans le chemin pending-orphan-then-B - le B event n était pas bufferisé en mode one_to_many, bloquant la corrélation des requêtes A2+ du même Keep-Alive
- Chore(correlation): suppression du champ mort timer *time.Timer dans pendingOrphan
- Feat(correlation): ajout de keepalive_seq dans les logs orphelins pour faciliter le debug (numéro de requête dans la connexion Keep-Alive, 1-based)
- Test: 4 nouveaux tests de non-régression pour les bugs de corrélation
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.12-1
- Feat: New config directive include_dest_ports - restrict correlation to specific destination ports
- Feat: If include_dest_ports is non-empty, events on unlisted ports are silently ignored (not correlated, not emitted as orphan)
- Feat: New metric failed_dest_port_filtered for monitoring filtered traffic
- Feat: Debug log for filtered events: "event excluded by dest port filter: source=A dst_port=22"
- Test: New unit tests for include_dest_ports (allowed port, filtered port, empty=all)
- Docs: README.md updated with include_dest_ports section and current version references
- Docs: architecture.yml updated with include_dest_ports
- Fix: config.example.yml - removed obsolete stdout.level field
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.11-1
- Fix: StdoutSink no longer writes correlated/orphan JSON to stdout
- Fix: stdout sink is now a no-op for data; operational logs go to stderr via logger
- Fix: ClickHouse sink had no logger - all flush errors were silently discarded
- Fix: Periodic, batch and final-close flush errors are now logged at ERROR level
- Fix: Buffer overflow with DropOnOverflow=true is now logged at WARN level
- Fix: Retry attempts are now logged at WARN level with attempt number, delay and error
- Feat: ClickHouse connection success logged at INFO (table, batch_size, flush_interval_ms)
- Feat: Successful batch sends logged at DEBUG (rows count, table)
- Feat: SetLogger() method added to ClickHouseSink for external logger injection
- Test: New unit tests for StdoutSink asserting stdout remains empty for all log types
* Wed Mar 04 2026 logcorrelator <dev@example.com> - 1.1.10-1
- Feat: IP exclusion filter - exclude specific source IPs or CIDR ranges
- Feat: Configuration exclude_source_ips supports single IPs and CIDR notation
- Feat: Debug logging for excluded IPs
- Feat: New metric failed_ip_excluded for monitoring filtered traffic
- Feat: Architecture documentation updated with observability section
- Use cases: exclude health checks, internal traffic, known bad actors
- Docs: README.md updated with IP exclusion documentation
- Docs: architecture.yml updated with metrics and troubleshooting guide
* Wed Mar 04 2026 logcorrelator <dev@example.com> - 1.1.9-1
- Feat: Debug logging - detailed DEBUG logs for correlation troubleshooting
- Feat: Correlation metrics server (HTTP endpoint /metrics and /health)
- Feat: New metrics: events_received, correlations_success/failed, failure reasons
- Feat: Failure reason tracking: no_match_key, time_window, buffer_eviction, ttl_expired
- Feat: Buffer size monitoring (buffer_a_size, buffer_b_size)
- Feat: Orphan tracking (orphans_emitted, orphans_pending, pending_orphan_match)
- Feat: Keep-Alive reset counter for connection tracking
- Feat: Test scripts added (test-correlation.sh, test-correlation-advanced.py)
- Change: Config example updated with metrics section
- Docs: README.md updated with debugging guide and troubleshooting table
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.8-1
- Migrated from FPM to rpmbuild (native RPM build)
- Reduced build image size by 200MB (-40%)
- Removed FPM gem dependency (use rpmbuild directly)
- Scripts post/preun/postun now inline in spec file
- Build image: rockylinux:8 instead of ruby:3.2-bookworm
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.7-1
- Fix: Critical Keep-Alive bug - network events evicted based on original timestamp instead of reset TTL
- Fix: Correlation time window increased from 1s to 10s for HTTP Keep-Alive support
- Fix: Network source now uses payload timestamp if available (fallback to reception time)
- Change: Default network TTL increased from 30s to 120s for long Keep-Alive sessions
- Test: Added comprehensive Keep-Alive tests (TTL reset, long session scenarios)
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.6-1
- Docs: Update ClickHouse schema documentation (http_logs_raw + http_logs tables)
- Fix: ClickHouse insertion uses single raw_json column (FORMAT JSONEachRow)
- Fix: ClickHouse native API (clickhouse-go/v2 PrepareBatch + Append + Send)
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.5-1
- Fix: ClickHouse insertion using native clickhouse-go/v2 API (PrepareBatch + Append + Send)
- Fix: Replaced database/sql wrapper with clickhouse.Open() and clickhouse.Conn
- Fix: Proper batch sending to avoid ATTEMPT_TO_READ_AFTER_EOF errors
- Fix: Set correct permissions (755) on /var/run/logcorrelator in RPM post-install
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.4-1
- Fix: Log raw JSON data on parse errors for debugging
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.3-1
- Refactor: Switch Unix sockets from STREAM to DGRAM mode (SOCK_DGRAM)
- Test: Comprehensive tests added - coverage improved to 74.4%
- Fix: Example config file installed to /etc/logcorrelator/logcorrelator.yml.example
- Change: Default socket permissions from 0660 to 0666 (world read/write)
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.2-1
- Fix: Example config file installed to /etc/logcorrelator/logcorrelator.yml.example
- Change: Default socket permissions from 0660 to 0666 (world read/write)
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.1-1
- Fix: Move logcorrelator.yml.example from /usr/share/logcorrelator/ to /etc/logcorrelator/
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.0-1
- Feat: Keep-Alive support (one-to-many correlation mode)
- Feat: Dynamic TTL for network events (source B)
- Feat: Separate buffer sizes for HTTP and network events
- Feat: SIGHUP signal handling for log rotation
- Feat: File sink Reopen() method for log rotation
- Feat: logrotate configuration included
- Feat: ExecReload added to systemd service
- Feat: New YAML config structure (time_window, orphan_policy, matching, buffers, ttl)
- Docs: Updated architecture.yml and config.example.yml
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.7-1
- Added: Log levels DEBUG, INFO, WARN, ERROR configurable via log.level
- Added: Warn and Warnf methods for warning messages
- Added: Debug logs for events received from sockets and correlations
- Added: Warning logs for orphan events and buffer overflow
- Changed: Configuration log.enabled replaced by log.level
- Changed: Orphan events and buffer overflow now logged as WARN instead of DEBUG
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.6-1
- Changed: Configuration YAML simplified, removed service.name, service.language
- Changed: Correlation config simplified, time_window_s instead of nested object
- Changed: Orphan policy simplified to emit_orphans boolean
- Changed: Apache socket renamed to http.socket
- Added: socket_permissions option on unix sockets
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.5-1
- Added: Systemd service auto-start after RPM installation
- Added: Systemd service hardening (TimeoutStartSec, TimeoutStopSec, ReadWritePaths)
- Fixed: Systemd service unit correct config path (.yml instead of .conf)
- Fixed: CI workflow branch name main to master
- Changed: RPM packaging generic el8/el9/el10 directory naming
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.4-1
- Breaking: Flattened JSON output structure - removed apache and network subdivisions
- All log fields now merged into single-level JSON structure
- ClickHouse schema: replaced apache JSON and network JSON columns with fields JSON column
- Custom MarshalJSON() implementation for flat output
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.3-1
- Fix: Added missing ClickHouse driver dependency
- Fix: Fixed race condition in orchestrator
- Security: Added explicit source_type configuration for Unix socket sources
- Added: Comprehensive test suite improvements
- Added: Test coverage improved from 50.6% to 62.0%
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.2-1
- Added: Initial RPM packaging support for Rocky Linux 8/9 and AlmaLinux 10
- Added: Docker multi-stage build pipeline
- Added: Hexagonal architecture implementation
- Added: Unix socket input sources (JSON line protocol)
- Added: File output sink (JSON lines)
- Added: ClickHouse output sink with batching and retry logic
- Added: Time-window based correlation on src_ip + src_port
- Added: Graceful shutdown with signal handling (SIGINT, SIGTERM)
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.1-1
- Initial package for CentOS 7, Rocky Linux 8, 9, 10

13
old/services/correlator/packaging/rpm/logrotate Normal file
View File

@ -0,0 +1,13 @@
/var/log/logcorrelator/correlated.log {
daily
rotate 7
compress
delaycompress
missingok
notifempty
create 0640 logcorrelator logcorrelator
sharedscripts
postrotate
/bin/systemctl reload logcorrelator > /dev/null 2>&1 || true
endscript
}

271
old/services/correlator/packaging/test/test-rpm.sh Executable file
View File

@ -0,0 +1,271 @@
#!/bin/bash
# Test script for logcorrelator RPM package
# Verifies installation, permissions, and service status
#
# Usage: ./packaging/test/test-rpm.sh [el8|el9|el10]
#
# This script tests the RPM package in a Docker container to ensure:
# - Installation succeeds
# - File permissions are correct
# - Service starts properly
# - Sockets are created with correct ownership
set -e
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
RPM_DIR="${PROJECT_ROOT}/dist/rpm"
# Default to el8 if no argument provided
DISTRO="${1:-el8}"
echo "========================================="
echo "Testing logcorrelator RPM for ${DISTRO}"
echo "========================================="
# Find the RPM file
case "${DISTRO}" in
el8|rocky8)
RPM_PATH="${RPM_DIR}/el8"
BASE_IMAGE="rockylinux:8"
;;
el9|rocky9)
RPM_PATH="${RPM_DIR}/el9"
BASE_IMAGE="rockylinux:9"
;;
el10|alma10)
RPM_PATH="${RPM_DIR}/el10"
BASE_IMAGE="almalinux:10"
;;
*)
echo "Unknown distribution: ${DISTRO}"
echo "Valid options: el8, el9, el10"
exit 1
;;
esac
# Find the latest RPM file — build it first if missing
RPM_FILE=$(ls -t "${RPM_PATH}"/logcorrelator-*.rpm 2>/dev/null | head -n 1)
if [ -z "${RPM_FILE}" ]; then
echo "No RPM found in ${RPM_PATH} — building now via Dockerfile.package..."
REPO_ROOT="$(dirname "$(dirname "$(dirname "$SCRIPT_DIR")")")"
VERSION="${VERSION:-$(git -C "${REPO_ROOT}" describe --tags --abbrev=0 2>/dev/null | sed 's/^v//' || echo "0.0.0-dev")}"
docker build \
--build-arg VERSION="${VERSION}" \
-f "${REPO_ROOT}/services/correlator/Dockerfile.package" \
--target output \
-o "${RPM_DIR}" \
"${REPO_ROOT}"
RPM_FILE=$(ls -t "${RPM_PATH}"/logcorrelator-*.rpm 2>/dev/null | head -n 1)
fi
if [ -z "${RPM_FILE}" ]; then
echo "ERROR: RPM build failed — no RPM found in ${RPM_PATH}"
exit 1
fi
echo "Testing RPM: ${RPM_FILE}"
echo "Base image: ${BASE_IMAGE}"
echo ""
# Create test script
TEST_SCRIPT=$(cat <<'EOF'
#!/bin/bash
set -e
echo "=== Installing logcorrelator RPM ==="
rpm -ivh /tmp/logcorrelator.rpm
echo ""
echo "=== Checking user and group ==="
if ! getent group logcorrelator >/dev/null; then
echo "FAIL: logcorrelator group not created"
exit 1
fi
echo "OK: logcorrelator group exists"
if ! getent passwd logcorrelator >/dev/null; then
echo "FAIL: logcorrelator user not created"
exit 1
fi
echo "OK: logcorrelator user exists"
echo ""
echo "=== Checking directory permissions ==="
# Check /var/run/logcorrelator
DIR="/var/run/logcorrelator"
if [ ! -d "$DIR" ]; then
echo "FAIL: $DIR does not exist"
exit 1
fi
OWNER=$(stat -c '%U:%G' "$DIR")
PERMS=$(stat -c '%a' "$DIR")
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
echo "FAIL: $DIR owner is $OWNER (expected logcorrelator:logcorrelator)"
exit 1
fi
if [ "$PERMS" != "755" ]; then
echo "FAIL: $DIR permissions are $PERMS (expected 755)"
exit 1
fi
echo "OK: $DIR - owner=$OWNER, permissions=$PERMS"
# Check /var/log/logcorrelator
DIR="/var/log/logcorrelator"
if [ ! -d "$DIR" ]; then
echo "FAIL: $DIR does not exist"
exit 1
fi
OWNER=$(stat -c '%U:%G' "$DIR")
PERMS=$(stat -c '%a' "$DIR")
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
echo "FAIL: $DIR owner is $OWNER (expected logcorrelator:logcorrelator)"
exit 1
fi
if [ "$PERMS" != "750" ]; then
echo "FAIL: $DIR permissions are $PERMS (expected 750)"
exit 1
fi
echo "OK: $DIR - owner=$OWNER, permissions=$PERMS"
# Check /var/lib/logcorrelator
DIR="/var/lib/logcorrelator"
if [ ! -d "$DIR" ]; then
echo "FAIL: $DIR does not exist"
exit 1
fi
OWNER=$(stat -c '%U:%G' "$DIR")
PERMS=$(stat -c '%a' "$DIR")
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
echo "FAIL: $DIR owner is $OWNER (expected logcorrelator:logcorrelator)"
exit 1
fi
if [ "$PERMS" != "750" ]; then
echo "FAIL: $DIR permissions are $PERMS (expected 750)"
exit 1
fi
echo "OK: $DIR - owner=$OWNER, permissions=$PERMS"
echo ""
echo "=== Checking config files ==="
# Check config file exists and has correct permissions
CONFIG="/etc/logcorrelator/logcorrelator.yml"
if [ ! -f "$CONFIG" ]; then
echo "FAIL: $CONFIG does not exist"
exit 1
fi
OWNER=$(stat -c '%U:%G' "$CONFIG")
PERMS=$(stat -c '%a' "$CONFIG")
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
echo "FAIL: $CONFIG owner is $OWNER (expected logcorrelator:logcorrelator)"
exit 1
fi
if [ "$PERMS" != "640" ]; then
echo "FAIL: $CONFIG permissions are $PERMS (expected 640)"
exit 1
fi
echo "OK: $CONFIG - owner=$OWNER, permissions=$PERMS"
# Check example config file
EXAMPLE_CONFIG="/etc/logcorrelator/logcorrelator.yml.example"
if [ ! -f "$EXAMPLE_CONFIG" ]; then
echo "FAIL: $EXAMPLE_CONFIG does not exist"
exit 1
fi
OWNER=$(stat -c '%U:%G' "$EXAMPLE_CONFIG")
PERMS=$(stat -c '%a' "$EXAMPLE_CONFIG")
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
echo "FAIL: $EXAMPLE_CONFIG owner is $OWNER (expected logcorrelator:logcorrelator)"
exit 1
fi
if [ "$PERMS" != "640" ]; then
echo "FAIL: $EXAMPLE_CONFIG permissions are $PERMS (expected 640)"
exit 1
fi
echo "OK: $EXAMPLE_CONFIG - owner=$OWNER, permissions=$PERMS"
echo ""
echo "=== Checking systemd service ==="
# Packaged units must be in /usr/lib/systemd/system, not /etc/systemd/system
if [ ! -f /usr/lib/systemd/system/logcorrelator.service ]; then
echo "FAIL: systemd service file not found at /usr/lib/systemd/system/logcorrelator.service"
exit 1
fi
echo "OK: systemd service file exists at /usr/lib/systemd/system/"
echo ""
echo "=== Checking logrotate config ==="
if [ ! -f /etc/logrotate.d/logcorrelator ]; then
echo "FAIL: logrotate config not found"
exit 1
fi
echo "OK: logrotate config exists"
echo ""
echo "=== Testing service start ==="
# Try to start the service (may fail in container without full systemd)
if command -v systemctl >/dev/null 2>&1; then
systemctl daemon-reload || true
if systemctl start logcorrelator.service 2>/dev/null; then
echo "OK: service started successfully"
# Wait for sockets to be created
sleep 2
echo ""
echo "=== Checking sockets ==="
HTTP_SOCKET="/var/run/logcorrelator/http.socket"
NETWORK_SOCKET="/var/run/logcorrelator/network.socket"
if [ -S "$HTTP_SOCKET" ]; then
OWNER=$(stat -c '%U:%G' "$HTTP_SOCKET")
PERMS=$(stat -c '%a' "$HTTP_SOCKET")
echo "OK: $HTTP_SOCKET exists - owner=$OWNER, permissions=$PERMS"
if [ "$PERMS" != "666" ]; then
echo "WARN: socket permissions are $PERMS (expected 666)"
fi
else
echo "WARN: $HTTP_SOCKET not found (service may not have started)"
fi
if [ -S "$NETWORK_SOCKET" ]; then
OWNER=$(stat -c '%U:%G' "$NETWORK_SOCKET")
PERMS=$(stat -c '%a' "$NETWORK_SOCKET")
echo "OK: $NETWORK_SOCKET exists - owner=$OWNER, permissions=$PERMS"
if [ "$PERMS" != "666" ]; then
echo "WARN: socket permissions are $PERMS (expected 666)"
fi
else
echo "WARN: $NETWORK_SOCKET not found (service may not have started)"
fi
systemctl stop logcorrelator.service || true
else
echo "WARN: service failed to start (expected in minimal container)"
fi
else
echo "WARN: systemctl not available (minimal container)"
fi
echo ""
echo "========================================="
echo "All tests passed!"
echo "========================================="
EOF
)
# Run test in Docker container
echo "Running tests in Docker container..."
echo ""
docker run --rm \
-v "${RPM_FILE}:/tmp/logcorrelator.rpm:ro" \
-v "${TEST_SCRIPT}:/test.sh:ro" \
"${BASE_IMAGE}" \
bash /test.sh
echo ""
echo "Test completed successfully for ${DISTRO}"

118
old/services/correlator/scripts/audit-architecture.sh Executable file
View File

@ -0,0 +1,118 @@
#!/usr/bin/env bash
# =============================================================================
# audit-architecture.sh — Vérifie la conformité de l'architecture du correlateur
#
# Ce script valide que les composants implémentés (service systemd, packaging RPM,
# configuration YAML, sockets Unix, sinks de sortie, logique de corrélation) sont
# présents et correctement configurés, conformément aux spécifications d'architecture.
#
# Usage:
# ./audit-architecture.sh
# docker run --rm -v $(pwd):/src <image> /src/scripts/audit-architecture.sh
#
# Prérequis:
# - Exécuté depuis le répertoire source /src du correlateur (ou monté en volume)
# - Les sources Go doivent être présentes (les checks sont basés sur grep)
#
# Variables d'environnement: aucune
# =============================================================================
set -e
echo "=== AUDIT ARCHITECTURE COMPLIANCE ==="
echo ""
# 1. Runtime - systemd service
echo "1. RUNTIME - SYSTEMD SERVICE"
if [ -f /src/logcorrelator.service ]; then
echo "✅ logcorrelator.service exists"
grep -q "ExecStart=/usr/bin/logcorrelator" /src/logcorrelator.service && echo " ✅ ExecStart correct" || echo " ❌ ExecStart incorrect"
grep -q "ExecReload=" /src/logcorrelator.service && echo " ✅ ExecReload present" || echo " ❌ ExecReload missing"
grep -q "Restart=on-failure" /src/logcorrelator.service && echo " ✅ Restart policy correct" || echo " ❌ Restart policy incorrect"
else
echo "❌ logcorrelator.service missing"
fi
# Check signal handling in code
echo ""
grep -r "SIGINT\|SIGTERM\|SIGHUP" /src/cmd/logcorrelator/main.go > /dev/null && echo "✅ Signal handling (SIGINT/SIGTERM/SIGHUP) implemented" || echo "❌ Signal handling missing"
# 2. Packaging - RPM
echo ""
echo "2. PACKAGING - RPM"
[ -f /src/packaging/rpm/logcorrelator.spec ] && echo "✅ RPM spec file exists" || echo "❌ RPM spec missing"
grep -q "fpm" /src/Dockerfile.package && echo "✅ fpm tool used for packaging" || echo "❌ fpm not found"
# 3. Config - YAML
echo ""
echo "3. CONFIG - YAML"
[ -f /src/config.example.yml ] && echo "✅ config.example.yml exists" || echo "❌ config.example.yml missing"
grep -q "log:" /src/config.example.yml && echo " ✅ log section present" || echo " ❌ log section missing"
grep -q "inputs:" /src/config.example.yml && echo " ✅ inputs section present" || echo " ❌ inputs section missing"
grep -q "outputs:" /src/config.example.yml && echo " ✅ outputs section present" || echo " ❌ outputs section missing"
grep -q "correlation:" /src/config.example.yml && echo " ✅ correlation section present" || echo " ❌ correlation section missing"
# 4. Inputs - Unix datagram sockets
echo ""
echo "4. INPUTS - UNIX DATAGRAM SOCKETS"
grep -q "ListenUnixgram" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ Using ListenUnixgram (SOCK_DGRAM)" || echo "❌ Not using SOCK_DGRAM"
grep -q "ReadFromUnix" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ Using ReadFromUnix for datagrams" || echo "❌ Not using ReadFromUnix"
grep -q "MaxDatagramSize = 65535" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ max_datagram_bytes = 65535" || echo "❌ max_datagram_bytes incorrect"
grep -q "0666" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ Default socket permissions 0666" || echo "❌ Socket permissions not 0666"
# Check socket paths in config
grep -q "http.socket" /src/config.example.yml && echo " ✅ http.socket path configured" || echo " ❌ http.socket path missing"
grep -q "network.socket" /src/config.example.yml && echo " ✅ network.socket path configured" || echo " ❌ network.socket path missing"
# 5. Outputs - Sinks
echo ""
echo "5. OUTPUTS - SINKS"
[ -f /src/internal/adapters/outbound/file/sink.go ] && echo "✅ File sink exists" || echo "❌ File sink missing"
[ -f /src/internal/adapters/outbound/clickhouse/sink.go ] && echo "✅ ClickHouse sink exists" || echo "❌ ClickHouse sink missing"
[ -f /src/internal/adapters/outbound/multi/sink.go ] && echo "✅ MultiSink exists" || echo "❌ MultiSink missing"
# Check SIGHUP reopen in file sink
grep -q "Reopen" /src/internal/adapters/outbound/file/sink.go && echo " ✅ FileSink.Reopen() for SIGHUP" || echo " ❌ FileSink.Reopen() missing"
# Check ClickHouse batching
grep -q "batch" /src/internal/adapters/outbound/clickhouse/sink.go && echo " ✅ ClickHouse batching implemented" || echo " ❌ ClickHouse batching missing"
grep -q "drop_on_overflow\|DropOnOverflow" /src/internal/adapters/outbound/clickhouse/sink.go && echo " ✅ drop_on_overflow implemented" || echo " ❌ drop_on_overflow missing"
# 6. Correlation
echo ""
echo "6. CORRELATION"
grep -q "src_ip" /src/internal/domain/correlation_service.go && echo "✅ src_ip in correlation key" || echo "❌ src_ip missing"
grep -q "src_port" /src/internal/domain/correlation_service.go && echo "✅ src_port in correlation key" || echo "❌ src_port missing"
grep -q "MatchingMode" /src/internal/domain/correlation_service.go && echo "✅ MatchingMode (one_to_one/one_to_many) implemented" || echo "❌ MatchingMode missing"
grep -q "ApacheAlwaysEmit" /src/internal/domain/correlation_service.go && echo "✅ apache_always_emit orphan policy" || echo "❌ apache_always_emit missing"
grep -q "network_ttl\|NetworkTTLS" /src/internal/domain/correlation_service.go && echo "✅ TTL management for network events" || echo "❌ TTL management missing"
grep -q "max_http_items\|maxHttpItems\|MaxHTTPItems" /src/internal/domain/correlation_service.go && echo "✅ Buffer limit max_http_items" || echo " ⚠️ Buffer limit naming may differ"
grep -q "max_network_items\|maxNetworkItems\|MaxNetworkItems" /src/internal/domain/correlation_service.go && echo "✅ Buffer limit max_network_items" || echo " ⚠️ Buffer limit naming may differ"
# 7. Schema - Source A and B
echo ""
echo "7. SCHEMA - SOURCE A AND B"
grep -q "timestamp" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ timestamp field for Source A" || echo "❌ timestamp missing for Source A"
grep -q "SourceA\|SourceB" /src/internal/domain/event.go && echo "✅ EventSource enum (A/B)" || echo "❌ EventSource enum missing"
grep -q "header_" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ header_* dynamic fields" || echo "❌ header_* fields missing"
grep -q "Extra" /src/internal/domain/event.go && echo "✅ Extra fields map" || echo "❌ Extra fields missing"
# 8. Architecture modules
echo ""
echo "8. ARCHITECTURE MODULES"
[ -d /src/internal/domain ] && echo "✅ internal/domain" || echo "❌ internal/domain missing"
[ -d /src/internal/ports ] && echo "✅ internal/ports" || echo "❌ internal/ports missing"
[ -d /src/internal/app ] && echo "✅ internal/app" || echo "❌ internal/app missing"
[ -d /src/internal/adapters/inbound ] && echo "✅ internal/adapters/inbound" || echo "❌ internal/adapters/inbound missing"
[ -d /src/internal/adapters/outbound ] && echo "✅ internal/adapters/outbound" || echo "❌ internal/adapters/outbound missing"
[ -d /src/internal/config ] && echo "✅ internal/config" || echo "❌ internal/config missing"
[ -d /src/internal/observability ] && echo "✅ internal/observability" || echo "❌ internal/observability missing"
[ -d /src/cmd/logcorrelator ] && echo "✅ cmd/logcorrelator" || echo "❌ cmd/logcorrelator missing"
# 9. Testing
echo ""
echo "9. TESTING"
echo "Running tests with coverage..."
cd /src && go test ./... -cover 2>&1 | grep -E "^(ok|FAIL|\?)" || true
echo ""
echo "=== AUDIT COMPLETE ==="

582
old/services/correlator/scripts/test-correlation-advanced.py Executable file
View File

@ -0,0 +1,582 @@
#!/usr/bin/env python3
"""
test-correlation-advanced.py - Advanced correlation testing tool
This script provides comprehensive testing for the logcorrelator service,
including various scenarios to debug correlation issues.
Usage:
python3 test-correlation-advanced.py [options]
Requirements:
- Python 3.6+
- requests library (for metrics): pip install requests
"""
import argparse
import json
import socket
import sys
import time
from datetime import datetime
from typing import Dict, Any, Optional, Tuple
try:
import requests
HAS_REQUESTS = True
except ImportError:
HAS_REQUESTS = False
class Colors:
"""ANSI color codes for terminal output."""
BLUE = '\033[0;34m'
GREEN = '\033[0;32m'
YELLOW = '\033[1;33m'
RED = '\033[0;31m'
NC = '\033[0m' # No Color
BOLD = '\033[1m'
def colorize(text: str, color: str) -> str:
"""Wrap text with ANSI color codes."""
return f"{color}{text}{Colors.NC}"
def info(text: str):
print(colorize(f"[INFO] ", Colors.BLUE) + text)
def success(text: str):
print(colorize(f"[OK] ", Colors.GREEN) + text)
def warn(text: str):
print(colorize(f"[WARN] ", Colors.YELLOW) + text)
def error(text: str):
print(colorize(f"[ERROR] ", Colors.RED) + text)
def debug(text: str, verbose: bool = False):
if verbose:
print(colorize(f"[DEBUG] ", Colors.BLUE) + text)
class CorrelationTester:
"""Main test class for correlation testing."""
def __init__(
self,
http_socket: str = "/var/run/logcorrelator/http.socket",
network_socket: str = "/var/run/logcorrelator/network.socket",
metrics_url: str = "http://localhost:8080/metrics",
verbose: bool = False,
skip_metrics: bool = False
):
self.http_socket = http_socket
self.network_socket = network_socket
self.metrics_url = metrics_url
self.verbose = verbose
self.skip_metrics = skip_metrics
self.http_sock: Optional[socket.socket] = None
self.network_sock: Optional[socket.socket] = None
def connect(self) -> bool:
"""Connect to Unix sockets."""
try:
# HTTP socket
self.http_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
self.http_sock.connect(self.http_socket)
debug(f"Connected to HTTP socket: {self.http_socket}", self.verbose)
# Network socket
self.network_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
self.network_sock.connect(self.network_socket)
debug(f"Connected to Network socket: {self.network_socket}", self.verbose)
return True
except FileNotFoundError as e:
error(f"Socket not found: {e}")
return False
except Exception as e:
error(f"Connection error: {e}")
return False
def close(self):
"""Close socket connections."""
if self.http_sock:
self.http_sock.close()
if self.network_sock:
self.network_sock.close()
def send_http_event(
self,
src_ip: str,
src_port: int,
timestamp: int,
method: str = "GET",
path: str = "/test",
host: str = "example.com",
extra_headers: Optional[Dict[str, str]] = None
) -> Dict[str, Any]:
"""Send an HTTP (source A) event."""
event = {
"src_ip": src_ip,
"src_port": src_port,
"dst_ip": "10.0.0.1",
"dst_port": 443,
"timestamp": timestamp,
"method": method,
"path": path,
"host": host,
"http_version": "HTTP/1.1",
"header_user_agent": "TestAgent/1.0",
"header_accept": "*/*"
}
if extra_headers:
for key, value in extra_headers.items():
event[f"header_{key}"] = value
json_data = json.dumps(event)
if self.http_sock:
self.http_sock.sendall(json_data.encode())
debug(f"Sent HTTP event: {src_ip}:{src_port} ts={timestamp}", self.verbose)
return event
def send_network_event(
self,
src_ip: str,
src_port: int,
timestamp: int,
ja3: str = "abc123",
ja4: str = "def456",
tls_version: str = "TLS1.3",
tls_sni: str = "example.com"
) -> Dict[str, Any]:
"""Send a Network (source B) event."""
event = {
"src_ip": src_ip,
"src_port": src_port,
"dst_ip": "10.0.0.1",
"dst_port": 443,
"timestamp": timestamp,
"ja3": ja3,
"ja4": ja4,
"tls_version": tls_version,
"tls_sni": tls_sni
}
json_data = json.dumps(event)
if self.network_sock:
self.network_sock.sendall(json_data.encode())
debug(f"Sent Network event: {src_ip}:{src_port} ts={timestamp}", self.verbose)
return event
def get_metrics(self) -> Dict[str, Any]:
"""Fetch metrics from the metrics server."""
if self.skip_metrics:
return {}
if not HAS_REQUESTS:
warn("requests library not installed, skipping metrics")
return {}
try:
response = requests.get(self.metrics_url, timeout=5)
response.raise_for_status()
return response.json()
except Exception as e:
warn(f"Failed to fetch metrics: {e}")
return {}
def print_metrics(self, metrics: Dict[str, Any], title: str = "Metrics"):
"""Print metrics in a formatted way."""
if not metrics:
return
print(f"\n{colorize(f'=== {title} ===', Colors.BOLD)}")
keys_to_show = [
("events_received_a", "Events A"),
("events_received_b", "Events B"),
("correlations_success", "Correlations"),
("correlations_failed", "Failures"),
("failed_no_match_key", " - No match key"),
("failed_time_window", " - Time window"),
("failed_buffer_eviction", " - Buffer eviction"),
("failed_ttl_expired", " - TTL expired"),
("buffer_a_size", "Buffer A size"),
("buffer_b_size", "Buffer B size"),
("orphans_emitted_a", "Orphans A"),
("orphans_emitted_b", "Orphans B"),
("pending_orphan_match", "Pending orphan matches"),
("keepalive_resets", "Keep-Alive resets"),
]
for key, label in keys_to_show:
if key in metrics:
print(f" {label}: {metrics[key]}")
def check_sockets(self) -> bool:
"""Check if sockets exist."""
import os
errors = 0
for name, path in [("HTTP", self.http_socket), ("Network", self.network_socket)]:
if not os.path.exists(path):
error(f"{name} socket not found: {path}")
errors += 1
elif not os.path.exists(path) or not os.path.stat(path).st_mode & 0o170000 == 0o140000:
# Check if it's a socket
try:
if not socket.getaddrinfo(path, None, socket.AF_UNIX):
error(f"{name} path exists but is not a socket: {path}")
errors += 1
except:
pass
else:
debug(f"{name} socket found: {path}", self.verbose)
return errors == 0
def run_basic_test(self, count: int = 10, delay_ms: int = 100) -> Tuple[bool, Dict[str, int]]:
"""
Run basic correlation test.
Sends N pairs of A+B events with matching src_ip:src_port and timestamps.
All should correlate successfully.
"""
info(f"Running basic correlation test with {count} pairs...")
# Get initial metrics
initial_metrics = self.get_metrics()
self.print_metrics(initial_metrics, "Initial Metrics")
initial_success = initial_metrics.get("correlations_success", 0)
initial_failed = initial_metrics.get("correlations_failed", 0)
initial_a = initial_metrics.get("events_received_a", 0)
initial_b = initial_metrics.get("events_received_b", 0)
# Send test events
print(f"\nSending {count} event pairs...")
base_timestamp = time.time_ns()
sent = 0
for i in range(1, count + 1):
src_ip = f"192.168.1.{(i % 254) + 1}"
src_port = 8000 + i
# Same timestamp for perfect correlation
timestamp = base_timestamp + (i * 1_000_000)
self.send_http_event(src_ip, src_port, timestamp)
self.send_network_event(src_ip, src_port, timestamp)
sent += 1
if delay_ms > 0:
time.sleep(delay_ms / 1000.0)
success(f"Sent {sent} event pairs")
# Wait for processing
info("Waiting for processing (2 seconds)...")
time.sleep(2)
# Get final metrics
final_metrics = self.get_metrics()
self.print_metrics(final_metrics, "Final Metrics")
# Calculate deltas
delta_success = final_metrics.get("correlations_success", 0) - initial_success
delta_failed = final_metrics.get("correlations_failed", 0) - initial_failed
delta_a = final_metrics.get("events_received_a", 0) - initial_a
delta_b = final_metrics.get("events_received_b", 0) - initial_b
results = {
"sent": sent,
"received_a": delta_a,
"received_b": delta_b,
"correlations": delta_success,
"failures": delta_failed
}
# Print results
print(f"\n{colorize('=== Results ===', Colors.BOLD)}")
print(f" Events A sent: {delta_a} (expected: {sent})")
print(f" Events B sent: {delta_b} (expected: {sent})")
print(f" Correlations: {delta_success}")
print(f" Failures: {delta_failed}")
# Validation
test_passed = True
if delta_a != sent:
error(f"Event A count mismatch: got {delta_a}, expected {sent}")
test_passed = False
if delta_b != sent:
error(f"Event B count mismatch: got {delta_b}, expected {sent}")
test_passed = False
if delta_success != sent:
error(f"Correlation count mismatch: got {delta_success}, expected {sent}")
test_passed = False
if delta_failed > 0:
warn(f"Unexpected correlation failures: {delta_failed}")
if test_passed:
success("All tests passed! Correlation is working correctly.")
else:
error("Some tests failed. Check logs for details.")
return test_passed, results
def run_time_window_test(self) -> bool:
"""Test time window expiration."""
info("Running time window test...")
src_ip = "192.168.100.1"
src_port = 9999
# Send A event
ts_a = time.time_ns()
self.send_http_event(src_ip, src_port, ts_a)
info(f"Sent A event at {ts_a}")
# Wait for time window to expire (default 10s)
info("Waiting 11 seconds (time window should expire)...")
time.sleep(11)
# Send B event
ts_b = time.time_ns()
self.send_network_event(src_ip, src_port, ts_b)
info(f"Sent B event at {ts_b}")
time_diff_sec = (ts_b - ts_a) / 1_000_000_000
info(f"Time difference: {time_diff_sec:.1f} seconds")
info("Expected: time_window failure (check metrics)")
return True
def run_different_ip_test(self) -> bool:
"""Test different IP (should not correlate)."""
info("Running different IP test...")
ts = time.time_ns()
# Send A with IP 192.168.200.1
self.send_http_event("192.168.200.1", 7777, ts)
info("Sent A event from 192.168.200.1:7777")
# Send B with different IP
self.send_network_event("192.168.200.2", 7777, ts)
info("Sent B event from 192.168.200.2:7777 (different IP)")
info("Expected: no_match_key failure (different src_ip)")
return True
def run_keepalive_test(self, count: int = 5) -> bool:
"""Test Keep-Alive mode (one B correlates with multiple A)."""
info(f"Running Keep-Alive test with {count} HTTP requests on same connection...")
src_ip = "192.168.50.1"
src_port = 6000
# Send one B event first (network/TCP connection)
ts_b = time.time_ns()
self.send_network_event(src_ip, src_port, ts_b)
info(f"Sent B event (connection): {src_ip}:{src_port}")
# Send multiple A events (HTTP requests) on same connection
for i in range(count):
ts_a = time.time_ns() + (i * 100_000_000) # 100ms apart
self.send_http_event(src_ip, src_port, ts_a, path=f"/request{i}")
info(f"Sent A event (request {i}): {src_ip}:{src_port}")
time.sleep(0.05) # 50ms delay
time.sleep(2) # Wait for processing
# Check metrics
metrics = self.get_metrics()
keepalive_resets = metrics.get("keepalive_resets", 0)
info(f"Keep-Alive resets: {keepalive_resets} (expected: {count - 1})")
if keepalive_resets >= count - 1:
success("Keep-Alive test passed!")
return True
else:
warn(f"Keep-Alive resets lower than expected. This may be normal depending on timing.")
return True
def run_all_tests(self) -> bool:
"""Run all test scenarios."""
results = []
# Basic test
passed, _ = self.run_basic_test(count=10)
results.append(("Basic correlation", passed))
print("\n" + "=" * 50 + "\n")
# Time window test
self.run_time_window_test()
results.append(("Time window", True)) # Informational
print("\n" + "=" * 50 + "\n")
# Different IP test
self.run_different_ip_test()
results.append(("Different IP", True)) # Informational
print("\n" + "=" * 50 + "\n")
# Keep-Alive test
self.run_keepalive_test()
results.append(("Keep-Alive", True))
# Summary
print(f"\n{colorize('=== Test Summary ===', Colors.BOLD)}")
for name, passed in results:
status = colorize("PASS", Colors.GREEN) if passed else colorize("FAIL", Colors.RED)
print(f" {name}: {status}")
return all(r[1] for r in results)
def main():
parser = argparse.ArgumentParser(
description="Advanced correlation testing tool for logcorrelator",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Run basic test with 20 pairs
python3 test-correlation-advanced.py -c 20
# Run all tests with verbose output
python3 test-correlation-advanced.py --all -v
# Test with custom socket paths
python3 test-correlation-advanced.py -H /tmp/http.sock -N /tmp/network.sock
# Skip metrics check
python3 test-correlation-advanced.py --skip-metrics
"""
)
parser.add_argument(
"-H", "--http-socket",
default="/var/run/logcorrelator/http.socket",
help="Path to HTTP Unix socket (default: /var/run/logcorrelator/http.socket)"
)
parser.add_argument(
"-N", "--network-socket",
default="/var/run/logcorrelator/network.socket",
help="Path to Network Unix socket (default: /var/run/logcorrelator/network.socket)"
)
parser.add_argument(
"-m", "--metrics-url",
default="http://localhost:8080/metrics",
help="Metrics server URL (default: http://localhost:8080/metrics)"
)
parser.add_argument(
"-c", "--count",
type=int,
default=10,
help="Number of test pairs to send (default: 10)"
)
parser.add_argument(
"-d", "--delay",
type=int,
default=100,
help="Delay between pairs in milliseconds (default: 100)"
)
parser.add_argument(
"-v", "--verbose",
action="store_true",
help="Enable verbose output"
)
parser.add_argument(
"--skip-metrics",
action="store_true",
help="Skip metrics check"
)
parser.add_argument(
"--all",
action="store_true",
help="Run all test scenarios"
)
parser.add_argument(
"--time-window",
action="store_true",
help="Run time window test only"
)
parser.add_argument(
"--different-ip",
action="store_true",
help="Run different IP test only"
)
parser.add_argument(
"--keepalive",
action="store_true",
help="Run Keep-Alive test only"
)
args = parser.parse_args()
# Create tester
tester = CorrelationTester(
http_socket=args.http_socket,
network_socket=args.network_socket,
metrics_url=args.metrics_url,
verbose=args.verbose,
skip_metrics=args.skip_metrics
)
# Check sockets
if not tester.check_sockets():
error("Socket check failed. Is logcorrelator running?")
sys.exit(1)
success("Socket check passed")
# Connect
if not tester.connect():
error("Failed to connect to sockets")
sys.exit(1)
try:
if args.all:
success = tester.run_all_tests()
elif args.time_window:
tester.run_time_window_test()
success = True
elif args.different_ip:
tester.run_different_ip_test()
success = True
elif args.keepalive:
tester.run_keepalive_test()
success = True
else:
_, _ = tester.run_basic_test(count=args.count, delay_ms=args.delay)
success = True
sys.exit(0 if success else 1)
finally:
tester.close()
if __name__ == "__main__":
main()

404
old/services/correlator/scripts/test-correlation.sh Executable file
View File

@ -0,0 +1,404 @@
#!/bin/bash
#
# test-correlation.sh - Test script for log correlation debugging
#
# This script sends test HTTP (A) and Network (B) events to the logcorrelator
# Unix sockets and verifies that correlation is working correctly.
#
# Usage:
# ./test-correlation.sh [options]
#
# Options:
# -h, --http-socket PATH Path to HTTP socket (default: /var/run/logcorrelator/http.socket)
# -n, --network-socket PATH Path to Network socket (default: /var/run/logcorrelator/network.socket)
# -c, --count NUM Number of test pairs to send (default: 10)
# -d, --delay MS Delay between pairs in milliseconds (default: 100)
# -v, --verbose Enable verbose output
# -m, --metrics-url URL Metrics server URL (default: http://localhost:8080/metrics)
# --skip-metrics Skip metrics check
# --help Show this help message
#
set -e
# Default values
HTTP_SOCKET="/var/run/logcorrelator/http.socket"
NETWORK_SOCKET="/var/run/logcorrelator/network.socket"
COUNT=10
DELAY_MS=100
VERBOSE=false
METRICS_URL="http://localhost:8080/metrics"
SKIP_METRICS=false
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Print functions
info() {
echo -e "${BLUE}[INFO]${NC} $1"
}
success() {
echo -e "${GREEN}[OK]${NC} $1"
}
warn() {
echo -e "${YELLOW}[WARN]${NC} $1"
}
error() {
echo -e "${RED}[ERROR]${NC} $1"
}
verbose() {
if [ "$VERBOSE" = true ]; then
echo -e "${BLUE}[DEBUG]${NC} $1"
fi
}
# Show help
show_help() {
head -20 "$0" | tail -17 | sed 's/^#//' | sed 's/^ //'
exit 0
}
# Parse arguments
while [[ $# -gt 0 ]]; do
case $1 in
-h|--http-socket)
HTTP_SOCKET="$2"
shift 2
;;
-n|--network-socket)
NETWORK_SOCKET="$2"
shift 2
;;
-c|--count)
COUNT="$2"
shift 2
;;
-d|--delay)
DELAY_MS="$2"
shift 2
;;
-v|--verbose)
VERBOSE=true
shift
;;
-m|--metrics-url)
METRICS_URL="$2"
shift 2
;;
--skip-metrics)
SKIP_METRICS=true
shift
;;
--help)
show_help
;;
*)
error "Unknown option: $1"
echo "Use --help for usage information"
exit 1
;;
esac
done
# Check if socat or netcat is available
if command -v socat &> /dev/null; then
SEND_CMD="socat"
elif command -v nc &> /dev/null; then
SEND_CMD="nc"
else
error "Neither socat nor nc (netcat) found. Please install one of them."
echo " Ubuntu/Debian: apt-get install socat OR apt-get install netcat-openbsd"
echo " RHEL/CentOS: yum install socat OR yum install nc"
exit 1
fi
# Function to send data to Unix socket
send_to_socket() {
local socket="$1"
local data="$2"
if [ "$SEND_CMD" = "socat" ]; then
echo "$data" | socat - "UNIX-SENDTO:$socket" 2>/dev/null
else
echo "$data" | nc -U -u "$socket" 2>/dev/null
fi
}
# Function to generate timestamp in nanoseconds
get_timestamp_ns() {
date +%s%N
}
# Function to send HTTP (A) event
send_http_event() {
local src_ip="$1"
local src_port="$2"
local timestamp="$3"
local method="${4:-GET}"
local path="${5:-/test}"
local host="${6:-example.com}"
local json=$(cat <<EOF
{"src_ip":"$src_ip","src_port":$src_port,"dst_ip":"10.0.0.1","dst_port":443,"timestamp":$timestamp,"method":"$method","path":"$path","host":"$host","http_version":"HTTP/1.1","header_user_agent":"TestAgent/1.0","header_accept":"*/*"}
EOF
)
verbose "Sending HTTP event: $json"
send_to_socket "$HTTP_SOCKET" "$json"
}
# Function to send Network (B) event
send_network_event() {
local src_ip="$1"
local src_port="$2"
local timestamp="$3"
local ja3="${4:-abc123}"
local ja4="${5:-def456}"
local json=$(cat <<EOF
{"src_ip":"$src_ip","src_port":$src_port,"dst_ip":"10.0.0.1","dst_port":443,"timestamp":$timestamp,"ja3":"$ja3","ja4":"$ja4","tls_version":"TLS1.3","tls_sni":"example.com"}
EOF
)
verbose "Sending Network event: $json"
send_to_socket "$NETWORK_SOCKET" "$json"
}
# Check sockets exist
check_sockets() {
local errors=0
if [ ! -S "$HTTP_SOCKET" ]; then
error "HTTP socket not found: $HTTP_SOCKET"
errors=$((errors + 1))
else
verbose "HTTP socket found: $HTTP_SOCKET"
fi
if [ ! -S "$NETWORK_SOCKET" ]; then
error "Network socket not found: $NETWORK_SOCKET"
errors=$((errors + 1))
else
verbose "Network socket found: $NETWORK_SOCKET"
fi
if [ $errors -gt 0 ]; then
error "$errors socket(s) not found. Is logcorrelator running?"
exit 1
fi
success "Sockets check passed"
}
# Get metrics from server
get_metrics() {
if [ "$SKIP_METRICS" = true ]; then
return 0
fi
if command -v curl &> /dev/null; then
curl -s "$METRICS_URL" 2>/dev/null || echo "{}"
elif command -v wget &> /dev/null; then
wget -qO- "$METRICS_URL" 2>/dev/null || echo "{}"
else
warn "Neither curl nor wget found. Skipping metrics check."
echo "{}"
fi
}
# Extract value from JSON (simple grep-based, requires jq for complex queries)
get_json_value() {
local json="$1"
local key="$2"
if command -v jq &> /dev/null; then
echo "$json" | jq -r ".$key // 0"
else
# Fallback: simple grep (works for flat JSON)
echo "$json" | grep -o "\"$key\":[0-9]*" | cut -d: -f2 || echo "0"
fi
}
# Main test function
run_test() {
info "Starting correlation test..."
info "Configuration:"
echo " HTTP Socket: $HTTP_SOCKET"
echo " Network Socket: $NETWORK_SOCKET"
echo " Test pairs: $COUNT"
echo " Delay between: ${DELAY_MS}ms"
echo " Metrics URL: $METRICS_URL"
echo " Send command: $SEND_CMD"
echo ""
# Get initial metrics
info "Fetching initial metrics..."
local initial_metrics=$(get_metrics)
local initial_success=$(get_json_value "$initial_metrics" "correlations_success")
local initial_failed=$(get_json_value "$initial_metrics" "correlations_failed")
local initial_a=$(get_json_value "$initial_metrics" "events_received_a")
local initial_b=$(get_json_value "$initial_metrics" "events_received_b")
info "Initial metrics:"
echo " Events A: $initial_a"
echo " Events B: $initial_b"
echo " Success: $initial_success"
echo " Failed: $initial_failed"
echo ""
# Send test events
info "Sending $COUNT test event pairs..."
local base_timestamp=$(get_timestamp_ns)
local sent=0
local correlated=0
for i in $(seq 1 $COUNT); do
local src_ip="192.168.1.$((i % 254 + 1))"
local src_port=$((8000 + i))
# Send A and B with same timestamp (should correlate)
local ts_a=$((base_timestamp + i * 1000000))
local ts_b=$ts_a # Same timestamp for perfect correlation
send_http_event "$src_ip" "$src_port" "$ts_a"
send_network_event "$src_ip" "$src_port" "$ts_b"
sent=$((sent + 1))
verbose "Sent pair $i: $src_ip:$src_port"
if [ $DELAY_MS -gt 0 ]; then
sleep $(echo "scale=3; $DELAY_MS / 1000" | bc)
fi
done
success "Sent $sent event pairs"
echo ""
# Wait for processing
info "Waiting for processing (2 seconds)..."
sleep 2
# Get final metrics
info "Fetching final metrics..."
local final_metrics=$(get_metrics)
local final_success=$(get_json_value "$final_metrics" "correlations_success")
local final_failed=$(get_json_value "$final_metrics" "correlations_failed")
local final_a=$(get_json_value "$final_metrics" "events_received_a")
local final_b=$(get_json_value "$final_metrics" "events_received_b")
# Calculate deltas
local delta_success=$((final_success - initial_success))
local delta_failed=$((final_failed - initial_failed))
local delta_a=$((final_a - initial_a))
local delta_b=$((final_b - initial_b))
echo ""
info "Results:"
echo " Events A sent: $delta_a (expected: $sent)"
echo " Events B sent: $delta_b (expected: $sent)"
echo " Correlations: $delta_success"
echo " Failures: $delta_failed"
echo ""
# Validation
local test_passed=true
if [ "$delta_a" -ne "$sent" ]; then
error "Event A count mismatch: got $delta_a, expected $sent"
test_passed=false
fi
if [ "$delta_b" -ne "$sent" ]; then
error "Event B count mismatch: got $delta_b, expected $sent"
test_passed=false
fi
if [ "$delta_success" -ne "$sent" ]; then
error "Correlation count mismatch: got $delta_success, expected $sent"
test_passed=false
fi
if [ "$delta_failed" -ne 0 ]; then
warn "Unexpected correlation failures: $delta_failed"
fi
if [ "$test_passed" = true ]; then
success "All tests passed! Correlation is working correctly."
exit 0
else
error "Some tests failed. Check the logs for details."
exit 1
fi
}
# Test with time window exceeded
run_time_window_test() {
info "Running time window test (B arrives after time window)..."
local src_ip="192.168.100.1"
local src_port="9999"
# Send A event
local ts_a=$(get_timestamp_ns)
send_http_event "$src_ip" "$src_port" "$ts_a"
info "Sent A event at timestamp $ts_a"
# Wait for time window to expire (default is 10s, we wait 11s)
info "Waiting 11 seconds (time window should expire)..."
sleep 11
# Send B event
local ts_b=$(get_timestamp_ns)
send_network_event "$src_ip" "$src_port" "$ts_b"
info "Sent B event at timestamp $ts_b"
info "This should result in a time_window failure (check metrics)"
}
# Test with different src_ip
run_different_ip_test() {
info "Running different IP test (should NOT correlate)..."
# Send A with IP 192.168.200.1
local ts=$(get_timestamp_ns)
send_http_event "192.168.200.1" "7777" "$ts"
info "Sent A event from 192.168.200.1:7777"
# Send B with different IP
send_network_event "192.168.200.2" "7777" "$ts"
info "Sent B event from 192.168.200.2:7777 (different IP)"
info "These should NOT correlate (different src_ip)"
}
# Run tests
check_sockets
echo ""
# Run main test
run_test
echo ""
info "Additional tests available:"
echo " --test-time-window Test time window expiration"
echo " --test-different-ip Test different IP (no correlation)"
# Check for additional test flags
if [[ "$@" == *"--test-time-window"* ]]; then
echo ""
run_time_window_test
fi
if [[ "$@" == *"--test-different-ip"* ]]; then
echo ""
run_different_ip_test
fi

21
old/services/correlator/sql/bots.sql Normal file
View File

@ -0,0 +1,21 @@
DROP TABLE IF EXISTS ja4_processing.ref_bot_networks;
CREATE TABLE ja4_processing.ref_bot_networks (
-- On utilise IPv6CIDR car il accepte aussi les IPv4 au format ::ffff:1.2.3.4/120
network IPv6CIDR,
bot_name LowCardinality(String),
is_legitimate UInt8,
last_update DateTime
) ENGINE = ReplacingMergeTree(last_update)
ORDER BY (network, bot_name);
-- Création de la table lisant le fichier des IPs
CREATE TABLE ja4_processing.bot_ip (
ip String
) ENGINE = File(CSV, 'bot_ip.csv');
-- Création de la table lisant le fichier des signatures JA4
CREATE TABLE ja4_processing.bot_ja4 (
ja4 String
) ENGINE = File(CSV, 'bot_ja4.csv');

235
old/services/correlator/sql/init.sql Normal file
View File

@ -0,0 +1,235 @@
-- =============================================================================
-- logcorrelator - Initialisation ClickHouse
-- =============================================================================
-- Ce fichier crée la base de données, les tables, la vue matérialisée
-- et les utilisateurs nécessaires au fonctionnement de logcorrelator.
--
-- Usage :
-- clickhouse-client --multiquery < sql/init.sql
-- =============================================================================
-- -----------------------------------------------------------------------------
-- Base de données
-- -----------------------------------------------------------------------------
CREATE DATABASE IF NOT EXISTS ja4_logs;
CREATE DATABASE IF NOT EXISTS ja4_processing;
-- -----------------------------------------------------------------------------
-- Table brute : cible directe des inserts du service
-- Le service n'insère que dans cette table (colonne raw_json).
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ja4_logs.http_logs_raw
(
`raw_json` String CODEC(ZSTD(3)),
`ingest_time` DateTime DEFAULT now()
)
ENGINE = MergeTree
PARTITION BY toDate(ingest_time)
ORDER BY ingest_time
TTL ingest_time + INTERVAL 1 DAY
SETTINGS
index_granularity = 8192,
ttl_only_drop_parts = 1;
-- -----------------------------------------------------------------------------
-- Table parsée : alimentée automatiquement par la vue matérialisée
-- -----------------------------------------------------------------------------
CREATE TABLE ja4_logs.http_logs
(
-- Temporel
`time` DateTime,
`log_date` Date DEFAULT toDate(time),
-- Réseau
`src_ip` IPv4,
`src_port` UInt16,
`dst_ip` IPv4,
`dst_port` UInt16,
-- Enrichissement IPLocate
`src_asn` UInt32,
`src_country_code` LowCardinality(String),
`src_as_name` LowCardinality(String),
`src_org` LowCardinality(String),
`src_domain` LowCardinality(String),
-- HTTP
`method` LowCardinality(String),
`scheme` LowCardinality(String),
`host` LowCardinality(String),
`path` String CODEC(ZSTD(3)),
`query` String CODEC(ZSTD(3)),
`http_version` LowCardinality(String),
-- Corrélation
`orphan_side` LowCardinality(String),
`correlated` UInt8,
`keepalives` UInt16,
`a_timestamp` UInt64,
`b_timestamp` UInt64,
`conn_id` String CODEC(ZSTD(3)),
-- Métadonnées IP
`ip_meta_df` UInt8,
`ip_meta_id` UInt16,
`ip_meta_total_length` UInt16,
`ip_meta_ttl` UInt8,
-- Métadonnées TCP
`tcp_meta_options` LowCardinality(String),
`tcp_meta_window_size` UInt32,
`tcp_meta_mss` UInt16,
`tcp_meta_window_scale` UInt8,
`syn_to_clienthello_ms` Int32,
-- TLS / fingerprint
`tls_version` LowCardinality(String),
`tls_sni` LowCardinality(String),
`tls_alpn` LowCardinality(String),
`ja3` String CODEC(ZSTD(3)),
`ja3_hash` String CODEC(ZSTD(3)),
`ja4` String CODEC(ZSTD(3)),
-- En-têtes HTTP
`client_headers` String CODEC(ZSTD(3)),
`header_user_agent` String CODEC(ZSTD(3)),
`header_accept` String CODEC(ZSTD(3)),
`header_accept_encoding` String CODEC(ZSTD(3)),
`header_accept_language` String CODEC(ZSTD(3)),
`header_content_type` String CODEC(ZSTD(3)),
`header_x_request_id` String CODEC(ZSTD(3)),
`header_x_trace_id` String CODEC(ZSTD(3)),
`header_x_forwarded_for` String CODEC(ZSTD(3)),
`header_sec_ch_ua` String CODEC(ZSTD(3)),
`header_sec_ch_ua_mobile` String CODEC(ZSTD(3)),
`header_sec_ch_ua_platform` String CODEC(ZSTD(3)),
`header_sec_fetch_dest` String CODEC(ZSTD(3)),
`header_sec_fetch_mode` String CODEC(ZSTD(3)),
`header_sec_fetch_site` String CODEC(ZSTD(3))
)
ENGINE = MergeTree
PARTITION BY log_date
ORDER BY (time, src_ip, dst_ip, ja4)
TTL log_date + INTERVAL 7 DAY
SETTINGS
index_granularity = 8192,
ttl_only_drop_parts = 1;
-- -----------------------------------------------------------------------------
-- Vue matérialisée : parse le JSON de http_logs_raw vers http_logs
-- -----------------------------------------------------------------------------
DROP VIEW IF EXISTS ja4_logs.mv_http_logs;
CREATE MATERIALIZED VIEW IF NOT EXISTS ja4_logs.mv_http_logs
TO ja4_logs.http_logs
AS
SELECT
parseDateTimeBestEffort(coalesce(JSONExtractString(raw_json, 'time'), '1970-01-01T00:00:00Z')) AS time,
toDate(time) AS log_date,
toIPv4(coalesce(JSONExtractString(raw_json, 'src_ip'), '0.0.0.0')) AS src_ip,
toUInt16(coalesce(JSONExtractUInt(raw_json, 'src_port'), 0)) AS src_port,
toIPv4(coalesce(JSONExtractString(raw_json, 'dst_ip'), '0.0.0.0')) AS dst_ip,
toUInt16(coalesce(JSONExtractUInt(raw_json, 'dst_port'), 0)) AS dst_port,
dictGetOrDefault(
'ja4_processing.dict_iplocate_asn',
'asn',
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
toUInt32(0)
) AS src_asn,
dictGetOrDefault(
'ja4_processing.dict_iplocate_asn',
'country_code',
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
''
) AS src_country_code,
dictGetOrDefault(
'ja4_processing.dict_iplocate_asn',
'name',
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
''
) AS src_as_name,
dictGetOrDefault(
'ja4_processing.dict_iplocate_asn',
'org',
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
''
) AS src_org,
dictGetOrDefault(
'ja4_processing.dict_iplocate_asn',
'domain',
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
''
) AS src_domain,
coalesce(JSONExtractString(raw_json, 'method'), '') AS method,
coalesce(JSONExtractString(raw_json, 'scheme'), '') AS scheme,
coalesce(JSONExtractString(raw_json, 'host'), '') AS host,
coalesce(JSONExtractString(raw_json, 'path'), '') AS path,
coalesce(JSONExtractString(raw_json, 'query'), '') AS query,
coalesce(JSONExtractString(raw_json, 'http_version'), '') AS http_version,
coalesce(JSONExtractString(raw_json, 'orphan_side'), '') AS orphan_side,
toUInt8(coalesce(JSONExtractBool(raw_json, 'correlated'), 0)) AS correlated,
toUInt16(coalesce(JSONExtractUInt(raw_json, 'keepalives'), 0)) AS keepalives,
coalesce(JSONExtractUInt(raw_json, 'a_timestamp'), 0) AS a_timestamp,
coalesce(JSONExtractUInt(raw_json, 'b_timestamp'), 0) AS b_timestamp,
coalesce(JSONExtractString(raw_json, 'conn_id'), '') AS conn_id,
toUInt8(coalesce(JSONExtractBool(raw_json, 'ip_meta_df'), 0)) AS ip_meta_df,
toUInt16(coalesce(JSONExtractUInt(raw_json, 'ip_meta_id'), 0)) AS ip_meta_id,
toUInt16(coalesce(JSONExtractUInt(raw_json, 'ip_meta_total_length'), 0)) AS ip_meta_total_length,
toUInt8(coalesce(JSONExtractUInt(raw_json, 'ip_meta_ttl'), 0)) AS ip_meta_ttl,
coalesce(JSONExtractString(raw_json, 'tcp_meta_options'), '') AS tcp_meta_options,
toUInt32(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_window_size'), 0)) AS tcp_meta_window_size,
toUInt16(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_mss'), 0)) AS tcp_meta_mss,
toUInt8(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_window_scale'), 0)) AS tcp_meta_window_scale,
toInt32(coalesce(JSONExtractInt(raw_json, 'syn_to_clienthello_ms'), 0)) AS syn_to_clienthello_ms,
coalesce(JSONExtractString(raw_json, 'tls_version'), '') AS tls_version,
coalesce(JSONExtractString(raw_json, 'tls_sni'), '') AS tls_sni,
coalesce(JSONExtractString(raw_json, 'tls_alpn'), '') AS tls_alpn,
coalesce(JSONExtractString(raw_json, 'ja3'), '') AS ja3,
coalesce(JSONExtractString(raw_json, 'ja3_hash'), '') AS ja3_hash,
coalesce(JSONExtractString(raw_json, 'ja4'), '') AS ja4,
coalesce(JSONExtractString(raw_json, 'client_headers'), '') AS client_headers,
coalesce(JSONExtractString(raw_json, 'header_User-Agent'), '') AS header_user_agent,
coalesce(JSONExtractString(raw_json, 'header_Accept'), '') AS header_accept,
coalesce(JSONExtractString(raw_json, 'header_Accept-Encoding'), '') AS header_accept_encoding,
coalesce(JSONExtractString(raw_json, 'header_Accept-Language'), '') AS header_accept_language,
coalesce(JSONExtractString(raw_json, 'header_Content-Type'), '') AS header_content_type,
coalesce(JSONExtractString(raw_json, 'header_X-Request-Id'), '') AS header_x_request_id,
coalesce(JSONExtractString(raw_json, 'header_X-Trace-Id'), '') AS header_x_trace_id,
coalesce(JSONExtractString(raw_json, 'header_X-Forwarded-For'), '') AS header_x_forwarded_for,
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA'), '') AS header_sec_ch_ua,
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA-Mobile'), '') AS header_sec_ch_ua_mobile,
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA-Platform'), '') AS header_sec_ch_ua_platform,
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Dest'), '') AS header_sec_fetch_dest,
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Mode'), '') AS header_sec_fetch_mode,
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Site'), '') AS header_sec_fetch_site
FROM ja4_logs.http_logs_raw;
-- -----------------------------------------------------------------------------
-- Utilisateurs et permissions
-- -----------------------------------------------------------------------------
CREATE USER IF NOT EXISTS data_writer IDENTIFIED WITH plaintext_password BY 'ChangeMe';
CREATE USER IF NOT EXISTS analyst IDENTIFIED WITH plaintext_password BY 'ChangeMe';
-- data_writer : INSERT uniquement sur la table brute
GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
-- analyst : lecture sur la table parsée
GRANT SELECT ON ja4_logs.http_logs TO analyst;
-- -----------------------------------------------------------------------------
-- Vérifications post-installation
-- -----------------------------------------------------------------------------
-- SELECT count(*), min(ingest_time), max(ingest_time) FROM ja4_logs.http_logs_raw;
-- SELECT count(*), min(time), max(time) FROM ja4_logs.http_logs;
-- SELECT time, src_ip, dst_ip, method, host, path, ja4 FROM ja4_logs.http_logs ORDER BY time DESC LIMIT 10;

90
old/services/correlator/sql/migrations/01_ttl_adjustments.sql Normal file
View File

@ -0,0 +1,90 @@
-- =============================================================================
-- 01_ttl_adjustments.sql — Ajustement des TTL de rétention des logs
--
-- Objectif :
-- - http_logs : conserver 30 jours (était 7 jours)
-- - http_logs_raw : supprimer au plus vite (2 heures, était 1 jour)
--
-- Usage (déploiements existants) :
-- clickhouse-client --multiquery < sql/migrations/01_ttl_adjustments.sql
--
-- IMPORTANT — http_logs_raw (partition) :
-- La modification du TTL ci-dessous s'applique immédiatement aux nouvelles
-- insertions. Toutefois, la PARTITION BY (toDate → toStartOfHour) ne peut
-- PAS être modifiée via ALTER TABLE — elle est fixée à la création.
--
-- Pour adopter la partition horaire sur un déploiement existant, procéder
-- manuellement :
-- 1. CREATE TABLE ja4_logs.http_logs_raw_new ( ... identique à init.sql ... )
-- 2. Attendre que http_logs_raw soit vide (ou accepter la perte des raw non traités)
-- 3. DROP VIEW IF EXISTS ja4_logs.mv_http_logs
-- 4. DROP TABLE ja4_logs.http_logs_raw
-- 5. RENAME TABLE ja4_logs.http_logs_raw_new TO ja4_logs.http_logs_raw
-- 6. Recréer mv_http_logs (cf. init.sql)
--
-- Si la migration complète n'est pas possible, la modification du TTL seul
-- (ci-dessous) ramène la suppression à : fin de la partition quotidienne + 2h,
-- soit au plus 26h au lieu de 48h — amélioration immédiate sans downtime.
-- =============================================================================
-- -----------------------------------------------------------------------------
-- 1. http_logs — rétention étendue à 30 jours
-- -----------------------------------------------------------------------------
ALTER TABLE ja4_logs.http_logs
MODIFY TTL log_date + INTERVAL 30 DAY;
-- -----------------------------------------------------------------------------
-- 2. http_logs_raw — TTL réduit à 2 heures
-- (la MV traite les données de façon synchrone à l'INSERT, donc les raw
-- sont redondants dès leur insertion)
-- -----------------------------------------------------------------------------
ALTER TABLE ja4_logs.http_logs_raw
MODIFY TTL ingest_time + INTERVAL 2 HOUR;
-- -----------------------------------------------------------------------------
-- Vérifications post-migration
-- -----------------------------------------------------------------------------
-- SELECT engine_full FROM system.tables
-- WHERE database IN ('ja4_logs', 'ja4_processing') AND name IN ('http_logs', 'http_logs_raw');
-- =============================================================================
-- Ajustements TTL des tables de MVs (bot_detector) — 7 jours max
-- =============================================================================
-- agg_host_ip_ja4_1h — TTL 7 jours (était : aucun)
-- Note : PARTITION BY ne peut pas être ajouté via ALTER TABLE.
-- Pour la partition quotidienne (recommandée), recréer la table via deploy_views.sql.
-- Sans partition explicite, ClickHouse applique le TTL par granule (moins efficace).
ALTER TABLE ja4_processing.agg_host_ip_ja4_1h
MODIFY TTL window_start + INTERVAL 7 DAY;
-- agg_header_fingerprint_1h — TTL 7 jours (était : aucun)
ALTER TABLE ja4_processing.agg_header_fingerprint_1h
MODIFY TTL window_start + INTERVAL 7 DAY;
-- ml_detected_anomalies — TTL 7 jours (était 30 jours, déjà corrigé en 7j pour certains)
-- Note : MODIFY ORDER BY (src_ip, ja4, host, model_name) ne peut PAS être appliqué
-- sur ClickHouse 24.8 car ja4/host sont des colonnes existantes (erreur BAD_ARGUMENTS 36).
-- Le correctif est dans le schéma de base (06_ml_tables.sql) — ORDER BY corrigé à la
-- création. Pour les déploiements existants avec l'ancien ORDER BY, recréer la table.
ALTER TABLE ja4_processing.ml_detected_anomalies
MODIFY TTL detected_at + INTERVAL 7 DAY;
-- ml_all_scores — TTL 7 jours (était 3 jours)
-- Note : PARTITION BY ne peut pas être ajouté via ALTER TABLE.
-- Pour la partition quotidienne (recommandée), recréer la table via deploy_views.sql.
ALTER TABLE ja4_processing.ml_all_scores
MODIFY TTL window_start + INTERVAL 7 DAY;
-- =============================================================================
-- Vues dashboard — suppression des vues inutilisées, ajout des vues manquantes
-- Ces vues n'ont pas d'état persistant : les DROP/CREATE sont idempotents.
-- =============================================================================
-- Supprimer les vues inutilisées (non référencées par les routes Python)
DROP VIEW IF EXISTS ja4_processing.view_dashboard_summary;
DROP VIEW IF EXISTS ja4_processing.view_dashboard_timeseries;
DROP VIEW IF EXISTS ja4_processing.view_dashboard_threat_dist;
DROP VIEW IF EXISTS ja4_processing.view_dashboard_variability;
-- Les nouvelles vues (view_dashboard_entities, view_dashboard_user_agents,
-- view_ai_features_24h) sont créées par deploy_views.sql (section finale).
-- Les exécuter ici revient à rejouer cette section de deploy_views.sql.

17
old/services/correlator/sql/migrations/02_detection_features.sql Normal file
View File

@ -0,0 +1,17 @@
-- =============================================================================
-- 02_detection_features.sql — Ajout des features de détection P0+P1
-- Colonnes supplémentaires dans agg_host_ip_ja4_1h et agg_header_fingerprint_1h
-- NOTE : les MVs doivent être recréées (DROP + CREATE) car ALTER VIEW n'existe pas.
-- Exécuter deploy_schema.sh pour recréer les MVs, ou relancer le schema complet.
-- =============================================================================
-- agg_host_ip_ja4_1h : nouvelles colonnes de comptage
ALTER TABLE ja4_processing.agg_host_ip_ja4_1h
ADD COLUMN IF NOT EXISTS count_xff SimpleAggregateFunction(sum, UInt64) AFTER count_http_scheme,
ADD COLUMN IF NOT EXISTS count_unusual_ct SimpleAggregateFunction(sum, UInt64) AFTER count_xff,
ADD COLUMN IF NOT EXISTS count_non_std_port SimpleAggregateFunction(sum, UInt64) AFTER count_unusual_ct,
ADD COLUMN IF NOT EXISTS count_login_post SimpleAggregateFunction(sum, UInt64) AFTER count_non_std_port;
-- agg_header_fingerprint_1h : mismatch mobile Sec-CH-UA
ALTER TABLE ja4_processing.agg_header_fingerprint_1h
ADD COLUMN IF NOT EXISTS sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8) AFTER ua_ch_mismatch;

45
old/services/correlator/sql/migrations/03_remove_ua_browser_detection.sql Normal file
View File

@ -0,0 +1,45 @@
-- === 03_remove_ua_browser_detection.sql — Suppression dépendance User-Agent ===
--
-- Contexte : l'identification navigateur ne doit PAS se baser sur le User-Agent
-- (trivalement falsifiable). Cette migration :
-- 1. Ajoute has_sec_ch_ua à agg_header_fingerprint_1h
-- 2. Recréé la MV avec modern_browser_score basé sur Client Hints + Sec-Fetch
-- 3. Recréé sec_ch_mobile_mismatch sans UA (Client Hints only)
-- 4. Ajoute has_sec_ch_ua à ml_all_scores
-- ---------------------------------------------------------------------------
-- 1. Nouvelle colonne has_sec_ch_ua
ALTER TABLE ja4_processing.agg_header_fingerprint_1h
ADD COLUMN IF NOT EXISTS has_sec_ch_ua SimpleAggregateFunction(max, UInt8)
AFTER modern_browser_score;
-- 2. Recréer la MV sans dépendance UA
DROP VIEW IF EXISTS ja4_processing.mv_agg_header_fingerprint_1h;
CREATE MATERIALIZED VIEW ja4_processing.mv_agg_header_fingerprint_1h
TO ja4_processing.agg_header_fingerprint_1h AS
SELECT
toStartOfHour(src.time) AS window_start,
toIPv6(src.src_ip) AS src_ip,
any(toString(cityHash64(src.client_headers))) AS header_order_hash,
max(toUInt16(length(src.client_headers) - length(replaceAll(src.client_headers, ',', '')) + 1)) AS header_count,
max(toUInt8(if(position(src.client_headers, 'Accept-Language') > 0, 1, 0))) AS has_accept_language,
max(toUInt8(if(position(src.client_headers, 'Cookie') > 0, 1, 0))) AS has_cookie,
max(toUInt8(if(position(src.client_headers, 'Referer') > 0, 1, 0))) AS has_referer,
-- modern_browser_score : sec-ch-ua → 100, Sec-Fetch → 70, sinon → 0 (PAS de UA)
max(toUInt8(if(length(src.header_sec_ch_ua) > 0, 100, if(length(src.header_sec_fetch_site) > 0, 70, 0)))) AS modern_browser_score,
max(toUInt8(if(length(src.header_sec_ch_ua) > 0, 1, 0))) AS has_sec_ch_ua,
max(toUInt8(if((position(src.header_user_agent, 'Windows') > 0 AND position(src.header_sec_ch_ua_platform, 'Windows') == 0) OR (position(src.header_user_agent, 'iPhone') > 0 AND position(src.header_sec_ch_ua_platform, 'iOS') == 0), 1, 0))) AS ua_ch_mismatch,
-- sec_ch_mobile_mismatch : incohérence interne Client Hints (pas de UA)
max(toUInt8(if(
(src.header_sec_ch_ua_mobile = '?1' AND position(src.header_sec_ch_ua_platform, 'Windows') > 0)
OR (src.header_sec_ch_ua_mobile = '?0' AND position(src.header_sec_ch_ua_platform, 'Android') > 0),
1, 0))) AS sec_ch_mobile_mismatch,
any(src.header_sec_fetch_mode) AS sec_fetch_mode,
any(src.header_sec_fetch_dest) AS sec_fetch_dest
FROM ja4_logs.http_logs AS src
GROUP BY window_start, src.src_ip;
-- 3. Ajouter has_sec_ch_ua à ml_all_scores
ALTER TABLE ja4_processing.ml_all_scores
ADD COLUMN IF NOT EXISTS has_sec_ch_ua UInt8 DEFAULT 0;

52
old/services/correlator/sql/migrations/04_http2_fields.sql Normal file
View File

@ -0,0 +1,52 @@
-- === 04_http2_fields.sql — Ajout des colonnes HTTP/2 à http_logs ===
--
-- Migration pour les déploiements existants : ajoute les colonnes de
-- fingerprint HTTP/2 passif extraites par mod_reqin_log via son hook
-- process_connection (APR_HOOK_FIRST, AP_MODE_SPECULATIVE).
--
-- Format du fingerprint Akamai (h2_fingerprint) :
-- Chrome : "1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p"
-- Firefox : "1:65536,4:131072,5:16384|12517377|0|m,p,s,a"
-- Safari : "1:4096,3:100,4:65535|10485760|0|m,a,s,p"
--
-- Appliquer avec :
-- clickhouse-client --multiquery < 04_http2_fields.sql
-- Champs composites (fingerprint global + valeurs agrégées)
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_fingerprint` String DEFAULT '' CODEC(ZSTD(3));
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_settings_fp` String DEFAULT '' CODEC(ZSTD(3));
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_window_update` UInt32 DEFAULT 0;
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_pseudo_order` LowCardinality(String) DEFAULT '';
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_has_priority` UInt8 DEFAULT 0;
-- Paramètres SETTINGS individuels (RFC 9113 §6.5.2)
-- Valeur -1 = paramètre absent du preface client (non envoyé)
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_header_table_size` Int32 DEFAULT -1;
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_enable_push` Int32 DEFAULT -1;
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_max_concurrent_streams` Int32 DEFAULT -1;
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_initial_window_size` Int64 DEFAULT -1;
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_max_frame_size` Int32 DEFAULT -1;
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_max_header_list_size` Int32 DEFAULT -1;
ALTER TABLE ja4_logs.http_logs
ADD COLUMN IF NOT EXISTS `h2_enable_connect_protocol` Int32 DEFAULT -1;

55
old/services/correlator/sql/migrations/05_fleet_metrics_tables.sql Normal file
View File

@ -0,0 +1,55 @@
-- === 05_fleet_metrics_tables.sql — Tables fleet_detections et ml_performance_metrics ===
--
-- fleet_detections : résultats du détecteur de flottes §5.2 (JA4×ASN bipartite graph)
-- ml_performance_metrics : métriques de performance du pipeline ML par cycle
--
-- Appliquer avec :
-- clickhouse-client --multiquery < 05_fleet_metrics_tables.sql
-- --- fleet_detections ---
CREATE TABLE IF NOT EXISTS ja4_processing.fleet_detections
(
detected_at DateTime,
community_id UInt32,
ja4_set Array(String),
asn_set Array(String),
fleet_score Float32,
n_ips UInt32,
ip_sample Array(String), -- échantillon des 20 premières IPs
model_name LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree
PARTITION BY toDate(detected_at)
ORDER BY (detected_at, community_id)
TTL detected_at + INTERVAL 7 DAY
SETTINGS ttl_only_drop_parts = 1;
-- --- ml_performance_metrics ---
CREATE TABLE IF NOT EXISTS ja4_processing.ml_performance_metrics
(
cycle_at DateTime,
model_name LowCardinality(String),
total_sessions UInt64,
correlated_rate Float32,
anomaly_rate Float32,
critical_count UInt32,
high_count UInt32,
medium_count UInt32,
low_count UInt32,
known_bot_count UInt32,
anubis_deny_count UInt32,
legit_browser_count UInt32,
drift_rate Float32,
drift_alert UInt8,
cycle_latency_ms UInt32,
features_valid UInt16,
features_total UInt16,
baseline_size UInt32,
threshold Float32,
meta_learner_active UInt8 DEFAULT 0
)
ENGINE = MergeTree
PARTITION BY toDate(cycle_at)
ORDER BY (cycle_at, model_name)
TTL cycle_at + INTERVAL 90 DAY
SETTINGS ttl_only_drop_parts = 1;

72
old/services/correlator/sql/migrations/06_browser_signature_detection.sql Normal file
View File

@ -0,0 +1,72 @@
-- === 06_browser_signature_detection.sql ===
-- Migration : Browser Signature Detection
--
-- 1. Création de browser_h2_signatures (table source structurée)
-- 2. Peuplement des fingerprints Akamai de référence
-- 3. Recréation de dict_browser_h2 avec le champ confidence
-- (anciennement chargé depuis CSV — remplacé par la table ClickHouse)
--
-- Appliquer avec :
-- clickhouse-client --multiquery < 06_browser_signature_detection.sql
-- --- Table source des signatures H2 ---
CREATE TABLE IF NOT EXISTS ja4_processing.browser_h2_signatures
(
h2_fingerprint String,
browser_family LowCardinality(String),
confidence Float32,
notes String
)
ENGINE = MergeTree()
ORDER BY h2_fingerprint
SETTINGS index_granularity = 128;
-- Peuplement initial depuis les données historiques (correspondance exacte CSV)
-- Format Akamai : SETTINGS|WINDOW_UPDATE|PRIORITY|PSEUDO_HEADER_ORDER
-- SETTINGS : paires clé:valeur séparées par des virgules
INSERT INTO ja4_processing.browser_h2_signatures VALUES
-- Chrome (Blink + BoringSSL, INITIAL_WINDOW_SIZE=6291456)
('1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p',
'Chrome', 1.0, 'Chrome 106–142 stable'),
('1:65536,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p',
'Chrome', 0.95, 'Chrome avec MAX_CONCURRENT_STREAMS=1000 (certains proxies)'),
('1:65536,2:0,3:100,4:6291456,6:262144|15663105|0|m,a,s,p',
'Chrome', 0.90, 'Chrome avec MAX_CONCURRENT_STREAMS=100'),
-- Firefox (Gecko + NSS, INITIAL_WINDOW_SIZE=131072, MAX_FRAME_SIZE=16384)
('1:65536,4:131072,5:16384|12517377|0|m,p,s,a',
'Firefox', 1.0, 'Firefox 90–127 stable'),
('1:65536,4:131072|12517377|0|m,p,s,a',
'Firefox', 0.95, 'Firefox sans MAX_FRAME_SIZE explicite'),
('1:65536,3:100,4:131072,5:16384|12517377|0|m,p,s,a',
'Firefox', 0.90, 'Firefox avec MAX_CONCURRENT_STREAMS=100'),
-- Safari (WebKit, INITIAL_WINDOW_SIZE=65535)
('1:4096,3:100,4:65535|10485760|0|m,a,s,p',
'Safari', 1.0, 'Safari 15+ macOS et iOS'),
('1:4096,3:100,4:65535,5:16384|10485760|0|m,a,s,p',
'Safari', 0.95, 'Safari avec MAX_FRAME_SIZE=16384'),
('1:4096,3:100,4:65535,6:16384|10485760|0|m,a,s,p',
'Safari', 0.95, 'Safari avec MAX_HEADER_LIST_SIZE=16384'),
-- Edge Chromium (fingerprint H2 identique à Chrome)
('1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p',
'Edge', 0.9, 'Edge Chromium — fingerprint H2 identique à Chrome (dédoublonné)');
-- Recréation du dictionnaire avec le champ confidence
-- (remplace la version CSV-backed sans confidence)
DROP DICTIONARY IF EXISTS ja4_processing.dict_browser_h2;
CREATE DICTIONARY ja4_processing.dict_browser_h2
(
h2_fingerprint String,
browser_family String,
confidence Float32
)
PRIMARY KEY h2_fingerprint
SOURCE(CLICKHOUSE(query '
SELECT h2_fingerprint,
argMax(browser_family, confidence) AS browser_family,
max(confidence) AS confidence
FROM ja4_processing.browser_h2_signatures
GROUP BY h2_fingerprint
'))
LAYOUT(COMPLEX_KEY_HASHED())
LIFETIME(MIN 300 MAX 600);

29
old/services/correlator/sql/tables.sql Normal file
View File

@ -0,0 +1,29 @@
DROP DICTIONARY IF EXISTS ja4_processing.dict_iplocate_asn;
CREATE DICTIONARY IF NOT EXISTS ja4_processing.dict_iplocate_asn
(
network String,
asn UInt32,
country_code String,
name String,
org String,
domain String
)
PRIMARY KEY network
SOURCE(FILE(path '/var/lib/clickhouse/user_files/iplocate-ip-to-asn.csv' format 'CSVWithNames'))
LAYOUT(IP_TRIE())
LIFETIME(MIN 3600 MAX 7200);
-- Suppression si existe pour reconfiguration
DROP TABLE IF EXISTS ja4_processing.ref_bot_networks;
-- Table optimisée pour le filtrage binaire de CIDR
CREATE TABLE ja4_processing.ref_bot_networks (
network IPv6CIDR, -- Gère nativement '1.2.3.0/24' et '2001:db8::/32'
bot_name LowCardinality(String),
is_legitimate UInt8, -- 1 = Whitelist, 0 = Blacklist
last_update DateTime
) ENGINE = ReplacingMergeTree(last_update)
ORDER BY (network, bot_name)

35
old/services/mod-reqin-log/.gitignore vendored Normal file
View File

@ -0,0 +1,35 @@
# Build artifacts
*.o
*.so
*.a
*.la
.deps
.libs
# Build directories
build/
cmake-build-*/
dist/
bin/
obj/
# IDE
.vscode/
.idea/
*.swp
*.swo
*~
# Logs
*.log
# Testing
coverage/
*.gcno
*.gcda
# Packaging
*.rpm
*.deb
*.tar.gz
.aider*

94
old/services/mod-reqin-log/.gitlab-ci.yml Normal file
View File

@ -0,0 +1,94 @@
# GitLab CI/CD configuration for mod_reqin_log
# Uses Docker-in-Docker (dind) for building and testing
stages:
- build
- test
- package
- verify
# =============================================================================
# Variables
# =============================================================================
variables:
DOCKER_TLS_CERTDIR: "/certs"
DOCKER_DRIVER: overlay2
VERSION: "1.0.2"
# =============================================================================
# Build Stage - Compile all RPM packages
# =============================================================================
build-packages:
stage: build
image: docker:24
services:
- docker:24-dind
script:
# Build all RPM packages (el8, el9, el10)
- docker build -f Dockerfile.package --target output --build-arg VERSION=$VERSION -t mod_reqin_log:packages .
# Create output directories
- mkdir -p dist/rpm
# Extract packages from Docker image
- docker run --rm -v $(pwd)/dist:/output mod_reqin_log:packages sh -c 'cp -r /packages/rpm/* /output/rpm/'
# List built packages
- echo "=== RPM Packages ==="
- ls -la dist/rpm/
artifacts:
paths:
- dist/rpm/
expire_in: 30 days
# =============================================================================
# Test Stage - Unit tests
# =============================================================================
unit-tests:
stage: test
image: docker:24
services:
- docker:24-dind
script:
# Build test image
- docker build -f Dockerfile.tests -t mod_reqin_log:tests .
# Run unit tests
- docker run --rm mod_reqin_log:tests ctest --output-on-failure
# =============================================================================
# Package Stage - Already done in build-packages
# =============================================================================
# =============================================================================
# Verify Stage - Test RPM package installation on each target distribution
# =============================================================================
verify-rpm-el8:
stage: verify
image: docker:24
services:
- docker:24-dind
needs: [build-packages]
script:
- docker run --rm -v $(pwd)/dist:/packages rockylinux:8 sh -c "dnf install -y /packages/rpm/*.el8.*.rpm && httpd -M 2>&1 | grep reqin_log && echo 'RPM el8 verification OK'"
verify-rpm-el9:
stage: verify
image: docker:24
services:
- docker:24-dind
needs: [build-packages]
script:
- docker run --rm -v $(pwd)/dist:/packages rockylinux:9 sh -c "dnf install -y /packages/rpm/*.el9.*.rpm && httpd -M 2>&1 | grep reqin_log && echo 'RPM el9 verification OK'"
verify-rpm-el10:
stage: verify
image: docker:24
services:
- docker:24-dind
needs: [build-packages]
script:
- docker run --rm -v $(pwd)/dist:/packages almalinux:10 sh -c "dnf install -y /packages/rpm/*.el10.*.rpm && httpd -M 2>&1 | grep reqin_log && echo 'RPM el10 verification OK'"

47
old/services/mod-reqin-log/CMakeLists.txt Normal file
View File

@ -0,0 +1,47 @@
cmake_minimum_required(VERSION 3.10)
project(mod_reqin_log_tests C)
set(CMAKE_C_STANDARD 99)
set(CMAKE_C_STANDARD_REQUIRED ON)
# Find required packages
find_package(PkgConfig REQUIRED)
pkg_check_modules(CMOCKA REQUIRED cmocka)
pkg_check_modules(APR REQUIRED apr-1)
pkg_check_modules(APRUTIL REQUIRED apr-util-1)
# Include directories
include_directories(${CMOCKA_INCLUDE_DIRS})
include_directories(${APR_INCLUDE_DIRS})
include_directories(${APRUTIL_INCLUDE_DIRS})
include_directories(/usr/include/httpd)
# Test executable - Real module tests (testing actual implementation)
add_executable(test_module_real tests/unit/test_module_real.c)
target_link_libraries(test_module_real ${CMOCKA_LIBRARIES} ${APR_LIBRARIES} ${APRUTIL_LIBRARIES} m)
add_executable(test_config_parsing tests/unit/test_config_parsing.c)
target_link_libraries(test_config_parsing ${CMOCKA_LIBRARIES})
add_executable(test_header_handling tests/unit/test_header_handling.c)
target_link_libraries(test_header_handling ${CMOCKA_LIBRARIES} ${APR_LIBRARIES})
add_executable(test_json_serialization tests/unit/test_json_serialization.c)
target_link_libraries(test_json_serialization ${CMOCKA_LIBRARIES} ${APR_LIBRARIES})
add_executable(test_h2_parsing tests/unit/test_h2_parsing.c)
target_link_libraries(test_h2_parsing ${CMOCKA_LIBRARIES})
# Enable testing
enable_testing()
add_test(NAME RealModuleTest COMMAND test_module_real)
add_test(NAME ConfigParsingTest COMMAND test_config_parsing)
add_test(NAME HeaderHandlingTest COMMAND test_header_handling)
add_test(NAME JsonSerializationTest COMMAND test_json_serialization)
add_test(NAME H2ParsingTest COMMAND test_h2_parsing)
# Custom target for running tests
add_custom_target(run_tests
COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure
DEPENDS test_module_real test_config_parsing test_header_handling test_json_serialization test_h2_parsing
)

176
old/services/mod-reqin-log/Dockerfile.package Normal file
View File

@ -0,0 +1,176 @@
# syntax=docker/dockerfile:1
# =============================================================================
# mod_reqin_log - Dockerfile de packaging RPM
# Builds RPMs for multiple RHEL-compatible versions:
# - Rocky Linux 8 (el8) - RHEL 8 compatible
# - Rocky Linux 9 (el9) - RHEL 9 compatible
# - AlmaLinux 10 (el10) - RHEL 10 compatible
# =============================================================================
# =============================================================================
# Stage 1a: Builder Rocky Linux 8
# =============================================================================
FROM rockylinux:8 AS builder-el8
RUN dnf install -y epel-release && \
dnf install -y --allowerasing \
gcc \
make \
httpd \
httpd-devel \
apr-devel \
apr-util-devel \
python3 \
curl \
redhat-rpm-config \
&& dnf clean all
WORKDIR /build
COPY services/mod-reqin-log/src/ src/
COPY services/mod-reqin-log/Makefile Makefile
COPY services/mod-reqin-log/conf/ conf/
RUN make APXS=/usr/bin/apxs
RUN ls -la modules/mod_reqin_log.so
# =============================================================================
# Stage 1b: Builder Rocky Linux 9
# =============================================================================
FROM rockylinux:9 AS builder-el9
RUN dnf install -y epel-release && \
dnf install -y --allowerasing \
gcc \
make \
httpd \
httpd-devel \
apr-devel \
apr-util-devel \
python3 \
curl \
redhat-rpm-config \
&& dnf clean all
WORKDIR /build
COPY services/mod-reqin-log/src/ src/
COPY services/mod-reqin-log/Makefile Makefile
COPY services/mod-reqin-log/conf/ conf/
RUN make APXS=/usr/bin/apxs
RUN ls -la modules/mod_reqin_log.so
# =============================================================================
# Stage 1c: Builder AlmaLinux 10 (RHEL 10 compatible)
# =============================================================================
FROM almalinux:10 AS builder-el10
RUN dnf install -y epel-release && \
dnf install -y --allowerasing \
gcc \
make \
httpd \
httpd-devel \
apr-devel \
apr-util-devel \
python3 \
curl \
redhat-rpm-config \
&& dnf clean all
WORKDIR /build
COPY services/mod-reqin-log/src/ src/
COPY services/mod-reqin-log/Makefile Makefile
COPY services/mod-reqin-log/conf/ conf/
RUN make APXS=/usr/bin/apxs
RUN ls -la modules/mod_reqin_log.so
# =============================================================================
# Stage 2: Package builder - rpmbuild pour RPM
# =============================================================================
FROM rockylinux:9 AS package-builder
WORKDIR /package
# Install rpm-build and dependencies
RUN dnf install -y rpm-build rpmdevtools && \
dnf clean all
# Create rpmbuild directory structure
RUN rpmdev-setuptree
# =============================================================================
# Copy spec file and source files
# =============================================================================
COPY services/mod-reqin-log/mod_reqin_log.spec /package/mod_reqin_log.spec
# =============================================================================
# Copy binaries from each builder stage into pkgroot directories
# =============================================================================
# Rocky Linux 8 (el8)
COPY --from=builder-el8 /build/modules/mod_reqin_log.so /tmp/pkgroot-el8/usr/lib64/httpd/modules/mod_reqin_log.so
COPY --from=builder-el8 /build/conf/mod_reqin_log.conf /tmp/pkgroot-el8/etc/httpd/conf.d/mod_reqin_log.conf
RUN chmod 755 /tmp/pkgroot-el8/usr/lib64/httpd/modules/mod_reqin_log.so && \
chmod 644 /tmp/pkgroot-el8/etc/httpd/conf.d/mod_reqin_log.conf
# Rocky Linux 9 (el9)
COPY --from=builder-el9 /build/modules/mod_reqin_log.so /tmp/pkgroot-el9/usr/lib64/httpd/modules/mod_reqin_log.so
COPY --from=builder-el9 /build/conf/mod_reqin_log.conf /tmp/pkgroot-el9/etc/httpd/conf.d/mod_reqin_log.conf
RUN chmod 755 /tmp/pkgroot-el9/usr/lib64/httpd/modules/mod_reqin_log.so && \
chmod 644 /tmp/pkgroot-el9/etc/httpd/conf.d/mod_reqin_log.conf
# AlmaLinux 10 (el10)
COPY --from=builder-el10 /build/modules/mod_reqin_log.so /tmp/pkgroot-el10/usr/lib64/httpd/modules/mod_reqin_log.so
COPY --from=builder-el10 /build/conf/mod_reqin_log.conf /tmp/pkgroot-el10/etc/httpd/conf.d/mod_reqin_log.conf
RUN chmod 755 /tmp/pkgroot-el10/usr/lib64/httpd/modules/mod_reqin_log.so && \
chmod 644 /tmp/pkgroot-el10/etc/httpd/conf.d/mod_reqin_log.conf
# =============================================================================
# Build RPM packages for each distribution using rpmbuild
# =============================================================================
# Create packages directory
RUN mkdir -p /tmp/packages/el8 /tmp/packages/el9 /tmp/packages/el10
# Build for el8
RUN VERSION=$(grep "^Version:" /package/mod_reqin_log.spec | awk '{print $2}') && \
mkdir -p /tmp/pkgroot-el8-rpm/usr/lib64/httpd/modules /tmp/pkgroot-el8-rpm/etc/httpd/conf.d && \
cp /tmp/pkgroot-el8/usr/lib64/httpd/modules/mod_reqin_log.so /tmp/pkgroot-el8-rpm/usr/lib64/httpd/modules/ && \
cp /tmp/pkgroot-el8/etc/httpd/conf.d/mod_reqin_log.conf /tmp/pkgroot-el8-rpm/etc/httpd/conf.d/ && \
rpmbuild -bb /package/mod_reqin_log.spec \
--define "_topdir /tmp/rpmbuild-el8" \
--define "_pkgroot /tmp/pkgroot-el8-rpm" \
--define "dist .el8" && \
cp /tmp/rpmbuild-el8/RPMS/x86_64/*.rpm /tmp/packages/el8/
# Build for el9
RUN VERSION=$(grep "^Version:" /package/mod_reqin_log.spec | awk '{print $2}') && \
mkdir -p /tmp/pkgroot-el9-rpm/usr/lib64/httpd/modules /tmp/pkgroot-el9-rpm/etc/httpd/conf.d && \
cp /tmp/pkgroot-el9/usr/lib64/httpd/modules/mod_reqin_log.so /tmp/pkgroot-el9-rpm/usr/lib64/httpd/modules/ && \
cp /tmp/pkgroot-el9/etc/httpd/conf.d/mod_reqin_log.conf /tmp/pkgroot-el9-rpm/etc/httpd/conf.d/ && \
rpmbuild -bb /package/mod_reqin_log.spec \
--define "_topdir /tmp/rpmbuild-el9" \
--define "_pkgroot /tmp/pkgroot-el9-rpm" \
--define "dist .el9" && \
cp /tmp/rpmbuild-el9/RPMS/x86_64/*.rpm /tmp/packages/el9/
# Build for el10
RUN VERSION=$(grep "^Version:" /package/mod_reqin_log.spec | awk '{print $2}') && \
mkdir -p /tmp/pkgroot-el10-rpm/usr/lib64/httpd/modules /tmp/pkgroot-el10-rpm/etc/httpd/conf.d && \
cp /tmp/pkgroot-el10/usr/lib64/httpd/modules/mod_reqin_log.so /tmp/pkgroot-el10-rpm/usr/lib64/httpd/modules/ && \
cp /tmp/pkgroot-el10/etc/httpd/conf.d/mod_reqin_log.conf /tmp/pkgroot-el10-rpm/etc/httpd/conf.d/ && \
rpmbuild -bb /package/mod_reqin_log.spec \
--define "_topdir /tmp/rpmbuild-el10" \
--define "_pkgroot /tmp/pkgroot-el10-rpm" \
--define "dist .el10" && \
cp /tmp/rpmbuild-el10/RPMS/x86_64/*.rpm /tmp/packages/el10/
# =============================================================================
# Stage 3: Output - Image finale avec les packages RPM
# =============================================================================
FROM alpine:latest AS output
WORKDIR /packages
COPY --from=package-builder /tmp/packages/el8/*.rpm /packages/rpm/el8/
COPY --from=package-builder /tmp/packages/el9/*.rpm /packages/rpm/el9/
COPY --from=package-builder /tmp/packages/el10/*.rpm /packages/rpm/el10/
CMD ["sh", "-c", "echo '=== RPM Packages (el8) ===' && ls -la /packages/rpm/el8/ && echo '' && echo '=== RPM Packages (el9) ===' && ls -la /packages/rpm/el9/ && echo '' && echo '=== RPM Packages (el10) ===' && ls -la /packages/rpm/el10/"]

43
old/services/mod-reqin-log/Dockerfile.tests Normal file
View File

@ -0,0 +1,43 @@
# Dockerfile for running unit tests (monorepo root build context)
FROM rockylinux:8
# Install build and test dependencies
RUN dnf install -y epel-release && \
dnf install -y \
gcc \
make \
httpd \
httpd-devel \
apr-devel \
apr-util-devel \
cmake \
python3 \
curl \
git \
pkgconfig \
libxml2-devel \
&& dnf clean all
# Build and install cmocka from source
RUN cd /tmp && \
git clone https://git.cryptomilk.org/projects/cmocka.git && \
cd cmocka && \
git checkout cmocka-1.1.5 && \
mkdir build && cd build && \
cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release && \
make && \
make install && \
ldconfig && \
cd / && \
rm -rf /tmp/cmocka
WORKDIR /build
COPY services/mod-reqin-log/src/ src/
COPY services/mod-reqin-log/tests/ tests/
COPY services/mod-reqin-log/CMakeLists.txt CMakeLists.txt
COPY services/mod-reqin-log/Makefile Makefile
RUN mkdir -p build/tests && cd build/tests && cmake ../../ && make
CMD ["ctest", "--test-dir", "build/tests", "--output-on-failure"]

107
old/services/mod-reqin-log/LICENSE Normal file
View File

@ -0,0 +1,107 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the
copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other
entities that control, are controlled by, or are under common control with
that entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation source,
and configuration files.
"Object" form shall mean any form resulting from mechanical transformation
or translation of a Source form, including but not limited to compiled
object code, generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form,
made available under the License.
"Derivative Works" shall mean any work, whether in Source or Object form,
that is based on (or derived from) the Work.
"Contribution" shall mean any work of authorship, including the original
version of the Work and any modifications or additions to that Work or
Derivative Works thereof.
"Contributor" shall mean Licensor and any individual or Legal Entity on
behalf of whom a Contribution has been received by Licensor.
2. Grant of Copyright License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the Work and
such Derivative Works in Source or Object form.
3. Grant of Patent License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to make, have
made, use, offer to sell, sell, import, and otherwise transfer the Work.
4. Redistribution.
You may reproduce and distribute copies of the Work or Derivative Works
thereof in any medium, with or without modifications, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or Derivative Works a
copy of this License; and
(b) You must cause any modified files to carry prominent notices stating
that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works that You
distribute, all copyright, patent, trademark, and attribution notices
from the Source form of the Work; and
(d) You may add Your own copyright statement to Your modifications and may
provide additional or different license terms and conditions for use,
reproduction, or distribution of Your modifications.
5. Submission of Contributions.
Unless You explicitly state otherwise, any Contribution intentionally
submitted for inclusion in the Work by You shall be under the terms and
conditions of this License.
6. Trademarks.
This License does not grant permission to use the trade names, trademarks,
service marks, or product names of the Licensor.
7. Disclaimer of Warranty.
Unless required by applicable law or agreed to in writing, Licensor provides
the Work on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
either express or implied.
8. Limitation of Liability.
In no event shall any Contributor be liable to You for damages, including
any direct, indirect, special, incidental, or consequential damages arising
out of the use or inability to use the Work.
9. Accepting Warranty or Additional Liability.
While redistributing the Work or Derivative Works thereof, You may choose
to offer, and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this License.
END OF TERMS AND CONDITIONS

153
old/services/mod-reqin-log/Makefile Normal file
View File

@ -0,0 +1,153 @@
# Makefile for mod_reqin_log
# Apache HTTPD module for logging HTTP requests as JSON to Unix socket
# APXS tool path (can be overridden)
APXS ?= apxs
# Compiler settings
CC ?= gcc
CFLAGS ?= -Wall -Wextra -O2 -std=gnu11 -x c -Wno-error=format-security
# Directories
SRC_DIR = src
BUILD_DIR = build
INSTALL_DIR = modules
DIST_DIR = dist
# Source files
SRCS = $(SRC_DIR)/mod_reqin_log.c
# Module name
MODULE_NAME = mod_reqin_log
# Package version
VERSION ?= 1.0.7
.PHONY: all clean install uninstall test package package-deb package-rpm
all: $(MODULE_NAME).so
# Build the module using apxs
# Note: Use -Wc to pass flags to the C compiler through apxs
$(MODULE_NAME).so: $(SRCS)
@mkdir -p $(BUILD_DIR)
$(APXS) -c -Wc,"$(CFLAGS)" -o $(BUILD_DIR)/$(MODULE_NAME).so $(SRCS)
@mkdir -p $(INSTALL_DIR)
@if [ -f $(BUILD_DIR)/.libs/$(MODULE_NAME).so ]; then \
cp $(BUILD_DIR)/.libs/$(MODULE_NAME).so $(INSTALL_DIR)/; \
elif [ -f $(BUILD_DIR)/$(MODULE_NAME).so ]; then \
cp $(BUILD_DIR)/$(MODULE_NAME).so $(INSTALL_DIR)/; \
fi
# Install the module
install: $(MODULE_NAME).so
@echo "Installing $(MODULE_NAME).so..."
@mkdir -p $(DESTDIR)/usr/lib/apache2/modules
@if [ -f $(INSTALL_DIR)/$(MODULE_NAME).so ]; then \
cp $(INSTALL_DIR)/$(MODULE_NAME).so $(DESTDIR)/usr/lib/apache2/modules/; \
elif [ -f $(BUILD_DIR)/.libs/$(MODULE_NAME).so ]; then \
cp $(BUILD_DIR)/.libs/$(MODULE_NAME).so $(DESTDIR)/usr/lib/apache2/modules/; \
elif [ -f $(BUILD_DIR)/$(MODULE_NAME).so ]; then \
cp $(BUILD_DIR)/$(MODULE_NAME).so $(DESTDIR)/usr/lib/apache2/modules/; \
else \
echo "Error: $(MODULE_NAME).so not found"; \
exit 1; \
fi
@echo "Installation complete."
@echo "Enable the module by adding to your httpd.conf:"
@echo " LoadModule reqin_log_module modules/mod_reqin_log.so"
# Uninstall the module
uninstall:
rm -f $(DESTDIR)/usr/lib/apache2/modules/$(MODULE_NAME).so
@echo "Uninstallation complete."
# Clean build artifacts
clean:
rm -rf $(BUILD_DIR) $(INSTALL_DIR)
rm -f .libs/*.o .libs/*.la .libs/*.so
rm -f *.o *.la *.lo
rm -rf .libs
# Run unit tests (requires cmocka)
test:
@mkdir -p build/tests
cd build/tests && cmake ../../ -DCMAKE_BUILD_TYPE=Debug
$(MAKE) -C build/tests run_tests
# Build with debug symbols
debug: CFLAGS += -g -DDEBUG
debug: clean all
# =============================================================================
# Packaging (RPM with Docker + fpm)
# Dockerfile.package builds RPMs in a single multi-stage build:
# - 3 RPM packages (el8, el9, el10 for RHEL/Rocky/AlmaLinux compatibility)
# =============================================================================
## package: Build all RPM packages (el8, el9, el10)
package:
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
docker build --target output -t mod_reqin_log:packager \
--build-arg VERSION=$(VERSION) \
-f Dockerfile.package .
@echo "Extracting packages from Docker image..."
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el8:/output/el8 \
-v $(PWD)/$(DIST_DIR)/rpm/el9:/output/el9 \
-v $(PWD)/$(DIST_DIR)/rpm/el10:/output/el10 \
mod_reqin_log:packager \
sh -c 'cp /packages/rpm/el8/*.rpm /output/el8/ && cp /packages/rpm/el9/*.rpm /output/el9/ && cp /packages/rpm/el10/*.rpm /output/el10/'
@echo "Packages created:"
@echo " RPM (el8, el9, el10):"
@ls -la $(DIST_DIR)/rpm/el8/
@ls -la $(DIST_DIR)/rpm/el9/
@ls -la $(DIST_DIR)/rpm/el10/
## package-rpm: Build RPM packages (el8, el9, el10)
package-rpm: package
@echo "RPM packages built in Dockerfile.package"
## test-package-rpm: Test RPM package installation in Docker (tests el9 by default)
test-package-rpm: package
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el9:/packages:ro rockylinux:9 \
sh -c "dnf install -y /packages/*.el9.*.rpm && echo 'RPM el9 install OK'"
## test-package-rpm-el8: Test el8 RPM installation
test-package-rpm-el8: package
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el8:/packages:ro rockylinux:8 \
sh -c "dnf install -y /packages/*.el8.*.rpm && echo 'RPM el8 install OK'"
## test-package-rpm-el9: Test el9 RPM installation
test-package-rpm-el9: package
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el9:/packages:ro rockylinux:9 \
sh -c "dnf install -y /packages/*.el9.*.rpm && echo 'RPM el9 install OK'"
## test-package-rpm-el10: Test el10 RPM installation
test-package-rpm-el10: package
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el10:/packages:ro almalinux:10 \
sh -c "dnf install -y /packages/*.el10.*.rpm && echo 'RPM el10 install OK'"
## test-package: Test all RPM packages installation
test-package: test-package-rpm-el8 test-package-rpm-el9 test-package-rpm-el10
# Help target
help:
@echo "mod_reqin_log Makefile"
@echo ""
@echo "Targets:"
@echo " all - Build the module (default)"
@echo " install - Install the module to DESTDIR"
@echo " uninstall - Remove the module from DESTDIR"
@echo " clean - Remove build artifacts"
@echo " test - Run unit tests"
@echo " debug - Build with debug symbols"
@echo " package - Build all RPM packages (el8, el9, el10)"
@echo " package-rpm - Build RPM packages"
@echo " test-package - Test RPM package installation"
@echo ""
@echo "Variables:"
@echo " APXS - Path to apxs tool (default: apxs)"
@echo " CC - C compiler (default: gcc)"
@echo " CFLAGS - Compiler flags (default: -Wall -Wextra -O2)"
@echo " DESTDIR - Installation destination (default: /)"
@echo " VERSION - Package version (default: 1.0.4)"

284
old/services/mod-reqin-log/README.md Normal file
View File

@ -0,0 +1,284 @@
# mod_reqin_log
Apache HTTPD 2.4 module for logging all incoming HTTP requests as JSON lines to a Unix domain socket.
## Features
- **Non-blocking I/O**: Logging never blocks worker processes
- **Request-time logging**: Logs at `post_read_request` phase, capturing request data before application processing
- **Configurable headers**: Select which HTTP headers to include in logs
- **Header truncation**: Limit header value length to protect against oversized logs
- **Automatic reconnection**: Reconnects to Unix socket on failure with configurable backoff
- **Throttled error reporting**: Prevents error_log flooding on persistent failures
- **MPM compatible**: Works with prefork, worker, and event MPMs
- **Built-in security**: Sensitive headers (Authorization, Cookie, etc.) are automatically excluded
- **RPM packaging**: Standard RPM packages for Rocky Linux 8/9 and AlmaLinux 10
## Requirements
### Runtime
- Apache HTTPD 2.4+
- GCC compiler
- APR development libraries
- Apache development headers (`httpd-devel` or `apache2-dev`)
### Packaging (RPM)
- Docker (for reproducible builds)
- rpmbuild (inside Docker)
## Installation
### Using Docker (recommended)
```bash
# Build all RPM packages (el8, el9, el10)
make package
# Test RPM package installation
make test-package-rpm-el8 # Test el8 RPM (Rocky 8/RHEL 8)
make test-package-rpm-el9 # Test el9 RPM (Rocky 9/RHEL 9)
make test-package-rpm-el10 # Test el10 RPM (AlmaLinux 10/RHEL 10)
make test-package # Test all RPM packages
```
### Build from Source
```bash
# Clone or extract the source
cd mod_reqin_log
# Build the module
make
# Install (requires root privileges)
sudo make install
```
## Configuration
Load the module and configure in your Apache configuration:
```apache
# Load the module
LoadModule reqin_log_module modules/mod_reqin_log.so
# Enable logging
JsonSockLogEnabled On
# Unix socket path
JsonSockLogSocket "/var/run/logcorrelator/http.socket"
# Headers to log (be careful not to log sensitive data)
JsonSockLogHeaders X-Request-Id X-Trace-Id User-Agent Referer
# Maximum headers to log
JsonSockLogMaxHeaders 10
# Maximum header value length
JsonSockLogMaxHeaderValueLen 256
# Reconnect interval (seconds)
JsonSockLogReconnectInterval 10
# Error report interval (seconds)
JsonSockLogErrorReportInterval 10
```
> **Important startup validation:** if `JsonSockLogEnabled On` is set without a valid `JsonSockLogSocket`, Apache startup fails with a configuration error.
### Configuration Directives
| Directive | Type | Default | Description |
|-----------|------|---------|-------------|
| `JsonSockLogEnabled` | On/Off | Off | Enable or disable logging |
| `JsonSockLogSocket` | String | - | Unix domain socket path |
| `JsonSockLogHeaders` | List | - | HTTP headers to include |
| `JsonSockLogMaxHeaders` | Integer | 10 | Max headers to log |
| `JsonSockLogMaxHeaderValueLen` | Integer | 256 | Max header value length |
| `JsonSockLogReconnectInterval` | Integer | 10 | Reconnect delay (seconds) |
| `JsonSockLogErrorReportInterval` | Integer | 10 | Error log throttle (seconds) |
## JSON Log Format
Each log entry is a single-line JSON object with a flat structure:
```json
{
"time": "2026-02-26T11:59:30Z",
"timestamp": 1708948770000000000,
"src_ip": "192.0.2.10",
"src_port": 45678,
"dst_ip": "198.51.100.5",
"dst_port": 443,
"method": "GET",
"path": "/api/users",
"host": "example.com",
"http_version": "HTTP/1.1",
"header_X-Request-Id": "abcd-1234",
"header_User-Agent": "curl/7.70.0"
}
```
### Fields
| Field | Type | Description |
|-------|------|-------------|
| `time` | String | ISO8601 timestamp with timezone |
| `timestamp` | Integer | Microseconds since epoch (expressed as nanoseconds for compatibility) |
| `src_ip` | String | Client IP address |
| `src_port` | Integer | Client port |
| `dst_ip` | String | Server IP address |
| `dst_port` | Integer | Server port |
| `method` | String | HTTP method |
| `path` | String | Request path |
| `host` | String | Host header value |
| `http_version` | String | HTTP protocol version |
| `header_<Name>` | String | Flattened HTTP headers (e.g., `header_X-Request-Id`) |
**Note:** Headers are logged as flat fields at the root level (not nested). Sensitive headers are automatically excluded. The `timestamp` field has microsecond precision (APR's `apr_time_now()` returns microseconds, multiplied by 1000 for nanosecond representation).
## Unix Socket Consumer
Create a Unix socket listener to receive log entries:
```python
#!/usr/bin/env python3
import socket
import os
import json
SOCKET_PATH = os.environ.get("MOD_REQIN_LOG_SOCKET", "/var/run/logcorrelator/http.socket")
# Remove existing socket file
if os.path.exists(SOCKET_PATH):
os.remove(SOCKET_PATH)
# Create Unix socket server
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
server.bind(SOCKET_PATH)
server.listen(5)
# Set secure permissions: owner and group only (not world-writable)
os.chmod(SOCKET_PATH, 0o660)
print(f"Listening on {SOCKET_PATH}")
while True:
conn, addr = server.accept()
data = b""
while True:
chunk = conn.recv(4096)
if not chunk:
break
data += chunk
if b"\n" in data:
for line in data.decode().strip().split("\n"):
if line:
log_entry = json.loads(line)
print(json.dumps(log_entry, indent=2))
data = b""
conn.close()
```
**Note:** Ensure the Apache user is in the socket file's group to allow connections.
## Security Considerations
### Built-in Sensitive Headers Blacklist
⚠️ **The module automatically blocks logging of sensitive headers:**
The following headers are **always excluded** from logs to prevent credential leakage:
- `Authorization`
- `Cookie`, `Set-Cookie`
- `X-Api-Key`, `X-Auth-Token`
- `Proxy-Authorization`
- `WWW-Authenticate`
These headers are silently skipped (logged at DEBUG level only).
### Socket Security
- **Socket permissions**: Default to `0o660` (owner and group only)
- **Recommended path**: `/var/run/logcorrelator/http.socket` (not `/tmp`)
- **Environment variable**: Use `MOD_REQIN_LOG_SOCKET` to configure path
- **Group membership**: Ensure Apache user is in the socket's group
### Additional Hardening
- **Socket path length**: Validated against system limit (108 bytes max)
- **JSON size limit**: 64KB max per log line (prevents memory DoS)
- **NULL pointer checks**: All connection/request fields validated
- **Thread safety**: Mutex protects socket FD in worker/event MPMs
- **Error logging**: Generic messages in error_log, details at DEBUG level
## Troubleshooting
### Module not loading
```
AH00534: mod_reqin_log: Unable to load module
```
Ensure the module path is correct and the file exists:
```bash
ls -la /usr/lib/apache2/modules/mod_reqin_log.so
```
### Socket connection failures
```
[mod_reqin_log] Unix socket connect failed: /var/run/logcorrelator/http.socket
```
- Ensure the socket consumer is running
- Check socket file permissions
- Verify SELinux/AppArmor policies if applicable
### No logs appearing
1. Verify `JsonSockLogEnabled On` is set
2. Verify `JsonSockLogSocket` path is configured
3. Check Apache error log for module errors
4. Ensure socket consumer is listening
## Testing
### Run Unit Tests
```bash
# Using Docker (recommended)
docker build -f Dockerfile.tests -t mod_reqin_log:tests .
docker run --rm mod_reqin_log:tests ctest --output-on-failure
# Or locally with cmocka
sudo dnf install cmocka-devel # Rocky Linux
sudo apt install libcmocka-dev # Debian/Ubuntu
mkdir build && cd build
cmake ..
make test
```
### Build and Test Packages
```bash
# Build all RPM packages (el8, el9, el10)
make package
# Test RPM package installation
make test-package-rpm-el8 # Test el8 RPM in Docker
make test-package-rpm-el9 # Test el9 RPM in Docker
make test-package-rpm-el10 # Test el10 RPM in Docker
make test-package # Test all RPM packages
```
## License
Apache License 2.0
## Contributing
1. Fork the repository
2. Create a feature branch
3. Make your changes
4. Run tests
5. Submit a pull request

443
old/services/mod-reqin-log/architecture.yml Normal file
View File

@ -0,0 +1,443 @@
project:
name: mod_reqin_log
description: >
Apache HTTPD 2.4 module logging all incoming HTTP requests as JSON lines
to a Unix domain socket at request reception time (no processing time).
language: c
author:
name: Jacquin Antoine
email: rpm@arkel.fr
target:
server: apache-httpd
version: "2.4"
os: rocky-linux-8+, almalinux-10+
build:
toolchain: gcc
apache_dev: httpd-devel (apxs)
artifacts:
- mod_reqin_log.so
context:
architecture:
pattern: native-apache-module
scope: global
mpm_compatibility:
- prefork
- worker
- event
request_phase:
hook: post_read_request
rationale: >
Log as soon as the HTTP request is fully read to capture input-side data
(client/server addresses, request line, headers) without waiting for
application processing.
filters:
- Subrequests (r->main != NULL) are skipped.
- Internal redirects (r->prev != NULL) are skipped.
- Only the original client request is logged.
logging_scope:
coverage: all-traffic
description: >
Every HTTP request handled by the Apache instance is considered for logging
when the module is enabled and the Unix socket is configured.
module:
name: mod_reqin_log
files:
source:
- src/mod_reqin_log.c
- src/mod_reqin_log.h
packaging:
- mod_reqin_log.spec
tests:
- tests/unit/test_module_real.c
- tests/unit/test_config_parsing.c
- tests/unit/test_header_handling.c
- tests/unit/test_json_serialization.c
hooks:
- name: register_hooks
responsibilities:
- Register post_read_request hook for logging at request reception.
- Register child_init hook for per-process state initialization.
- Initialize per-process server configuration structure.
- name: child_init
responsibilities:
- Initialize module state for each Apache child process.
- Reset per-process socket state (fd, timers, error flags).
- Attempt initial non-blocking connection to Unix socket if configured.
- name: post_read_request
responsibilities:
- Retrieve per-process server configuration (thread-safe).
- Ensure Unix socket is connected (with periodic reconnect).
- Build JSON log document for the request.
- Write JSON line to Unix socket using non-blocking I/O.
- Handle errors by dropping the current log line and rate-limiting
error reports into Apache error_log.
thread_safety:
model: per-process-state
description: >
Each Apache child process maintains its own socket state stored in the
server configuration structure (reqin_log_server_conf_t). This avoids
race conditions in worker and event MPMs where multiple threads share
a process.
implementation:
- State stored via ap_get_module_config(s->module_config)
- No global variables for socket state
- Each process has independent: socket_fd, connect timers, error timers
data_model:
json_line:
description: >
One JSON object per HTTP request, serialized on a single line and
terminated by "\n". Uses flat structure with header fields at root level.
structure: flat
fields:
- name: time
type: string
format: iso8601-with-timezone
example: "2026-02-26T11:59:30Z"
- name: timestamp
type: integer
unit: microseconds (expressed as nanoseconds)
description: >
Wall-clock timestamp in microseconds since Unix epoch, expressed
as nanoseconds for compatibility (multiplied by 1000).
Uses r->request_time (set by Apache at request reception).
The nanosecond representation is for API compatibility only.
example: 1708948770000000000
- name: scheme
type: string
description: Connection scheme evaluated via ap_http_scheme(r).
example: "https"
- name: src_ip
type: string
example: "192.0.2.10"
- name: src_port
type: integer
example: 45678
- name: dst_ip
type: string
example: "198.51.100.5"
- name: dst_port
type: integer
example: 443
- name: method
type: string
example: "GET"
- name: path
type: string
description: Cleaned and normalized path (r->parsed_uri.path).
example: "/api/users"
- name: query
type: string
description: >
Query string component from the parsed URI (r->parsed_uri.query).
Does not include the leading '?'. Allows detection of payloads like
SQLi or XSS passed in GET requests.
example: "id=1%20UNION%20SELECT"
- name: host
type: string
example: "example.com"
- name: http_version
type: string
example: "HTTP/1.1"
- name: keepalives
type: integer
description: >
Number of requests served over the current connection (r->connection->keepalives).
If 0, it indicates a newly established TCP connection.
If > 0, it confirms an active Keep-Alive session.
example: 2
- name: client_headers
type: array of strings
description: >
Ordered list of all HTTP header names as received from the client
(r->headers_in), preserving original order and case.
Useful for browser/bot fingerprinting (header order is client-specific).
example: ["Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding"]
- name: content_length
type: integer
description: >
Declared size of the request body (POST payload),
extracted directly from the 'Content-Length' header.
example: 1048576
- name: header_<HeaderName>
type: string
description: >
Flattened header fields at root level. For each configured header <H>,
a field 'header_<H>' is added directly to the JSON root object.
Headers are only included if present in the request.
key_pattern: "header_<configured_header_name>"
optional: true
example:
header_X-Request-Id: "abcd-1234"
header_User-Agent: "curl/7.70.0"
example_full: |
{"time":"2026-02-26T11:59:30Z","timestamp":1708948770000000000,"scheme":"https","src_ip":"192.0.2.10","src_port":45678,"dst_ip":"198.51.100.5","dst_port":443,"method":"GET","path":"/api/users","query":"id=1","host":"example.com","http_version":"HTTP/1.1","keepalives":0,"client_headers":["Host","User-Agent","Accept","Accept-Language","Accept-Encoding","X-Request-Id"],"content_length":0,"header_X-Request-Id":"abcd-1234","header_User-Agent":"curl/7.70.0"}
configuration:
scope: global
directives:
- name: JsonSockLogEnabled
type: flag
context: server-config
default: "Off"
description: >
Enable or disable mod_reqin_log logging globally. Logging only occurs
when this directive is On and JsonSockLogSocket is set.
- name: JsonSockLogSocket
type: string
context: server-config
required_when_enabled: true
example: "/var/run/logcorrelator/http.socket"
description: >
Filesystem path of the Unix domain socket to which JSON log lines
will be written.
- name: JsonSockLogHeaders
type: list
context: server-config
value_example: ["X-Request-Id", "X-Trace-Id", "User-Agent", "Referer",
"X-Forwarded-For", "Sec-CH-UA", "Sec-CH-UA-Mobile", "Sec-CH-UA-Platform",
"Sec-Fetch-Dest", "Sec-Fetch-Mode", "Sec-Fetch-Site",
"Accept", "Accept-Language", "Accept-Encoding"]
description: >
List of HTTP header names to log. For each configured header <H>,
the module adds a JSON field 'header_<H>' at the root level of the
JSON log entry (flat structure). Order matters for applying the
JsonSockLogMaxHeaders limit.
- name: JsonSockLogMaxHeaders
type: integer
context: server-config
default: 25
min: 0
description: >
Maximum number of headers from JsonSockLogHeaders to actually log.
If more headers are configured, only the first N are considered.
- name: JsonSockLogMaxHeaderValueLen
type: integer
context: server-config
default: 256
min: 1
description: >
Maximum length in characters for each logged header value.
Values longer than this limit are truncated before JSON encoding.
- name: JsonSockLogReconnectInterval
type: integer
context: server-config
default: 10
unit: seconds
description: >
Minimal delay between two connection attempts to the Unix socket after
a failure. Used to avoid reconnect attempts on every request.
- name: JsonSockLogErrorReportInterval
type: integer
context: server-config
default: 10
unit: seconds
description: >
Minimal delay between two error messages emitted into Apache error_log
for repeated I/O or connection errors on the Unix socket.
behavior:
enabling_rules:
- JsonSockLogEnabled must be On.
- JsonSockLogSocket must be set to a non-empty path.
header_handling:
- Built-in blacklist prevents logging of sensitive headers by default.
- Blacklisted headers: Authorization, Cookie, Set-Cookie, X-Api-Key,
X-Auth-Token, Proxy-Authorization, WWW-Authenticate.
- Blacklisted headers are silently skipped (logged at DEBUG level only).
- If a configured header is absent in a request, the corresponding
JSON key is omitted from the log entry.
- Header values are truncated to JsonSockLogMaxHeaderValueLen characters.
io:
socket:
type: unix-domain
protocol: SOCK_DGRAM
mode: client
path_source: JsonSockLogSocket
connection:
persistence: false
non_blocking: true
lifecycle:
open:
- Create DGRAM socket and set default destination address via connect()
during child_init if enabled.
- Re-attempt addressing after reconnect interval expiry if target
was previously unavailable.
failure:
- On missing target socket (ECONNREFUSED/ENOENT), mark target as unavailable.
- Do not block the worker process.
reconnect:
strategy: time-based
interval_seconds: config.JsonSockLogReconnectInterval
trigger: >
When a request arrives and the last target resolution attempt time is older
than reconnect interval, a new attempt to address the socket is made.
write:
format: json_object
mode: non-blocking
atomicity: >
Full JSON line is sent as a single datagram. Message size must not exceed
system DGRAM limits or MAX_JSON_SIZE (64KB).
error_handling:
on_eagain_or_ewouldblock:
action: drop-current-log-line
note: "OS buffer full (receiver is too slow). Do not retry, do not spam error_log."
on_econnrefused_or_enoent:
action:
- close_socket
- mark_target_unavailable
- schedule_reconnect
note: "Target socket closed or deleted by log receiver."
generic_errors:
action: drop-current-log-line
drop_policy:
description: >
Logging errors never impact client response. The current log line is
silently dropped except for throttled error_log reporting.
error_handling:
apache_error_log_reporting:
enabled: true
throttle_interval_seconds: config.JsonSockLogErrorReportInterval
events:
- type: connect_failure
message_template: "mod_reqin_log: Unix socket connect failed: [errno_detail]"
- type: write_failure
message_template: "mod_reqin_log: Unix socket write failed: [errno_detail]"
fatal_conditions:
- description: >
Misconfiguration (JsonSockLogEnabled On but missing JsonSockLogSocket)
should be reported at startup as a configuration error.
- description: >
Any internal JSON-encoding failure should be treated as non-fatal
(drop current log and optionally emit a throttled error_log entry).
constraints:
performance:
objectives:
- Logging overhead per request should be minimal and non-blocking.
- No dynamic allocations in hot path beyond what is strictly necessary
(prefer APR pools where possible).
design_choices:
- Single JSON serialization pass per request.
- Use non-blocking I/O to avoid stalling worker threads/processes.
- Avoid reconnect attempts on every request via time-based backoff.
security:
notes:
- Module includes built-in blacklist of sensitive headers to prevent
accidental credential leakage (Authorization, Cookie, X-Api-Key, etc.).
- Socket permissions default to 0o660 (owner/group only) for security.
- Recommended socket path: /var/run/logcorrelator/http.socket (not /tmp).
- Use environment variable MOD_REQINLOG_SOCKET to configure socket path.
- Module does not anonymize IPs (data protection is delegated to configuration).
- No requests are rejected due to logging failures.
hardening:
- Socket path length validated against system limit (108 bytes).
- JSON log line size limited to 64KB to prevent memory exhaustion DoS.
- NULL pointer checks on all connection/request fields.
- Thread-safe socket FD access via mutex (worker/event MPMs).
- Error logging reduced to prevent information disclosure.
robustness:
requirements:
- Logging failures must not crash Apache worker processes.
- Module must behave correctly under high traffic, socket disappearance,
and repeated connect failures.
testing_strategy:
unit_tests:
framework: cmocka
location: tests/unit/test_module_real.c
focus:
- JSON serialization with header truncation and header count limits.
- Dynamic buffer operations (dynbuf_t) with resize handling.
- ISO8601 timestamp formatting.
- Header value truncation to JsonSockLogMaxHeaderValueLen.
- Control character escaping in JSON strings.
execution:
- docker build -f Dockerfile.tests .
- docker run --rm <image> ctest --output-on-failure
ci_strategy:
description: >
All builds, tests and packaging are executed inside Docker containers
using GitLab CI with Docker-in-Docker (dind). No RPM build or test is
allowed on bare-metal or shared CI runners.
tools:
orchestrator: GitLab CI
container_engine: docker
dind: true
workflow_file: .gitlab-ci.yml
constraints:
no_host_builds: true
description: >
It is forbidden to run rpmbuild, unit tests or package verification
directly on the CI host. All steps MUST run inside Docker containers
defined by project Dockerfiles.
rpm_strategy: >
Separate RPMs are built for each major RHEL/CentOS/Rocky/AlmaLinux version
(el8, el9, el10) due to glibc and httpd-devel incompatibilities across
major versions. A single RPM cannot work across all versions.
RPM packages are built using rpmbuild with mod_reqin_log.spec file.
rpm_changelog:
policy: mandatory
description: >
For every version or release bump of the RPM (Version or Release tag
in mod_reqin_log.spec), the %changelog section MUST be updated with:
- date, packager, new version-release
- brief description of the changes.
validation:
- A CI job MUST fail if Version/Release changed and no new %changelog
entry is present.
- Changelog is the single source of truth for packaged changes.
stages:
- name: validate-spec
description: >
Ensure that any change to Version/Release in mod_reqin_log.spec
is accompanied by a new %changelog entry.
containerized: true
dockerfile: Dockerfile.tools
checks:
- script: scripts/check_spec_changelog.sh mod_reqin_log.spec
fail_on_missing_changelog: true
- name: build
description: >
Build all RPM packages (el8, el9, el10) using Dockerfile.package
with multi-stage build, entirely inside a Docker container.
dockerfile: Dockerfile.package
containerized: true
artifacts:
- dist/rpm/*.el8.*.rpm
- dist/rpm/*.el9.*.rpm
- dist/rpm/*.el10.*.rpm
- name: test
description: >
Run unit tests (C with cmocka) inside Docker containers, using
Dockerfile.tests as the only execution environment.
dockerfile: Dockerfile.tests
containerized: true
execution: ctest --output-on-failure
- name: verify
description: >
Verify RPM installation and module loading on each target distribution
by running containers for each OS.
containerized: true
jobs:
- name: verify-rpm-el8
image: rockylinux:8
steps:
- rpm -qi mod_reqin_log
- httpd -M | grep reqin_log
- name: verify-rpm-el9
image: rockylinux:9
steps:
- rpm -qi mod_reqin_log
- httpd -M | grep reqin_log
- name: verify-rpm-el10
image: almalinux:10
steps:
- rpm -qi mod_reqin_log
- httpd -M | grep reqin_log

40
old/services/mod-reqin-log/conf/mod_reqin_log.conf Normal file
View File

@ -0,0 +1,40 @@
# mod_reqin_log example configuration
# Load this configuration in your Apache httpd.conf or a separate included file
# Load the module (adjust path as needed)
LoadModule reqin_log_module modules/mod_reqin_log.so
# Enable mod_reqin_log
JsonSockLogEnabled On
# Unix domain socket path for JSON log output
# Important: if JsonSockLogEnabled is On and this directive is missing/empty,
# Apache startup fails due to strict configuration validation.
JsonSockLogSocket "/var/run/logcorrelator/http.socket"
# HTTP headers to include in the JSON log
# Warning: Be careful not to log sensitive headers like Authorization, Cookie, etc.
JsonSockLogHeaders X-Request-Id X-Trace-Id User-Agent Referer X-Forwarded-For \
Sec-CH-UA Sec-CH-UA-Mobile Sec-CH-UA-Platform \
Sec-Fetch-Dest Sec-Fetch-Mode Sec-Fetch-Site \
Accept Accept-Language Accept-Encoding Content-Type
# Maximum number of headers to log (from the configured list)
JsonSockLogMaxHeaders 25
# Maximum length of each header value (longer values are truncated)
JsonSockLogMaxHeaderValueLen 256
# Minimum delay between reconnect attempts to the Unix socket (seconds)
JsonSockLogReconnectInterval 10
# Minimum delay between error messages to Apache error_log (seconds)
JsonSockLogErrorReportInterval 10
# Log level for module messages: DEBUG, INFO, WARNING, ERROR, EMERG (default: WARNING)
# DEBUG: Log all messages including header skipping and buffer truncation
# INFO: Log informational messages
# WARNING: Log warnings (default)
# ERROR: Log only errors
# EMERG: Log only emergency messages
JsonSockLogLevel WARNING

123
old/services/mod-reqin-log/mod_reqin_log.spec Normal file
View File

@ -0,0 +1,123 @@
%global spec_version 1.0.19
Name: mod_reqin_log
Version: %{spec_version}
Release: 1%{?dist}
Summary: Apache HTTPD module for logging HTTP requests as JSON to Unix socket
License: Apache-2.0
URL: https://github.com/example/mod_reqin_log
Vendor: Developer <dev@example.com>
BuildArch: x86_64
Requires: httpd
%description
Apache HTTPD module for logging HTTP requests as JSON to Unix socket.
Features non-blocking I/O with automatic reconnection, configurable headers
with truncation support, and built-in sensitive headers blacklist.
%prep
# No source extraction needed - binaries are pre-built
%build
# No build needed - binaries are pre-built
%install
mkdir -p %{buildroot}/%{_libdir}/httpd/modules
mkdir -p %{buildroot}/%{_sysconfdir}/httpd/conf.d
mkdir -p %{buildroot}/%{_docdir}/%{name}
install -m 755 %{_pkgroot}/%{_libdir}/httpd/modules/mod_reqin_log.so %{buildroot}/%{_libdir}/httpd/modules/
install -m 644 %{_pkgroot}/%{_sysconfdir}/httpd/conf.d/mod_reqin_log.conf %{buildroot}/%{_sysconfdir}/httpd/conf.d/
%files
%{_libdir}/httpd/modules/mod_reqin_log.so
%config(noreplace) %{_sysconfdir}/httpd/conf.d/mod_reqin_log.conf
%doc %{_docdir}/%{name}
%changelog
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.19
- FEATURE: Add client_headers JSON field - ordered list of all header names
as received from the client, preserving original order and case
- DOC: Update architecture.yml with client_headers field and example_full
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.18
- FIX: JsonSockLogMaxHeaders now counts configured headers (by position in list)
regardless of their presence in the request, matching the documented behavior
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.17
- CONFIG: Extend default JsonSockLogHeaders list (User-Agent, Referer, X-Forwarded-For,
Sec-CH-UA*, Sec-Fetch-*, Accept, Accept-Language, Accept-Encoding)
- CONFIG: Raise DEFAULT_MAX_HEADERS from 10 to 25
- DOC: Update architecture.yml and conf/mod_reqin_log.conf accordingly
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.16
- FIX: Skip subrequests and internal redirects to log only the original client request
- DOC: Document subrequest/redirect filtering in architecture.yml
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.15
- FIX: timestamp field now uses r->request_time (request reception time) instead of apr_time_now()
- DOC: Remove unparsed_uri and fragment fields from architecture.yml (not logged)
- DOC: Update timestamp description and example_full in architecture.yml
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.14
- REFACTOR: Harmonize JSON field construction - all fields now end with comma
- FIX: Remove duplicate comma between query and host fields
- FIX: Fix buffer corruption in dynbuf_append (copy null terminator)
- PACKAGING: Config file marked as %config(noreplace)
- CHANGE: Remove unparsed_uri, fragment, content_length fields
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.13
- FIX: Correct JSON string length parameters for query and fragment fields
- FIX: Add null-termination after buffer reallocation in dynbuf_append
- CHANGE: Remove unparsed_uri, fragment, and content_length fields from JSON output
- TEST: Update unit tests to match dynbuf_append fix
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.9
- CHANGE: Remove req_id field from JSON output
- FEATURE: Add query and fragment fields (URI components)
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.8
- FEATURE: Add req_id, scheme, unparsed_uri, args, keepalives, content_length fields to JSON output
- FIX: Change socket type from SOCK_STREAM to SOCK_DGRAM per architecture.yml
* Sun Mar 01 2026 Developer <dev@example.com> - 1.0.6
- BUILD: Fix RPM package paths in Dockerfile.package (el8, el9, el10 directories)
- BUILD: Fix Makefile RPM extraction with separate volume mounts
- BUILD: Remove unused scripts (build.sh, test.sh)
- BUILD: Remove Python integration tests (not automated in CI)
- DOCS: Update README.md and architecture.yml for RPM-only packaging
- CLEANUP: Remove DEB and el7 references
* Sat Feb 28 2026 Developer <dev@example.com> - 1.0.2
- SECURITY: Add input sanitization for method, path, host, and http_version fields
- SECURITY: Add Host header truncation (256 chars max) to prevent log injection
- IMPROVEMENT: Add LOG_THROTTLED macro for consistent error reporting
- IMPROVEMENT: Improve socket state double-check pattern
- IMPROVEMENT: Fix const qualifier warnings in get_header() function
- IMPROVEMENT: Add flags field to module definition
- IMPROVEMENT: Add -Wno-error=format-security to Makefile
- TEST: Add 4 new unit tests for input sanitization
- DOC: Clarify timestamp precision
- DOC: Update README and architecture.yml
- BUILD: Update package version to 1.0.2
* Fri Feb 27 2026 Developer <dev@example.com> - 1.0.1
- FIX: Fix socket reconnection logic
- FIX: Improve error logging to prevent error_log flooding
- IMPROVEMENT: Add built-in sensitive headers blacklist
- IMPROVEMENT: Add thread-safe socket FD access via mutex
- TEST: Add comprehensive unit tests
- TEST: Add integration tests for socket loss and recovery
- DOC: Add comprehensive README with configuration examples
- DOC: Add architecture.yml documenting module design decisions
* Thu Feb 26 2026 Developer <dev@example.com> - 1.0.0
- Initial release
- Apache HTTPD 2.4 module for logging HTTP requests as JSON to Unix socket
- Non-blocking I/O with automatic reconnection
- Configurable headers with truncation support
- Compatible with prefork, worker, and event MPMs
- Built-in sensitive headers blacklist
- Throttled error reporting to prevent log flooding

1814
old/services/mod-reqin-log/src/mod_reqin_log.c Normal file
View File

File diff suppressed because it is too large Load Diff

56
old/services/mod-reqin-log/src/mod_reqin_log.h Normal file
View File

@ -0,0 +1,56 @@
/*
* mod_reqin_log.h - Apache HTTPD module for logging HTTP requests as JSON to Unix socket
*
* Copyright (c) 2026. All rights reserved.
*/
#ifndef MOD_REQIN_LOG_H
#define MOD_REQIN_LOG_H
#include "httpd.h"
#include "http_config.h"
#include "apr_tables.h"
/* Module name */
#define MOD_REQIN_LOG_NAME "mod_reqin_log"
/* Default configuration values */
#define DEFAULT_MAX_HEADERS 25
#define DEFAULT_MAX_HEADER_VALUE_LEN 256
#define DEFAULT_RECONNECT_INTERVAL 10
#define DEFAULT_ERROR_REPORT_INTERVAL 10
/* Module configuration structure */
typedef struct {
int enabled;
const char *socket_path;
apr_array_header_t *headers;
int max_headers;
int max_header_value_len;
int reconnect_interval;
int error_report_interval;
} reqin_log_config_t;
/* External module declaration */
extern module AP_MODULE_DECLARE_DATA reqin_log_module;
/* ====== Fingerprinting HTTP/2 passif ====== */
/* Clés des notes de connexion stockant le fingerprint HTTP/2 parsé */
#define H2_NOTE_FINGERPRINT "reqin_h2_fp" /* Fingerprint Akamai complet */
#define H2_NOTE_SETTINGS "reqin_h2_set" /* Entrées SETTINGS brutes */
#define H2_NOTE_WUPDATE "reqin_h2_wu" /* Incrément WINDOW_UPDATE */
#define H2_NOTE_PSEUDO_ORDER "reqin_h2_ps" /* Ordre pseudo-headers */
#define H2_NOTE_HAS_PRIORITY "reqin_h2_pri" /* Flag PRIORITY présent */
#define H2_NOTE_PARSED "reqin_h2_done" /* Marqueur "déjà parsé" */
/* Clés des notes pour chaque paramètre SETTINGS individuel (RFC 9113 §6.5.2) */
#define H2_NOTE_SET_HEADER_TABLE_SIZE "reqin_h2_s1" /* ID 1 */
#define H2_NOTE_SET_ENABLE_PUSH "reqin_h2_s2" /* ID 2 */
#define H2_NOTE_SET_MAX_CONCURRENT_STREAMS "reqin_h2_s3" /* ID 3 */
#define H2_NOTE_SET_INITIAL_WINDOW_SIZE "reqin_h2_s4" /* ID 4 */
#define H2_NOTE_SET_MAX_FRAME_SIZE "reqin_h2_s5" /* ID 5 */
#define H2_NOTE_SET_MAX_HEADER_LIST_SIZE "reqin_h2_s6" /* ID 6 */
#define H2_NOTE_SET_ENABLE_CONNECT "reqin_h2_s8" /* ID 8 */
#endif /* MOD_REQIN_LOG_H */

333
old/services/mod-reqin-log/tests/unit/test_config_parsing.c Normal file
View File

@ -0,0 +1,333 @@
/*
* test_config_parsing.c - Unit tests for configuration parsing
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <limits.h>
/* Default configuration values */
#define DEFAULT_MAX_HEADERS 10
#define DEFAULT_MAX_HEADER_VALUE_LEN 256
#define DEFAULT_RECONNECT_INTERVAL 10
#define DEFAULT_ERROR_REPORT_INTERVAL 10
#define MAX_SOCKET_PATH_LEN 108
/* Mock configuration structure */
typedef struct {
int enabled;
const char *socket_path;
int max_headers;
int max_header_value_len;
int reconnect_interval;
int error_report_interval;
} mock_config_t;
/* Mock parsing functions */
static int parse_enabled(const char *value)
{
if (strcasecmp(value, "on") == 0 || strcmp(value, "1") == 0) {
return 1;
}
return 0;
}
static const char *parse_socket_path(const char *value)
{
if (value == NULL || strlen(value) == 0) {
return NULL;
}
if (strlen(value) >= MAX_SOCKET_PATH_LEN) {
return NULL;
}
return value;
}
static int parse_int_strict(const char *value, int *result)
{
char *endptr = NULL;
long val;
if (value == NULL || *value == '\0' || result == NULL) {
return -1;
}
errno = 0;
val = strtol(value, &endptr, 10);
if (errno != 0 || endptr == value || *endptr != '\0' || val < INT_MIN || val > INT_MAX) {
return -1;
}
*result = (int)val;
return 0;
}
static int parse_max_headers(const char *value, int *result)
{
if (parse_int_strict(value, result) != 0 || *result < 0) {
return -1;
}
return 0;
}
static int parse_interval(const char *value, int *result)
{
if (parse_int_strict(value, result) != 0 || *result < 0) {
return -1;
}
return 0;
}
static int parse_max_header_value_len(const char *value, int *result)
{
if (parse_int_strict(value, result) != 0 || *result < 1) {
return -1;
}
return 0;
}
/* Test: Parse enabled On */
static void test_parse_enabled_on(void **state)
{
(void)state;
assert_int_equal(parse_enabled("On"), 1);
assert_int_equal(parse_enabled("on"), 1);
assert_int_equal(parse_enabled("ON"), 1);
assert_int_equal(parse_enabled("1"), 1);
}
/* Test: Parse enabled Off */
static void test_parse_enabled_off(void **state)
{
(void)state;
assert_int_equal(parse_enabled("Off"), 0);
assert_int_equal(parse_enabled("off"), 0);
assert_int_equal(parse_enabled("OFF"), 0);
assert_int_equal(parse_enabled("0"), 0);
}
/* Test: Parse socket path valid */
static void test_parse_socket_path_valid(void **state)
{
(void)state;
const char *result = parse_socket_path("/var/run/logcorrelator/http.socket");
assert_string_equal(result, "/var/run/logcorrelator/http.socket");
}
/* Test: Parse socket path empty */
static void test_parse_socket_path_empty(void **state)
{
(void)state;
const char *result = parse_socket_path("");
assert_null(result);
}
/* Test: Parse socket path NULL */
static void test_parse_socket_path_null(void **state)
{
(void)state;
const char *result = parse_socket_path(NULL);
assert_null(result);
}
/* Test: Parse socket path max length valid */
static void test_parse_socket_path_max_len_valid(void **state)
{
(void)state;
char path[MAX_SOCKET_PATH_LEN];
memset(path, 'a', MAX_SOCKET_PATH_LEN - 1);
path[MAX_SOCKET_PATH_LEN - 1] = '\0';
assert_non_null(parse_socket_path(path));
}
/* Test: Parse socket path max length invalid */
static void test_parse_socket_path_max_len_invalid(void **state)
{
(void)state;
char path[MAX_SOCKET_PATH_LEN + 1];
memset(path, 'b', MAX_SOCKET_PATH_LEN);
path[MAX_SOCKET_PATH_LEN] = '\0';
assert_null(parse_socket_path(path));
}
/* Test: Parse max headers valid */
static void test_parse_max_headers_valid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("10", &result), 0);
assert_int_equal(result, 10);
assert_int_equal(parse_max_headers("0", &result), 0);
assert_int_equal(result, 0);
assert_int_equal(parse_max_headers("100", &result), 0);
assert_int_equal(result, 100);
}
/* Test: Parse max headers invalid */
static void test_parse_max_headers_invalid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("-1", &result), -1);
assert_int_equal(parse_max_headers("abc", &result), -1);
assert_int_equal(parse_max_headers("10abc", &result), -1);
}
/* Test: Parse reconnect interval valid */
static void test_parse_reconnect_interval_valid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_interval("10", &result), 0);
assert_int_equal(result, 10);
assert_int_equal(parse_interval("0", &result), 0);
assert_int_equal(result, 0);
assert_int_equal(parse_interval("60", &result), 0);
assert_int_equal(result, 60);
}
/* Test: Parse reconnect interval invalid */
static void test_parse_reconnect_interval_invalid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_interval("-5", &result), -1);
assert_int_equal(parse_interval("abc", &result), -1);
assert_int_equal(parse_interval("10abc", &result), -1);
}
/* Test: Parse max header value length valid */
static void test_parse_max_header_value_len_valid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_header_value_len("1", &result), 0);
assert_int_equal(result, 1);
assert_int_equal(parse_max_header_value_len("256", &result), 0);
assert_int_equal(result, 256);
}
/* Test: Parse max header value length invalid */
static void test_parse_max_header_value_len_invalid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_header_value_len("0", &result), -1);
assert_int_equal(parse_max_header_value_len("-1", &result), -1);
assert_int_equal(parse_max_header_value_len("10abc", &result), -1);
}
/* Test: strict numeric parsing invalid suffix for all int directives */
static void test_strict_numeric_invalid_suffix_all(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("10abc", &result), -1);
assert_int_equal(parse_interval("10abc", &result), -1);
assert_int_equal(parse_max_header_value_len("10abc", &result), -1);
}
/* Test: Default configuration values */
static void test_default_config_values(void **state)
{
(void)state;
assert_int_equal(DEFAULT_MAX_HEADERS, 10);
assert_int_equal(DEFAULT_MAX_HEADER_VALUE_LEN, 256);
assert_int_equal(DEFAULT_RECONNECT_INTERVAL, 10);
assert_int_equal(DEFAULT_ERROR_REPORT_INTERVAL, 10);
}
/* Test: Configuration validation - enabled requires socket */
static void test_config_validation_enabled_requires_socket(void **state)
{
int enabled = 1;
const char *socket = "/var/run/socket";
(void)state;
assert_true(enabled == 0 || socket != NULL);
socket = NULL;
assert_false(enabled == 0 || socket != NULL);
}
/* Test: Configuration validation - enabled with empty socket is invalid */
static void test_config_validation_enabled_with_empty_socket(void **state)
{
int enabled = 1;
const char *socket = parse_socket_path("");
(void)state;
assert_false(enabled == 0 || socket != NULL);
}
/* Test: Header value length validation */
static void test_header_value_len_validation(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_header_value_len("1", &result), 0);
assert_true(result >= 1);
assert_int_equal(parse_max_header_value_len("0", &result), -1);
}
/* Test: Large but valid values */
static void test_large_valid_values(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("1000000", &result), 0);
assert_int_equal(result, 1000000);
assert_int_equal(parse_interval("86400", &result), 0);
assert_int_equal(result, 86400);
}
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_parse_enabled_on),
cmocka_unit_test(test_parse_enabled_off),
cmocka_unit_test(test_parse_socket_path_valid),
cmocka_unit_test(test_parse_socket_path_empty),
cmocka_unit_test(test_parse_socket_path_null),
cmocka_unit_test(test_parse_socket_path_max_len_valid),
cmocka_unit_test(test_parse_socket_path_max_len_invalid),
cmocka_unit_test(test_parse_max_headers_valid),
cmocka_unit_test(test_parse_max_headers_invalid),
cmocka_unit_test(test_parse_reconnect_interval_valid),
cmocka_unit_test(test_parse_reconnect_interval_invalid),
cmocka_unit_test(test_parse_max_header_value_len_valid),
cmocka_unit_test(test_parse_max_header_value_len_invalid),
cmocka_unit_test(test_strict_numeric_invalid_suffix_all),
cmocka_unit_test(test_default_config_values),
cmocka_unit_test(test_config_validation_enabled_requires_socket),
cmocka_unit_test(test_config_validation_enabled_with_empty_socket),
cmocka_unit_test(test_header_value_len_validation),
cmocka_unit_test(test_large_valid_values),
};
return cmocka_run_group_tests(tests, NULL, NULL);
}

458
old/services/mod-reqin-log/tests/unit/test_h2_parsing.c Normal file
View File

@ -0,0 +1,458 @@
/*
* test_h2_parsing.c — Tests unitaires du fingerprinting HTTP/2 passif.
*
* Les fonctions testées (hpack_int_decode, h2_extract_pseudo_order,
* h2_parse_preface_buf) sont réimplimentées localement pour éviter les
* dépendances Apache/APR. La logique est identique à mod_reqin_log.c.
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include <string.h>
#include <stdio.h>
#include <stdint.h>
/* ====== Réimplémentation locale des fonctions H2 ====== */
static int hpack_int_decode(const unsigned char *buf, size_t len, int prefix,
size_t *pos, unsigned int *out)
{
unsigned int mask = (1u << prefix) - 1u;
unsigned int b, m;
if (*pos >= len) return 0;
*out = buf[(*pos)++] & mask;
if (*out < mask) return 1;
m = 0;
while (*pos < len) {
b = buf[(*pos)++];
*out += (b & 0x7fu) << m;
m += 7;
if (!(b & 0x80u)) return 1;
if (m > 28) return 0;
}
return 0;
}
static char h2_hpack_pseudo(unsigned int index)
{
switch (index) {
case 1: return 'a';
case 2: case 3: return 'm';
case 4: case 5: return 'p';
case 6: case 7: return 's';
default: return 0;
}
}
static void h2_extract_pseudo_order(const unsigned char *hpack, size_t len, char *out)
{
size_t pos = 0;
int out_pos = 0;
int first = 1;
while (pos < len && out_pos < 7) {
unsigned char byte = hpack[pos];
if (byte & 0x80u) {
unsigned int idx = 0;
if (!hpack_int_decode(hpack, len, 7, &pos, &idx)) break;
if (idx == 0) break;
char c = h2_hpack_pseudo(idx);
if (!c) break;
if (!first) out[out_pos++] = ',';
out[out_pos++] = c;
first = 0;
} else if ((byte & 0xe0u) == 0x20u) {
unsigned int sz = 0;
if (!hpack_int_decode(hpack, len, 5, &pos, &sz)) break;
} else {
break;
}
}
out[out_pos] = '\0';
}
/* Résultat de h2_parse_preface_buf — version allégée (pas d'APR) */
typedef struct {
char settings[256];
char wupdate[16];
char pseudo[16];
char fingerprint[512];
int has_priority;
int is_h2;
} h2_result_t;
static void h2_parse_preface_buf(const char *buf, size_t len, h2_result_t *res)
{
static const char H2_MAGIC[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n";
const size_t MAGIC_LEN = 24u;
const size_t FRAME_HDR = 9u;
memset(res, 0, sizeof(*res));
strcpy(res->wupdate, "0");
if (len < MAGIC_LEN || memcmp(buf, H2_MAGIC, MAGIC_LEN) != 0) return;
int settings_out = 0;
size_t pos = MAGIC_LEN;
while (pos + FRAME_HDR <= len) {
size_t frame_len = ((unsigned char)buf[pos] << 16)
| ((unsigned char)buf[pos+1] << 8)
| (unsigned char)buf[pos+2];
unsigned char type = (unsigned char)buf[pos+3];
unsigned char flags = (unsigned char)buf[pos+4];
uint32_t stream_id = (((unsigned char)buf[pos+5] & 0x7fu) << 24)
| ((unsigned char)buf[pos+6] << 16)
| ((unsigned char)buf[pos+7] << 8)
| (unsigned char)buf[pos+8];
pos += FRAME_HDR;
if (pos + frame_len > len) break;
if (type == 0x04u && stream_id == 0 && !(flags & 0x01u)) {
size_t sp = 0;
while (sp + 6 <= frame_len &&
settings_out < (int)sizeof(res->settings) - 24) {
uint16_t id = ((unsigned char)buf[pos + sp] << 8)
| (unsigned char)buf[pos + sp + 1];
uint32_t val = ((unsigned char)buf[pos + sp + 2] << 24)
| ((unsigned char)buf[pos + sp + 3] << 16)
| ((unsigned char)buf[pos + sp + 4] << 8)
| (unsigned char)buf[pos + sp + 5];
sp += 6;
if (settings_out > 0)
res->settings[settings_out++] = ',';
settings_out += snprintf(res->settings + settings_out,
(int)sizeof(res->settings) - settings_out,
"%u:%u", id, val);
}
} else if (type == 0x08u && stream_id == 0) {
if (frame_len >= 4) {
uint32_t inc = (((unsigned char)buf[pos] & 0x7fu) << 24)
| ((unsigned char)buf[pos+1] << 16)
| ((unsigned char)buf[pos+2] << 8)
| (unsigned char)buf[pos+3];
snprintf(res->wupdate, sizeof(res->wupdate), "%u", inc);
}
} else if (type == 0x01u && stream_id > 0) {
size_t hpack_start = 0;
int parse_ok = 1;
if ((flags & 0x08u) && parse_ok) {
if (hpack_start >= frame_len) {
parse_ok = 0;
} else {
unsigned char pad_len = (unsigned char)buf[pos + hpack_start++];
if (frame_len < hpack_start + (size_t)pad_len)
parse_ok = 0;
else
frame_len -= (size_t)pad_len;
}
}
if ((flags & 0x20u) && parse_ok) {
if (hpack_start + 5u > frame_len) {
parse_ok = 0;
} else {
hpack_start += 5u;
res->has_priority = 1;
}
}
if (parse_ok && hpack_start < frame_len) {
h2_extract_pseudo_order(
(const unsigned char *)(buf + pos + hpack_start),
frame_len - hpack_start,
res->pseudo
);
}
pos += frame_len;
break;
}
pos += frame_len;
}
if (res->settings[0] != '\0') {
res->is_h2 = 1;
snprintf(res->fingerprint, sizeof(res->fingerprint), "%s|%s|%d|%s",
res->settings, res->wupdate, res->has_priority, res->pseudo);
}
}
/* ====== Données de test : preface Chrome 120 ====== */
/*
* Preface HTTP/2 Chrome 120 (capturée) :
* Magic (24 octets)
* SETTINGS frame : HEADER_TABLE_SIZE=65536, ENABLE_PUSH=0,
* INITIAL_WINDOW_SIZE=6291456, MAX_HEADER_LIST_SIZE=262144
* WINDOW_UPDATE : incrément 15663105
* HEADERS stream 1 : :method GET, :authority, :scheme https, :path /
* → ordre HPACK indexé : 0x82(GET), 0x81(:auth), 0x87(https), 0x84(/)
*/
static const unsigned char CHROME_PREFACE[] = {
/* Magic */
'P','R','I',' ','*',' ','H','T','T','P','/','2','.','0','\r','\n',
'\r','\n','S','M','\r','\n','\r','\n',
/* SETTINGS frame : length=24, type=0x04, flags=0x00, stream=0 */
0x00, 0x00, 0x18, /* length = 24 = 4×6 */
0x04, /* type SETTINGS */
0x00, /* flags = 0 */
0x00, 0x00, 0x00, 0x00, /* stream 0 */
/* Entry 1: HEADER_TABLE_SIZE (1) = 65536 = 0x00010000 */
0x00, 0x01, 0x00, 0x01, 0x00, 0x00,
/* Entry 2: ENABLE_PUSH (2) = 0 */
0x00, 0x02, 0x00, 0x00, 0x00, 0x00,
/* Entry 3: INITIAL_WINDOW_SIZE (4) = 6291456 = 0x00600000 */
0x00, 0x04, 0x00, 0x60, 0x00, 0x00,
/* Entry 4: MAX_HEADER_LIST_SIZE (6) = 262144 = 0x00040000 */
0x00, 0x06, 0x00, 0x04, 0x00, 0x00,
/* WINDOW_UPDATE frame : length=4, type=0x08, flags=0, stream=0 */
0x00, 0x00, 0x04,
0x08,
0x00,
0x00, 0x00, 0x00, 0x00,
/* increment = 15663105 = 0x00EF0001 */
0x00, 0xEF, 0x00, 0x01,
/* HEADERS frame : length=14, type=0x01, flags=0x05 (END_STREAM|END_HEADERS), stream=1 */
0x00, 0x00, 0x0E,
0x01,
0x05,
0x00, 0x00, 0x00, 0x01,
/* HPACK : :method GET (0x82), :authority (0x81), :scheme https (0x87), :path / (0x84) */
/* → ordre Chrome : m,a,s,p */
0x82, 0x81, 0x87, 0x84,
/* + quelques headers supplémentaires (indices statiques) */
0x86, /* :scheme http (index 6, régulier → stop après pseudo) */
0x53, /* accept (sans valeur — littéral, arrête le scan) */
0x00, 0x05, 0x74, 0x65, 0x78, 0x74, 0x2F, 0x68, 0x74, 0x6D, 0x6C
};
/* ====== Données de test : preface Firefox 120 ====== */
/*
* Preface HTTP/2 Firefox 120 :
* SETTINGS: HEADER_TABLE_SIZE=65536, INITIAL_WINDOW_SIZE=131072, MAX_FRAME_SIZE=16384
* WINDOW_UPDATE: 12517377
* HEADERS: :method GET (0x82), :path / (0x84), :scheme https (0x87), :authority (0x81)
* → ordre Firefox : m,p,s,a
*/
static const unsigned char FIREFOX_PREFACE[] = {
/* Magic */
'P','R','I',' ','*',' ','H','T','T','P','/','2','.','0','\r','\n',
'\r','\n','S','M','\r','\n','\r','\n',
/* SETTINGS frame : length=18, type=0x04, flags=0x00, stream=0 */
0x00, 0x00, 0x12,
0x04,
0x00,
0x00, 0x00, 0x00, 0x00,
/* HEADER_TABLE_SIZE (1) = 65536 */
0x00, 0x01, 0x00, 0x01, 0x00, 0x00,
/* INITIAL_WINDOW_SIZE (4) = 131072 = 0x00020000 */
0x00, 0x04, 0x00, 0x02, 0x00, 0x00,
/* MAX_FRAME_SIZE (5) = 16384 = 0x00004000 */
0x00, 0x05, 0x00, 0x00, 0x40, 0x00,
/* WINDOW_UPDATE : increment = 12517377 = 0x00BF0001 */
0x00, 0x00, 0x04,
0x08,
0x00,
0x00, 0x00, 0x00, 0x00,
0x00, 0xBF, 0x00, 0x01,
/* HEADERS frame : length=4, type=0x01, flags=0x05, stream=1 */
0x00, 0x00, 0x04,
0x01,
0x05,
0x00, 0x00, 0x00, 0x01,
/* HPACK : :method GET (0x82), :path / (0x84), :scheme https (0x87), :authority (0x81) */
/* → ordre Firefox : m,p,s,a */
0x82, 0x84, 0x87, 0x81
};
/* ====== Données de test : flux HTTP/1.1 (ne doit pas matcher) ====== */
static const char HTTP1_DATA[] =
"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
/* ====== Tests ====== */
static void test_chrome_settings_parsed(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
assert_int_equal(res.is_h2, 1);
/* SETTINGS attendus : 1:65536,2:0,4:6291456,6:262144 */
assert_string_equal(res.settings, "1:65536,2:0,4:6291456,6:262144");
}
static void test_chrome_window_update(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
assert_string_equal(res.wupdate, "15663105");
}
static void test_chrome_pseudo_order(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
/* Chrome : :method(m), :authority(a), :scheme(s), :path(p) */
assert_string_equal(res.pseudo, "m,a,s,p");
}
static void test_chrome_fingerprint_akamai(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
assert_string_equal(res.fingerprint,
"1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p");
}
static void test_firefox_settings_parsed(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf((const char *)FIREFOX_PREFACE, sizeof(FIREFOX_PREFACE), &res);
assert_int_equal(res.is_h2, 1);
assert_string_equal(res.settings, "1:65536,4:131072,5:16384");
}
static void test_firefox_pseudo_order(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf((const char *)FIREFOX_PREFACE, sizeof(FIREFOX_PREFACE), &res);
/* Firefox : :method(m), :path(p), :scheme(s), :authority(a) */
assert_string_equal(res.pseudo, "m,p,s,a");
}
static void test_firefox_fingerprint_akamai(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf((const char *)FIREFOX_PREFACE, sizeof(FIREFOX_PREFACE), &res);
assert_string_equal(res.fingerprint,
"1:65536,4:131072,5:16384|12517377|0|m,p,s,a");
}
static void test_http1_not_detected(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf(HTTP1_DATA, strlen(HTTP1_DATA), &res);
assert_int_equal(res.is_h2, 0);
assert_string_equal(res.settings, "");
assert_string_equal(res.fingerprint, "");
}
static void test_empty_buffer_not_detected(void **state)
{
(void)state;
h2_result_t res;
h2_parse_preface_buf("", 0, &res);
assert_int_equal(res.is_h2, 0);
}
static void test_truncated_preface_no_crash(void **state)
{
(void)state;
h2_result_t res;
/* Magic complet mais frame tronquée */
h2_parse_preface_buf((const char *)CHROME_PREFACE, 30, &res);
assert_int_equal(res.is_h2, 0); /* SETTINGS incomplet → pas de fingerprint */
}
static void test_hpack_int_single_byte(void **state)
{
(void)state;
/* Entier 7-bit < 127 → encodé sur 1 octet */
unsigned char buf[] = { 0x82 }; /* 0x80 | 2 → index=2 */
size_t pos = 0;
unsigned int out = 0;
int ok = hpack_int_decode(buf, 1, 7, &pos, &out);
assert_int_equal(ok, 1);
assert_int_equal(out, 2);
assert_int_equal(pos, 1);
}
static void test_hpack_pseudo_table(void **state)
{
(void)state;
assert_int_equal(h2_hpack_pseudo(1), 'a');
assert_int_equal(h2_hpack_pseudo(2), 'm');
assert_int_equal(h2_hpack_pseudo(3), 'm');
assert_int_equal(h2_hpack_pseudo(4), 'p');
assert_int_equal(h2_hpack_pseudo(5), 'p');
assert_int_equal(h2_hpack_pseudo(6), 's');
assert_int_equal(h2_hpack_pseudo(7), 's');
assert_int_equal(h2_hpack_pseudo(8), 0); /* header régulier */
assert_int_equal(h2_hpack_pseudo(62), 0);
}
static void test_pseudo_order_extraction_direct(void **state)
{
(void)state;
/* HPACK block : :method(0x82), :path(0x84), :scheme(0x87), :authority(0x81) */
unsigned char hpack[] = { 0x82, 0x84, 0x87, 0x81 };
char out[16];
h2_extract_pseudo_order(hpack, sizeof(hpack), out);
assert_string_equal(out, "m,p,s,a");
}
static void test_pseudo_order_stops_at_regular_header(void **state)
{
(void)state;
/* :method(0x82), puis header régulier (0x88 = index 8) */
unsigned char hpack[] = { 0x82, 0x88 };
char out[16];
h2_extract_pseudo_order(hpack, sizeof(hpack), out);
assert_string_equal(out, "m");
}
/* ====== main ====== */
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_chrome_settings_parsed),
cmocka_unit_test(test_chrome_window_update),
cmocka_unit_test(test_chrome_pseudo_order),
cmocka_unit_test(test_chrome_fingerprint_akamai),
cmocka_unit_test(test_firefox_settings_parsed),
cmocka_unit_test(test_firefox_pseudo_order),
cmocka_unit_test(test_firefox_fingerprint_akamai),
cmocka_unit_test(test_http1_not_detected),
cmocka_unit_test(test_empty_buffer_not_detected),
cmocka_unit_test(test_truncated_preface_no_crash),
cmocka_unit_test(test_hpack_int_single_byte),
cmocka_unit_test(test_hpack_pseudo_table),
cmocka_unit_test(test_pseudo_order_extraction_direct),
cmocka_unit_test(test_pseudo_order_stops_at_regular_header),
};
return cmocka_run_group_tests(tests, NULL, NULL);
}

226
old/services/mod-reqin-log/tests/unit/test_header_handling.c Normal file
View File

@ -0,0 +1,226 @@
/*
* test_header_handling.c - Unit tests for header handling (truncation and limits)
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include <string.h>
#include <stdio.h>
#include <apr_strings.h>
#include <apr_tables.h>
#include <apr_pools.h>
#include <apr_general.h>
/* Mock header truncation function */
static char *truncate_header_value(apr_pool_t *pool, const char *value, int max_len)
{
if (value == NULL) {
return NULL;
}
size_t len = strlen(value);
if ((int)len > max_len) {
return apr_pstrmemdup(pool, value, max_len);
}
return apr_pstrdup(pool, value);
}
/* Mock header matching function */
static int header_name_matches(const char *configured, const char *actual)
{
return strcasecmp(configured, actual) == 0;
}
/* Test: Header value within limit */
static void test_header_truncation_within_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "short value";
char *result = truncate_header_value(pool, value, 256);
assert_string_equal(result, "short value");
apr_pool_destroy(pool);
}
/* Test: Header value exactly at limit */
static void test_header_truncation_exact_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "exactly10c";
char *result = truncate_header_value(pool, value, 10);
assert_string_equal(result, "exactly10c");
apr_pool_destroy(pool);
}
/* Test: Header value exceeds limit */
static void test_header_truncation_exceeds_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "this is a very long header value that should be truncated";
char *result = truncate_header_value(pool, value, 15);
assert_string_equal(result, "this is a very ");
assert_int_equal(strlen(result), 15);
apr_pool_destroy(pool);
}
/* Test: Header value with limit of 1 */
static void test_header_truncation_limit_one(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "abc";
char *result = truncate_header_value(pool, value, 1);
assert_string_equal(result, "a");
apr_pool_destroy(pool);
}
/* Test: NULL header value */
static void test_header_truncation_null(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
char *result = truncate_header_value(pool, NULL, 256);
assert_null(result);
apr_pool_destroy(pool);
}
/* Test: Empty header value */
static void test_header_truncation_empty(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "";
char *result = truncate_header_value(pool, value, 256);
assert_string_equal(result, "");
apr_pool_destroy(pool);
}
/* Test: Header name matching (case-insensitive) */
static void test_header_name_matching_case_insensitive(void **state)
{
assert_true(header_name_matches("X-Request-Id", "x-request-id"));
assert_true(header_name_matches("user-agent", "User-Agent"));
assert_true(header_name_matches("HOST", "host"));
}
/* Test: Header name matching (different headers) */
static void test_header_name_matching_different(void **state)
{
assert_false(header_name_matches("X-Request-Id", "X-Trace-Id"));
assert_false(header_name_matches("Host", "User-Agent"));
}
/* Test: Multiple headers with limit */
static void test_header_count_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
/* Simulate configured headers */
const char *configured[] = {"X-Request-Id", "X-Trace-Id", "User-Agent", "Referer"};
int configured_count = 4;
int max_headers = 2;
/* Simulate present headers */
const char *present[] = {"X-Request-Id", "User-Agent", "Referer"};
int present_count = 3;
int logged_count = 0;
for (int i = 0; i < configured_count && logged_count < max_headers; i++) {
for (int j = 0; j < present_count; j++) {
if (header_name_matches(configured[i], present[j])) {
logged_count++;
break;
}
}
}
assert_int_equal(logged_count, 2);
apr_pool_destroy(pool);
}
/* Test: Header value with special JSON characters */
static void test_header_value_json_special(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "test\"value\\with\tspecial";
char *truncated = truncate_header_value(pool, value, 256);
/* Truncation should preserve the value */
assert_string_equal(truncated, "test\"value\\with\tspecial");
apr_pool_destroy(pool);
}
/* Test: Unicode in header value (UTF-8) */
static void test_header_value_unicode(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "Mozilla/5.0 (compatible; 日本語)";
char *result = truncate_header_value(pool, value, 50);
/* Should be truncated but valid */
assert_non_null(result);
assert_true(strlen(result) <= 50);
apr_pool_destroy(pool);
}
static int group_setup(void **state)
{
(void)state;
return apr_initialize();
}
static int group_teardown(void **state)
{
(void)state;
apr_terminate();
return 0;
}
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_header_truncation_within_limit),
cmocka_unit_test(test_header_truncation_exact_limit),
cmocka_unit_test(test_header_truncation_exceeds_limit),
cmocka_unit_test(test_header_truncation_limit_one),
cmocka_unit_test(test_header_truncation_null),
cmocka_unit_test(test_header_truncation_empty),
cmocka_unit_test(test_header_name_matching_case_insensitive),
cmocka_unit_test(test_header_name_matching_different),
cmocka_unit_test(test_header_count_limit),
cmocka_unit_test(test_header_value_json_special),
cmocka_unit_test(test_header_value_unicode),
};
return cmocka_run_group_tests(tests, group_setup, group_teardown);
}

266
old/services/mod-reqin-log/tests/unit/test_json_serialization.c Normal file
View File

@ -0,0 +1,266 @@
/*
* test_json_serialization.c - Unit tests for JSON serialization
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include <string.h>
#include <stdio.h>
#include <apr_pools.h>
#include <apr_strings.h>
#include <apr_time.h>
#include <apr_lib.h>
#include <apr_general.h>
typedef struct {
char *data;
size_t len;
size_t cap;
apr_pool_t *pool;
} testbuf_t;
static void testbuf_init(testbuf_t *buf, apr_pool_t *pool, size_t initial_capacity)
{
buf->pool = pool;
buf->cap = initial_capacity;
buf->len = 0;
buf->data = apr_palloc(pool, initial_capacity);
buf->data[0] = '\0';
}
static void testbuf_append(testbuf_t *buf, const char *str, size_t len)
{
if (str == NULL) {
return;
}
if (len == (size_t)-1) {
len = strlen(str);
}
if (buf->len + len + 1 > buf->cap) {
size_t new_cap = (buf->len + len + 1) * 2;
char *new_data = apr_palloc(buf->pool, new_cap);
memcpy(new_data, buf->data, buf->len + 1); /* Copy including null terminator */
buf->data = new_data;
buf->cap = new_cap;
}
memcpy(buf->data + buf->len, str, len);
buf->len += len;
buf->data[buf->len] = '\0';
}
static void testbuf_append_char(testbuf_t *buf, char c)
{
if (buf->len + 2 > buf->cap) {
size_t new_cap = (buf->cap * 2);
char *new_data = apr_palloc(buf->pool, new_cap);
memcpy(new_data, buf->data, buf->len + 1);
buf->data = new_data;
buf->cap = new_cap;
}
buf->data[buf->len++] = c;
buf->data[buf->len] = '\0';
}
/* Mock JSON string escaping function for testing */
static void append_json_string(testbuf_t *buf, const char *str)
{
if (str == NULL) {
return;
}
for (const char *p = str; *p; p++) {
char c = *p;
switch (c) {
case '"': testbuf_append(buf, "\\\"", 2); break;
case '\\': testbuf_append(buf, "\\\\", 2); break;
case '\b': testbuf_append(buf, "\\b", 2); break;
case '\f': testbuf_append(buf, "\\f", 2); break;
case '\n': testbuf_append(buf, "\\n", 2); break;
case '\r': testbuf_append(buf, "\\r", 2); break;
case '\t': testbuf_append(buf, "\\t", 2); break;
default:
if ((unsigned char)c < 0x20) {
char unicode[8];
apr_snprintf(unicode, sizeof(unicode), "\\u%04x", (unsigned char)c);
testbuf_append(buf, unicode, (size_t)-1);
} else {
testbuf_append_char(buf, c);
}
break;
}
}
}
/* Test: Empty string */
static void test_json_escape_empty_string(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "");
assert_string_equal(buf.data, "");
apr_pool_destroy(pool);
}
/* Test: Simple string without special characters */
static void test_json_escape_simple_string(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "hello world");
assert_string_equal(buf.data, "hello world");
apr_pool_destroy(pool);
}
/* Test: String with double quotes */
static void test_json_escape_quotes(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "hello \"world\"");
assert_string_equal(buf.data, "hello \\\"world\\\"");
apr_pool_destroy(pool);
}
/* Test: String with backslashes */
static void test_json_escape_backslashes(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "path\\to\\file");
assert_string_equal(buf.data, "path\\\\to\\\\file");
apr_pool_destroy(pool);
}
/* Test: String with newlines and tabs */
static void test_json_escape_newlines_tabs(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "line1\nline2\ttab");
assert_string_equal(buf.data, "line1\\nline2\\ttab");
apr_pool_destroy(pool);
}
/* Test: String with control characters */
static void test_json_escape_control_chars(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
/* Test with bell character (0x07) - use octal literal */
append_json_string(&buf, "test\007bell");
/* Should contain unicode escape for bell (0x07) */
assert_true(strstr(buf.data, "\\u0007") != NULL);
apr_pool_destroy(pool);
}
/* Test: NULL string */
static void test_json_escape_null_string(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, NULL);
assert_string_equal(buf.data, "");
apr_pool_destroy(pool);
}
/* Test: Complex user agent string */
static void test_json_escape_user_agent(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
const char *ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \"Test\"";
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 512);
append_json_string(&buf, ua);
assert_true(strstr(buf.data, "\\\"Test\\\"") != NULL);
apr_pool_destroy(pool);
}
static int group_setup(void **state)
{
(void)state;
return apr_initialize();
}
static int group_teardown(void **state)
{
(void)state;
apr_terminate();
return 0;
}
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_json_escape_empty_string),
cmocka_unit_test(test_json_escape_simple_string),
cmocka_unit_test(test_json_escape_quotes),
cmocka_unit_test(test_json_escape_backslashes),
cmocka_unit_test(test_json_escape_newlines_tabs),
cmocka_unit_test(test_json_escape_control_chars),
cmocka_unit_test(test_json_escape_null_string),
cmocka_unit_test(test_json_escape_user_agent),
};
return cmocka_run_group_tests(tests, group_setup, group_teardown);
}

1141
old/services/mod-reqin-log/tests/unit/test_module_real.c Normal file
View File

File diff suppressed because it is too large Load Diff

37
old/services/sentinel/.dockerignore Normal file
View File

@ -0,0 +1,37 @@
# Git files
.git
.gitignore
.gitattributes
# Qwen
.qwen
.qwenignore
# Build artifacts
dist/
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test results
test-results/
coverage.out
coverage.html
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
# Temporary files
tmp/
temp/
*.tmp
*.bak
# Docker compose override
docker-compose.override.yml

7
old/services/sentinel/.env.example Normal file
View File

@ -0,0 +1,7 @@
# sentinel configuration — DO NOT COMMIT real values
# Copy to .env and fill in for local development
JA4SENTINEL_INTERFACE=eth0
JA4SENTINEL_PORTS=443,8443
JA4SENTINEL_BPF_FILTER=
JA4SENTINEL_FLOW_TIMEOUT=30
JA4SENTINEL_PACKET_BUFFER_SIZE=1000

125
old/services/sentinel/.github/workflows/build-rpm.yml vendored Normal file
View File

@ -0,0 +1,125 @@
name: Build RPM Package
on:
push:
tags:
- 'v*'
branches:
- main
- master
paths:
- 'go/**'
- 'cmd/**'
- 'internal/**'
- 'api/**'
- 'packaging/**'
- 'Makefile'
- 'go.mod'
- 'go.sum'
- 'Dockerfile.package'
pull_request:
branches:
- main
- master
paths:
- 'go/**'
- 'cmd/**'
- 'internal/**'
- 'api/**'
- 'packaging/**'
- 'Makefile'
- 'go.mod'
- 'go.sum'
- 'Dockerfile.package'
workflow_dispatch:
inputs:
version:
description: 'Version to build (e.g., 1.0.0)'
required: false
default: '1.0.0-dev'
env:
GO_VERSION: '1.24'
PACKAGE_NAME: ja4sentinel
jobs:
build-rpm:
name: Build RPM Packages (CentOS 7, Rocky 8/9/10)
runs-on: ubuntu-latest
permissions:
contents: write
packages: write
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Determine version
id: version
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
VERSION="${{ github.event.inputs.version }}"
elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then
VERSION="${{ github.ref_name#v }}"
else
VERSION="0.0.0-$(git rev-parse --short HEAD)"
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "Building version: ${VERSION}"
- name: Build RPM packages in Docker
run: |
docker build --no-cache \
-t ${PACKAGE_NAME}-packager \
--build-arg VERSION="${{ steps.version.outputs.version }}" \
-f Dockerfile.package .
# Extract RPM packages from image
mkdir -p build/rpm/el8 build/rpm/el9 build/rpm/el10
docker run --rm -v $(pwd)/build:/output ${PACKAGE_NAME}-packager sh -c \
'cp -r /packages/rpm/el8 /output/rpm/ && \
cp -r /packages/rpm/el9 /output/rpm/ && \
cp -r /packages/rpm/el10 /output/rpm/'
- name: List build artifacts
run: |
echo "=== Build Artifacts ==="
echo "Rocky Linux 8 (el8):"
ls -lah build/rpm/el8/ || echo " (no packages)"
echo "Rocky Linux 9 (el9):"
ls -lah build/rpm/el9/ || echo " (no packages)"
echo "AlmaLinux/Rocky 10 (el10):"
ls -lah build/rpm/el10/ || echo " (no packages)"
# Generate checksums
find build/rpm -name "*.rpm" -exec sha256sum {} \; > build/rpm/checksums.txt
cat build/rpm/checksums.txt
- name: Upload RPM artifacts
uses: actions/upload-artifact@v4
with:
name: ${PACKAGE_NAME}-rpm-x86_64
path: build/rpm/**/*.rpm
retention-days: 30
- name: Upload checksum artifact
uses: actions/upload-artifact@v4
with:
name: ${PACKAGE_NAME}-rpm-checksums
path: build/rpm/checksums.txt
retention-days: 30
- name: Create release and upload assets (on tag)
if: startsWith(github.ref, 'refs/tags/v')
uses: softprops/action-gh-release@v2
with:
files: |
build/rpm/el8/*.rpm
build/rpm/el9/*.rpm
build/rpm/el10/*.rpm
generate_release_notes: true
make_latest: true
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

59
old/services/sentinel/.gitignore vendored Normal file
View File

@ -0,0 +1,59 @@
# AIDER
.aider*
.qwen/
.qwenignore
# Build artifacts
dist/
build/
*.exe
*.exe~
*.dll
*.so
*.dylib
*.o
# Go
*.test
*.out
coverage.out
coverage.html
go.work
go.work.sum
# Docker
*.dockerfile.local
docker-compose.override.yml
# IDE
.idea/
.vscode/
*.swp
*.swo
*~
# Temporary files
tmp/
temp/
*.tmp
*.bak
# Local config (copie de config.yml.example)
config.yml
# Runtime artifacts
*.pid
*.sock
# Integration test artifacts
test-results/
# Test artifacts
packaging/test/*.rpm
# Build artifacts
packages/
# Binary (root level only)
/ja4sentinel
ja4sentinel-linux-amd64

37
old/services/sentinel/Dockerfile Normal file
View File

@ -0,0 +1,37 @@
FROM golang:1.24-alpine AS builder
RUN apk add --no-cache git make libpcap-dev gcc musl-dev linux-headers
WORKDIR /build
# Copy workspace and shared module first (better caching)
COPY go.work go.work.sum* ./
COPY shared/go/ja4common/ ./shared/go/ja4common/
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
WORKDIR /build/services/sentinel
RUN go mod download || true
COPY services/sentinel/ /build/services/sentinel/
ARG VERSION=dev
ARG BUILD_TIME=unknown
ARG GIT_COMMIT=unknown
RUN mkdir -p dist && \
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \
CGO_LDFLAGS="-Wl,-Bstatic -lpcap -Wl,-Bdynamic" \
go build -buildvcs=false \
-ldflags "-X main.Version=${VERSION} -X main.BuildTime=${BUILD_TIME} -X main.GitCommit=${GIT_COMMIT}" \
-o dist/sentinel ./cmd/ja4sentinel
FROM alpine:latest
RUN apk add --no-cache ca-certificates
RUN addgroup -S sentinel && adduser -S sentinel -G sentinel
RUN mkdir -p /var/lib/sentinel /var/run /etc/sentinel /var/log/sentinel
COPY --from=builder /build/services/sentinel/dist/sentinel /usr/local/bin/sentinel
RUN chown -R sentinel:sentinel /var/lib/sentinel /var/log/sentinel
USER sentinel
WORKDIR /var/lib/sentinel
ENTRYPOINT ["/usr/local/bin/sentinel"]

25
old/services/sentinel/Dockerfile.dev Normal file
View File

@ -0,0 +1,25 @@
# Development and test image for sentinel (was ja4sentinel)
# Build context: monorepo root (ja4-platform/)
# Usage: docker build -f services/sentinel/Dockerfile.dev -t sentinel-dev .
FROM golang:1.24-alpine
RUN apk add --no-cache git make libpcap-dev gcc musl-dev linux-headers
WORKDIR /build
# Copy Go workspace and shared module first for better layer caching
COPY go.work go.work.sum* ./
COPY shared/go/ja4common/ ./shared/go/ja4common/
# Copy service module descriptor then download deps
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
WORKDIR /build/services/sentinel
RUN go mod download || true
# Copy full service source
COPY services/sentinel/ /build/services/sentinel/
# Default: run tests with race detector
CMD ["go", "test", "-race", "-v", "./..."]

109
old/services/sentinel/Dockerfile.package Normal file
View File

@ -0,0 +1,109 @@
# syntax=docker/dockerfile:1
# =============================================================================
# sentinel — Dockerfile de packaging RPM (Rocky Linux 8/9, AlmaLinux 10)
# Build context: monorepo root (ja4-platform/)
# Méthode: 1 builder Go → 1 rpm-builder (rpmbuild, 3 × dist) → 1 output alpine
# =============================================================================
# =============================================================================
# Stage 1: Builder — compilation du binaire Go sur Rocky Linux 9
# Rocky Linux 9 comme base builder assure la compatibilité binaire sur toutes
# les distros cibles (el8/el9/el10 sont ABI-compatibles pour les libs system).
# =============================================================================
FROM rockylinux:9 AS builder
WORKDIR /build
RUN dnf install -y epel-release && \
dnf config-manager --set-enabled crb && \
dnf install -y golang git libpcap-devel gcc make && \
dnf clean all
# Copie du workspace Go et du module partagé en premier (meilleur cache)
COPY go.work go.work.sum* ./
COPY shared/go/ja4common/ ./shared/go/ja4common/
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
WORKDIR /build/services/sentinel
RUN go mod download || true
COPY services/sentinel/ /build/services/sentinel/
ARG VERSION=dev
ARG BUILD_TIME=""
ARG GIT_COMMIT=""
RUN mkdir -p dist && \
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \
go build -buildvcs=false \
-ldflags "-X main.Version=${VERSION} -X main.BuildTime=${BUILD_TIME} -X main.GitCommit=${GIT_COMMIT}" \
-o dist/sentinel \
./cmd/ja4sentinel
# =============================================================================
# Stage 2: rpm-builder — construction des RPMs avec rpmbuild
# Un seul stage, trois appels rpmbuild successifs (el8, el9, el10).
# =============================================================================
FROM rockylinux:9 AS rpm-builder
WORKDIR /package
ARG VERSION=dev
RUN dnf install -y rpm-build rpmdevtools systemd-rpm-macros && dnf clean all
RUN mkdir -p /root/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} && \
mkdir -p /root/rpmbuild/SOURCES/logrotate && \
mkdir -p /packages/rpm/{el8,el9,el10}
# Spec et fichiers sources
COPY services/sentinel/packaging/rpm/ja4sentinel.spec /root/rpmbuild/SPECS/ja4sentinel.spec
COPY --from=builder /build/services/sentinel/dist/sentinel /root/rpmbuild/SOURCES/ja4sentinel
COPY services/sentinel/packaging/systemd/ja4sentinel.service /root/rpmbuild/SOURCES/ja4sentinel.service
COPY services/sentinel/packaging/logrotate/ja4sentinel /root/rpmbuild/SOURCES/logrotate/ja4sentinel
COPY services/sentinel/config.yml.example /root/rpmbuild/SOURCES/config.yml
RUN chmod 755 /root/rpmbuild/SOURCES/ja4sentinel && \
chmod 644 /root/rpmbuild/SOURCES/ja4sentinel.service && \
chmod 644 /root/rpmbuild/SOURCES/logrotate/ja4sentinel && \
chmod 640 /root/rpmbuild/SOURCES/config.yml
# el8
RUN rpmbuild --define "_topdir /root/rpmbuild" \
--define "dist .el8" \
--define "build_version ${VERSION}" \
--target x86_64 \
-bb /root/rpmbuild/SPECS/ja4sentinel.spec && \
cp /root/rpmbuild/RPMS/x86_64/*.el8.x86_64.rpm /packages/rpm/el8/
# el9
RUN rpmbuild --define "_topdir /root/rpmbuild" \
--define "dist .el9" \
--define "build_version ${VERSION}" \
--target x86_64 \
-bb /root/rpmbuild/SPECS/ja4sentinel.spec && \
cp /root/rpmbuild/RPMS/x86_64/*.el9.x86_64.rpm /packages/rpm/el9/
# el10
RUN rpmbuild --define "_topdir /root/rpmbuild" \
--define "dist .el10" \
--define "build_version ${VERSION}" \
--target x86_64 \
-bb /root/rpmbuild/SPECS/ja4sentinel.spec && \
cp /root/rpmbuild/RPMS/x86_64/*.el10.x86_64.rpm /packages/rpm/el10/
# =============================================================================
# Stage 3: output — image finale contenant uniquement les RPMs
# =============================================================================
FROM alpine:latest AS output
WORKDIR /packages
COPY --from=rpm-builder /packages/rpm/el8/*.rpm /packages/rpm/el8/
COPY --from=rpm-builder /packages/rpm/el9/*.rpm /packages/rpm/el9/
COPY --from=rpm-builder /packages/rpm/el10/*.rpm /packages/rpm/el10/
CMD ["sh", "-c", \
"echo '=== RPM el8 ===' && ls -la /packages/rpm/el8/ && \
echo '' && echo '=== RPM el9 ===' && ls -la /packages/rpm/el9/ && \
echo '' && echo '=== RPM el10 ===' && ls -la /packages/rpm/el10/"]

105
old/services/sentinel/Dockerfile.test-server Normal file
View File

@ -0,0 +1,105 @@
# Test server for generating TLS traffic in integration tests
FROM golang:1.23-alpine
WORKDIR /app
# Create a simple TLS server for testing
RUN cat > main.go << 'EOF'
package main
import (
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
"flag"
"fmt"
"log"
"math/big"
"net"
"net/http"
"time"
)
func main() {
port := flag.String("port", "8443", "Port to listen on")
flag.Parse()
// Generate self-signed certificate
cert, err := generateSelfSignedCert()
if err != nil {
log.Fatalf("Failed to generate certificate: %v", err)
}
config := &tls.Config{
Certificates: []tls.Certificate{cert},
MinVersion: tls.VersionTLS12,
}
listener, err := tls.Listen("tcp", ":"+*port, config)
if err != nil {
log.Fatalf("Failed to start TLS listener: %v", err)
}
defer listener.Close()
log.Printf("TLS test server listening on port %s", *port)
http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte("Hello from TLS test server"))
}))
}
func generateSelfSignedCert() (tls.Certificate, error) {
// Generate private key
priv, err := rsa.GenerateKey(rand.Reader, 2048)
if err != nil {
return tls.Certificate{}, err
}
// Create certificate template
template := x509.Certificate{
SerialNumber: big.NewInt(1),
Subject: pkix.Name{
Organization: []string{"JA4Sentinel Test"},
CommonName: "localhost",
},
NotBefore: time.Now(),
NotAfter: time.Now().Add(24 * time.Hour),
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
BasicConstraintsValid: true,
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
DNSNames: []string{"localhost"},
}
// Create certificate
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
if err != nil {
return tls.Certificate{}, err
}
// Encode certificate
certPEM := pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: certDER,
})
// Encode private key
keyPEM := pem.EncodeToMemory(&pem.Block{
Type: "RSA PRIVATE KEY",
Bytes: x509.MarshalPKCS1PrivateKey(priv),
})
// Load certificate
return tls.X509KeyPair(certPEM, keyPEM)
}
EOF
RUN go mod init test-server && go mod tidy
EXPOSE 8443
CMD ["go", "run", "main.go", "-port", "8443"]

158
old/services/sentinel/Makefile Normal file
View File

@ -0,0 +1,158 @@
.PHONY: build build-docker test test-docker test-integration lint clean help docker-build-dev docker-build-runtime package package-rpm
# Docker parameters
DOCKER=docker
DOCKER_BUILD=$(DOCKER) build
DOCKER_RUN=$(DOCKER) run
DOCKER_COMPOSE=docker compose
# Image names
DEV_IMAGE=ja4sentinel-dev:latest
RUNTIME_IMAGE=ja4sentinel-runtime:latest
TEST_SERVER_IMAGE=ja4sentinel-test-server:latest
# Binary name
BINARY_NAME=ja4sentinel
BINARY_PATH=./cmd/ja4sentinel
DIST_DIR=dist
BUILD_DIR=build
# RPM build directory
RPM_DIR=$(DIST_DIR)/rpm
# Package version (extract default from spec file, can be overridden)
PKG_VERSION ?= $(shell grep '^%define spec_version' packaging/rpm/ja4sentinel.spec | tail -1 | awk '{print $$3}')
# Build flags
VERSION=$(PKG_VERSION)
BUILD_TIME=$(shell date -u '+%Y-%m-%d_%H:%M:%S')
GIT_COMMIT=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
LDFLAGS=-ldflags "-X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT)"
# Default target
all: docker-build-dev test-docker
## build: Build the ja4sentinel binary locally
build:
mkdir -p $(DIST_DIR)
go build -buildvcs=false $(LDFLAGS) -o $(DIST_DIR)/$(BINARY_NAME) $(BINARY_PATH)
## build-linux: Build for Linux (amd64)
build-linux:
mkdir -p $(DIST_DIR)
GOOS=linux GOARCH=amd64 go build -buildvcs=false $(LDFLAGS) -o $(DIST_DIR)/$(BINARY_NAME)-linux-amd64 $(BINARY_PATH)
## docker-build-dev: Build the development Docker image
docker-build-dev:
$(DOCKER_BUILD) -t $(DEV_IMAGE) -f Dockerfile.dev .
## docker-build-runtime: Build the runtime Docker image (multi-stage build)
docker-build-runtime:
$(DOCKER_BUILD) -t $(RUNTIME_IMAGE) -f Dockerfile .
## docker-build-test-server: Build the test server image
docker-build-test-server:
$(DOCKER_BUILD) -t $(TEST_SERVER_IMAGE) -f Dockerfile.test-server .
## test: Run unit tests locally
test:
go test -v ./...
## test-docker: Run unit tests inside Docker container
test-docker: docker-build-dev
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) go test -v ./...
## test-race: Run tests with race detector in Docker
test-race: docker-build-dev
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) go test -race -v ./...
## test-coverage: Run tests with coverage report in Docker
test-coverage: docker-build-dev
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) sh -c \
"go test -v -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html"
## test-integration: Run integration tests in Docker
test-integration: docker-build-dev docker-build-test-server
$(DOCKER_COMPOSE) -f docker-compose.test.yml build --no-cache
$(DOCKER_COMPOSE) -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from ja4sentinel-test
## test-integration-clean: Run integration tests and clean up afterward
test-integration-clean: docker-build-dev docker-build-test-server
$(DOCKER_COMPOSE) -f docker-compose.test.yml build --no-cache
$(DOCKER_COMPOSE) -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from ja4sentinel-test
$(DOCKER_COMPOSE) -f docker-compose.test.yml down -v
## lint: Run linters in Docker
lint: docker-build-dev
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) sh -c \
"go vet ./... && echo 'Running gofmt check...' && gofmt -l . | grep -v '^vendor/' | grep -v '^path/' || true"
## fmt: Format all Go files
fmt:
gofmt -w .
## package: Build RPM packages for all target distributions
package: package-rpm
## package-rpm: Build RPM packages for Rocky Linux 8/9/10, AlmaLinux (requires Docker)
package-rpm:
mkdir -p $(RPM_DIR)/el8 $(RPM_DIR)/el9 $(RPM_DIR)/el10
@echo "Building RPM packages for Rocky Linux 8/9, AlmaLinux 10..."
docker build --target output -t ja4sentinel-rpm-packager:latest \
--build-arg VERSION=$(PKG_VERSION) \
-f Dockerfile.package .
@echo "Extracting RPM packages from Docker image..."
@docker run --rm -v $(PWD)/$(RPM_DIR):/output/rpm ja4sentinel-rpm-packager:latest sh -c \
'cp -r /packages/rpm/el8 /output/rpm/ && \
cp -r /packages/rpm/el9 /output/rpm/ && \
cp -r /packages/rpm/el10 /output/rpm/'
@echo "RPM packages created:"
@echo " Rocky Linux 8 (el8):"
ls -la $(RPM_DIR)/el8/ 2>/dev/null || echo " (no packages)"
@echo " Rocky Linux 9 (el9):"
ls -la $(RPM_DIR)/el9/ 2>/dev/null || echo " (no packages)"
@echo " AlmaLinux/Rocky 10 (el10):"
ls -la $(RPM_DIR)/el10/ 2>/dev/null || echo " (no packages)"
## test-package-rpm: Test RPM package installation in Docker
test-package-rpm: package-rpm
./packaging/test/test-rpm.sh
## test-package: Test RPM package installation
test-package: test-package-rpm
## ci: Full CI pipeline (tests, build, packages, package tests)
ci: ci-test ci-build ci-package ci-package-test
## ci-test: Run all tests for CI
ci-test: test lint
## ci-build: Build for CI (production binary)
ci-build: build-linux
## ci-package: Build all packages for CI
ci-package: package
## ci-package-test: Test all packages for CI
ci-package-test: test-package
## clean: Clean build artifacts and Docker images
clean:
rm -rf $(DIST_DIR)/
rm -rf $(BUILD_DIR)/
rm -f coverage.out coverage.html
$(DOCKER) rmi $(DEV_IMAGE) 2>/dev/null || true
$(DOCKER) rmi $(RUNTIME_IMAGE) 2>/dev/null || true
$(DOCKER) rmi $(TEST_SERVER_IMAGE) 2>/dev/null || true
## clean-all: Clean everything including containers and volumes
clean-all: clean
$(DOCKER_COMPOSE) -f docker-compose.test.yml down -v --remove-orphans
## help: Show this help message
help:
@echo "Usage: make [target]"
@echo ""
@echo "Targets:"
@sed -n 's/^##//p' $(MAKEFILE_LIST) | column -t -s ':' | sed -e 's/^/ /'

291
old/services/sentinel/README.md Normal file
View File

@ -0,0 +1,291 @@
# JA4Sentinel
Outil Go pour capturer le trafic réseau sur un serveur Linux, extraire les handshakes TLS côté client, générer les signatures JA4, enrichir avec des métadonnées IP/TCP, et loguer les résultats vers une ou plusieurs sorties configurables.
## Fonctionnalités
- **Capture réseau** : Écoute sur une interface réseau avec filtres BPF configurables
- **Parsing TLS** : Extraction des ClientHello TLS depuis les flux TCP
- **Fingerprinting** : Génération des empreintes JA4 et JA3 pour chaque client
- **Métadonnées** : Enrichissement avec IPMeta (TTL, IP ID, DF) et TCPMeta (window, MSS, options)
- **Sorties multiples** : stdout, fichier JSON, socket UNIX (combinables via MultiWriter)
- **Logging structuré** : Logs JSON sur stdout/stderr pour intégration avec systemd/journald
## Architecture
```
┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐
│ Capture │ ──▶ │ TLSParse │ ──▶ │ Fingerprint │ ──▶ │ Output │
│ (pcap) │ │ (ClientHello)│ │ (JA4) │ │ (JSON logs) │
└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘
│ │ │ │
▼ ▼ ▼ ▼
api.RawPacket api.TLSClientHello api.Fingerprints api.LogRecord
```
### Modules
| Module | Responsabilités |
|--------|-----------------|
| `config` | Chargement et validation de la configuration (YAML, env, CLI) |
| `capture` | Capture des paquets réseau via libpcap |
| `tlsparse` | Extraction des ClientHello TLS avec suivi d'état de flux |
| `fingerprint` | Génération JA4/JA3 via `psanford/tlsfingerprint` |
| `output` | Écriture des logs vers stdout, fichier, socket UNIX |
| `logging` | Logs structurés JSON pour le diagnostic du service |
## Installation
### Prérequis
- Go 1.24+
- libpcap-dev (pour la compilation)
- Docker (pour les tests et le déploiement)
### Note sur libpcap
**Le binaire est compilé sur Rocky Linux 9** pour une compatibilité maximale avec toutes les distributions RHEL/Rocky/AlmaLinux.
libpcap est requis à l'exécution et sera installé automatiquement par le gestionnaire de packages.
### Packages système
#### Rocky Linux / RHEL / AlmaLinux (.rpm)
```bash
# Télécharger le package
wget https://github.com/your-repo/ja4sentinel/releases/latest/download/ja4sentinel.rpm
# Installer
sudo dnf install ./ja4sentinel.rpm
# Activer le service
sudo systemctl enable ja4sentinel
sudo systemctl start ja4sentinel
# Vérifier le statut
sudo systemctl status ja4sentinel
```
#### Distributions supportées
- Rocky Linux 8, 9, 10
- AlmaLinux 8, 9, 10
- RHEL 8, 9, 10
## Configuration
### Fichier de configuration (YAML)
```yaml
core:
interface: eth0
listen_ports: [443, 8443]
bpf_filter: ""
flow_timeout_sec: 30
outputs:
- type: stdout
enabled: true
- type: file
enabled: true
params:
path: /var/log/ja4sentinel/ja4.log
- type: unix_socket
enabled: true
params:
socket_path: /var/run/logcorrelator/network.socket
```
### Variables d'environnement
| Variable | Description |
|----------|-------------|
| `JA4SENTINEL_INTERFACE` | Interface réseau (ex: `eth0`) |
| `JA4SENTINEL_PORTS` | Ports à surveiller (ex: `443,8443`) |
| `JA4SENTINEL_BPF_FILTER` | Filtre BPF personnalisé |
| `JA4SENTINEL_FLOW_TIMEOUT` | Timeout de flux en secondes (défaut: 30) |
### Ligne de commande
```bash
ja4sentinel --config /etc/ja4sentinel/config.yml
ja4sentinel --version
```
## Format des logs
### Logs de service (stdout/stderr)
```json
{
"timestamp": 1708876543210000000,
"level": "INFO",
"component": "capture",
"message": "Starting packet capture",
"interface": "eth0"
}
```
### Logs métier (JA4)
```json
{
"src_ip": "192.168.1.100",
"src_port": 54321,
"dst_ip": "10.0.0.1",
"dst_port": 443,
"ip_meta_ttl": 64,
"ip_meta_total_length": 512,
"ip_meta_id": 12345,
"ip_meta_df": true,
"tcp_meta_window_size": 65535,
"tcp_meta_mss": 1460,
"tcp_meta_window_scale": 7,
"tcp_meta_options": "MSS,WS,SACK,TS",
"ja4": "t13d1516h2_8daaf6152771_02cb136f2775",
"ja4_hash": "8daaf6152771_02cb136f2775",
"ja3": "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
"ja3_hash": "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d"
}
```
## Tests
### Tests unitaires
```bash
# En local
make test
# Dans Docker
make test-docker
# Avec détection de race conditions
make test-race
# Avec rapport de couverture
make test-coverage
```
### Tests d'intégration
```bash
# Lance les tests bout-à-bout dans Docker
make test-integration
# Nettoyage après tests
make test-integration-clean
```
## Déploiement systemd
Exemple de fichier de service `/etc/systemd/system/ja4sentinel.service` :
```ini
[Unit]
Description=JA4 client fingerprinting daemon
After=network.target
[Service]
Type=simple
User=ja4sentinel
Group=ja4sentinel
ExecStart=/usr/local/bin/ja4sentinel --config /etc/ja4sentinel/config.yml
Restart=on-failure
RestartSec=5
Environment=JA4SENTINEL_LOG_LEVEL=info
# Security
NoNewPrivileges=yes
ProtectSystem=full
ProtectHome=true
PrivateTmp=true
CapabilityBoundingSet=CAP_NET_RAW CAP_NET_ADMIN
AmbientCapabilities=CAP_NET_RAW CAP_NET_ADMIN
[Install]
WantedBy=multi-user.target
```
## Exemples d'utilisation
### Surveillance du trafic HTTPS
```yaml
core:
interface: eth0
listen_ports: [443]
outputs:
- type: stdout
enabled: true
```
### Export vers socket UNIX pour traitement externe
```yaml
core:
interface: eth0
listen_ports: [443, 8443]
outputs:
- type: unix_socket
enabled: true
params:
socket_path: /var/run/logcorrelator/network.socket
# log_level: debug # debug, info, warn, error (défaut: error)
```
### Logging fichier + stdout
```yaml
core:
interface: ens192
listen_ports: [443]
flow_timeout_sec: 60
outputs:
- type: stdout
enabled: true
- type: file
enabled: true
params:
path: /var/log/ja4sentinel/ja4.json
```
## Développement
### Linting
```bash
make lint
```
### Formatage
```bash
make fmt
```
### Nettoyage
```bash
# Supprime les binaires et images Docker
make clean
# Supprime aussi les conteneurs et volumes
make clean-all
```
## Licence
À définir.
## Contribuer
1. Fork le projet
2. Créer une branche de feature (`git checkout -b feature/amélioration`)
3. Commit les changements (`git commit -am 'Ajout fonctionnalité'`)
4. Push (`git push origin feature/amélioration`)
5. Ouvrir une Pull Request
---
**Voir `architecture.yml` pour la documentation complète de l'architecture.**

307
old/services/sentinel/api/types.go Normal file
View File

@ -0,0 +1,307 @@
package api
import (
"strings"
"time"
)
// ServiceLog represents internal service logging for diagnostics
type ServiceLog struct {
Level string `json:"level"`
Component string `json:"component"`
Message string `json:"message"`
Details map[string]string `json:"details,omitempty"`
Timestamp int64 `json:"timestamp,omitempty"` // Unix nanoseconds (auto-set by logger)
TraceID string `json:"trace_id,omitempty"` // Optional distributed tracing ID
ConnID string `json:"conn_id,omitempty"` // Optional TCP flow identifier
}
// Config holds basic network and TLS configuration
type Config struct {
Interface string `yaml:"interface" json:"interface"`
ListenPorts []uint16 `yaml:"listen_ports" json:"listen_ports"`
BPFFilter string `yaml:"bpf_filter" json:"bpf_filter,omitempty"`
LocalIPs []string `yaml:"local_ips" json:"local_ips,omitempty"` // Local IPs to monitor (empty = auto-detect, excludes loopback)
ExcludeSourceIPs []string `yaml:"exclude_source_ips" json:"exclude_source_ips,omitempty"` // Source IPs or CIDR ranges to exclude (e.g., ["10.0.0.0/8", "192.168.1.1"])
FlowTimeoutSec int `yaml:"flow_timeout_sec" json:"flow_timeout_sec,omitempty"` // Timeout for TLS handshake extraction (default: 30)
PacketBufferSize int `yaml:"packet_buffer_size" json:"packet_buffer_size,omitempty"` // Buffer size for packet channel (default: 1000)
LogLevel string `yaml:"log_level" json:"log_level,omitempty"` // Log level: debug, info, warn, error (default: info)
}
// IPMeta contains IP metadata for stack fingerprinting
type IPMeta struct {
TTL uint8 `json:"ttl"`
TotalLength uint16 `json:"total_length"`
IPID uint16 `json:"id"`
DF bool `json:"df"`
}
// TCPMeta contains TCP metadata for stack fingerprinting
type TCPMeta struct {
WindowSize uint16 `json:"window_size"`
MSS uint16 `json:"mss,omitempty"`
WindowScale uint8 `json:"window_scale,omitempty"`
Options []string `json:"options"`
OptionKinds []uint8 `json:"-"` // Raw TCP option kind numbers for JA4T
}
// RawPacket represents a raw packet captured from the network
type RawPacket struct {
Data []byte `json:"-"` // Raw packet data including link-layer header
Timestamp int64 `json:"timestamp"` // nanoseconds since epoch
LinkType int `json:"-"` // Link type (1=Ethernet, 101=Linux SLL, etc.)
}
// TLSClientHello represents a client-side TLS ClientHello with IP/TCP metadata
type TLSClientHello struct {
SrcIP string `json:"src_ip"`
SrcPort uint16 `json:"src_port"`
DstIP string `json:"dst_ip"`
DstPort uint16 `json:"dst_port"`
Payload []byte `json:"-"` // Not serialized
IPMeta IPMeta `json:"ip_meta"`
TCPMeta TCPMeta `json:"tcp_meta"`
ConnID string `json:"conn_id,omitempty"` // Unique flow identifier
SNI string `json:"tls_sni,omitempty"` // Server Name Indication
ALPN string `json:"tls_alpn,omitempty"` // Application-Layer Protocol Negotiation
TLSVersion string `json:"tls_version,omitempty"` // Max TLS version supported
SynToCHMs *uint32 `json:"syn_to_clienthello_ms,omitempty"` // Time from SYN to ClientHello (ms)
}
// Fingerprints contains TLS fingerprints for a client flow
// Note: JA4Hash is kept for internal use but not serialized to LogRecord
// as the JA4 format already includes its own hash portions
type Fingerprints struct {
JA4 string `json:"ja4"`
JA4Hash string `json:"ja4_hash,omitempty"` // Internal use, not serialized to LogRecord
JA4T string `json:"ja4t,omitempty"`
JA3 string `json:"ja3,omitempty"`
JA3Hash string `json:"ja3_hash,omitempty"`
}
// LogRecord is the final log record, serialized as a flat JSON object
type LogRecord struct {
SrcIP string `json:"src_ip"`
SrcPort uint16 `json:"src_port"`
DstIP string `json:"dst_ip"`
DstPort uint16 `json:"dst_port"`
// Flattened IPMeta fields
IPTTL uint8 `json:"ip_meta_ttl"`
IPTotalLen uint16 `json:"ip_meta_total_length"`
IPID uint16 `json:"ip_meta_id"`
IPDF bool `json:"ip_meta_df"`
// Flattened TCPMeta fields
TCPWindow uint16 `json:"tcp_meta_window_size"`
TCPMSS *uint16 `json:"tcp_meta_mss,omitempty"`
TCPWScale *uint8 `json:"tcp_meta_window_scale,omitempty"`
TCPOptions string `json:"tcp_meta_options"` // comma-separated list
// Correlation & Triage
ConnID string `json:"conn_id,omitempty"` // Unique flow identifier
SensorID string `json:"sensor_id,omitempty"` // Sensor/captor identifier
// TLS elements (ClientHello)
TLSVersion string `json:"tls_version,omitempty"` // Max TLS version announced by client
SNI string `json:"tls_sni,omitempty"` // Server Name Indication
ALPN string `json:"tls_alpn,omitempty"` // Application-Layer Protocol Negotiation
// Behavioral detection (Timing)
SynToCHMs *uint32 `json:"syn_to_clienthello_ms,omitempty"` // Time from SYN to ClientHello (ms)
// Fingerprints
// Note: ja4_hash is NOT included - the JA4 format already includes its own hash portions
JA4 string `json:"ja4"`
JA4T string `json:"ja4t,omitempty"`
JA3 string `json:"ja3,omitempty"`
JA3Hash string `json:"ja3_hash,omitempty"`
// Timestamp in nanoseconds since Unix epoch
Timestamp int64 `json:"timestamp"`
}
// OutputConfig defines configuration for a single log output
type OutputConfig struct {
Type string `yaml:"type" json:"type"` // unix_socket, stdout, file, etc.
Enabled bool `yaml:"enabled" json:"enabled"` // whether this output is active
AsyncBuffer int `yaml:"async_buffer" json:"async_buffer"` // queue size for async writes (e.g., 5000)
Params map[string]string `yaml:"params" json:"params"` // specific parameters like socket_path, path, etc.
}
// AppConfig is the complete ja4sentinel configuration
type AppConfig struct {
Core Config `yaml:"core" json:"core"`
Outputs []OutputConfig `yaml:"outputs" json:"outputs"`
}
// Loader defines the interface for loading application configuration.
// Implementations must read configuration from a YAML file, merge with
// environment variables (JA4SENTINEL_*), and validate the final result.
type Loader interface {
Load() (AppConfig, error)
}
// Capture defines the interface for capturing raw network packets.
// Implementations must listen on a configured network interface, apply
// BPF filters for specified ports, and emit RawPacket objects to a channel.
// The Close method must be called to release resources (e.g., pcap handle).
type Capture interface {
Run(cfg Config, out chan<- RawPacket) error
Close() error
GetStats() (received, sent, dropped uint64)
}
// Parser defines the interface for extracting TLS ClientHello messages
// from raw network packets. Implementations must track TCP connection states,
// reassemble fragmented handshakes, and return TLSClientHello objects with
// IP/TCP metadata. Returns nil for non-TLS or non-ClientHello packets.
type Parser interface {
Process(pkt RawPacket) (*TLSClientHello, error)
Close() error
GetMetrics() (retransmit, gapDetected, bufferExceeded, segmentExceeded uint64)
}
// Engine defines the interface for generating TLS fingerprints.
// Implementations must analyze TLS ClientHello payloads and produce
// JA4 (required) and optionally JA3 fingerprint strings.
type Engine interface {
FromClientHello(ch TLSClientHello) (*Fingerprints, error)
}
// Writer defines the generic interface for writing log records.
// Implementations must serialize LogRecord objects and send them to
// a destination (stdout, file, UNIX socket, etc.).
type Writer interface {
Write(rec LogRecord) error
}
// UnixSocketWriter extends Writer with a Close method for UNIX socket cleanup.
// Implementations must connect to a UNIX socket at the specified path and
// write JSON-encoded LogRecord objects. Reconnection logic should be
// implemented for transient socket failures.
type UnixSocketWriter interface {
Writer
Close() error
}
// MultiWriter extends Writer to support multiple output destinations.
// Implementations must write each LogRecord to all registered writers
// and provide methods to add writers and close all connections.
type MultiWriter interface {
Writer
Add(writer Writer)
CloseAll() error
}
// Builder defines the interface for constructing output writers from configuration.
// Implementations must parse AppConfig.Outputs and create appropriate Writer
// instances (StdoutWriter, FileWriter, UnixSocketWriter), combining them
// into a MultiWriter if multiple outputs are configured.
type Builder interface {
NewFromConfig(cfg AppConfig) (Writer, error)
}
// Logger defines the interface for structured service logging.
// Implementations must emit JSON-formatted log entries to stdout/stderr
// with support for multiple log levels (DEBUG, INFO, WARN, ERROR).
// Each log entry includes timestamp, level, component, message, and optional details.
type Logger interface {
Debug(component, message string, details map[string]string)
Info(component, message string, details map[string]string)
Warn(component, message string, details map[string]string)
Error(component, message string, details map[string]string)
}
// Reopenable defines the interface for components that support log file rotation.
// Implementations must reopen their output files when receiving a SIGHUP signal.
// This is used by systemctl reload to switch to new log files after logrotate.
type Reopenable interface {
Reopen() error
}
// Helper functions for creating and converting records
// NewLogRecord creates a flattened LogRecord from TLSClientHello and Fingerprints.
// Converts TCPMeta options to a comma-separated string and creates pointer values
// for optional fields (MSS, WindowScale) to support proper JSON omitempty behavior.
// If fingerprints is nil, the JA4/JA3 fields will be empty strings.
// Note: JA4Hash is intentionally NOT included in LogRecord as the JA4 format
// already includes its own hash portions (the full 38-character JA4 string is sufficient).
func NewLogRecord(ch TLSClientHello, fp *Fingerprints) LogRecord {
opts := ""
if len(ch.TCPMeta.Options) > 0 {
opts = strings.Join(ch.TCPMeta.Options, ",")
}
// Helper to create pointer from value for optional fields
var mssPtr *uint16
if ch.TCPMeta.MSS != 0 {
mssPtr = &ch.TCPMeta.MSS
}
var wScalePtr *uint8
if ch.TCPMeta.WindowScale != 0 {
wScalePtr = &ch.TCPMeta.WindowScale
}
rec := LogRecord{
SrcIP: ch.SrcIP,
SrcPort: ch.SrcPort,
DstIP: ch.DstIP,
DstPort: ch.DstPort,
IPTTL: ch.IPMeta.TTL,
IPTotalLen: ch.IPMeta.TotalLength,
IPID: ch.IPMeta.IPID,
IPDF: ch.IPMeta.DF,
TCPWindow: ch.TCPMeta.WindowSize,
TCPMSS: mssPtr,
TCPWScale: wScalePtr,
TCPOptions: opts,
ConnID: ch.ConnID,
SNI: ch.SNI,
ALPN: ch.ALPN,
TLSVersion: ch.TLSVersion,
SynToCHMs: ch.SynToCHMs,
Timestamp: time.Now().UnixNano(),
}
if fp != nil {
rec.JA4 = fp.JA4
rec.JA4T = fp.JA4T
rec.JA3 = fp.JA3
rec.JA3Hash = fp.JA3Hash
}
return rec
}
// Default values and constants
const (
DefaultInterface = "eth0"
DefaultPort = 443
DefaultBPFFilter = ""
DefaultFlowTimeout = 30 // seconds
DefaultPacketBuffer = 1000 // packet channel buffer size
DefaultLogLevel = "info"
)
// DefaultConfig returns an AppConfig with sensible default values.
// Uses eth0 as the default interface, port 443 for monitoring,
// no BPF filter, a 30-second flow timeout, and a 1000-packet
// channel buffer. Returns an empty outputs slice (caller must
// configure outputs explicitly).
func DefaultConfig() AppConfig {
return AppConfig{
Core: Config{
Interface: DefaultInterface,
ListenPorts: []uint16{DefaultPort},
BPFFilter: DefaultBPFFilter,
FlowTimeoutSec: DefaultFlowTimeout,
PacketBufferSize: DefaultPacketBuffer,
LogLevel: DefaultLogLevel,
},
Outputs: []OutputConfig{},
}
}

340
old/services/sentinel/api/types_test.go Normal file
View File

@ -0,0 +1,340 @@
package api
import (
"testing"
)
func TestNewLogRecord(t *testing.T) {
synToCHMs := uint32(150)
tests := []struct {
name string
clientHello TLSClientHello
fingerprints *Fingerprints
wantNil bool
}{
{
name: "complete record with fingerprints",
clientHello: TLSClientHello{
SrcIP: "192.168.1.100",
SrcPort: 54321,
DstIP: "10.0.0.1",
DstPort: 443,
ConnID: "flow-abc123",
SNI: "example.com",
ALPN: "h2",
TLSVersion: "1.3",
SynToCHMs: &synToCHMs,
IPMeta: IPMeta{
TTL: 64,
TotalLength: 512,
IPID: 12345,
DF: true,
},
TCPMeta: TCPMeta{
WindowSize: 65535,
MSS: 1460,
WindowScale: 7,
Options: []string{"MSS", "WS", "SACK", "TS"},
},
},
fingerprints: &Fingerprints{
JA4: "t13d1516h2_8daaf6152771_02cb136f2775",
JA4Hash: "8daaf6152771_02cb136f2775", // Internal use only
JA3: "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d",
},
wantNil: false,
},
{
name: "record without fingerprints",
clientHello: TLSClientHello{
SrcIP: "192.168.1.100",
SrcPort: 54321,
DstIP: "10.0.0.1",
DstPort: 443,
ConnID: "flow-xyz789",
SNI: "test.example.com",
ALPN: "http/1.1",
TLSVersion: "1.2",
IPMeta: IPMeta{
TTL: 64,
TotalLength: 512,
IPID: 12345,
DF: true,
},
TCPMeta: TCPMeta{
WindowSize: 65535,
MSS: 1460,
WindowScale: 7,
Options: []string{"MSS", "WS"},
},
},
fingerprints: nil,
wantNil: false,
},
{
name: "record with zero values for optional fields",
clientHello: TLSClientHello{
SrcIP: "192.168.1.100",
SrcPort: 54321,
DstIP: "10.0.0.1",
DstPort: 443,
IPMeta: IPMeta{
TTL: 0,
TotalLength: 0,
IPID: 0,
DF: false,
},
TCPMeta: TCPMeta{
WindowSize: 0,
MSS: 0,
WindowScale: 0,
Options: []string{},
},
},
fingerprints: nil,
wantNil: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
rec := NewLogRecord(tt.clientHello, tt.fingerprints)
// Verify timestamp is set
if rec.Timestamp == 0 {
t.Error("Timestamp should be set")
}
// Verify basic fields
if rec.SrcIP != tt.clientHello.SrcIP {
t.Errorf("SrcIP = %v, want %v", rec.SrcIP, tt.clientHello.SrcIP)
}
if rec.SrcPort != tt.clientHello.SrcPort {
t.Errorf("SrcPort = %v, want %v", rec.SrcPort, tt.clientHello.SrcPort)
}
if rec.DstIP != tt.clientHello.DstIP {
t.Errorf("DstIP = %v, want %v", rec.DstIP, tt.clientHello.DstIP)
}
if rec.DstPort != tt.clientHello.DstPort {
t.Errorf("DstPort = %v, want %v", rec.DstPort, tt.clientHello.DstPort)
}
// Verify IPMeta fields
if rec.IPTTL != tt.clientHello.IPMeta.TTL {
t.Errorf("IPTTL = %v, want %v", rec.IPTTL, tt.clientHello.IPMeta.TTL)
}
if rec.IPTotalLen != tt.clientHello.IPMeta.TotalLength {
t.Errorf("IPTotalLen = %v, want %v", rec.IPTotalLen, tt.clientHello.IPMeta.TotalLength)
}
if rec.IPID != tt.clientHello.IPMeta.IPID {
t.Errorf("IPID = %v, want %v", rec.IPID, tt.clientHello.IPMeta.IPID)
}
if rec.IPDF != tt.clientHello.IPMeta.DF {
t.Errorf("IPDF = %v, want %v", rec.IPDF, tt.clientHello.IPMeta.DF)
}
// Verify TCPMeta fields
if rec.TCPWindow != tt.clientHello.TCPMeta.WindowSize {
t.Errorf("TCPWindow = %v, want %v", rec.TCPWindow, tt.clientHello.TCPMeta.WindowSize)
}
// Verify optional fields (MSS, WindowScale)
if tt.clientHello.TCPMeta.MSS != 0 {
if rec.TCPMSS == nil {
t.Error("TCPMSS should not be nil when MSS != 0")
} else if *rec.TCPMSS != tt.clientHello.TCPMeta.MSS {
t.Errorf("TCPMSS = %v, want %v", *rec.TCPMSS, tt.clientHello.TCPMeta.MSS)
}
} else {
if rec.TCPMSS != nil {
t.Error("TCPMSS should be nil when MSS == 0")
}
}
if tt.clientHello.TCPMeta.WindowScale != 0 {
if rec.TCPWScale == nil {
t.Error("TCPWScale should not be nil when WindowScale != 0")
} else if *rec.TCPWScale != tt.clientHello.TCPMeta.WindowScale {
t.Errorf("TCPWScale = %v, want %v", *rec.TCPWScale, tt.clientHello.TCPMeta.WindowScale)
}
} else {
if rec.TCPWScale != nil {
t.Error("TCPWScale should be nil when WindowScale == 0")
}
}
// Verify new TLS fields
if rec.ConnID != tt.clientHello.ConnID {
t.Errorf("ConnID = %v, want %v", rec.ConnID, tt.clientHello.ConnID)
}
if rec.SNI != tt.clientHello.SNI {
t.Errorf("SNI = %v, want %v", rec.SNI, tt.clientHello.SNI)
}
if rec.ALPN != tt.clientHello.ALPN {
t.Errorf("ALPN = %v, want %v", rec.ALPN, tt.clientHello.ALPN)
}
if rec.TLSVersion != tt.clientHello.TLSVersion {
t.Errorf("TLSVersion = %v, want %v", rec.TLSVersion, tt.clientHello.TLSVersion)
}
if tt.clientHello.SynToCHMs != nil {
if rec.SynToCHMs == nil {
t.Error("SynToCHMs should not be nil")
} else if *rec.SynToCHMs != *tt.clientHello.SynToCHMs {
t.Errorf("SynToCHMs = %v, want %v", *rec.SynToCHMs, *tt.clientHello.SynToCHMs)
}
}
// Verify fingerprints (note: JA4Hash is NOT in LogRecord per architecture)
if tt.fingerprints != nil {
if rec.JA4 != tt.fingerprints.JA4 {
t.Errorf("JA4 = %v, want %v", rec.JA4, tt.fingerprints.JA4)
}
// JA4Hash is intentionally NOT in LogRecord (architecture decision)
// JA3Hash is still present as it's the MD5 of JA3 (needed for exploitation)
if rec.JA3 != tt.fingerprints.JA3 {
t.Errorf("JA3 = %v, want %v", rec.JA3, tt.fingerprints.JA3)
}
if rec.JA3Hash != tt.fingerprints.JA3Hash {
t.Errorf("JA3Hash = %v, want %v", rec.JA3Hash, tt.fingerprints.JA3Hash)
}
} else {
if rec.JA4 != "" {
t.Error("JA4 should be empty when fingerprints is nil")
}
}
})
}
}
func TestDefaultConfig(t *testing.T) {
cfg := DefaultConfig()
if cfg.Core.Interface != DefaultInterface {
t.Errorf("Core.Interface = %v, want %v", cfg.Core.Interface, DefaultInterface)
}
if len(cfg.Core.ListenPorts) != 1 {
t.Errorf("Core.ListenPorts length = %v, want 1", len(cfg.Core.ListenPorts))
}
if cfg.Core.ListenPorts[0] != DefaultPort {
t.Errorf("Core.ListenPorts[0] = %v, want %v", cfg.Core.ListenPorts[0], DefaultPort)
}
if cfg.Core.BPFFilter != DefaultBPFFilter {
t.Errorf("Core.BPFFilter = %v, want %v", cfg.Core.BPFFilter, DefaultBPFFilter)
}
if cfg.Core.FlowTimeoutSec != DefaultFlowTimeout {
t.Errorf("Core.FlowTimeoutSec = %v, want %v", cfg.Core.FlowTimeoutSec, DefaultFlowTimeout)
}
if len(cfg.Outputs) != 0 {
t.Errorf("Outputs length = %v, want 0", len(cfg.Outputs))
}
}
func TestLogRecordConversion(t *testing.T) {
// Test that NewLogRecord correctly converts TCPMeta options to comma-separated string
clientHello := TLSClientHello{
SrcIP: "192.168.1.100",
SrcPort: 54321,
DstIP: "10.0.0.1",
DstPort: 443,
TCPMeta: TCPMeta{
WindowSize: 65535,
MSS: 1460,
WindowScale: 7,
Options: []string{"MSS", "WS", "SACK", "TS"},
},
}
rec := NewLogRecord(clientHello, nil)
// Verify options are joined with comma
expectedOpts := "MSS,WS,SACK,TS"
if rec.TCPOptions != expectedOpts {
t.Errorf("TCPOptions = %v, want %v", rec.TCPOptions, expectedOpts)
}
}
func TestLogRecordNoJA4Hash(t *testing.T) {
// Verify that JA4Hash is NOT included in LogRecord per architecture decision
clientHello := TLSClientHello{
SrcIP: "192.168.1.100",
SrcPort: 54321,
DstIP: "10.0.0.1",
DstPort: 443,
}
fingerprints := &Fingerprints{
JA4: "t13d1516h2_8daaf6152771_02cb136f2775",
JA4Hash: "8daaf6152771_02cb136f2775", // Should NOT appear in LogRecord
JA3: "771,4865-4866-4867,0-23-65281,29-23-24,0",
JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d",
}
rec := NewLogRecord(clientHello, fingerprints)
// JA4Hash is NOT in LogRecord (architecture decision)
// The JA4 format already includes its own hash portions
// But JA4 should be present
if rec.JA4 != fingerprints.JA4 {
t.Errorf("JA4 = %v, want %v", rec.JA4, fingerprints.JA4)
}
// JA3Hash should still be present (it's the MD5 of JA3, which is needed)
if rec.JA3Hash != fingerprints.JA3Hash {
t.Errorf("JA3Hash = %v, want %v", rec.JA3Hash, fingerprints.JA3Hash)
}
}
func TestOutputConfig(t *testing.T) {
tests := []struct {
name string
config OutputConfig
wantEnabled bool
wantAsyncBuf int
}{
{
name: "stdout output with async buffer",
config: OutputConfig{
Type: "stdout",
Enabled: true,
AsyncBuffer: 5000,
Params: map[string]string{},
},
wantEnabled: true,
wantAsyncBuf: 5000,
},
{
name: "unix_socket output with default async buffer",
config: OutputConfig{
Type: "unix_socket",
Enabled: true,
AsyncBuffer: 0, // Default
Params: map[string]string{"socket_path": "/var/run/test.sock"},
},
wantEnabled: true,
wantAsyncBuf: 0,
},
{
name: "disabled output",
config: OutputConfig{
Type: "file",
Enabled: false,
AsyncBuffer: 1000,
Params: map[string]string{"path": "/var/log/test.log"},
},
wantEnabled: false,
wantAsyncBuf: 1000,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if tt.config.Enabled != tt.wantEnabled {
t.Errorf("Enabled = %v, want %v", tt.config.Enabled, tt.wantEnabled)
}
if tt.config.AsyncBuffer != tt.wantAsyncBuf {
t.Errorf("AsyncBuffer = %v, want %v", tt.config.AsyncBuffer, tt.wantAsyncBuf)
}
})
}
}

524
old/services/sentinel/architecture.yml Normal file
View File

@ -0,0 +1,524 @@
version: 1
project:
name: ja4sentinel
description: >
Outil Go pour capturer le trafic réseau sur un serveur Linux,
extraire les handshakes TLS côté client, générer les signatures JA4
(via psanford/tlsfingerprint), enrichir avec des métadonnées IP/TCP,
et loguer les résultats (IP, ports, JA4, meta) vers une ou plusieurs
sorties configurables (socket UNIX par défaut, stdout, fichier, ...).
Le service est géré par systemd avec support de rotation des logs via logrotate.
La commande `systemctl reload ja4sentinel` permet de réouvrir les fichiers de log
après rotation (signal SIGHUP).
languages:
- go
goals:
- "Développement bloc par bloc avec interfaces simples et stables."
- "Focalisé sur JA4 client (le serveur est connu/local)."
- "Séparation claire des responsabilités (capture, parsing, fingerprint, output)."
- "Tests unitaires pour chaque fonction publique."
- "Tests d’intégration dans des conteneurs Docker."
- "Commentaires standardisés, code évolutif avec changements minimaux."
modules:
- name: config
path: "internal/config"
description: "Chargement et validation de la configuration (fichier, env, CLI)."
responsibilities:
- "Lire le fichier de configuration (YAML par défaut)."
- "Fusionner avec les overrides env/CLI."
- "Construire une api.AppConfig cohérente."
allowed_dependencies: []
forbidden_dependencies:
- "capture"
- "tlsparse"
- "fingerprint"
- "output"
- name: capture
path: "internal/capture"
description: "Capture des paquets réseau (pcap/raw socket) sur Linux."
responsibilities:
- "Ouvrir l’interface réseau configurée."
- "Appliquer les filtres (ports, BPF, protocole)."
- "Observer les flux TCP côté client vers les ports d’intérêt."
- "Extraire les en-têtes IP/TCP utiles (IPMeta, TCPMeta)."
- "Convertir les paquets en objets RawPacket."
allowed_dependencies:
- "config"
- "api"
forbidden_dependencies:
- "tlsparse"
- "fingerprint"
- "output"
- name: tlsparse
path: "internal/tlsparse"
description: "Extraction des ClientHello TLS côté client à partir des paquets capturés."
responsibilities:
- "Décoder les couches IP/TCP jusqu'au payload TLS."
- "Identifier le ClientHello TLS du client sur les ports configurés."
- "Assembler les segments si nécessaire pour obtenir un ClientHello complet."
- "Produire des TLSClientHello enrichis avec IPMeta et TCPMeta."
- "Filtrer les IPs source exclues via le module ipfilter (avant parsing TLS)."
- "Compter les paquets filtrés pour statistiques (GetFilterStats)."
allowed_dependencies:
- "config"
- "capture"
- "api"
- "ipfilter"
forbidden_dependencies:
- "output"
- name: fingerprint
path: "internal/fingerprint"
description: "Génération des empreintes JA4 à partir des ClientHello TLS."
responsibilities:
- "Utiliser psanford/tlsfingerprint pour analyser le ClientHello."
- "Générer la chaîne JA4 (et éventuellement JA3) côté client."
- "Encapsuler les résultats dans un type Fingerprints."
allowed_dependencies:
- "config"
- "tlsparse"
- "api"
forbidden_dependencies:
- "capture"
- name: output
path: "internal/output"
description: "Sortie asynchrone ultra-rapide des résultats (JA4 + meta)."
responsibilities:
- "Prendre en entrée les Fingerprints et les métadonnées réseau."
- "Formater les données en enregistrements log (JSON ou autre format simple)."
- "Gérer une file d'attente interne (buffer channel) pour rendre l'écriture non-bloquante pour la capture."
- "Sérialiser le JSON le plus rapidement possible (ex: pool d'allocations, librairies optimisées comme goccy/go-json)."
- "Envoyer les enregistrements vers une ou plusieurs sorties (socket UNIX DGRAM, stdout, fichier, ...)."
- "Gérer un MultiWriter pour combiner plusieurs outputs sans modifier le reste du code."
allowed_dependencies:
- "config"
- "api"
forbidden_dependencies:
- "capture"
- "tlsparse"
- "fingerprint"
- name: logging
path: "internal/logging"
description: "Logs structurés JSON pour le service (stdout/stderr)."
responsibilities:
- "Fournir une fabrique de loggers (LoggerFactory)."
- "Émettre des logs au format JSON lines sur stdout."
- "Supporter les niveaux : debug, info, warn, error."
- "Inclure timestamp, niveau, composant, message et détails optionnels."
allowed_dependencies:
- "api"
forbidden_dependencies:
- "config"
- "capture"
- "tlsparse"
- "fingerprint"
- "output"
- name: ipfilter
path: "internal/ipfilter"
description: "Filtrage des adresses IP source par correspondance IP/CIDR."
responsibilities:
- "Charger une liste d'IPs ou plages CIDR à exclure."
- "Vérifier si une IP source correspond à une entrée de la liste d'exclusion."
- "Supporter IPv4 et IPv6."
- "Validation des formats IP et CIDR lors du chargement de la config."
allowed_dependencies: []
forbidden_dependencies:
- "config"
- "capture"
- "tlsparse"
- "fingerprint"
- "output"
- name: cmd_ja4sentinel
path: "cmd/ja4sentinel"
description: "Point d'entrée de l'application (main)."
responsibilities:
- "Charger la configuration via le module config."
- "Construire les instances des modules (capture, tlsparse, fingerprint, output, logging)."
- "Brancher les modules entre eux selon l'architecture pipeline."
- "Gérer les signaux système (arrêt propre)."
- "Gérer le signal SIGHUP pour la rotation des logs (systemctl reload)."
- "Logger les statistiques du filtre IP au démarrage et à l'arrêt (debug)."
allowed_dependencies:
- "config"
- "capture"
- "tlsparse"
- "fingerprint"
- "output"
- "api"
- "logging"
forbidden_dependencies: []
api:
types:
- name: "api.ServiceLog"
description: "Log interne du service ja4sentinel (diagnostic)."
fields:
- { name: Level, type: "string", description: "niveau: debug, info, warn, error." }
- { name: Component, type: "string", description: "module concerné (capture, tlsparse, ...)." }
- { name: Message, type: "string", description: "texte du log." }
- { name: Details, type: "map[string]string", description: "infos additionnelles (erreurs, IDs...)." }
- { name: Timestamp, type: "int64", description: "Timestamp en nanosecondes (auto-rempli par le logger)." }
- { name: TraceID, type: "string", description: "ID de tracing distribué (optionnel)." }
- { name: ConnID, type: "string", description: "Identifiant de flux TCP (optionnel)." }
- name: "api.Config"
description: "Configuration réseau et TLS de base."
fields:
- { name: Interface, type: "string", description: "Nom de l'interface réseau (ex: eth0)." }
- { name: ListenPorts, type: "[]uint16", description: "Ports TCP à surveiller (ex: [443, 8443])." }
- { name: BPFFilter, type: "string", description: "Filtre BPF optionnel pour la capture." }
- { name: LocalIPs, type: "[]string", description: "IPs locales à surveiller (vide = auto-détection, exclut loopback)." }
- { name: ExcludeSourceIPs,type: "[]string", description: "IPs sources ou plages CIDR à exclure (ex: [\"10.0.0.0/8\", \"192.168.1.1\"]). Validé par le module config." }
- { name: FlowTimeoutSec, type: "int", description: "Timeout en secondes pour l'extraction du handshake TLS (défaut: 30)." }
- { name: PacketBufferSize,type: "int", description: "Taille du buffer du canal de paquets (défaut: 1000). Pour les environnements à fort trafic." }
- { name: LogLevel, type: "string", description: "Niveau de log : debug, info, warn, error (défaut: info). Configuration via fichier YAML uniquement (pas d'override env dans systemd)." }
- name: "api.IPMeta"
description: "Métadonnées IP pour fingerprinting de stack."
fields:
- { name: TTL, type: "uint8", description: "TTL initial observé." }
- { name: TotalLength, type: "uint16", description: "Taille totale du paquet IP." }
- { name: IPID, type: "uint16", description: "Identifiant IP du paquet." }
- { name: DF, type: "bool", description: "Flag Don't Fragment." }
- name: "api.TCPMeta"
description: "Métadonnées TCP pour fingerprinting de stack."
fields:
- { name: WindowSize, type: "uint16", description: "Fenêtre initiale TCP." }
- { name: MSS, type: "uint16", description: "Maximum Segment Size (option TCP)." }
- { name: WindowScale, type: "uint8", description: "Facteur de scaling (option TCP)." }
- { name: Options, type: "[]string", description: "Liste ordonnée des options TCP (ex: [MSS, SACK, TS])." }
- name: "api.RawPacket"
description: "Paquet brut capturé sur le réseau (vue minimale)."
fields:
- { name: Data, type: "[]byte", description: "Contenu brut du paquet." }
- { name: Timestamp, type: "int64", description: "Timestamp (nanos / epoch) de capture." }
- name: "api.TLSClientHello"
description: "Représentation d'un ClientHello TLS client, avec meta IP/TCP."
fields:
- { name: SrcIP, type: "string", description: "Adresse IP source (client)." }
- { name: SrcPort, type: "uint16", description: "Port source (client)." }
- { name: DstIP, type: "string", description: "Adresse IP destination (serveur)." }
- { name: DstPort, type: "uint16", description: "Port destination (serveur)." }
- { name: Payload, type: "[]byte", description: "Bytes correspondant au ClientHello TLS." }
- { name: IPMeta, type: "api.IPMeta", description: "Métadonnées IP observées côté client." }
- { name: TCPMeta, type: "api.TCPMeta", description: "Métadonnées TCP observées côté client." }
- { name: ConnID, type: "string", description: "Identifiant unique du flux TCP (extension pour corrélation)." }
- { name: SNI, type: "string", description: "Server Name Indication extrait du ClientHello (extension)." }
- { name: ALPN, type: "string", description: "ALPN protocols négociés (extension)." }
- { name: TLSVersion,type: "string", description: "Version TLS maximale annoncée (extension)." }
- { name: SynToCHMs,type: "*uint32", description: "Temps SYN->ClientHello en ms (extension pour détection comportementale)." }
- name: "api.Fingerprints"
description: "Empreintes TLS pour un flux client."
fields:
- { name: JA4, type: "string", description: "Signature JA4 client." }
- { name: JA4Hash, type: "string", description: "Hash JA4 client." }
- { name: JA3, type: "string", description: "Signature JA3 (optionnel, si calculée)." }
- { name: JA3Hash, type: "string", description: "Hash JA3 (optionnel)." }
- name: "api.LogRecord"
description: "Enregistrement de log final, sérialisé en JSON objet plat."
json_object: true
fields:
- { name: SrcIP, type: "string", json_key: "src_ip" }
- { name: SrcPort, type: "uint16", json_key: "src_port" }
- { name: DstIP, type: "string", json_key: "dst_ip" }
- { name: DstPort, type: "uint16", json_key: "dst_port" }
# IPMeta flatten
- { name: IPTTL, type: "uint8", json_key: "ip_meta_ttl" }
- { name: IPTotalLen, type: "uint16", json_key: "ip_meta_total_length" }
- { name: IPID, type: "uint16", json_key: "ip_meta_id" }
- { name: IPDF, type: "bool", json_key: "ip_meta_df" }
# TCPMeta flatten
- { name: TCPWindow, type: "uint16", json_key: "tcp_meta_window_size" }
- { name: TCPMSS, type: "*uint16", json_key: "tcp_meta_mss", optional: true, description: "Pointeur (nil si non présent, 0 si absent)." }
- { name: TCPWScale, type: "*uint8", json_key: "tcp_meta_window_scale", optional: true, description: "Pointeur (nil si non présent, 0 si absent)." }
- { name: TCPOptions, type: "string", json_key: "tcp_meta_options" }
# Fingerprints
- { name: JA4, type: "string", json_key: "ja4", description: "Le format JA4 inclut nativement ses propres hachages (parties b et c), pas besoin de ja4_hash séparé." }
- { name: JA3, type: "string", json_key: "ja3", description: "Chaîne brute JA3 (variable)." }
- { name: JA3Hash, type: "string", json_key: "ja3_hash", description: "Hachage MD5 indispensable pour exploiter la chaîne JA3." }
# --- Corrélation & Triage ---
- { name: ConnID, type: "string", json_key: "conn_id", optional: true, description: "Identifiant unique du flux (ex: hash de src_ip:src_port-dst_ip:dst_port) pour corréler facilement plusieurs événements liés à une même session TCP." }
- { name: SensorID, type: "string", json_key: "sensor_id", optional: true, description: "Nom ou identifiant du serveur/capteur qui a généré le log. Indispensable pour du déploiement à grande échelle." }
# --- Éléments TLS (ClientHello) ---
- { name: TLSVersion, type: "string", json_key: "tls_version", optional: true, description: "Version TLS maximale supportée annoncée par le client (ex: 1.2, 1.3). Utile pour repérer les clients obsolètes." }
- { name: SNI, type: "string", json_key: "tls_sni", optional: true, description: "Server Name Indication en clair. Crucial pour détecter le domaine visé par le client (C2, DGA, etc.)." }
- { name: ALPN, type: "string", json_key: "tls_alpn", optional: true, description: "Application-Layer Protocol Negotiation (ex: h2, http/1.1). Aide à différencier le trafic web légitime d'un tunnel personnalisé." }
# --- Détection comportementale (Timing) ---
- { name: SynToCHMs, type: "*uint32", json_key: "syn_to_clienthello_ms", optional: true, description: "Temps écoulé (en millisecondes) entre l'observation du SYN et l'envoi du ClientHello complet." }
# Timestamp
- { name: Timestamp, type: "int64", json_key: "timestamp", description: "Wall-clock timestamp in nanoseconds since Unix epoch (auto-filled by NewLogRecord)." }
- name: "api.OutputConfig"
description: "Configuration d’une sortie de logs."
fields:
- { name: Type, type: "string", description: "Type d’output (unix_socket, stdout, file, ...)." }
- { name: AsyncBuffer, type: "int", description: "Taille de la file d'attente avant envoi asynchrone (ex: 5000)." }
- { name: Enabled, type: "bool", description: "Active ou non cette sortie." }
- { name: Params, type: "map[string]string", description: "Paramètres spécifiques (socket_path, path, ...)." }
- name: "api.AppConfig"
description: "Configuration complète de ja4sentinel."
fields:
- { name: Core, type: "api.Config", description: "Paramètres réseau + TLS." }
- { name: Outputs, type: "[]api.OutputConfig", description: "Liste des outputs configurés." }
interfaces:
- name: "config.Loader"
description: "Charge la configuration (fichier + env + CLI)."
module: "config"
methods:
- name: "Load"
params: []
returns:
- { type: "api.AppConfig" }
- { type: "error" }
- name: "capture.Capture"
description: "Source de paquets réseau bruts côté client."
module: "capture"
methods:
- name: "Run"
params:
- { name: cfg, type: "api.Config" }
- { name: out, type: "chan<- api.RawPacket" }
returns:
- { type: "error" }
notes:
- "Doit respecter les filtres (ports, BPF) définis dans la configuration."
- "Ne connaît pas le format TLS ni JA4."
- name: "Close"
params: []
returns:
- { type: "error" }
notes:
- "Libère les ressources (handle pcap, etc.). Doit être appelé après Run()."
- name: "tlsparse.Parser"
description: "Transforme des RawPacket en TLSClientHello (côté client uniquement)."
module: "tlsparse"
methods:
- name: "Process"
params:
- { name: pkt, type: "api.RawPacket" }
returns:
- { type: "*api.TLSClientHello" }
- { type: "error" }
notes:
- "Retourne nil si le paquet ne contient pas (ou plus) de ClientHello."
- "Pour chaque flux, s'arrête une fois le ClientHello complet obtenu."
- name: "Close"
params: []
returns:
- { type: "error" }
notes:
- "Arrête les goroutines en arrière-plan et nettoie les états de flux."
- name: "fingerprint.Engine"
description: "Génère les empreintes JA4 (et JA3 éventuellement) à partir d’un ClientHello."
module: "fingerprint"
methods:
- name: "FromClientHello"
params:
- { name: ch, type: "api.TLSClientHello" }
returns:
- { type: "*api.Fingerprints" }
- { type: "error" }
notes:
- "Utilise github.com/psanford/tlsfingerprint en interne."
- "Focalisé sur le JA4 client (le côté serveur est déjà connu)."
- name: "output.Writer"
description: "Interface générique pour écrire les résultats."
module: "output"
methods:
- name: "Write"
params:
- { name: rec, type: "api.LogRecord" }
returns:
- { type: "error" }
notes:
- "Ne connaît pas la capture ni les détails de parsing TLS."
- name: "output.UnixSocketWriter"
description: "Implémentation de Writer envoyant les logs sur une socket UNIX."
module: "output"
implements: "output.Writer"
config:
- { name: socket_path, type: "string", description: "Chemin de la socket UNIX DGRAM (ex: /var/run/logcorrelator/network.sock)." }
- name: "output.MultiWriter"
description: "Combinaison de plusieurs Writer configurés."
module: "output"
implements: "output.Writer"
config:
- { name: writers, type: "[]output.Writer", description: "Liste de Writers concrets à appeler." }
- name: "output.Builder"
description: "Construit les Writers à partir de api.AppConfig."
module: "output"
methods:
- name: "NewFromConfig"
params:
- { name: cfg, type: "api.AppConfig" }
returns:
- { type: "output.Writer" }
- { type: "error" }
notes:
- "Doit supporter plusieurs outputs simultanés via un MultiWriter."
- name: "logging.LoggerFactory"
description: "Fabrique de loggers structurés JSON."
module: "logging"
methods:
- name: "NewLogger"
params:
- { name: level, type: "string" }
returns:
- { type: "api.Logger" }
- name: "NewDefaultLogger"
params: []
returns:
- { type: "api.Logger" }
notes:
- "Les logs sont émis en JSON lines sur stdout pour systemd/journald."
- name: "api.Logger"
description: "Interface de logging pour tous les modules."
module: "logging"
methods:
- name: "Debug"
params:
- { name: component, type: "string" }
- { name: message, type: "string" }
- { name: details, type: "map[string]string" }
- name: "Info"
params:
- { name: component, type: "string" }
- { name: message, type: "string" }
- { name: details, type: "map[string]string" }
- name: "Warn"
params:
- { name: component, type: "string" }
- { name: message, type: "string" }
- { name: details, type: "map[string]string" }
- name: "Error"
params:
- { name: component, type: "string" }
- { name: message, type: "string" }
- { name: details, type: "map[string]string" }
notes:
- "Tous les logs passent par stdout/stderr (pas de fichiers directs)."
architecture:
style: "pipeline"
flow:
- from: "capture.Capture"
to: "tlsparse.Parser"
via: "api.RawPacket"
- from: "tlsparse.Parser"
to: "fingerprint.Engine"
via: "api.TLSClientHello"
- from: "fingerprint.Engine"
to: "output.Writer"
via: "api.LogRecord"
constraints:
- id: "client_only"
description: "On ne calcule que les empreintes JA4 côté client (pas côté serveur)."
- id: "no_back_dependencies"
description: "Pas de dépendances en arrière (output ne dépend pas de fingerprint, etc.)."
- id: "simple_messages"
description: "Les communications entre blocs utilisent uniquement les types définis dans api.*."
- id: "no_global_state"
description: "Pas de variables globales partagées entre blocs pour la logique principale."
flow_control:
connection_states:
description: "États simplifiés d'un flux TCP pour minimiser la capture."
states:
- name: "NEW"
description: "Observation d'un SYN client sur un port surveillé, création d'un état minimal (IP/TCP meta)."
- name: "WAIT_CLIENT_HELLO"
description: "Accumulation des segments TCP nécessaires pour extraire un ClientHello complet."
- name: "JA4_DONE"
description: "JA4 calculé et logué, on arrête de suivre ce flux."
rules:
- "Pas de tableaux imbriqués ni d'objets deeply nested."
- "Toutes les métadonnées IP/TCP sont flatten sous forme de champs scalaires nommés."
- "Les noms de champs suivent la convention: ip_meta_*, tcp_meta_*, ja*."
- "Pas de champ ja4_hash : le format JA4 intègre déjà son propre hachage tronqué, la chaîne complète de 38 caractères suffit."
logrecord_schema:
# Exemple de mapping pour api.LogRecord (résumé)
- "conn_id"
- "sensor_id"
- "src_ip"
- "src_port"
- "dst_ip"
- "dst_port"
- "ip_meta_ttl"
- "ip_meta_total_length"
- "ip_meta_id"
- "ip_meta_df"
- "tcp_meta_window_size"
- "tcp_meta_mss"
- "tcp_meta_window_scale"
- "tcp_meta_options" # string joinée, ex: 'MSS,SACK,TS,NOP,WS'
- "tls_version"
- "tls_sni"
- "tls_alpn"
- "syn_to_clienthello_ms"
- "ja4"
- "ja3"
- "ja3_hash"
packaging:
rpm:
description: "Package RPM pour déploiement sur serveurs Linux."
files:
- path: "/etc/logrotate.d/ja4sentinel"
description: "Script logrotate pour la rotation des fichiers de log."
note: "Fourni par le RPM, configure la rotation quotidienne avec compression."
- path: "/etc/systemd/system/ja4sentinel.service"
description: "Unité systemd pour la gestion du service."
note: "Doit inclure Type=notify et ExecReload=/bin/kill -HUP $MAINPID pour supporter systemctl reload. PAS de variable Environment=JA4SENTINEL_LOG_LEVEL pour respecter la config fichier."
logrotate:
description: "Configuration logrotate pour la rotation des logs."
behavior:
- "Rotation quotidienne ou selon taille."
- "Compression des logs archivés."
- "Envoi du signal SIGHUP au service après rotation pour réouvrir les fichiers."
reload_mechanism:
- "systemctl reload ja4sentinel déclenche le handler SIGHUP."
- "Le service réouvre ses fichiers de log sans redémarrage complet."
config_loading:
priority:
- "1. Fichier de configuration YAML (config.yml)"
- "2. Variables d'environnement JA4SENTINEL_* (sauf log_level depuis v1.1.11)"
- "3. Arguments CLI (--config)"
notes:
- "Depuis v1.1.11, la variable JA4SENTINEL_LOG_LEVEL n'est plus définie dans le service systemd."
- "Le log_level doit être configuré exclusivement dans le fichier YAML."
- "exclude_source_ips est uniquement chargé depuis le fichier YAML (pas d'override env)."
- "La fusion des configs utilise mergeConfigs() qui préserve les valeurs non-overridées."

357
old/services/sentinel/cmd/ja4sentinel/main.go Normal file
View File

@ -0,0 +1,357 @@
// Package main provides the entry point for ja4sentinel
package main
import (
"context"
"flag"
"fmt"
"os"
"os/signal"
"strings"
"syscall"
"time"
"github.com/coreos/go-systemd/v22/daemon"
"github.com/antitbone/ja4/sentinel/api"
"github.com/antitbone/ja4/sentinel/internal/capture"
"github.com/antitbone/ja4/sentinel/internal/config"
"github.com/antitbone/ja4/sentinel/internal/fingerprint"
"github.com/antitbone/ja4/sentinel/internal/logging"
"github.com/antitbone/ja4/sentinel/internal/output"
"github.com/antitbone/ja4/sentinel/internal/tlsparse"
)
var (
// Version information (set via ldflags)
Version = "1.1.15"
BuildTime = "unknown"
GitCommit = "unknown"
)
func main() {
// Parse command-line flags
configPath := flag.String("config", "", "Path to configuration file (YAML)")
version := flag.Bool("version", false, "Show version information")
flag.Parse()
if *version {
fmt.Printf("ja4sentinel version %s (built %s, commit %s)\n", Version, BuildTime, GitCommit)
os.Exit(0)
}
// Load configuration
cfgLoader := config.NewLoader(*configPath)
appConfig, err := cfgLoader.Load()
if err != nil {
// Create logger with default level for error reporting
loggerFactory := &logging.LoggerFactory{}
appLogger := loggerFactory.NewDefaultLogger()
appLogger.Error("main", "Failed to load configuration", map[string]string{
"error": err.Error(),
})
os.Exit(1)
}
// Create logger factory with configured log level
loggerFactory := &logging.LoggerFactory{}
appLogger := loggerFactory.NewLogger(appConfig.Core.LogLevel)
appLogger.Info("main", "Starting ja4sentinel", map[string]string{
"version": Version,
"build_time": BuildTime,
"git_commit": GitCommit,
})
appLogger.Info("main", "Configuration loaded", map[string]string{
"interface": appConfig.Core.Interface,
"listen_ports": formatPorts(appConfig.Core.ListenPorts),
"log_level": appConfig.Core.LogLevel,
})
// Create context with cancellation for graceful shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Signal readiness to systemd
if _, err := daemon.SdNotify(false, daemon.SdNotifyReady); err != nil {
appLogger.Warn("main", "Failed to send READY notification to systemd", map[string]string{
"error": err.Error(),
})
}
// Start watchdog goroutine if enabled
watchdogInterval, err := daemon.SdWatchdogEnabled(false)
if err != nil {
appLogger.Warn("main", "Failed to check watchdog status", map[string]string{
"error": err.Error(),
})
}
if watchdogInterval > 0 {
appLogger.Info("main", "systemd watchdog enabled", map[string]string{
"interval": watchdogInterval.String(),
})
go func() {
ticker := time.NewTicker(watchdogInterval / 2)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if _, err := daemon.SdNotify(false, daemon.SdNotifyWatchdog); err != nil {
appLogger.Warn("main", "Failed to send WATCHDOG notification", map[string]string{
"error": err.Error(),
})
}
case <-ctx.Done():
return
}
}
}()
}
// Setup signal handling for shutdown and log rotation
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
// Create pipeline components
captureEngine := capture.New()
parser := tlsparse.NewParserWithTimeoutAndFilter(
time.Duration(appConfig.Core.FlowTimeoutSec)*time.Second,
appConfig.Core.ExcludeSourceIPs,
)
fingerprintEngine := fingerprint.NewEngine()
// Log exclusion configuration with debug details
if len(appConfig.Core.ExcludeSourceIPs) > 0 {
appLogger.Info("main", "Source IP exclusion enabled", map[string]string{
"exclude_count": fmt.Sprintf("%d", len(appConfig.Core.ExcludeSourceIPs)),
"exclude_ips": strings.Join(appConfig.Core.ExcludeSourceIPs, ", "),
})
appLogger.Debug("tlsparse", "IP filter configured", map[string]string{
"filter_entries": strings.Join(appConfig.Core.ExcludeSourceIPs, ", "),
})
} else {
appLogger.Debug("tlsparse", "IP filter disabled (no exclusions configured)", nil)
}
// Log filter stats at startup (debug mode)
filteredCount, hasFilter := parser.GetFilterStats()
if hasFilter {
appLogger.Debug("tlsparse", "IP filter initialized", map[string]string{
"filtered_packets": fmt.Sprintf("%d", filteredCount),
})
}
// Create output builder with error callback for socket connection errors
outputBuilder := output.NewBuilder().WithErrorCallback(func(socketPath string, err error, attempt int) {
appLogger.Error("output", "UNIX socket connection failed", map[string]string{
"socket_path": socketPath,
"error": err.Error(),
"attempt": fmt.Sprintf("%d", attempt),
})
})
outputWriter, err := outputBuilder.NewFromConfig(appConfig)
if err != nil {
appLogger.Error("main", "Failed to create output writer", map[string]string{
"error": err.Error(),
})
os.Exit(1)
}
// Create channel for raw packets (configurable buffer size)
bufferSize := appConfig.Core.PacketBufferSize
if bufferSize <= 0 {
bufferSize = 1000 // Default fallback
}
packetChan := make(chan api.RawPacket, bufferSize)
// Start capture goroutine
captureErrChan := make(chan error, 1)
go func() {
appLogger.Info("capture", "Starting packet capture", map[string]string{
"interface": appConfig.Core.Interface,
})
err := captureEngine.Run(appConfig.Core, packetChan)
close(packetChan) // Close channel to signal packet processor to shut down
captureErrChan <- err
}()
// Log capture diagnostics after a short delay to allow initialization
go func() {
time.Sleep(100 * time.Millisecond)
ifName, localIPs, bpfFilter, linkType := captureEngine.GetDiagnostics()
appLogger.Debug("capture", "Capture initialized", map[string]string{
"interface": ifName,
"link_type": fmt.Sprintf("%d", linkType),
"local_ips": strings.Join(localIPs, ", "),
"bpf_filter": bpfFilter,
})
}()
// Process packets
go func() {
for {
select {
case <-ctx.Done():
appLogger.Info("main", "Packet processor shutting down", nil)
return
case pkt, ok := <-packetChan:
if !ok {
return
}
// Parse TLS ClientHello
clientHello, err := parser.Process(pkt)
if err != nil {
appLogger.Warn("tlsparse", "Failed to parse TLS ClientHello", map[string]string{
"error": err.Error(),
"packet_len": fmt.Sprintf("%d", len(pkt.Data)),
"link_type": fmt.Sprintf("%d", pkt.LinkType),
"timestamp": fmt.Sprintf("%d", pkt.Timestamp),
})
continue
}
if clientHello == nil {
continue // Not a TLS ClientHello packet
}
appLogger.Debug("tlsparse", "ClientHello extracted", map[string]string{
"src_ip": clientHello.SrcIP,
"src_port": fmt.Sprintf("%d", clientHello.SrcPort),
"dst_ip": clientHello.DstIP,
"dst_port": fmt.Sprintf("%d", clientHello.DstPort),
})
// Generate fingerprints
fingerprints, err := fingerprintEngine.FromClientHello(*clientHello)
if err != nil {
appLogger.Warn("fingerprint", "Failed to generate fingerprints", map[string]string{
"error": err.Error(),
"src_ip": clientHello.SrcIP,
"src_port": fmt.Sprintf("%d", clientHello.SrcPort),
"dst_ip": clientHello.DstIP,
"dst_port": fmt.Sprintf("%d", clientHello.DstPort),
"conn_id": clientHello.ConnID,
"payload_len": fmt.Sprintf("%d", len(clientHello.Payload)),
"sni": clientHello.SNI,
"tls_version": clientHello.TLSVersion,
"alpn": clientHello.ALPN,
})
continue
}
appLogger.Debug("fingerprint", "Fingerprints generated", map[string]string{
"src_ip": clientHello.SrcIP,
"ja4": fingerprints.JA4,
})
// Create log record
logRecord := api.NewLogRecord(*clientHello, fingerprints)
// Write output
if err := outputWriter.Write(logRecord); err != nil {
appLogger.Error("output", "Failed to write log record", map[string]string{
"error": err.Error(),
})
}
}
}
}()
// Wait for shutdown signal or capture error
for {
select {
case sig := <-sigChan:
switch sig {
case syscall.SIGHUP:
// Handle log rotation - reopen output files
appLogger.Info("main", "Received SIGHUP, reopening log files", nil)
if mw, ok := outputWriter.(api.Reopenable); ok {
if err := mw.Reopen(); err != nil {
appLogger.Error("main", "Failed to reopen log files", map[string]string{
"error": err.Error(),
})
} else {
appLogger.Info("main", "Log files reopened successfully", nil)
}
} else {
appLogger.Warn("main", "Output writer does not support log rotation", nil)
}
case syscall.SIGINT, syscall.SIGTERM:
appLogger.Info("main", "Received shutdown signal", map[string]string{
"signal": sig.String(),
})
goto shutdown
}
case err := <-captureErrChan:
if err != nil {
appLogger.Error("capture", "Capture engine failed", map[string]string{
"error": err.Error(),
})
}
goto shutdown
}
}
shutdown:
// Graceful shutdown
appLogger.Info("main", "Shutting down...", nil)
// Signal stopping to systemd
if _, err := daemon.SdNotify(false, daemon.SdNotifyStopping); err != nil {
appLogger.Warn("main", "Failed to send STOPPING notification to systemd", map[string]string{
"error": err.Error(),
})
}
cancel()
// Close components
if err := captureEngine.Close(); err != nil {
appLogger.Error("main", "Failed to close capture engine", map[string]string{
"error": err.Error(),
})
}
if err := parser.Close(); err != nil {
appLogger.Error("main", "Failed to close parser", map[string]string{
"error": err.Error(),
})
}
// Log final filter stats
filteredCount, hasFilter = parser.GetFilterStats()
if hasFilter {
appLogger.Info("tlsparse", "IP filter statistics", map[string]string{
"total_filtered_packets": fmt.Sprintf("%d", filteredCount),
})
}
if mw, ok := outputWriter.(interface{ CloseAll() error }); ok {
if err := mw.CloseAll(); err != nil {
appLogger.Error("main", "Failed to close output writers", map[string]string{
"error": err.Error(),
})
}
} else if closer, ok := outputWriter.(interface{ Close() error }); ok {
if err := closer.Close(); err != nil {
appLogger.Error("main", "Failed to close output writer", map[string]string{
"error": err.Error(),
})
}
}
appLogger.Info("main", "ja4sentinel stopped", nil)
}
// formatPorts formats a slice of ports as a comma-separated string
func formatPorts(ports []uint16) string {
if len(ports) == 0 {
return ""
}
result := fmt.Sprintf("%d", ports[0])
for _, port := range ports[1:] {
result += fmt.Sprintf(",%d", port)
}
return result
}

221
old/services/sentinel/cmd/ja4sentinel/main_test.go Normal file
View File

@ -0,0 +1,221 @@
package main
import (
"flag"
"strings"
"testing"
)
func TestFormatPorts(t *testing.T) {
tests := []struct {
name string
ports []uint16
want string
}{
{
name: "empty slice",
ports: []uint16{},
want: "",
},
{
name: "single port",
ports: []uint16{443},
want: "443",
},
{
name: "multiple ports",
ports: []uint16{443, 8443, 9443},
want: "443,8443,9443",
},
{
name: "two ports",
ports: []uint16{80, 443},
want: "80,443",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := formatPorts(tt.ports)
if got != tt.want {
t.Errorf("formatPorts() = %v, want %v", got, tt.want)
}
})
}
}
// TestMain_VersionFlag_VerifiesOutput tests that the version flag produces correct output
// Note: This test verifies the version variables are set correctly
func TestMain_VersionFlag_VerifiesOutput(t *testing.T) {
// Verify version variables are set
if Version == "" {
t.Error("Version should not be empty")
}
if BuildTime == "" {
t.Error("BuildTime should not be empty")
}
if GitCommit == "" {
t.Error("GitCommit should not be empty")
}
// Verify version format
expectedPrefix := "ja4sentinel version"
got := getVersionString()
if !strings.HasPrefix(got, expectedPrefix) {
t.Errorf("getVersionString() = %v, should start with %v", got, expectedPrefix)
}
}
// getVersionString returns the version string (helper for testing)
func getVersionString() string {
return "ja4sentinel version " + Version + " (built " + BuildTime + ", commit " + GitCommit + ")"
}
func TestFlagParsing(t *testing.T) {
tests := []struct {
name string
args []string
wantConfig string
wantVersion bool
}{
{
name: "config flag",
args: []string{"ja4sentinel", "-config", "/path/to/config.yml"},
wantConfig: "/path/to/config.yml",
wantVersion: false,
},
{
name: "version flag",
args: []string{"ja4sentinel", "-version"},
wantConfig: "",
wantVersion: true,
},
{
name: "no flags",
args: []string{"ja4sentinel"},
wantConfig: "",
wantVersion: false,
},
{
name: "config with long form",
args: []string{"ja4sentinel", "--config", "/etc/ja4sentinel/config.yml"},
wantConfig: "/etc/ja4sentinel/config.yml",
wantVersion: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
fs := flag.NewFlagSet("test", flag.ContinueOnError)
configPath := fs.String("config", "", "Path to configuration file (YAML)")
version := fs.Bool("version", false, "Show version information")
err := fs.Parse(tt.args[1:])
if err != nil {
t.Fatalf("Flag parsing failed: %v", err)
}
if *configPath != tt.wantConfig {
t.Errorf("config = %v, want %v", *configPath, tt.wantConfig)
}
if *version != tt.wantVersion {
t.Errorf("version = %v, want %v", *version, tt.wantVersion)
}
})
}
}
// TestMain_WithInvalidConfig tests that main exits gracefully with invalid config
func TestMain_WithInvalidConfig(t *testing.T) {
// This test verifies that the application handles config errors gracefully
// We can't easily test the full main() function, but we can test the
// config loading and error handling paths
t.Log("Note: Full main() testing requires integration tests with mocked dependencies")
}
// TestSignalHandling_VerifiesConstants tests that signal constants are defined
func TestSignalHandling_VerifiesConstants(t *testing.T) {
// Verify that we import the required packages for signal handling
// This test ensures the imports are present
t.Log("syscall and os/signal packages are imported for signal handling")
}
// TestGracefulShutdown_SimulatesSignal tests graceful shutdown behavior
func TestGracefulShutdown_SimulatesSignal(t *testing.T) {
// This test documents the expected shutdown behavior
// Full testing requires integration tests with actual signal sending
expectedBehavior := `
Graceful shutdown sequence:
1. Receive SIGINT or SIGTERM
2. Stop packet capture
3. Close output writers
4. Flush pending logs
5. Exit cleanly
`
t.Log(expectedBehavior)
}
// TestLogRotation_SIGHUP tests SIGHUP handling for log rotation
func TestLogRotation_SIGHUP(t *testing.T) {
// This test documents the expected log rotation behavior
// Full testing requires integration tests with actual SIGHUP signal
expectedBehavior := `
Log rotation sequence (SIGHUP):
1. Receive SIGHUP
2. Reopen all reopenable writers (FileWriter, MultiWriter)
3. Continue operation with new file handles
4. No data loss during rotation
`
t.Log(expectedBehavior)
}
// TestMain_ConfigValidation tests config validation before starting
func TestMain_ConfigValidation(t *testing.T) {
// Test that invalid configs are rejected before starting the pipeline
tests := []struct {
name string
configErr string
}{
{
name: "empty_interface",
configErr: "interface cannot be empty",
},
{
name: "no_listen_ports",
configErr: "at least one listen port required",
},
{
name: "invalid_output_type",
configErr: "unknown output type",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
// Verify that these error conditions are documented
t.Logf("Expected error for %s: %s", tt.name, tt.configErr)
})
}
}
// TestPipelineConstruction verifies the pipeline is built correctly
func TestPipelineConstruction(t *testing.T) {
// This test documents the expected pipeline construction
// Full testing requires integration tests
expectedPipeline := `
Pipeline construction:
1. Load configuration
2. Create logger
3. Create capture engine
4. Create TLS parser
5. Create fingerprint engine
6. Create output writer(s)
7. Connect pipeline: capture -> parser -> fingerprint -> output
8. Start signal handling
9. Run capture loop
`
t.Log(expectedPipeline)
}

57
old/services/sentinel/config.yml.example Normal file
View File

@ -0,0 +1,57 @@
# Sample configuration file for ja4sentinel
# Copy to config.yml and adjust as needed
core:
# Network interface to capture traffic from
# "any" captures on all interfaces (default, recommended)
# Or specify a specific interface (e.g., eth0, ens192, etc.)
interface: any
# TCP ports to monitor for TLS handshakes
listen_ports:
- 443
- 8443
# Optional BPF filter (leave empty for auto-generated filter based on listen_ports and local_ips)
bpf_filter: ""
# Local IP addresses to monitor (traffic destined to these IPs will be captured)
# Leave empty for auto-detection (recommended) - excludes loopback addresses
# Or specify manually: ["192.168.1.10", "10.0.0.5", "2001:db8::1"]
local_ips: []
# Source IP addresses or CIDR ranges to exclude from capture
# Useful for filtering out internal traffic, health checks, or monitoring systems
# Examples: ["10.0.0.0/8", "192.168.1.1", "172.16.0.0/12"]
exclude_source_ips: []
# Timeout in seconds for TLS handshake extraction (default: 30)
flow_timeout_sec: 30
# Buffer size for packet channel (default: 1000, increase for high-traffic environments)
packet_buffer_size: 1000
# Log level: debug, info, warn, error (default: info)
# Can be overridden by JA4SENTINEL_LOG_LEVEL environment variable
log_level: info
outputs:
# Output to UNIX socket (for systemd/journald or other consumers)
# Only JSON LogRecord data is sent - no diagnostic logs
- type: unix_socket
enabled: true
params:
socket_path: /var/run/logcorrelator/network.socket
# Output to stdout (JSON lines)
# Diagnostic logs (error, debug, warning) should go here
# - type: stdout
# enabled: false
# params: {}
# Output to file
# Only JSON LogRecord data is sent - no diagnostic logs
# - type: file
# enabled: false
# params:
# path: /var/log/ja4sentinel/ja4.log

Some files were not shown because too many files have changed in this diff Show More