From 25d49437142c6c10c7895c023869f73a86cc0593 Mon Sep 17 00:00:00 2001 From: toto Date: Tue, 3 Mar 2026 14:42:51 +0100 Subject: [PATCH] docs: README improvements - config, troubleshooting, structure - Update RPM version numbers to 1.1.6 - Fix config file name (.yml not .conf) - Add complete configuration example with current schema - Add ClickHouse DSN format documentation - Add Troubleshooting section (ClickHouse, MV, sockets, systemd) - Update project structure with accurate file names Co-authored-by: Qwen-Coder --- README.md | 159 ++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 118 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 4a59d6d..1a53a39 100644 --- a/README.md +++ b/README.md @@ -71,9 +71,9 @@ docker run -d \ make package-rpm # Installer le package RPM (Rocky Linux 8/9/10) -sudo dnf install -y dist/rpm/rocky8/logcorrelator-1.0.7-1.el8.x86_64.rpm -sudo dnf install -y dist/rpm/rocky9/logcorrelator-1.0.7-1.el9.x86_64.rpm -sudo dnf install -y dist/rpm/almalinux10/logcorrelator-1.0.7-1.el10.x86_64.rpm +sudo dnf install -y dist/rpm/el8/logcorrelator-1.1.6-1.el8.x86_64.rpm +sudo dnf install -y dist/rpm/el9/logcorrelator-1.1.6-1.el9.x86_64.rpm +sudo dnf install -y dist/rpm/el10/logcorrelator-1.1.6-1.el10.x86_64.rpm # Activer et démarrer le service sudo systemctl enable logcorrelator @@ -95,51 +95,70 @@ go build -o logcorrelator ./cmd/logcorrelator ## Configuration -La configuration utilise un fichier YAML : +La configuration utilise un fichier YAML. Voir `config.example.yml` pour un exemple complet. ```yaml -# Service configuration -service: - name: logcorrelator - language: go +# /etc/logcorrelator/logcorrelator.yml + +log: + level: INFO # DEBUG, INFO, WARN, ERROR -# Input sources (at least 2 required) inputs: unix_sockets: - - name: http_source - path: /var/run/logcorrelator/http.socket - format: json - - name: network_source - path: /var/run/logcorrelator/network.socket - format: json + # Source HTTP (A) : logs applicatifs en JSON + - name: http + path: /var/run/logcorrelator/http.sock + socket_permissions: "0666" + socket_type: dgram + + # Source réseau (B) : logs IP/TCP/JA3... en JSON + - name: network + path: /var/run/logcorrelator/network.sock + socket_permissions: "0666" + socket_type: dgram -# File output outputs: file: enabled: true path: /var/log/logcorrelator/correlated.log + format: json_lines -# ClickHouse output -outputs: clickhouse: - enabled: false - dsn: clickhouse://user:pass@localhost:9000/db - table: correlated_logs_http_network + enabled: true + dsn: clickhouse://data_writer:password@localhost:9000/mabase_prod + table: http_logs_raw + batch_size: 500 + flush_interval_ms: 200 + max_buffer_size: 5000 + drop_on_overflow: true -# Correlation configuration correlation: - key: - - src_ip - - src_port time_window: value: 1 unit: s orphan_policy: apache_always_emit: true network_emit: false + matching: + mode: one_to_many # Keep-Alive : un B peut corréler plusieurs A + buffers: + max_http_items: 10000 + max_network_items: 20000 + ttl: + network_ttl_s: 30 ``` -Exemple complet dans `config.example.yml`. +### Format du DSN ClickHouse + +``` +clickhouse://username:password@host:port/database +``` + +Exemple : `clickhouse://data_writer:MonMotDePasse@127.0.0.1:9000/mabase_prod` + +Ports courants : +- `9000` : port natif (recommandé pour le driver Go) +- `8123` : port HTTP (alternative) ## Format des logs @@ -479,26 +498,84 @@ journalctl -u logcorrelator -f ``` . -├── cmd/logcorrelator/ # Point d'entrée +├── cmd/logcorrelator/ # Point d'entrée ├── internal/ │ ├── adapters/ -│ │ ├── inbound/unixsocket/ +│ │ ├── inbound/unixsocket/ # Sources HTTP et réseau │ │ └── outbound/ -│ │ ├── clickhouse/ -│ │ ├── file/ -│ │ └── multi/ -│ ├── app/ # Orchestration -│ ├── config/ # Configuration -│ ├── domain/ # Domaine (corrélation) -│ ├── observability/ # Logging -│ └── ports/ # Interfaces -├── config.example.conf # Exemple de config -├── Dockerfile # Build multi-stage -├── build.sh # Script de build -├── test.sh # Script de tests -└── logcorrelator.service # Unité systemd +│ │ ├── clickhouse/ # Sink ClickHouse +│ │ ├── file/ # Sink fichier +│ │ └── multi/ # Multi-sink +│ ├── app/ # Orchestration +│ ├── config/ # Configuration YAML +│ ├── domain/ # Domaine (corrélation) +│ ├── observability/ # Logging interne +│ └── ports/ # Interfaces +├── config.example.yml # Exemple de config +├── Dockerfile # Build multi-stage +├── Dockerfile.package # Packaging RPM +├── Makefile # Commandes de build +├── architecture.yml # Spécification architecture +└── logcorrelator.service # Unité systemd ``` ## License MIT + +## Troubleshooting + +### ClickHouse : erreurs d'insertion + +**Erreur : `No such column timestamp`** +- Vérifiez que la table de destination est bien `http_logs_raw` (colonne unique `raw_json`) +- Le service envoie un JSON sérialisé dans `raw_json`, pas des colonnes séparées + +**Erreur : `ACCESS_DENIED`** +- Vérifiez les droits de l'utilisateur `data_writer` : + ```sql + GRANT INSERT(raw_json) ON mabase_prod.http_logs_raw TO data_writer; + GRANT SELECT(raw_json) ON mabase_prod.http_logs_raw TO data_writer; + ``` + +### Vue matérialisée ne fonctionne pas + +**Symptôme :** `http_logs_raw` a des données, mais `http_logs` est vide + +1. Vérifiez que la MV existe : + ```sql + SHOW CREATE TABLE mv_http_logs; + ``` + +2. Vérifiez les droits SELECT pour `data_writer` sur `http_logs_raw` + +3. Testez manuellement : + ```sql + INSERT INTO mabase_prod.http_logs + SELECT * FROM mabase_prod.mv_http_logs + WHERE time > now() - INTERVAL 1 HOUR; + ``` + +### Sockets Unix : permission denied + +**Erreur :** `permission denied` sur `/var/run/logcorrelator/*.sock` + +- Vérifiez que les sockets ont les permissions `0666` +- Vérifiez que l'utilisateur `logcorrelator` peut lire/écrire + +### Service systemd ne démarre pas + +1. Vérifiez les logs : + ```bash + journalctl -u logcorrelator -n 50 --no-pager + ``` + +2. Vérifiez la configuration : + ```bash + cat /etc/logcorrelator/logcorrelator.yml + ``` + +3. Testez manuellement : + ```bash + /usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml + ```