Feat: Détection menaces HTTP via vues ClickHouse + simplification shutdown

Nouvelles vues de détection (sql/views.sql) : - Identification hosts par IP/JA4 (view_host_identification, view_host_ja4_anomalies) - Détection brute force POST et query params variables - Header fingerprinting (ordre, headers modernes manquants, Sec-CH-UA) - ALPN mismatch detection (h2 déclaré mais HTTP/1.1 parlé) - Rate limiting & burst detection (50 req/min, 20 req/10s) - Path enumeration/scanning (paths sensibles) - Payload attacks (SQLi, XSS, path traversal) - JA4 botnet detection (même fingerprint sur 20+ IPs) - Correlation quality (orphan ratio >80%) ClickHouse (sql/init.sql) : - Compression ZSTD(3) sur champs texte (path, query, headers, ja3/ja4) - TTL automatique : 1 jour (raw) + 7 jours (http_logs) - Paramètre ttl_only_drop_parts = 1 Shutdown simplifié (internal/app/orchestrator.go) : - Suppression ShutdownTimeout et logique de flush/attente - Stop() = cancel() + Close() uniquement - systemd TimeoutStopSec gère l'arrêt forcé si besoin File output toggle (internal/config/*.go) : - Ajout champ Enabled dans FileOutputConfig - Le sink fichier n'est créé que si enabled && path != '' - Tests : TestValidate_FileOutputDisabled, TestLoadConfig_FileOutputDisabled RPM packaging (packaging/rpm/logcorrelator.spec) : - Changelog 1.1.18 → 1.1.22 - Suppression logcorrelator-tmpfiles.conf (redondant RuntimeDirectory=) Nettoyage : - idees.txt → idees/ (dossier) - Suppression 91.224.92.185.txt (logs exemple) Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
2026-03-11 18:28:07 +01:00
parent 5df2fd965b
commit 20ebe7240e
17 changed files with 1089 additions and 6598 deletions
--- a/sql/views.sql
+++ b/sql/views.sql
@ -0,0 +1,251 @@
+-- ============================================================================
+-- SCRIPT DE DÉPLOIEMENT DES VUES DE DÉTECTION DE BOTS & SPAM (CLICKHOUSE)
+-- ============================================================================
+
+-- ----------------------------------------------------------------------------
+-- 1. NETTOYAGE STRICT
+-- ----------------------------------------------------------------------------
+DROP TABLE IF EXISTS mabase_prod.ml_detected_anomalies;
+
+DROP VIEW IF EXISTS mabase_prod.view_ai_features_1h;
+DROP VIEW IF EXISTS mabase_prod.view_host_ip_ja4_rotation;
+DROP VIEW IF EXISTS mabase_prod.view_host_ja4_anomalies;
+DROP VIEW IF EXISTS mabase_prod.view_form_bruteforce_detected;
+DROP VIEW IF EXISTS mabase_prod.view_alpn_mismatch_detected;
+DROP VIEW IF EXISTS mabase_prod.view_tcp_spoofing_detected;
+
+DROP VIEW IF EXISTS mabase_prod.mv_agg_host_ip_ja4_1h;
+DROP TABLE IF EXISTS mabase_prod.agg_host_ip_ja4_1h;
+
+DROP VIEW IF EXISTS mabase_prod.mv_agg_header_fingerprint_1h;
+DROP TABLE IF EXISTS mabase_prod.agg_header_fingerprint_1h;
+
+
+-- ----------------------------------------------------------------------------
+-- 2. TABLES D'AGRÉGATION ET VUES MATÉRIALISÉES (TEMPS RÉEL)
+-- ----------------------------------------------------------------------------
+
+CREATE TABLE mabase_prod.agg_host_ip_ja4_1h (
+    window_start DateTime,
+    src_ip String,
+    ja4 String,
+    host String,
+    first_seen SimpleAggregateFunction(min, DateTime),
+    last_seen SimpleAggregateFunction(max, DateTime),
+    hits SimpleAggregateFunction(sum, UInt64),
+    count_post SimpleAggregateFunction(sum, UInt64),
+    uniq_paths AggregateFunction(uniq, String),
+    uniq_query_params AggregateFunction(uniq, String),
+    src_country_code SimpleAggregateFunction(any, String),
+    tcp_fingerprint SimpleAggregateFunction(any, String),
+    tcp_jitter_variance AggregateFunction(varPop, Float64),
+    tcp_window_size SimpleAggregateFunction(any, UInt32),
+    tcp_window_scale SimpleAggregateFunction(any, UInt32),
+    tcp_mss SimpleAggregateFunction(any, UInt32),
+    tcp_ttl SimpleAggregateFunction(any, UInt32),
+    http_version SimpleAggregateFunction(any, String),
+    first_ua SimpleAggregateFunction(any, String)
+) ENGINE = AggregatingMergeTree()
+ORDER BY (window_start, src_ip, ja4, host)
+TTL window_start + INTERVAL 7 DAY;
+
+CREATE MATERIALIZED VIEW mabase_prod.mv_agg_host_ip_ja4_1h 
+TO mabase_prod.agg_host_ip_ja4_1h AS
+SELECT
+    toStartOfHour(time) AS window_start,
+    src_ip,
+    ja4,
+    host,
+    min(time) AS first_seen,
+    max(time) AS last_seen,
+    count() AS hits,
+    sum(IF(method = 'POST', 1, 0)) AS count_post,
+    uniqState(path) AS uniq_paths,
+    uniqState(query) AS uniq_query_params,
+    any(src_country_code) AS src_country_code,
+    any(toString(cityHash64(concat(toString(tcp_meta_window_size), toString(tcp_meta_mss), toString(tcp_meta_window_scale), tcp_meta_options)))) AS tcp_fingerprint,
+    varPopState(toFloat64(syn_to_clienthello_ms)) AS tcp_jitter_variance,
+    any(tcp_meta_window_size) AS tcp_window_size,
+    any(tcp_meta_window_scale) AS tcp_window_scale,
+    any(tcp_meta_mss) AS tcp_mss,
+    any(ip_meta_ttl) AS tcp_ttl,
+    any(http_version) AS http_version,
+    any(header_user_agent) AS first_ua
+FROM mabase_prod.http_logs
+GROUP BY window_start, src_ip, ja4, host;
+
+
+CREATE TABLE mabase_prod.agg_header_fingerprint_1h (
+    window_start DateTime,
+    src_ip String,
+    header_order_hash SimpleAggregateFunction(any, String),
+    modern_browser_score SimpleAggregateFunction(max, UInt8),
+    sec_fetch_mode SimpleAggregateFunction(any, String),
+    sec_fetch_dest SimpleAggregateFunction(any, String),
+    count_site_none SimpleAggregateFunction(sum, UInt64)
+) ENGINE = AggregatingMergeTree()
+ORDER BY (window_start, src_ip)
+TTL window_start + INTERVAL 7 DAY;
+
+CREATE MATERIALIZED VIEW mabase_prod.mv_agg_header_fingerprint_1h 
+TO mabase_prod.agg_header_fingerprint_1h AS
+SELECT
+    toStartOfHour(time) AS window_start,
+    src_ip,
+    any(toString(cityHash64(client_headers))) AS header_order_hash,
+    max(toUInt8(if(length(header_sec_ch_ua) > 0, 100, if(length(header_user_agent) > 0, 50, 0)))) AS modern_browser_score,
+    any(header_sec_fetch_mode) AS sec_fetch_mode,
+    any(header_sec_fetch_dest) AS sec_fetch_dest,
+    sum(IF(header_sec_fetch_site = 'none', 1, 0)) AS count_site_none
+FROM mabase_prod.http_logs
+GROUP BY window_start, src_ip;
+
+
+-- ----------------------------------------------------------------------------
+-- 3. TABLE DE DESTINATION POUR LE MACHINE LEARNING
+-- ----------------------------------------------------------------------------
+CREATE TABLE mabase_prod.ml_detected_anomalies (
+    detected_at DateTime,
+    src_ip String,
+    ja4 String,
+    host String,
+    anomaly_score Float32,
+    reason String
+) ENGINE = MergeTree()
+ORDER BY (detected_at, src_ip, ja4)
+TTL detected_at + INTERVAL 30 DAY;
+
+
+-- ----------------------------------------------------------------------------
+-- 4. VUE DE FEATURE ENGINEERING POUR L'ISOLATION FOREST (RÉSOLUE)
+-- ----------------------------------------------------------------------------
+-- Utilisation de sous-requêtes agrégées (GROUP BY explicite) avant la jointure
+-- pour éviter les erreurs d'état et le produit cartésien.
+CREATE VIEW mabase_prod.view_ai_features_1h AS
+SELECT 
+    a.src_ip, 
+    a.ja4, 
+    a.host, 
+    a.hits, 
+    a.uniq_paths, 
+    a.uniq_query_params,
+    a.count_post,
+    
+    -- Indicateur de Corrélation L4/L7
+    IF(length(a.ja4) > 0 AND length(a.tcp_fingerprint) > 0, 1, 0) AS correlated,
+
+    -- DIMENSIONS COMPORTEMENTALES
+    (a.count_post / (a.hits + 1)) AS post_ratio,
+    (a.uniq_query_params / (a.uniq_paths + 1)) AS fuzzing_index,
+    (a.hits / (dateDiff('second', a.first_seen, a.last_seen) + 1)) AS hit_velocity,
+
+    -- DIMENSIONS TCP / L4
+    COALESCE(a.tcp_jitter_variance, 0) AS tcp_jitter_variance,
+    count() OVER (PARTITION BY a.tcp_fingerprint) AS tcp_shared_count,
+    a.tcp_window_size * exp2(a.tcp_window_scale) AS true_window_size,
+    IF(a.tcp_mss > 0, a.tcp_window_size / a.tcp_mss, 0) AS window_mss_ratio,
+
+    -- DIMENSIONS TLS / L5 (Mismatch)
+    IF(substring(a.ja4, 10, 2) = 'h2' AND a.http_version!= '2', 1, 0) AS alpn_http_mismatch,
+    IF(substring(a.ja4, 10, 2) = '00', 1, 0) AS is_alpn_missing,
+
+    -- DIMENSIONS HTTP / L7
+    COALESCE(h.modern_browser_score, 0) AS modern_browser_score,
+    IF(h.sec_fetch_mode = 'navigate' AND h.sec_fetch_dest!= 'document', 1, 0) AS is_fake_navigation,
+    (h.count_site_none / (a.hits + 1)) AS site_none_ratio
+
+FROM (
+    -- Consolidation des logs d'hôtes (Résolution du GROUP BY manquant)
+    SELECT
+        window_start, src_ip, ja4, host,
+        sum(hits) AS hits,
+        uniqMerge(uniq_paths) AS uniq_paths,
+        uniqMerge(uniq_query_params) AS uniq_query_params,
+        sum(count_post) AS count_post,
+        min(first_seen) AS first_seen,
+        max(last_seen) AS last_seen,
+        any(tcp_fingerprint) AS tcp_fingerprint,
+        varPopMerge(tcp_jitter_variance) AS tcp_jitter_variance,
+        any(tcp_window_size) AS tcp_window_size,
+        any(tcp_window_scale) AS tcp_window_scale,
+        any(tcp_mss) AS tcp_mss,
+        any(http_version) AS http_version
+    FROM mabase_prod.agg_host_ip_ja4_1h
+    WHERE window_start >= toStartOfHour(now() - INTERVAL 2 HOUR)
+    GROUP BY window_start, src_ip, ja4, host
+) a
+LEFT JOIN (
+    -- Consolidation des en-têtes
+    SELECT
+        window_start, src_ip,
+        max(modern_browser_score) AS modern_browser_score,
+        any(sec_fetch_mode) AS sec_fetch_mode,
+        any(sec_fetch_dest) AS sec_fetch_dest,
+        sum(count_site_none) AS count_site_none
+    FROM mabase_prod.agg_header_fingerprint_1h
+    WHERE window_start >= toStartOfHour(now() - INTERVAL 2 HOUR)
+    GROUP BY window_start, src_ip
+) h
+ON a.src_ip = h.src_ip AND a.window_start = h.window_start;
+
+
+-- ----------------------------------------------------------------------------
+-- 5. VUES DE DÉTECTION HEURISTIQUES STATIQUES (RÉSOLUES)
+-- ----------------------------------------------------------------------------
+
+CREATE VIEW mabase_prod.view_host_ip_ja4_rotation AS
+SELECT 
+    src_ip, 
+    uniqExact(ja4) AS distinct_ja4_count,
+    sum(hits) AS total_hits
+FROM mabase_prod.agg_host_ip_ja4_1h
+WHERE window_start >= toStartOfHour(now() - INTERVAL 1 HOUR)
+GROUP BY src_ip
+HAVING distinct_ja4_count >= 5 AND total_hits > 100;
+
+CREATE VIEW mabase_prod.view_host_ja4_anomalies AS
+SELECT 
+    ja4,
+    uniqExact(src_ip) AS unique_ips,
+    uniqExact(src_country_code) AS unique_countries,
+    uniqExact(host) AS targeted_hosts
+FROM mabase_prod.agg_host_ip_ja4_1h
+WHERE window_start >= toStartOfHour(now() - INTERVAL 1 HOUR)
+GROUP BY ja4
+HAVING unique_ips >= 20 AND targeted_hosts >= 3;
+
+-- Ajout du GROUP BY
+CREATE VIEW mabase_prod.view_form_bruteforce_detected AS
+SELECT 
+    src_ip, ja4, host, 
+    sum(hits) AS hits, 
+    uniqMerge(uniq_query_params) AS query_params_count
+FROM mabase_prod.agg_host_ip_ja4_1h
+WHERE window_start >= toStartOfHour(now() - INTERVAL 1 HOUR)
+GROUP BY src_ip, ja4, host
+HAVING query_params_count >= 10 AND hits >= 20;
+
+-- Ajout du GROUP BY
+CREATE VIEW mabase_prod.view_alpn_mismatch_detected AS
+SELECT 
+    src_ip, ja4, host, 
+    sum(hits) AS hits,
+    any(http_version) AS http_version
+FROM mabase_prod.agg_host_ip_ja4_1h
+WHERE window_start >= toStartOfHour(now() - INTERVAL 1 HOUR)
+  AND substring(ja4, 10, 2) IN ('h2', 'h3') 
+GROUP BY src_ip, ja4, host
+HAVING http_version = '1.1' AND hits >= 10;
+
+-- Ajout du GROUP BY
+CREATE VIEW mabase_prod.view_tcp_spoofing_detected AS
+SELECT 
+    src_ip, ja4, 
+    any(tcp_ttl) AS tcp_ttl, 
+    any(tcp_window_size) AS tcp_window_size,
+    any(first_ua) AS first_ua
+FROM mabase_prod.agg_host_ip_ja4_1h
+WHERE window_start >= toStartOfHour(now() - INTERVAL 1 HOUR)
+GROUP BY src_ip, ja4
+HAVING tcp_ttl <= 64 
+   AND (first_ua ILIKE '%Windows%' OR first_ua ILIKE '%iPhone%');