Feat: Détection menaces HTTP via vues ClickHouse + simplification shutdown
Nouvelles vues de détection (sql/views.sql) : - Identification hosts par IP/JA4 (view_host_identification, view_host_ja4_anomalies) - Détection brute force POST et query params variables - Header fingerprinting (ordre, headers modernes manquants, Sec-CH-UA) - ALPN mismatch detection (h2 déclaré mais HTTP/1.1 parlé) - Rate limiting & burst detection (50 req/min, 20 req/10s) - Path enumeration/scanning (paths sensibles) - Payload attacks (SQLi, XSS, path traversal) - JA4 botnet detection (même fingerprint sur 20+ IPs) - Correlation quality (orphan ratio >80%) ClickHouse (sql/init.sql) : - Compression ZSTD(3) sur champs texte (path, query, headers, ja3/ja4) - TTL automatique : 1 jour (raw) + 7 jours (http_logs) - Paramètre ttl_only_drop_parts = 1 Shutdown simplifié (internal/app/orchestrator.go) : - Suppression ShutdownTimeout et logique de flush/attente - Stop() = cancel() + Close() uniquement - systemd TimeoutStopSec gère l'arrêt forcé si besoin File output toggle (internal/config/*.go) : - Ajout champ Enabled dans FileOutputConfig - Le sink fichier n'est créé que si enabled && path != '' - Tests : TestValidate_FileOutputDisabled, TestLoadConfig_FileOutputDisabled RPM packaging (packaging/rpm/logcorrelator.spec) : - Changelog 1.1.18 → 1.1.22 - Suppression logcorrelator-tmpfiles.conf (redondant RuntimeDirectory=) Nettoyage : - idees.txt → idees/ (dossier) - Suppression 91.224.92.185.txt (logs exemple) Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
toto
121
sql/init.sql
121
sql/init.sql
@ -19,96 +19,101 @@ CREATE DATABASE IF NOT EXISTS mabase_prod;
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS mabase_prod.http_logs_raw
|
||||
(
|
||||
`raw_json` String,
|
||||
`raw_json` String CODEC(ZSTD(3)),
|
||||
`ingest_time` DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(ingest_time)
|
||||
ORDER BY ingest_time
|
||||
SETTINGS index_granularity = 8192;
|
||||
TTL ingest_time + INTERVAL 1 DAY
|
||||
SETTINGS
|
||||
index_granularity = 8192,
|
||||
ttl_only_drop_parts = 1;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Table parsée : alimentée automatiquement par la vue matérialisée
|
||||
-- -----------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE IF NOT EXISTS mabase_prod.http_logs
|
||||
CREATE TABLE mabase_prod.http_logs
|
||||
(
|
||||
-- Temporel
|
||||
`time` DateTime,
|
||||
`log_date` Date DEFAULT toDate(time),
|
||||
`time` DateTime,
|
||||
`log_date` Date DEFAULT toDate(time),
|
||||
|
||||
-- Réseau
|
||||
`src_ip` IPv4,
|
||||
`src_port` UInt16,
|
||||
`dst_ip` IPv4,
|
||||
`dst_port` UInt16,
|
||||
`src_ip` IPv4,
|
||||
`src_port` UInt16,
|
||||
`dst_ip` IPv4,
|
||||
`dst_port` UInt16,
|
||||
|
||||
-- Enrichissement IPLocate
|
||||
`src_asn` UInt32,
|
||||
`src_country_code` LowCardinality(String),
|
||||
`src_as_name` LowCardinality(String),
|
||||
`src_org` LowCardinality(String),
|
||||
`src_domain` LowCardinality(String),
|
||||
`src_asn` UInt32,
|
||||
`src_country_code` LowCardinality(String),
|
||||
`src_as_name` LowCardinality(String),
|
||||
`src_org` LowCardinality(String),
|
||||
`src_domain` LowCardinality(String),
|
||||
|
||||
-- HTTP
|
||||
`method` LowCardinality(String),
|
||||
`scheme` LowCardinality(String),
|
||||
`host` LowCardinality(String),
|
||||
`path` String,
|
||||
`query` String,
|
||||
`http_version` LowCardinality(String),
|
||||
`method` LowCardinality(String),
|
||||
`scheme` LowCardinality(String),
|
||||
`host` LowCardinality(String),
|
||||
`path` String CODEC(ZSTD(3)),
|
||||
`query` String CODEC(ZSTD(3)),
|
||||
`http_version` LowCardinality(String),
|
||||
|
||||
-- Corrélation
|
||||
`orphan_side` LowCardinality(String),
|
||||
`correlated` UInt8,
|
||||
`keepalives` UInt16,
|
||||
`a_timestamp` UInt64,
|
||||
`b_timestamp` UInt64,
|
||||
`conn_id` String,
|
||||
`orphan_side` LowCardinality(String),
|
||||
`correlated` UInt8,
|
||||
`keepalives` UInt16,
|
||||
`a_timestamp` UInt64,
|
||||
`b_timestamp` UInt64,
|
||||
`conn_id` String CODEC(ZSTD(3)),
|
||||
|
||||
-- Métadonnées IP
|
||||
`ip_meta_df` UInt8,
|
||||
`ip_meta_id` UInt16,
|
||||
`ip_meta_total_length` UInt16,
|
||||
`ip_meta_ttl` UInt8,
|
||||
`ip_meta_df` UInt8,
|
||||
`ip_meta_id` UInt16,
|
||||
`ip_meta_total_length` UInt16,
|
||||
`ip_meta_ttl` UInt8,
|
||||
|
||||
-- Métadonnées TCP
|
||||
`tcp_meta_options` LowCardinality(String),
|
||||
`tcp_meta_window_size` UInt32,
|
||||
`tcp_meta_mss` UInt16,
|
||||
`tcp_meta_window_scale` UInt8,
|
||||
`syn_to_clienthello_ms` Int32,
|
||||
`tcp_meta_options` LowCardinality(String),
|
||||
`tcp_meta_window_size` UInt32,
|
||||
`tcp_meta_mss` UInt16,
|
||||
`tcp_meta_window_scale` UInt8,
|
||||
`syn_to_clienthello_ms` Int32,
|
||||
|
||||
-- TLS / fingerprint
|
||||
`tls_version` LowCardinality(String),
|
||||
`tls_sni` LowCardinality(String),
|
||||
`tls_alpn` LowCardinality(String),
|
||||
`ja3` String,
|
||||
`ja3_hash` String,
|
||||
`ja4` String,
|
||||
`tls_version` LowCardinality(String),
|
||||
`tls_sni` LowCardinality(String),
|
||||
`tls_alpn` LowCardinality(String),
|
||||
`ja3` String CODEC(ZSTD(3)),
|
||||
`ja3_hash` String CODEC(ZSTD(3)),
|
||||
`ja4` String CODEC(ZSTD(3)),
|
||||
|
||||
-- En-têtes HTTP
|
||||
`client_headers` String,
|
||||
`header_user_agent` String,
|
||||
`header_accept` String,
|
||||
`header_accept_encoding` String,
|
||||
`header_accept_language` String,
|
||||
`header_content_type` String,
|
||||
`header_x_request_id` String,
|
||||
`header_x_trace_id` String,
|
||||
`header_x_forwarded_for` String,
|
||||
`header_sec_ch_ua` String,
|
||||
`header_sec_ch_ua_mobile` String,
|
||||
`header_sec_ch_ua_platform` String,
|
||||
`header_sec_fetch_dest` String,
|
||||
`header_sec_fetch_mode` String,
|
||||
`header_sec_fetch_site` String
|
||||
`client_headers` String CODEC(ZSTD(3)),
|
||||
`header_user_agent` String CODEC(ZSTD(3)),
|
||||
`header_accept` String CODEC(ZSTD(3)),
|
||||
`header_accept_encoding` String CODEC(ZSTD(3)),
|
||||
`header_accept_language` String CODEC(ZSTD(3)),
|
||||
`header_content_type` String CODEC(ZSTD(3)),
|
||||
`header_x_request_id` String CODEC(ZSTD(3)),
|
||||
`header_x_trace_id` String CODEC(ZSTD(3)),
|
||||
`header_x_forwarded_for` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua_mobile` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua_platform` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_dest` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_mode` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_site` String CODEC(ZSTD(3))
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY log_date
|
||||
ORDER BY (time, src_ip, dst_ip, ja4)
|
||||
SETTINGS index_granularity = 8192;
|
||||
|
||||
TTL log_date + INTERVAL 7 DAY
|
||||
SETTINGS
|
||||
index_granularity = 8192,
|
||||
ttl_only_drop_parts = 1;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vue matérialisée : parse le JSON de http_logs_raw vers http_logs
|
||||
|
||||
Reference in New Issue
Block a user