feat: split ClickHouse into dual configurable databases (ja4_logs / ja4_processing)
Architecture: - ja4_logs: raw log ingestion (http_logs_raw, http_logs, mv_http_logs) - ja4_processing: analytics, aggregation, ML, dictionaries, audit Configuration (env vars): - CLICKHOUSE_DB_LOGS (default: ja4_logs) - CLICKHOUSE_DB_PROCESSING (default: ja4_processing) Changes: - SQL migrations (10 files): all mabase_prod refs → ja4_logs or ja4_processing with correct cross-database references (MVs, views, dicts) - deploy_schema.sh: substitutes DB names from env vars at deploy time - Python shared settings: added CLICKHOUSE_DB_LOGS + CLICKHOUSE_DB_PROCESSING - Dashboard routes (19 files): replaced ~80 hardcoded mabase_prod refs with settings.CLICKHOUSE_DB_LOGS / settings.CLICKHOUSE_DB_PROCESSING - Bot-detector: DB → CLICKHOUSE_DB_PROCESSING, fetch_rules.py configurable - Correlator: DSN example updated to ja4_logs - Docker-compose + .env files: new env vars with defaults - All documentation updated (14 markdown files) All tests pass: sentinel 10/10, correlator 67.1%, bot-detector 11, dashboard 20, ja4_common 18 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
@ -1,50 +1,50 @@
|
||||
-- ============================================================================
|
||||
-- ARCHITECTURE DE DÉTECTION INTÉGRALE (v13 - bot_detector v11 + ml_all_scores)
|
||||
-- Base : mabase_prod | Fenêtre : 24h | Dédoublonnage par src_ip
|
||||
-- Base : ja4_processing | Fenêtre : 24h | Dédoublonnage par src_ip
|
||||
-- Modifications v11 : ajout campaign_id, raw_anomaly_score dans ml_detected_anomalies
|
||||
-- correction view_dashboard_variability (header_user_agent → reason)
|
||||
-- Modifications v12 : ajout table ml_all_scores (toutes les classifications, sans seuil)
|
||||
-- ============================================================================
|
||||
|
||||
-- 1. NETTOYAGE COMPLET
|
||||
DROP TABLE IF EXISTS mabase_prod.ml_all_scores;
|
||||
DROP DICTIONARY IF EXISTS mabase_prod.dict_bot_ip;
|
||||
DROP DICTIONARY IF EXISTS mabase_prod.dict_bot_ja4;
|
||||
DROP DICTIONARY IF EXISTS mabase_prod.dict_asn_reputation;
|
||||
DROP TABLE IF EXISTS mabase_prod.ml_detected_anomalies;
|
||||
DROP VIEW IF EXISTS mabase_prod.view_ip_recurrence;
|
||||
DROP VIEW IF EXISTS mabase_prod.view_ai_features_1h;
|
||||
DROP TABLE IF EXISTS ja4_processing.ml_all_scores;
|
||||
DROP DICTIONARY IF EXISTS ja4_processing.dict_bot_ip;
|
||||
DROP DICTIONARY IF EXISTS ja4_processing.dict_bot_ja4;
|
||||
DROP DICTIONARY IF EXISTS ja4_processing.dict_asn_reputation;
|
||||
DROP TABLE IF EXISTS ja4_processing.ml_detected_anomalies;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_ip_recurrence;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_ai_features_1h;
|
||||
-- Suppression des anciennes vues heuristiques
|
||||
DROP VIEW IF EXISTS mabase_prod.view_host_ip_ja4_rotation;
|
||||
DROP VIEW IF EXISTS mabase_prod.view_host_ja4_anomalies;
|
||||
DROP VIEW IF EXISTS mabase_prod.view_form_bruteforce_detected;
|
||||
DROP VIEW IF EXISTS mabase_prod.view_alpn_mismatch_detected;
|
||||
DROP VIEW IF EXISTS mabase_prod.view_tcp_spoofing_detected;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_host_ip_ja4_rotation;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_host_ja4_anomalies;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_form_bruteforce_detected;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_alpn_mismatch_detected;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_tcp_spoofing_detected;
|
||||
|
||||
DROP VIEW IF EXISTS mabase_prod.mv_agg_host_ip_ja4_1h;
|
||||
DROP TABLE IF EXISTS mabase_prod.agg_host_ip_ja4_1h;
|
||||
DROP VIEW IF EXISTS mabase_prod.mv_agg_header_fingerprint_1h;
|
||||
DROP TABLE IF EXISTS mabase_prod.agg_header_fingerprint_1h;
|
||||
DROP VIEW IF EXISTS ja4_processing.mv_agg_host_ip_ja4_1h;
|
||||
DROP TABLE IF EXISTS ja4_processing.agg_host_ip_ja4_1h;
|
||||
DROP VIEW IF EXISTS ja4_processing.mv_agg_header_fingerprint_1h;
|
||||
DROP TABLE IF EXISTS ja4_processing.agg_header_fingerprint_1h;
|
||||
|
||||
-- ============================================================================
|
||||
-- 2. DICTIONNAIRES DE RÉPUTATION EN RAM
|
||||
-- ============================================================================
|
||||
CREATE DICTIONARY mabase_prod.dict_bot_ip (prefix String, bot_name String)
|
||||
CREATE DICTIONARY ja4_processing.dict_bot_ip (prefix String, bot_name String)
|
||||
PRIMARY KEY prefix SOURCE(FILE(path '/var/lib/clickhouse/user_files/bot_ip.csv' format 'CSV'))
|
||||
LAYOUT(IP_TRIE()) LIFETIME(MIN 300 MAX 300);
|
||||
|
||||
CREATE DICTIONARY mabase_prod.dict_bot_ja4 (ja4 String, bot_name String)
|
||||
CREATE DICTIONARY ja4_processing.dict_bot_ja4 (ja4 String, bot_name String)
|
||||
PRIMARY KEY ja4 SOURCE(FILE(path '/var/lib/clickhouse/user_files/bot_ja4.csv' format 'CSV'))
|
||||
LAYOUT(COMPLEX_KEY_HASHED()) LIFETIME(MIN 300 MAX 300);
|
||||
|
||||
CREATE DICTIONARY mabase_prod.dict_asn_reputation (src_asn UInt64, label String)
|
||||
CREATE DICTIONARY ja4_processing.dict_asn_reputation (src_asn UInt64, label String)
|
||||
PRIMARY KEY src_asn SOURCE(FILE(path '/var/lib/clickhouse/user_files/asn_reputation.csv' format 'CSV'))
|
||||
LAYOUT(HASHED()) LIFETIME(MIN 300 MAX 300);
|
||||
|
||||
-- ============================================================================
|
||||
-- 3. TABLE D'AGRÉGATION COMPORTEMENTALE (L4 / L5 / L7)
|
||||
-- ============================================================================
|
||||
CREATE TABLE mabase_prod.agg_host_ip_ja4_1h
|
||||
CREATE TABLE ja4_processing.agg_host_ip_ja4_1h
|
||||
(
|
||||
window_start DateTime,
|
||||
src_ip IPv6, ja4 String, host String, src_asn UInt32,
|
||||
@ -98,8 +98,8 @@ ORDER BY (window_start, src_ip, ja4, host);
|
||||
-- ============================================================================
|
||||
-- 4. VUE MATÉRIALISÉE → agg_host_ip_ja4_1h
|
||||
-- ============================================================================
|
||||
CREATE MATERIALIZED VIEW mabase_prod.mv_agg_host_ip_ja4_1h
|
||||
TO mabase_prod.agg_host_ip_ja4_1h AS
|
||||
CREATE MATERIALIZED VIEW ja4_processing.mv_agg_host_ip_ja4_1h
|
||||
TO ja4_processing.agg_host_ip_ja4_1h AS
|
||||
SELECT
|
||||
toStartOfHour(src.time) AS window_start,
|
||||
toIPv6(src.src_ip) AS src_ip, src.ja4, src.host, src.src_asn,
|
||||
@ -135,13 +135,13 @@ SELECT
|
||||
sum(IF(length(src.header_accept) < 5, 1, 0)) AS count_generic_accept,
|
||||
sum(IF(src.http_version = 'HTTP/1.0', 1, 0)) AS count_http10,
|
||||
varPopState(toFloat64(src.ip_meta_df)) AS ip_df_var
|
||||
FROM mabase_prod.http_logs AS src
|
||||
FROM ja4_logs.http_logs AS src
|
||||
GROUP BY window_start, src_ip, ja4, host, src_asn;
|
||||
|
||||
-- ============================================================================
|
||||
-- 5. TABLE D'AGRÉGATION DES HEADERS (L7)
|
||||
-- ============================================================================
|
||||
CREATE TABLE mabase_prod.agg_header_fingerprint_1h
|
||||
CREATE TABLE ja4_processing.agg_header_fingerprint_1h
|
||||
(
|
||||
window_start DateTime,
|
||||
src_ip IPv6,
|
||||
@ -158,8 +158,8 @@ CREATE TABLE mabase_prod.agg_header_fingerprint_1h
|
||||
ENGINE = AggregatingMergeTree()
|
||||
ORDER BY (window_start, src_ip);
|
||||
|
||||
CREATE MATERIALIZED VIEW mabase_prod.mv_agg_header_fingerprint_1h
|
||||
TO mabase_prod.agg_header_fingerprint_1h AS
|
||||
CREATE MATERIALIZED VIEW ja4_processing.mv_agg_header_fingerprint_1h
|
||||
TO ja4_processing.agg_header_fingerprint_1h AS
|
||||
SELECT
|
||||
toStartOfHour(src.time) AS window_start,
|
||||
toIPv6(src.src_ip) AS src_ip,
|
||||
@ -172,13 +172,13 @@ SELECT
|
||||
max(toUInt8(if((position(src.header_user_agent, 'Windows') > 0 AND position(src.header_sec_ch_ua_platform, 'Windows') == 0) OR (position(src.header_user_agent, 'iPhone') > 0 AND position(src.header_sec_ch_ua_platform, 'iOS') == 0), 1, 0))) AS ua_ch_mismatch,
|
||||
any(src.header_sec_fetch_mode) AS sec_fetch_mode,
|
||||
any(src.header_sec_fetch_dest) AS sec_fetch_dest
|
||||
FROM mabase_prod.http_logs AS src
|
||||
FROM ja4_logs.http_logs AS src
|
||||
GROUP BY window_start, src.src_ip;
|
||||
|
||||
-- ============================================================================
|
||||
-- 6. TABLE DE RÉSULTATS ML — MENACES UNIQUEMENT (scores < seuil)
|
||||
-- ============================================================================
|
||||
CREATE TABLE mabase_prod.ml_detected_anomalies
|
||||
CREATE TABLE ja4_processing.ml_detected_anomalies
|
||||
(
|
||||
detected_at DateTime, src_ip IPv6, ja4 String, host String, bot_name String,
|
||||
anomaly_score Float32, threat_level String, model_name String, recurrence UInt32,
|
||||
@ -204,7 +204,7 @@ TTL detected_at + INTERVAL 30 DAY;
|
||||
-- ============================================================================
|
||||
-- 6b. TABLE DE TOUTES LES CLASSIFICATIONS (sans seuil, pour observabilité)
|
||||
-- ============================================================================
|
||||
CREATE TABLE mabase_prod.ml_all_scores
|
||||
CREATE TABLE ja4_processing.ml_all_scores
|
||||
(
|
||||
detected_at DateTime,
|
||||
window_start DateTime,
|
||||
@ -235,24 +235,24 @@ SETTINGS index_granularity = 8192;
|
||||
-- ============================================================================
|
||||
-- 7. VUE DE RÉCURRENCE
|
||||
-- ============================================================================
|
||||
CREATE OR REPLACE VIEW mabase_prod.view_ip_recurrence AS
|
||||
CREATE OR REPLACE VIEW ja4_processing.view_ip_recurrence AS
|
||||
SELECT src_ip, count() AS recurrence, min(detected_at) AS first_seen, max(detected_at) AS last_seen,
|
||||
min(anomaly_score) AS worst_score, argMin(threat_level, anomaly_score) AS worst_threat_level
|
||||
FROM mabase_prod.ml_detected_anomalies GROUP BY src_ip;
|
||||
FROM ja4_processing.ml_detected_anomalies GROUP BY src_ip;
|
||||
|
||||
-- ============================================================================
|
||||
-- 8. VUE IA PRINCIPALE (Avec CTE pour Entropie Temporelle)
|
||||
-- ============================================================================
|
||||
CREATE OR REPLACE VIEW mabase_prod.view_ai_features_1h AS
|
||||
CREATE OR REPLACE VIEW ja4_processing.view_ai_features_1h AS
|
||||
WITH base_data AS (
|
||||
SELECT
|
||||
a.window_start, a.src_ip, a.ja4, a.host,
|
||||
toString(a.src_asn) AS asn_number, a.src_as_name AS asn_org,
|
||||
a.src_org AS asn_detail, a.src_domain AS asn_domain, a.src_country_code AS country_code,
|
||||
dictGetOrDefault('mabase_prod.dict_asn_reputation', 'label', toUInt64(a.src_asn), 'unknown') AS asn_label,
|
||||
dictGetOrDefault('ja4_processing.dict_asn_reputation', 'label', toUInt64(a.src_asn), 'unknown') AS asn_label,
|
||||
COALESCE(
|
||||
nullIf(dictGetOrDefault('mabase_prod.dict_bot_ip', 'bot_name', a.src_ip, ''), ''),
|
||||
nullIf(dictGetOrDefault('mabase_prod.dict_bot_ja4', 'bot_name', tuple(a.ja4), ''), ''),
|
||||
nullIf(dictGetOrDefault('ja4_processing.dict_bot_ip', 'bot_name', a.src_ip, ''), ''),
|
||||
nullIf(dictGetOrDefault('ja4_processing.dict_bot_ja4', 'bot_name', tuple(a.ja4), ''), ''),
|
||||
''
|
||||
) AS bot_name,
|
||||
a.hits AS hits,
|
||||
@ -324,7 +324,7 @@ WITH base_data AS (
|
||||
sum(count_generic_accept) AS count_generic_accept,
|
||||
sum(count_http10) AS count_http10,
|
||||
varPopMerge(ip_df_var) AS ip_df_variance
|
||||
FROM mabase_prod.agg_host_ip_ja4_1h
|
||||
FROM ja4_processing.agg_host_ip_ja4_1h
|
||||
WHERE window_start >= now() - INTERVAL 24 HOUR
|
||||
GROUP BY window_start, src_ip, ja4, host, src_asn
|
||||
) a
|
||||
@ -335,7 +335,7 @@ WITH base_data AS (
|
||||
max(has_cookie) AS has_cookie, max(has_referer) AS has_referer,
|
||||
max(modern_browser_score) AS modern_browser_score, max(ua_ch_mismatch) AS ua_ch_mismatch,
|
||||
any(sec_fetch_mode) AS sec_fetch_mode, any(sec_fetch_dest) AS sec_fetch_dest
|
||||
FROM mabase_prod.agg_header_fingerprint_1h
|
||||
FROM ja4_processing.agg_header_fingerprint_1h
|
||||
WHERE window_start >= now() - INTERVAL 24 HOUR
|
||||
GROUP BY window_start, src_ip
|
||||
) h ON a.src_ip = h.src_ip AND a.window_start = h.window_start
|
||||
@ -352,7 +352,7 @@ FROM base_data;
|
||||
-- ============================================================================
|
||||
|
||||
-- Vue pour les métriques globales du dashboard
|
||||
CREATE OR REPLACE VIEW mabase_prod.view_dashboard_summary AS
|
||||
CREATE OR REPLACE VIEW ja4_processing.view_dashboard_summary AS
|
||||
SELECT
|
||||
count() AS total_detections,
|
||||
countIf(threat_level = 'CRITICAL') AS critical_count,
|
||||
@ -362,11 +362,11 @@ SELECT
|
||||
countIf(bot_name != '') AS known_bots_count,
|
||||
countIf(bot_name = '') AS anomalies_count,
|
||||
uniq(src_ip) AS unique_ips
|
||||
FROM mabase_prod.ml_detected_anomalies
|
||||
FROM ja4_processing.ml_detected_anomalies
|
||||
WHERE detected_at >= now() - INTERVAL 24 HOUR;
|
||||
|
||||
-- Vue pour la série temporelle (par heure)
|
||||
CREATE OR REPLACE VIEW mabase_prod.view_dashboard_timeseries AS
|
||||
CREATE OR REPLACE VIEW ja4_processing.view_dashboard_timeseries AS
|
||||
SELECT
|
||||
toStartOfHour(detected_at) AS hour,
|
||||
count() AS total,
|
||||
@ -374,25 +374,25 @@ SELECT
|
||||
countIf(threat_level = 'HIGH') AS high,
|
||||
countIf(threat_level = 'MEDIUM') AS medium,
|
||||
countIf(threat_level = 'LOW') AS low
|
||||
FROM mabase_prod.ml_detected_anomalies
|
||||
FROM ja4_processing.ml_detected_anomalies
|
||||
WHERE detected_at >= now() - INTERVAL 24 HOUR
|
||||
GROUP BY hour
|
||||
ORDER BY hour;
|
||||
|
||||
-- Vue pour la distribution des menaces
|
||||
CREATE OR REPLACE VIEW mabase_prod.view_dashboard_threat_dist AS
|
||||
CREATE OR REPLACE VIEW ja4_processing.view_dashboard_threat_dist AS
|
||||
SELECT
|
||||
threat_level,
|
||||
count() AS count,
|
||||
round(count() * 100.0 / sum(count()) OVER (), 2) AS percentage
|
||||
FROM mabase_prod.ml_detected_anomalies
|
||||
FROM ja4_processing.ml_detected_anomalies
|
||||
WHERE detected_at >= now() - INTERVAL 24 HOUR
|
||||
GROUP BY threat_level
|
||||
ORDER BY count DESC;
|
||||
|
||||
-- Vue pour la variabilité (utilisée par l'API)
|
||||
-- Note v12 : header_user_agent n'existe pas dans ml_detected_anomalies → remplacé par reason
|
||||
CREATE OR REPLACE VIEW mabase_prod.view_dashboard_variability AS
|
||||
CREATE OR REPLACE VIEW ja4_processing.view_dashboard_variability AS
|
||||
SELECT
|
||||
detected_at,
|
||||
src_ip,
|
||||
@ -407,5 +407,5 @@ SELECT
|
||||
anomaly_score,
|
||||
campaign_id,
|
||||
raw_anomaly_score
|
||||
FROM mabase_prod.ml_detected_anomalies
|
||||
FROM ja4_processing.ml_detected_anomalies
|
||||
WHERE detected_at >= now() - INTERVAL 24 HOUR;
|
||||
|
||||
Reference in New Issue
Block a user