logcorrelator/sql/mv1.sql

-- ============================================================================
-- PROJET : Moteur de Détection de Menaces HTTP
-- DESCRIPTION : Reconfiguration des vues d'agrégation, nouveauté et scoring.
-- DATE : 2026-03-08
-- ============================================================================

-- ----------------------------------------------------------------------------
-- 1. NETTOYAGE DES OBJETS EXISTANTS (Ordre inverse des dépendances)
-- ----------------------------------------------------------------------------
DROP VIEW IF EXISTS mabase_prod.live_threat_scores;
DROP VIEW IF EXISTS mabase_prod.mv_baseline_update;
DROP VIEW IF EXISTS mabase_prod.mv_novelty;
DROP VIEW IF EXISTS mabase_prod.mv_traffic_1d;
DROP VIEW IF EXISTS mabase_prod.mv_traffic_1h;
DROP VIEW IF EXISTS mabase_prod.mv_traffic_1m;

-- ----------------------------------------------------------------------------
-- 2. RECONSTRUCTION DE LA CHAÎNE DE ROLLUP (Aggrégations temporelles)
-- ----------------------------------------------------------------------------

-- MV 1 Minute : Transformation des logs bruts en métriques techniques
CREATE MATERIALIZED VIEW mabase_prod.mv_traffic_1m 
TO mabase_prod.agg_traffic_1m
AS SELECT
    toStartOfMinute(time) AS minute,
    host, src_ip, src_asn, src_country_code, ja4, ja3_hash, header_user_agent,
    countState() AS hits,
    uniqState(path) AS uniq_paths,
    avgState(syn_to_clienthello_ms) AS avg_syn_to_clienthello_ms,
    varPopState(syn_to_clienthello_ms) AS var_syn_to_clienthello_ms,
    avgState(toFloat64((length(client_headers) - length(replaceAll(client_headers, ',', ''))) + 1)) AS avg_headers_count,
    countIfState((header_user_agent ILIKE '%Chrome%') AND (ja4 NOT ILIKE 't13d%')) AS spoofing_ua_tls,
    countIfState((header_user_agent ILIKE '%Chrome%') AND (tls_alpn NOT ILIKE '%h2%')) AS spoofing_ua_alpn,
    countIfState((header_user_agent ILIKE '%Windows%') AND (ip_meta_ttl <= 64)) AS spoofing_os_ttl,
    countIfState((header_accept_language = '') OR (header_sec_ch_ua = '')) AS missing_human_headers,
    countIfState(method IN ('PUT', 'DELETE', 'OPTIONS', 'TRACE')) AS suspicious_methods,
    countIfState((length(query) > 200) OR match(query, '(%[0-9A-Fa-f]{2}){5,}')) AS suspicious_queries
FROM mabase_prod.http_logs
GROUP BY minute, host, src_ip, src_asn, src_country_code, ja4, ja3_hash, header_user_agent;

-- MV 1 Heure : Agrégation secondaire (Cascading)
CREATE MATERIALIZED VIEW mabase_prod.mv_traffic_1h 
TO mabase_prod.agg_traffic_1h
AS SELECT
    toStartOfHour(minute) AS hour,
    host, src_country_code, ja4, ja3_hash, header_user_agent,
    countMergeState(hits) AS hits,
    uniqMergeState(uniq_paths) AS uniq_paths,
    countIfMergeState(missing_human_headers) AS missing_human_headers,
    uniqState(src_ip) AS uniq_ips
FROM mabase_prod.agg_traffic_1m
GROUP BY hour, host, src_country_code, ja4, ja3_hash, header_user_agent;

-- MV 1 Jour : Agrégation tertiaire pour archivage et baseline
CREATE MATERIALIZED VIEW mabase_prod.mv_traffic_1d 
TO mabase_prod.agg_traffic_1d
AS SELECT
    toDate(hour) AS day,
    host, src_country_code, ja4, ja3_hash, header_user_agent,
    countMergeState(hits) AS hits,
    uniqMergeState(uniq_ips) AS uniq_ips,
    uniqMergeState(uniq_paths) AS uniq_paths,
    countIfMergeState(missing_human_headers) AS missing_human_headers
FROM mabase_prod.agg_traffic_1h
GROUP BY day, host, src_country_code, ja4, ja3_hash, header_user_agent;

-- ----------------------------------------------------------------------------
-- 3. RECONSTRUCTION DES BRIQUES D'INTELLIGENCE (Novelty & Baseline)
-- ----------------------------------------------------------------------------

-- MV Novelty : Détection de nouvelles empreintes (HTTP + TLS)
CREATE MATERIALIZED VIEW mabase_prod.mv_novelty
TO mabase_prod.agg_novelty
AS SELECT
    host,
    ja4,
    cityHash64(client_headers) AS http_fp,
    minState(time) AS first_seen,
    maxState(time) AS last_seen,
    countState() AS total_hits
FROM mabase_prod.http_logs
GROUP BY host, ja4, http_fp;

-- MV Baseline : Calcul statistique du trafic normal par JA4
CREATE MATERIALIZED VIEW mabase_prod.mv_baseline_update
TO mabase_prod.tbl_baseline_ja4_7d
AS SELECT
    ja4,
    quantile(0.99)(hourly_hits) AS p99_hits_per_hour,
    avg(hourly_hits) AS avg_hits_per_hour,
    now() AS last_update
FROM (
    SELECT
        ja4,
        toStartOfHour(minute) as hour,
        countMerge(hits) AS hourly_hits
    FROM mabase_prod.agg_traffic_1m
    WHERE minute >= now() - INTERVAL 7 DAY
    GROUP BY ja4, hour
)
GROUP BY ja4;

-- ----------------------------------------------------------------------------
-- 4. VUE DE SCORING FINAL (Verdict Temps Réel)
-- ----------------------------------------------------------------------------

CREATE VIEW mabase_prod.live_threat_scores AS
SELECT 
    T1.src_ip,
    T1.ja4,
    T1.src_asn,
    T1.src_country_code,
    (
        if(countMerge(T1.spoofing_ua_tls) > 0, 40, 0) +
        if(countMerge(T1.spoofing_os_ttl) > 0, 40, 0) +
        if(varPopMerge(T1.var_syn_to_clienthello_ms) < 1.0, 20, 0) +
        if(dateDiff('hour', minMerge(N.first_seen), now()) < 2, 30, 0) +
        if(countMerge(T1.hits) > coalesce(B.p99_hits_per_hour * 3, 1000), 50, 0)
    ) AS final_score,
    countMerge(T1.hits) AS current_hits,
    B.p99_hits_per_hour AS historical_baseline
FROM mabase_prod.agg_traffic_1m AS T1
LEFT JOIN mabase_prod.agg_novelty AS N ON T1.ja4 = N.ja4 AND T1.host = N.host
LEFT JOIN mabase_prod.tbl_baseline_ja4_7d AS B ON T1.ja4 = B.ja4
WHERE T1.minute >= now() - INTERVAL 5 MINUTE
GROUP BY T1.src_ip, T1.ja4, T1.src_asn, T1.src_country_code, B.p99_hits_per_hour;