From b409a70970b898c8a4f0aa58d0d1974b1de9c16f Mon Sep 17 00:00:00 2001 From: toto Date: Fri, 10 Apr 2026 00:59:57 +0200 Subject: [PATCH] fix(views): align SQL views with dashboard API expected columns - view_form_bruteforce_detected: add post_count, distinct_paths, first_seen, last_seen - view_host_ip_ja4_rotation: add host, distinct_ja4, ja4_list, window_start - view_ip_recurrence: add worst_threat alias + top_ja4, top_host columns All three views were missing columns referenced by /api/brute-force, /api/ja4-rotation and /api/recurrence endpoints, causing 500 errors on the Tactiques page. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- shared/clickhouse/06_ml_tables.sql | 15 ++++---- shared/clickhouse/11_views.sql | 58 +++++++++++++++++++----------- 2 files changed, 45 insertions(+), 28 deletions(-) diff --git a/shared/clickhouse/06_ml_tables.sql b/shared/clickhouse/06_ml_tables.sql index f07026d..296d17f 100644 --- a/shared/clickhouse/06_ml_tables.sql +++ b/shared/clickhouse/06_ml_tables.sql @@ -133,13 +133,14 @@ SETTINGS CREATE OR REPLACE VIEW ja4_processing.view_ip_recurrence AS SELECT src_ip, - count() AS recurrence, - min(detected_at) AS first_seen, - max(detected_at) AS last_seen, - max(anomaly_score) AS worst_score, - argMax(threat_level, anomaly_score) AS worst_threat_level + count() AS recurrence, + min(detected_at) AS first_seen, + max(detected_at) AS last_seen, + max(anomaly_score) AS worst_score, + argMax(threat_level, anomaly_score) AS worst_threat_level, + argMax(threat_level, anomaly_score) AS worst_threat, + argMax(ja4, anomaly_score) AS top_ja4, + argMax(host, hits) AS top_host FROM ja4_processing.ml_detected_anomalies --- Filtre temporel aligné sur le TTL de la table (30 jours) --- Évite de scanner les partitions expirées non encore supprimées par le TTL WHERE detected_at >= now() - INTERVAL 30 DAY GROUP BY src_ip; diff --git a/shared/clickhouse/11_views.sql b/shared/clickhouse/11_views.sql index 541eace..d708c48 100644 --- a/shared/clickhouse/11_views.sql +++ b/shared/clickhouse/11_views.sql @@ -21,28 +21,34 @@ -- - Volume élevé de requêtes POST vers un hôte donné (≥ 10 POST/heure) -- - Fenêtre glissante 24h depuis agg_host_ip_ja4_1h -- --- Colonnes utilisées par bruteforce.py et investigation_summary.py : --- src_ip, host, ja4, hits, query_params_count +-- Colonnes : +-- src_ip, host, ja4, hits, post_count, distinct_paths, first_seen, last_seen -- ----------------------------------------------------------------------------- CREATE OR REPLACE VIEW ja4_processing.view_form_bruteforce_detected AS SELECT src_ip, host, - -- JA4 avec le plus de hits pour ce couple IP+hôte (from subquery) argMax(ja4, ja4_hits) AS ja4, sum(ja4_hits) AS hits, - sum(ja4_posts) AS query_params_count + sum(ja4_posts) AS post_count, + -- Alias de compatibilité pour les anciens appels + sum(ja4_posts) AS query_params_count, + uniqExact(ja4) AS distinct_paths, + min(w_min) AS first_seen, + max(w_max) AS last_seen FROM ( SELECT src_ip, host, ja4, - sum(hits) AS ja4_hits, - sum(count_post) AS ja4_posts + sum(hits) AS ja4_hits, + sum(count_post) AS ja4_posts, + min(window_start) AS w_min, + max(window_start) AS w_max FROM ja4_processing.agg_host_ip_ja4_1h WHERE window_start >= now() - INTERVAL 24 HOUR GROUP BY src_ip, host, ja4 ) sub GROUP BY src_ip, host -HAVING query_params_count >= 10; +HAVING post_count >= 10; -- ----------------------------------------------------------------------------- @@ -51,25 +57,35 @@ HAVING query_params_count >= 10; -- Détecte les IPs qui changent de fingerprint JA4 (rotation de TLS ClientHello) -- — indicateur d'évasion de détection par les outils de bot. -- --- Colonnes utilisées par rotation.py et investigation_summary.py : --- src_ip, distinct_ja4_count, total_hits +-- Colonnes : +-- src_ip, host, distinct_ja4, distinct_ja4_count, ja4_list, +-- total_hits, window_start, first_seen, last_seen -- ----------------------------------------------------------------------------- CREATE OR REPLACE VIEW ja4_processing.view_host_ip_ja4_rotation AS SELECT src_ip, - -- Nombre de JA4 distincts émis par cette IP sur la fenêtre 24h - uniqExact(ja4) AS distinct_ja4_count, - sum(hits) AS total_hits, - -- Fenêtre temporelle pour le contexte - min(window_start) AS first_seen, - max(window_start) AS last_seen -FROM ja4_processing.agg_host_ip_ja4_1h -WHERE window_start >= now() - INTERVAL 24 HOUR - AND ja4 != '' + argMax(host, ja4_hits) AS host, + uniqExact(ja4) AS distinct_ja4, + uniqExact(ja4) AS distinct_ja4_count, + groupUniqArray(ja4) AS ja4_list, + sum(ja4_hits) AS total_hits, + max(w_max) AS window_start, + min(w_min) AS first_seen, + max(w_max) AS last_seen +FROM ( + SELECT + src_ip, host, ja4, + sum(hits) AS ja4_hits, + min(window_start) AS w_min, + max(window_start) AS w_max + FROM ja4_processing.agg_host_ip_ja4_1h + WHERE window_start >= now() - INTERVAL 24 HOUR + AND ja4 != '' + GROUP BY src_ip, host, ja4 +) sub GROUP BY src_ip --- Rotation = au moins 2 JA4 distincts -HAVING distinct_ja4_count >= 2 -ORDER BY distinct_ja4_count DESC; +HAVING distinct_ja4 >= 2 +ORDER BY distinct_ja4 DESC; -- -----------------------------------------------------------------------------