code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(views): align SQL views with dashboard API expected columns

Browse Source 
- view_form_bruteforce_detected: add post_count, distinct_paths, first_seen, last_seen
- view_host_ip_ja4_rotation: add host, distinct_ja4, ja4_list, window_start
- view_ip_recurrence: add worst_threat alias + top_ja4, top_host columns

All three views were missing columns referenced by /api/brute-force,
/api/ja4-rotation and /api/recurrence endpoints, causing 500 errors
on the Tactiques page.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-10 00:59:57 +02:00
parent 2f2c5e03bb
commit b409a70970
2 changed files with 45 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
shared/clickhouse/06_ml_tables.sql
View File

@ -133,13 +133,14 @@ SETTINGS
CREATE OR REPLACE VIEW ja4_processing.view_ip_recurrence AS
SELECT
src_ip,
count() AS recurrence,
min(detected_at) AS first_seen,
max(detected_at) AS last_seen,
max(anomaly_score) AS worst_score,
argMax(threat_level, anomaly_score) AS worst_threat_level
count() AS recurrence,
min(detected_at) AS first_seen,
max(detected_at) AS last_seen,
max(anomaly_score) AS worst_score,
argMax(threat_level, anomaly_score) AS worst_threat_level,
argMax(threat_level, anomaly_score) AS worst_threat,
argMax(ja4, anomaly_score) AS top_ja4,
argMax(host, hits) AS top_host
FROM ja4_processing.ml_detected_anomalies
-- Filtre temporel aligné sur le TTL de la table (30 jours)
-- Évite de scanner les partitions expirées non encore supprimées par le TTL
WHERE detected_at >= now() - INTERVAL 30 DAY
GROUP BY src_ip;

58
shared/clickhouse/11_views.sql
View File

@ -21,28 +21,34 @@
-- - Volume élevé de requêtes POST vers un hôte donné (≥ 10 POST/heure)
-- - Fenêtre glissante 24h depuis agg_host_ip_ja4_1h
--
-- Colonnes utilisées par bruteforce.py et investigation_summary.py :
-- src_ip, host, ja4, hits, query_params_count
-- Colonnes :
-- src_ip, host, ja4, hits, post_count, distinct_paths, first_seen, last_seen
-- -----------------------------------------------------------------------------
CREATE OR REPLACE VIEW ja4_processing.view_form_bruteforce_detected AS
SELECT
src_ip,
host,
-- JA4 avec le plus de hits pour ce couple IP+hôte (from subquery)
argMax(ja4, ja4_hits) AS ja4,
sum(ja4_hits) AS hits,
sum(ja4_posts) AS query_params_count
sum(ja4_posts) AS post_count,
-- Alias de compatibilité pour les anciens appels
sum(ja4_posts) AS query_params_count,
uniqExact(ja4) AS distinct_paths,
min(w_min) AS first_seen,
max(w_max) AS last_seen
FROM (
SELECT
src_ip, host, ja4,
sum(hits) AS ja4_hits,
sum(count_post) AS ja4_posts
sum(hits) AS ja4_hits,
sum(count_post) AS ja4_posts,
min(window_start) AS w_min,
max(window_start) AS w_max
FROM ja4_processing.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 24 HOUR
GROUP BY src_ip, host, ja4
) sub
GROUP BY src_ip, host
HAVING query_params_count >= 10;
HAVING post_count >= 10;
-- -----------------------------------------------------------------------------
@ -51,25 +57,35 @@ HAVING query_params_count >= 10;
-- Détecte les IPs qui changent de fingerprint JA4 (rotation de TLS ClientHello)
-- — indicateur d'évasion de détection par les outils de bot.
--
-- Colonnes utilisées par rotation.py et investigation_summary.py :
-- src_ip, distinct_ja4_count, total_hits
-- Colonnes :
-- src_ip, host, distinct_ja4, distinct_ja4_count, ja4_list,
-- total_hits, window_start, first_seen, last_seen
-- -----------------------------------------------------------------------------
CREATE OR REPLACE VIEW ja4_processing.view_host_ip_ja4_rotation AS
SELECT
src_ip,
-- Nombre de JA4 distincts émis par cette IP sur la fenêtre 24h
uniqExact(ja4) AS distinct_ja4_count,
sum(hits) AS total_hits,
-- Fenêtre temporelle pour le contexte
min(window_start) AS first_seen,
max(window_start) AS last_seen
FROM ja4_processing.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 24 HOUR
AND ja4 != ''
argMax(host, ja4_hits) AS host,
uniqExact(ja4) AS distinct_ja4,
uniqExact(ja4) AS distinct_ja4_count,
groupUniqArray(ja4) AS ja4_list,
sum(ja4_hits) AS total_hits,
max(w_max) AS window_start,
min(w_min) AS first_seen,
max(w_max) AS last_seen
FROM (
SELECT
src_ip, host, ja4,
sum(hits) AS ja4_hits,
min(window_start) AS w_min,
max(window_start) AS w_max
FROM ja4_processing.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 24 HOUR
AND ja4 != ''
GROUP BY src_ip, host, ja4
) sub
GROUP BY src_ip
-- Rotation = au moins 2 JA4 distincts
HAVING distinct_ja4_count >= 2
ORDER BY distinct_ja4_count DESC;
HAVING distinct_ja4 >= 2
ORDER BY distinct_ja4 DESC;
-- -----------------------------------------------------------------------------