diff --git a/services/dashboard/backend/routes/api.py b/services/dashboard/backend/routes/api.py index 0e05f53..cf5e1b1 100644 --- a/services/dashboard/backend/routes/api.py +++ b/services/dashboard/backend/routes/api.py @@ -1027,7 +1027,7 @@ async def campaign_detail(cid: int) -> dict[str, Any]: f"asn_org, asn_number, country_code, " f"browser_family, bot_name, detected_at, reason " f"FROM {_DB}.ml_detected_anomalies " - "WHERE campaign_id = {{cid:Int32}} " + "WHERE campaign_id = {cid:Int32} " "AND detected_at >= now() - INTERVAL 7 DAY " "ORDER BY anomaly_score ASC LIMIT 200", {"cid": cid}, @@ -1048,7 +1048,7 @@ async def campaign_detail(cid: int) -> dict[str, Any]: f"groupUniqArray(10)(host) AS host_list, " f"min(detected_at) AS first_seen, max(detected_at) AS last_seen " f"FROM {_DB}.ml_detected_anomalies " - "WHERE campaign_id = {{cid:Int32}} " + "WHERE campaign_id = {cid:Int32} " "AND detected_at >= now() - INTERVAL 7 DAY", {"cid": cid}, ) @@ -1057,7 +1057,7 @@ async def campaign_detail(cid: int) -> dict[str, Any]: f"SELECT toStartOfHour(detected_at) AS hour, " f"count() AS detections, uniqExact(src_ip) AS active_ips " f"FROM {_DB}.ml_detected_anomalies " - "WHERE campaign_id = {{cid:Int32}} " + "WHERE campaign_id = {cid:Int32} " "AND detected_at >= now() - INTERVAL 7 DAY " "GROUP BY hour ORDER BY hour", {"cid": cid}, @@ -1226,3 +1226,247 @@ async def ua_rotation() -> dict[str, Any]: except Exception as exc: logger.exception("ua-rotation query failed") return {"data": []} + + +# --------------------------------------------------------------------------- +# GET /api/ja4/{fingerprint} — JA4 fingerprint investigation detail +# --------------------------------------------------------------------------- +@router.get("/ja4/{fingerprint:path}") +async def ja4_detail(fingerprint: str) -> dict[str, Any]: + """Investigation complète d'une empreinte JA4 : IPs, scores, comportement.""" + params = {"ja4": fingerprint} + try: + # IPs utilisant cette empreinte (détections) + detections = query( + f"SELECT toString(src_ip) AS src_ip, anomaly_score, " + f"raw_anomaly_score, threat_level, hits, hit_velocity, " + f"host, asn_org, country_code, browser_family, bot_name, " + f"detected_at, campaign_id " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE ja4 = {ja4:String} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "ORDER BY detected_at DESC LIMIT 500", + params, + ) + + # Scores ML pour cette JA4 + all_scores = query( + f"SELECT toString(src_ip) AS src_ip, anomaly_score, " + f"raw_anomaly_score, ae_recon_error, xgb_prob, " + f"threat_level, model_name, host, hits, " + f"asn_org, country_code, browser_family, detected_at " + f"FROM {_DB}.ml_all_scores " + "WHERE ja4 = {ja4:String} " + "AND detected_at >= now() - INTERVAL 3 DAY " + "ORDER BY detected_at DESC LIMIT 500", + params, + ) + + # Profil agrégé + profile = query( + f"SELECT " + f"count() AS total_sessions, " + f"uniqExact(src_ip) AS unique_ips, " + f"uniqExact(host) AS unique_hosts, " + f"uniqExact(asn_org) AS unique_asns, " + f"avg(anomaly_score) AS avg_score, " + f"max(anomaly_score) AS max_score, " + f"avg(hits) AS avg_hits, " + f"avg(hit_velocity) AS avg_velocity, " + f"sum(hits) AS total_hits, " + f"groupUniqArray(20)(toString(src_ip)) AS ip_sample, " + f"groupUniqArray(10)(host) AS host_list, " + f"groupUniqArray(10)(asn_org) AS asn_list, " + f"groupUniqArray(10)(country_code) AS country_list, " + f"groupUniqArray(5)(browser_family) AS browser_list, " + f"groupUniqArray(5)(bot_name) AS bot_names, " + f"min(detected_at) AS first_seen, max(detected_at) AS last_seen, " + f"countIf(threat_level IN ('HIGH','CRITICAL')) AS threat_count, " + f"countIf(threat_level = 'KNOWN_BOT') AS known_bot_count, " + f"countIf(browser_family != '') AS browser_count " + f"FROM {_DB}.ml_all_scores " + "WHERE ja4 = {ja4:String} " + "AND detected_at >= now() - INTERVAL 7 DAY", + params, + ) + + # Timeline horaire + timeline = query( + f"SELECT toStartOfHour(detected_at) AS hour, " + f"count() AS sessions, uniqExact(src_ip) AS active_ips, " + f"avg(anomaly_score) AS avg_score " + f"FROM {_DB}.ml_all_scores " + "WHERE ja4 = {ja4:String} " + "AND detected_at >= now() - INTERVAL 3 DAY " + "GROUP BY hour ORDER BY hour", + params, + ) + + # Threat breakdown + threats = query( + f"SELECT threat_level, count() AS cnt " + f"FROM {_DB}.ml_all_scores " + "WHERE ja4 = {ja4:String} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "GROUP BY threat_level ORDER BY cnt DESC", + params, + ) + + # Trafic HTTP brut + http_logs = query( + f"SELECT time, toString(src_ip) AS src_ip, method, host, path, " + f"http_version, header_user_agent " + f"FROM {_DB_LOGS}.http_logs " + "WHERE ja4 = {ja4:String} " + "AND time >= now() - INTERVAL 1 DAY " + "ORDER BY time DESC LIMIT 200", + params, + ) + + # AI features pour cette JA4 + ai_features: list[dict] = [] + try: + ai_features = query( + f"SELECT * FROM {_DB}.view_ai_features_1h " + "WHERE ja4 = {ja4:String} LIMIT 20", + params, + ) + except Exception: + logger.debug("view_ai_features_1h unavailable for ja4=%s", fingerprint) + + return { + "ja4": fingerprint, + "profile": profile[0] if profile else {}, + "detections": detections, + "scores": all_scores, + "timeline": timeline, + "threats": threats, + "http_logs": http_logs, + "ai_features": ai_features, + } + except Exception as exc: + logger.exception("ja4 detail query failed for %s", fingerprint) + raise HTTPException(status_code=500, detail=str(exc)) + + +# --------------------------------------------------------------------------- +# GET /api/cluster/{cid} — Enhanced cluster investigation +# --------------------------------------------------------------------------- +@router.get("/cluster/{cid}") +async def cluster_detail(cid: int) -> dict[str, Any]: + """Investigation complète d'un cluster : profil, membres, graphe, timeline.""" + params = {"cid": cid} + try: + # Profil agrégé enrichi + profile = query( + f"SELECT " + f"count() AS total_members, " + f"uniqExact(src_ip) AS unique_ips, " + f"uniqExact(ja4) AS unique_ja4, " + f"uniqExact(host) AS unique_hosts, " + f"uniqExact(asn_org) AS unique_asns, " + f"avg(anomaly_score) AS avg_score, max(anomaly_score) AS max_score, " + f"min(anomaly_score) AS min_score, " + f"avg(hits) AS avg_hits, sum(hits) AS total_hits, " + f"avg(hit_velocity) AS avg_velocity, " + f"avg(fuzzing_index) AS avg_fuzzing, " + f"avg(post_ratio) AS avg_post_ratio, " + f"groupUniqArray(30)(toString(src_ip)) AS ip_list, " + f"groupUniqArray(20)(ja4) AS ja4_list, " + f"groupUniqArray(10)(host) AS host_list, " + f"groupUniqArray(10)(asn_org) AS asn_list, " + f"groupUniqArray(10)(country_code) AS country_list, " + f"groupUniqArray(5)(browser_family) AS browser_list, " + f"groupUniqArray(5)(bot_name) AS bot_names, " + f"min(detected_at) AS first_seen, max(detected_at) AS last_seen, " + f"countIf(threat_level IN ('HIGH','CRITICAL')) AS threat_count, " + f"countIf(threat_level = 'KNOWN_BOT') AS known_bot_count " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE campaign_id = {cid:Int32} " + "AND detected_at >= now() - INTERVAL 7 DAY", + params, + ) + + # Membres détaillés + members = query( + f"SELECT toString(src_ip) AS src_ip, ja4, host, " + f"anomaly_score, raw_anomaly_score, threat_level, " + f"hits, hit_velocity, fuzzing_index, post_ratio, " + f"asn_org, asn_number, country_code, " + f"browser_family, bot_name, detected_at, reason " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE campaign_id = {cid:Int32} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "ORDER BY anomaly_score ASC LIMIT 500", + params, + ) + + # Timeline horaire + timeline = query( + f"SELECT toStartOfHour(detected_at) AS hour, " + f"count() AS detections, uniqExact(src_ip) AS active_ips, " + f"avg(anomaly_score) AS avg_score " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE campaign_id = {cid:Int32} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "GROUP BY hour ORDER BY hour", + params, + ) + + # Répartition par JA4 (signature convergence) + ja4_breakdown = query( + f"SELECT ja4, count() AS sessions, " + f"uniqExact(src_ip) AS unique_ips, " + f"avg(anomaly_score) AS avg_score " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE campaign_id = {cid:Int32} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "GROUP BY ja4 ORDER BY sessions DESC", + params, + ) + + # Répartition par ASN (infrastructure) + asn_breakdown = query( + f"SELECT asn_org, count() AS sessions, " + f"uniqExact(src_ip) AS unique_ips " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE campaign_id = {cid:Int32} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "GROUP BY asn_org ORDER BY sessions DESC", + params, + ) + + # Répartition par host ciblé + host_breakdown = query( + f"SELECT host, count() AS sessions, " + f"avg(anomaly_score) AS avg_score " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE campaign_id = {cid:Int32} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "GROUP BY host ORDER BY sessions DESC", + params, + ) + + # Threat breakdown + threats = query( + f"SELECT threat_level, count() AS cnt " + f"FROM {_DB}.ml_detected_anomalies " + "WHERE campaign_id = {cid:Int32} " + "AND detected_at >= now() - INTERVAL 7 DAY " + "GROUP BY threat_level ORDER BY cnt DESC", + params, + ) + + return { + "campaign_id": cid, + "profile": profile[0] if profile else {}, + "members": members, + "timeline": timeline, + "ja4_breakdown": ja4_breakdown, + "asn_breakdown": asn_breakdown, + "host_breakdown": host_breakdown, + "threats": threats, + } + except Exception as exc: + logger.exception("cluster detail query failed for %s", cid) + raise HTTPException(status_code=500, detail=str(exc)) diff --git a/services/dashboard/backend/routes/pages.py b/services/dashboard/backend/routes/pages.py index 91c90c8..0e28ba7 100644 --- a/services/dashboard/backend/routes/pages.py +++ b/services/dashboard/backend/routes/pages.py @@ -61,3 +61,13 @@ async def network(request: Request): @router.get("/campaigns") async def campaigns_page(request: Request): return templates.TemplateResponse("campaigns.html", _ctx(request, "campaigns")) + + +@router.get("/ja4/{fingerprint:path}") +async def ja4_detail_page(request: Request, fingerprint: str): + return templates.TemplateResponse("ja4_detail.html", _ctx(request, "ja4_detail", ja4=fingerprint)) + + +@router.get("/cluster/{cid}") +async def cluster_detail_page(request: Request, cid: int): + return templates.TemplateResponse("cluster_detail.html", _ctx(request, "cluster_detail", cid=cid)) diff --git a/services/dashboard/backend/templates/base.html b/services/dashboard/backend/templates/base.html index 3385fd6..b1bbddf 100644 --- a/services/dashboard/backend/templates/base.html +++ b/services/dashboard/backend/templates/base.html @@ -291,11 +291,11 @@ } function fmtJA4(ja4) { if (!ja4) return ''; - return `${escapeHtml(ja4.substring(0,22))}…`; + return `${escapeHtml(ja4.substring(0,22))}…`; } function fmtJA4Full(ja4) { if (!ja4) return ''; - return `${escapeHtml(ja4)}`; + return `${escapeHtml(ja4)}`; } function fmtBotName(name) { if (!name) return ''; diff --git a/services/dashboard/backend/templates/campaigns.html b/services/dashboard/backend/templates/campaigns.html index 2cf0fb5..1f1c876 100644 --- a/services/dashboard/backend/templates/campaigns.html +++ b/services/dashboard/backend/templates/campaigns.html @@ -138,6 +138,7 @@ Détail campagne + Ouvrir ↗
@@ -507,6 +508,7 @@ async function selectCampaign(cid) { const panel = document.getElementById('detail-panel'); panel.classList.add('open'); document.getElementById('detail-cid').textContent = `#${cid}`; + document.getElementById('detail-link').href = `/cluster/${cid}`; try { const resp = await fetch(`/api/campaigns/${cid}`); diff --git a/services/dashboard/backend/templates/cluster_detail.html b/services/dashboard/backend/templates/cluster_detail.html new file mode 100644 index 0000000..14646ec --- /dev/null +++ b/services/dashboard/backend/templates/cluster_detail.html @@ -0,0 +1,264 @@ +{% extends "base.html" %} +{% block title %}JA4 SOC — Cluster #{{ cid }}{% endblock %} +{% block page_title %} + Cluster #{{ cid }} +
+

Investigation Cluster

+

Analyse complète d'un cluster détecté par HDBSCAN. Un cluster regroupe des IPs aux comportements similaires (features ML proches), indiquant une campagne coordonnée.

+

Workflow : Vérifiez la convergence JA4/ASN → analysez les cibles → classifiez le cluster entier.

+

Sources : ml_detected_anomalies WHERE campaign_id=…

+
+{% endblock %} +{% block header_actions %} +
+ ← Tous les clusters + + + +
+{% endblock %} +{% block content %} +
+ +
+ + +
+
+
+ + Timeline du cluster +
+

Activité temporelle

+

Détections et IPs actives par heure. Les bursts synchronisés confirment la coordination.

+

Source : ml_detected_anomalies GROUP BY hour

+
+
+
+
+
+
+
Menaces
+
+
+
+ + +
+
+
+ + Convergence JA4 +
+

Empreintes TLS du cluster

+

Répartition des JA4 dans le cluster. Une seule JA4 dominante = même outil/framework. Diversité JA4 = evasion ou rotation.

+

Action : Cliquez sur une JA4 pour l'investiguer.

+

Source : ml_detected_anomalies GROUP BY ja4

+
+
+
+
+
+
+
+
+
+
+ + Infrastructure (ASN) +
+

Systèmes autonomes

+

D'où viennent les IPs du cluster. Un seul ASN datacenter = hébergé. ASN ISP variés = botnet distribué.

+

Source : ml_detected_anomalies GROUP BY asn_org

+
+
+
+
+
+
+
Hosts ciblés
+
+
+
+ + +
+
+ + Membres du cluster +
+

IPs membres

+

Toutes les IPs rattachées à ce cluster par HDBSCAN. Triez par score pour prioriser l'investigation.

+

Action : Cliquez pour investiguer l'IP. Utilisez « Classifier tout le cluster » pour un traitement en masse.

+

Source : ml_detected_anomalies WHERE campaign_id=…

+
+
+ +
+
+ + + + + +
IPScoreThreatJA4HitsVelocityFuzzHostASNPaysBrowserBotDate
+
+
+
+{% endblock %} +{% block scripts %} + +{% endblock %} diff --git a/services/dashboard/backend/templates/ja4_detail.html b/services/dashboard/backend/templates/ja4_detail.html new file mode 100644 index 0000000..5cdb211 --- /dev/null +++ b/services/dashboard/backend/templates/ja4_detail.html @@ -0,0 +1,253 @@ +{% extends "base.html" %} +{% block title %}JA4 SOC — JA4 {{ ja4[:24] }}…{% endblock %} +{% block page_title %} + Empreinte JA4 +
+

Investigation JA4

+

Analyse complète d'une empreinte JA4 : toutes les IPs l'utilisant, distribution des menaces, timeline, comportement réseau et logs HTTP.

+

Workflow : Identifiez si l'empreinte est partagée par des bots (même JA4, IPs différentes) ou un navigateur légitime.

+

Sources : ml_all_scores, ml_detected_anomalies, http_logs

+
+{% endblock %} +{% block header_actions %} + {{ ja4 }} +{% endblock %} +{% block content %} +
+ +
+ + +
+
+
+ + Timeline activité +
+

Timeline JA4

+

Sessions et IPs actives par heure utilisant cette empreinte. Un pic indique une campagne coordonnée.

+

Source : ml_all_scores GROUP BY hour

+
+
+
+
+
+
+
+ + Distribution menaces +
+

Répartition des niveaux de menace

+

Proportion des sessions classifiées par niveau. Une JA4 avec 100% HIGH est probablement un outil automatisé.

+

Source : ml_all_scores WHERE ja4=…

+
+
+
+
+
+
+ + +
+
+
+ + IPs associées +
+

IPs utilisant cette empreinte

+

Liste des IP sources ayant présenté cette JA4. Cliquez pour investiguer une IP. Un grand nombre d'IPs avec la même JA4 suggère un botnet.

+

Source : ml_all_scores

+
+
+ +
+
+ + + + +
IPScoreThreatHitsHostASNPaysBrowserDate
+
+
+
+
+
Infrastructure (ASN)
+
+
+
+
Géographie
+
+
+
+
+ + +
+
+ + Logs HTTP récents +
+

Requêtes HTTP avec cette JA4

+

Échantillon du trafic brut (24h). Analysez les paths, User-Agents et méthodes pour confirmer l'automatisation.

+

Source : http_logs WHERE ja4=…

+
+
+
+
+ + +
TimeIPMethodHostPathHTTPUser-Agent
+
+
+ + + +
+{% endblock %} +{% block scripts %} + +{% endblock %}