feat(dashboard): SOC workflow overhaul — sidebar nav, doc tooltips, full-width layout

- base.html: collapsible sidebar navigation, doc tooltip system, JS helpers
  (fmtNum, fmtPct, fmtDuration, ecGrid, buildTable, docHTML)
- overview.html: SOC command center with stacked timeline, live alerts,
  campaigns panel, browser donut, 6 KPIs
- detections.html: threat color dots, raw score column, click-to-navigate rows
- network.html: JA4 rotation, brute-force, persistent threats tables, 6 KPIs
- ip_detail.html: ASN/country KPIs, AE/XGB/campaign columns, enriched features
- scores/traffic/features/models/classify: page_title blocks + doc tooltips
- api.py: 9 new endpoints (campaigns, brute-force, ja4-rotation, recurrence,
  cascade, alerts, timeline-detail, ua-rotation)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-09 00:29:34 +02:00
parent c994ad4466
commit 2d04288e95
11 changed files with 1137 additions and 592 deletions

View File

@ -1,70 +1,100 @@
{% extends "base.html" %}
{% block title %}JA4 SOC — IP {{ ip }}{% endblock %}
{% block page_title %}
<a href="/detections" class="text-gray-500 hover:text-gray-300 text-xs">&larr; Retour</a>
<span class="mx-2 text-gray-700">/</span>
Investigation IP : <span class="text-brand-500 font-mono">{{ ip }}</span>
{% endblock %}
{% block header_actions %}
<div class="flex gap-2">
<select id="cls-select" class="px-2 py-1 bg-gray-900 border border-gray-800 rounded text-xs text-gray-300">
<option value="bot">🤖 Bot</option><option value="legitimate">✅ Légitime</option><option value="suspicious">⚠️ Suspect</option>
</select>
<input type="text" id="cls-comment" placeholder="Commentaire…" class="px-2 py-1 bg-gray-900 border border-gray-800 rounded text-xs text-gray-300 w-40 focus:border-brand-500 focus:outline-none">
<button id="cls-btn" class="px-3 py-1 bg-brand-600 text-white rounded text-xs font-medium hover:bg-brand-500">Classifier</button>
<span id="cls-result" class="text-xs self-center"></span>
</div>
{% endblock %}
{% block content %}
<div class="space-y-6">
<div class="flex items-center gap-3">
<a href="/detections" class="text-gray-500 hover:text-gray-300">&larr; Retour</a>
<h2 class="text-lg font-semibold text-white">Investigation IP : <span class="text-brand-500">{{ ip }}</span></h2>
</div>
<!-- KPI Row -->
<div class="grid grid-cols-2 md:grid-cols-5 gap-4">
<div class="kpi-card"><div class="text-xs text-gray-500 mb-1">Détections</div><div class="text-xl font-bold text-red-400" id="ip-det-count"></div></div>
<div class="kpi-card"><div class="text-xs text-gray-500 mb-1">Pire score</div><div class="text-xl font-bold" id="ip-worst-score"></div></div>
<div class="kpi-card"><div class="text-xs text-gray-500 mb-1">Récurrence</div><div class="text-xl font-bold text-yellow-400" id="ip-recurrence"></div></div>
<div class="kpi-card"><div class="text-xs text-gray-500 mb-1">Requêtes HTTP</div><div class="text-xl font-bold text-gray-200" id="ip-http-count"></div></div>
<div class="kpi-card"><div class="text-xs text-gray-500 mb-1">Scores ML</div><div class="text-xl font-bold text-brand-500" id="ip-score-count"></div></div>
<div class="space-y-4">
<!-- KPIs -->
<div class="grid grid-cols-2 md:grid-cols-3 xl:grid-cols-6 gap-3">
<div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Détections</div><div class="text-xl font-bold text-red-400" id="ip-det-count"></div></div>
<div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Pire score</div><div class="text-xl font-bold" id="ip-worst-score"></div></div>
<div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Récurrence</div><div class="text-xl font-bold text-yellow-400" id="ip-recurrence"></div></div>
<div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Requêtes HTTP</div><div class="text-xl font-bold text-gray-200" id="ip-http-count"></div></div>
<div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Scores ML</div><div class="text-xl font-bold text-brand-500" id="ip-score-count"></div></div>
<div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">ASN / Pays</div><div class="text-sm font-medium" id="ip-asn-info"></div></div>
</div>
<!-- Charts Row: Radar + Score timeline -->
<!-- Radar + Score timeline -->
<div class="grid grid-cols-1 lg:grid-cols-2 gap-4">
<div class="bg-gray-900 rounded-xl p-5 border border-gray-800">
<h3 class="text-sm font-medium text-gray-400 mb-3">Profil comportemental (vs baseline)</h3>
<div id="radar-chart" style="height:320px"></div>
<div class="section-card">
<div class="section-header"><span class="section-title">Profil comportemental
<span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
<h4>Radar comportemental</h4>
<p>Compare cette IP aux profils moyens ISP (vert) et datacenter/bot (rouge). Les axes sont normalisés 0→1.</p>
<p><strong>Interprétation :</strong> Un profil proche du rouge indique un comportement bot. hit_velocity élevé + fuzzing élevé = scraping agressif.</p>
<p class="doc-source">Source : view_ai_features_1h</p>
</div></span>
</span></div>
<div class="section-body"><div id="radar-chart" style="height:300px"></div></div>
</div>
<div class="bg-gray-900 rounded-xl p-5 border border-gray-800">
<h3 class="text-sm font-medium text-gray-400 mb-3">Scores ML dans le temps</h3>
<div id="score-chart" style="height:320px"></div>
<div class="section-card">
<div class="section-header"><span class="section-title">Scores ML dans le temps
<span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
<h4>Historique des scores</h4>
<p>Évolution du score ML normalisé sur les derniers cycles. Un score stable élevé = bot persistant. Un pic soudain = changement de comportement.</p>
<p class="doc-source">Source : ml_all_scores</p>
</div></span>
</span></div>
<div class="section-body"><div id="score-chart" style="height:300px"></div></div>
</div>
</div>
<!-- Detections table -->
<div class="bg-gray-900 rounded-xl border border-gray-800 overflow-hidden">
<h3 class="text-sm font-medium text-gray-400 px-5 py-3 border-b border-gray-800">Détections</h3>
<div class="overflow-x-auto max-h-[40vh] overflow-y-auto">
<!-- Detections -->
<div class="section-card overflow-hidden">
<div class="section-header"><span class="section-title">Détections
<span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
<h4>Historique des détections</h4>
<p>Chaque ligne = une session classée comme anormale. Le score combiné utilise l'ensemble triple-voix (EIF + AE + XGBoost).</p>
<p class="doc-source">Source : ml_detected_anomalies</p>
</div></span>
</span></div>
<div class="overflow-x-auto" style="max-height:35vh; overflow-y:auto">
<table class="data-table"><thead><tr>
<th>Date</th><th>Score</th><th>Raw</th><th>Threat</th><th>JA4</th><th>Host</th><th>Hits</th><th>Raison</th>
<th>Date</th><th>Score</th><th>Raw</th><th>AE</th><th>XGB</th><th>Threat</th><th>JA4</th><th>Host</th><th>Hits</th><th>Campagne</th>
</tr></thead><tbody id="det-body"></tbody></table>
</div>
</div>
<!-- AI Features grid -->
<div class="bg-gray-900 rounded-xl border border-gray-800 overflow-hidden" id="features-section" style="display:none">
<h3 class="text-sm font-medium text-gray-400 px-5 py-3 border-b border-gray-800">Features AI</h3>
<div class="p-5 grid grid-cols-2 md:grid-cols-4 lg:grid-cols-6 gap-3 text-sm" id="features-grid"></div>
<div class="section-card overflow-hidden" id="features-section" style="display:none">
<div class="section-header"><span class="section-title">Features AI (dernière fenêtre)
<span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
<h4>Features ML détaillées</h4>
<p>72 features extraites : velocity, fuzzing, entropie, ratios, métriques TLS, etc. Valeurs élevées en rouge indiquent un comportement suspect.</p>
<p class="doc-source">Source : view_ai_features_1h</p>
</div></span>
</span></div>
<div class="p-4 grid grid-cols-2 md:grid-cols-4 lg:grid-cols-6 xl:grid-cols-8 gap-2 text-sm" id="features-grid"></div>
</div>
<!-- HTTP Logs -->
<div class="bg-gray-900 rounded-xl border border-gray-800 overflow-hidden">
<h3 class="text-sm font-medium text-gray-400 px-5 py-3 border-b border-gray-800">Dernières requêtes HTTP (100 max)</h3>
<div class="overflow-x-auto max-h-[40vh] overflow-y-auto">
<div class="section-card overflow-hidden">
<div class="section-header"><span class="section-title">Requêtes HTTP récentes (100 max)
<span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
<h4>Logs HTTP bruts</h4>
<p>Dernières requêtes de cette IP. Cherchez des patterns : scraping séquentiel, POST répétés, paths suspects, absence de referer.</p>
<p class="doc-source">Source : http_logs</p>
</div></span>
</span></div>
<div class="overflow-x-auto" style="max-height:35vh; overflow-y:auto">
<table class="data-table"><thead><tr>
<th>Time</th><th>Method</th><th>Host</th><th>Path</th><th>HTTP Ver</th><th>User-Agent</th><th>JA4</th>
<th>Time</th><th>Method</th><th>Host</th><th>Path</th><th>HTTP</th><th>User-Agent</th><th>JA4</th>
</tr></thead><tbody id="http-body"></tbody></table>
</div>
</div>
<!-- Classify -->
<div class="bg-gray-900 rounded-xl p-5 border border-gray-800">
<h3 class="text-sm font-medium text-gray-400 mb-3">Classifier cette IP</h3>
<div class="flex gap-3 items-center">
<select id="cls-select" class="px-3 py-2 bg-gray-800 border border-gray-700 rounded-lg text-sm text-gray-300">
<option value="bot">🤖 Bot</option><option value="legitimate">✅ Légitime</option><option value="suspicious">⚠️ Suspect</option>
</select>
<input type="text" id="cls-comment" placeholder="Commentaire (optionnel)" class="flex-1 px-3 py-2 bg-gray-800 border border-gray-700 rounded-lg text-sm text-gray-300 focus:border-brand-500 focus:outline-none">
<button id="cls-btn" class="px-4 py-2 bg-brand-600 text-white rounded-lg text-sm font-medium hover:bg-brand-700">Envoyer</button>
</div>
<div id="cls-result" class="mt-2 text-sm"></div>
</div>
</div>
{% endblock %}
{% block scripts %}
@ -94,37 +124,38 @@ async function loadIP() {
document.getElementById('ip-recurrence').textContent = d.recurrence[0].recurrence || 0;
document.getElementById('ip-worst-score').innerHTML = fmtScore(d.recurrence[0].worst_score);
}
// ASN info from first detection or score
const firstRow = d.detections?.[0] || d.scores?.[0] || {};
if (firstRow.asn_org || firstRow.country_code) {
document.getElementById('ip-asn-info').innerHTML =
(firstRow.asn_org ? fmtASN(firstRow.asn_org) : '') +
(firstRow.country_code ? ' ' + fmtCountry(firstRow.country_code) : '');
}
// Radar chart
// Radar
if (radar.features?.length && Object.keys(radar.ip_values).length) {
const labels = radar.features.map(f => f.replace('_',' '));
const labels = radar.features.map(f => f.replace(/_/g,' '));
const ipVals = radar.features.map(f => radar.ip_values[f] ?? 0);
const humanVals = radar.features.map(f => radar.human_baseline[f] ?? 0);
const botVals = radar.features.map(f => radar.bot_baseline[f] ?? 0);
// Normalize to 0-1
const maxVals = radar.features.map((f,i) => Math.max(ipVals[i], humanVals[i], botVals[i], 0.001));
const norm = (arr) => arr.map((v,i) => +(v/maxVals[i]).toFixed(3));
const maxVals = radar.features.map((_,i) => Math.max(ipVals[i], humanVals[i], botVals[i], 0.001));
const norm = arr => arr.map((v,i) => +(v/maxVals[i]).toFixed(3));
const ch = initChart('radar-chart');
ch.setOption(ecBase({
tooltip: ecTooltip({}),
legend: {data:['Cette IP','Humain moyen','Bot moyen'], bottom:0, textStyle:{color:EC_TEXT,fontSize:11}},
legend: {data:['Cette IP','ISP moyen','Bot moyen'], bottom:0, textStyle:{color:EC_TEXT,fontSize:10}},
radar: {
indicator: labels.map((l,i) => ({name:l, max:1})),
shape:'polygon',
indicator: labels.map(() => ({max:1})),
shape:'polygon', radius:'65%',
splitArea:{areaStyle:{color:['rgba(99,102,241,0.02)','rgba(99,102,241,0.04)']}},
splitLine:{lineStyle:{color:EC_GRID}},
axisLine:{lineStyle:{color:EC_GRID}},
axisName:{color:EC_TEXT,fontSize:10},
splitLine:{lineStyle:{color:EC_GRID}}, axisLine:{lineStyle:{color:EC_GRID}},
axisName:{color:EC_TEXT,fontSize:9,formatter:(_,i)=>labels[i.dimensionIndex]||''},
},
series: [{
type:'radar',
data: [
{value:norm(ipVals), name:'Cette IP', lineStyle:{color:'#f97316',width:2}, areaStyle:{color:'rgba(249,115,22,0.15)'}, itemStyle:{color:'#f97316'}},
{value:norm(humanVals), name:'Humain moyen', lineStyle:{color:'#22c55e',width:1,type:'dashed'}, areaStyle:{color:'rgba(34,197,94,0.05)'}, itemStyle:{color:'#22c55e'}},
{value:norm(botVals), name:'Bot moyen', lineStyle:{color:'#ef4444',width:1,type:'dashed'}, areaStyle:{color:'rgba(239,68,68,0.05)'}, itemStyle:{color:'#ef4444'}},
]
}]
series: [{type:'radar', data: [
{value:norm(ipVals), name:'Cette IP', lineStyle:{color:'#f97316',width:2}, areaStyle:{color:'rgba(249,115,22,0.15)'}, itemStyle:{color:'#f97316'}},
{value:norm(humanVals), name:'ISP moyen', lineStyle:{color:'#22c55e',width:1,type:'dashed'}, areaStyle:{color:'rgba(34,197,94,0.05)'}, itemStyle:{color:'#22c55e'}},
{value:norm(botVals), name:'Bot moyen', lineStyle:{color:'#ef4444',width:1,type:'dashed'}, areaStyle:{color:'rgba(239,68,68,0.05)'}, itemStyle:{color:'#ef4444'}},
]}]
}));
}
@ -134,8 +165,8 @@ async function loadIP() {
const ch = initChart('score-chart');
ch.setOption(ecBase({
tooltip: ecTooltip({trigger:'axis'}),
grid: {left:50,right:20,top:20,bottom:30},
xAxis: {type:'category', data:scores.map(s=>(s.detected_at||'').substring(11,16)), axisLine:{lineStyle:{color:EC_GRID}}, axisLabel:{color:EC_TEXT}},
grid: ecGrid(),
xAxis: {type:'category', data:scores.map(s=>(s.detected_at||'').substring(11,16)), axisLine:{lineStyle:{color:EC_GRID}}, axisLabel:{color:EC_TEXT,fontSize:10}},
yAxis: {type:'value', min:0, max:1, splitLine:{lineStyle:{color:EC_GRID,type:'dashed'}}, axisLabel:{color:EC_TEXT}},
series: [{
type:'line', data:scores.map(s=>s.anomaly_score), smooth:true,
@ -147,62 +178,61 @@ async function loadIP() {
// Detections table
document.getElementById('det-body').innerHTML = (d.detections||[]).map(row => `<tr>
<td class="text-xs whitespace-nowrap">${row.detected_at||''}</td>
<td class="text-[11px] whitespace-nowrap text-gray-400">${(row.detected_at||'').substring(0,16)}</td>
<td>${fmtScore(row.anomaly_score)}</td>
<td>${fmtScore(row.raw_anomaly_score)}</td>
<td>${fmtThreatLink(row.threat_level)}</td>
<td class="text-xs font-mono max-w-[100px] truncate">${fmtJA4(row.ja4)}</td>
<td class="text-xs max-w-[120px] truncate">${row.host||''}</td>
<td>${row.hits||0}</td>
<td class="text-xs max-w-[200px] truncate">${row.reason||''}</td>
</tr>`).join('') || '<tr><td colspan="8" class="text-center text-gray-500 py-4">Aucune détection</td></tr>';
<td class="font-mono text-[11px] text-gray-400">${parseFloat(row.raw_anomaly_score||0).toFixed(4)}</td>
<td class="font-mono text-[11px] text-gray-400">${parseFloat(row.ae_recon_error||0).toFixed(4)}</td>
<td class="font-mono text-[11px] text-gray-400">${parseFloat(row.xgb_prob||0).toFixed(4)}</td>
<td>${threatBadge(row.threat_level)}</td>
<td>${fmtJA4(row.ja4)}</td>
<td class="text-xs max-w-[100px] truncate">${escapeHtml(row.host||'')}</td>
<td class="font-mono text-xs">${row.hits||0}</td>
<td class="text-[11px] text-purple-400">${row.campaign_id && row.campaign_id!=='0' ? '#'+escapeHtml(String(row.campaign_id).substring(0,8)) : ''}</td>
</tr>`).join('') || '<tr><td colspan="10" class="text-center text-gray-500 py-4">Aucune détection</td></tr>';
// AI Features
if (d.ai_features?.length) {
const f = d.ai_features[0];
const grid = document.getElementById('features-grid');
const skip = new Set(['src_ip','window_start','ja4','host','bot_name','src_ip_str']);
grid.innerHTML = Object.entries(f).filter(([k])=>!skip.has(k)).map(([k,v]) => {
document.getElementById('features-grid').innerHTML = Object.entries(f).filter(([k])=>!skip.has(k)).map(([k,v]) => {
let val = typeof v === 'number' ? v.toFixed(4) : v;
let color = 'text-gray-200';
if (typeof v === 'number' && v > 0.7) color = 'text-red-400';
else if (typeof v === 'number' && v > 0.4) color = 'text-orange-400';
let display = `<span class="text-sm ${color} font-mono">${val}</span>`;
if (k === 'asn_org' && v) display = `<span class="text-sm">${fmtASN(v)}</span>`;
else if (k === 'country_code' && v) display = `<span class="text-sm">${fmtCountry(v)}</span>`;
else if (k === 'asn_label' && v) display = `<span class="text-sm">${fmtLabel(v)}</span>`;
return `<div class="bg-gray-800 rounded-lg p-2"><div class="text-[10px] text-gray-500 truncate">${k}</div>${display}</div>`;
let display = `<span class="${color} font-mono text-xs">${val}</span>`;
if (k === 'asn_org' && v) display = fmtASN(v);
else if (k === 'country_code' && v) display = fmtCountry(v);
else if (k === 'asn_label' && v) display = fmtLabel(v);
return `<div class="bg-gray-800/50 rounded p-2"><div class="text-[9px] text-gray-500 truncate">${escapeHtml(k)}</div>${display}</div>`;
}).join('');
document.getElementById('features-section').style.display = '';
}
// HTTP logs
document.getElementById('http-body').innerHTML = (d.http_logs||[]).map(row => `<tr>
<td class="text-xs whitespace-nowrap">${row.time||''}</td>
<td class="font-mono text-xs">${row.method||''}</td>
<td class="text-xs max-w-[120px] truncate">${row.host||''}</td>
<td class="text-xs max-w-[250px] truncate font-mono" title="${row.path||''}">${row.path||''}</td>
<td class="font-mono text-xs">${row.http_version||''}</td>
<td class="text-xs max-w-[200px] truncate">${row.header_user_agent||''}</td>
<td class="text-xs font-mono">${row.ja4||''}</td>
<td class="text-[11px] whitespace-nowrap text-gray-400">${row.time||''}</td>
<td><span class="font-mono text-xs ${row.method==='POST'?'text-orange-400':'text-gray-300'}">${escapeHtml(row.method||'')}</span></td>
<td class="text-xs max-w-[100px] truncate">${escapeHtml(row.host||'')}</td>
<td class="text-xs max-w-[200px] truncate font-mono" title="${escapeHtml(row.path||'')}">${escapeHtml(row.path||'')}</td>
<td class="font-mono text-[11px] text-gray-400">${escapeHtml(row.http_version||'')}</td>
<td class="text-xs max-w-[180px] truncate text-gray-400">${escapeHtml(row.header_user_agent||'')}</td>
<td class="font-mono text-[11px]">${escapeHtml(row.ja4||'')}</td>
</tr>`).join('') || '<tr><td colspan="7" class="text-center text-gray-500 py-4">Aucun log</td></tr>';
} catch(e) { console.error(e); }
}
// Classify button
document.getElementById('cls-btn').onclick = async () => {
try {
const r = await fetch('/api/classify', {method:'POST', headers:{'Content-Type':'application/json'},
body:JSON.stringify({src_ip:IP, classification:document.getElementById('cls-select').value, comment:document.getElementById('cls-comment').value})});
const d = await r.json();
document.getElementById('cls-result').innerHTML = r.ok
? `<span class="text-green-400">✓ Classifié : ${d.classification}</span>`
: `<span class="text-red-400">✗ Erreur : ${d.detail||'unknown'}</span>`;
} catch(e) { document.getElementById('cls-result').innerHTML = `<span class="text-red-400">✗ ${e}</span>`; }
? `<span class="text-green-400">✓ ${escapeHtml(d.classification)}</span>`
: `<span class="text-red-400">✗ ${escapeHtml(d.detail||'erreur')}</span>`;
} catch(e) { document.getElementById('cls-result').innerHTML = `<span class="text-red-400">✗ erreur</span>`; }
};
loadIP();
window.addEventListener('resize', () => Object.values(charts).forEach(c => c?.resize()));
</script>
{% endblock %}