feat(dashboard): rebuild SOC dashboard + fix ClickHouse SQL

Complete rewrite of the SOC dashboard using FastAPI + Jinja2 + htmx + Chart.js + Tailwind CSS.
Replaces the old React/Vite frontend with server-rendered templates.

Dashboard pages:
- Overview: KPIs, timeline chart, threat distribution, top IPs
- Detections: paginated/filterable anomaly table
- Scores: ml_all_scores with AE error & XGB prob columns
- Traffic: HTTP logs with method/host filters
- IP Investigation: full deep-dive (scores, features, HTTP logs, classify)
- Classification: SOC feedback form + history
- Features: AI + thesis feature stats
- Models: scoring stats + model metadata

API: 9 JSON endpoints with parameterized queries, sort whitelists

SQL fixes:
- 05_aggregation_tables: add deduplicate_merge_projection_mode
- 11_views: fix nested aggregate (argMax inside sum)
- 12_thesis_features: remove invalid 'let' bindings, fix groupArrayIf type

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-08 03:21:05 +02:00
parent 228ad7026a
commit b735bab5a5
120 changed files with 1444 additions and 24933 deletions

View File

@ -0,0 +1,65 @@
{% extends "base.html" %}
{% block title %}JA4 SOC — Modèles{% endblock %}
{% block content %}
<div class="space-y-6">
<h2 class="text-lg font-semibold text-white">État des modèles ML</h2>
<!-- Scoring stats from ClickHouse -->
<div class="bg-gray-900 rounded-xl border border-gray-800 overflow-hidden">
<h3 class="text-sm font-medium text-gray-400 px-5 py-3 border-b border-gray-800">Statistiques de scoring (7 derniers jours)</h3>
<div class="overflow-x-auto">
<table class="data-table"><thead><tr>
<th>Modèle</th><th>Sessions scorées</th><th>Premier scoring</th><th>Dernier scoring</th>
</tr></thead><tbody id="scoring-body"></tbody></table>
</div>
</div>
<!-- Model metadata files -->
<div class="bg-gray-900 rounded-xl border border-gray-800 overflow-hidden">
<h3 class="text-sm font-medium text-gray-400 px-5 py-3 border-b border-gray-800">Métadonnées des modèles</h3>
<div id="model-cards" class="p-5 space-y-4">
<span class="text-sm text-gray-500">Chargement...</span>
</div>
</div>
</div>
{% endblock %}
{% block scripts %}
<script>
async function loadModels() {
try {
const r = await fetch('/api/models'); const d = await r.json();
// Scoring stats table
document.getElementById('scoring-body').innerHTML = (d.scoring_stats||[]).map(row => `<tr>
<td class="font-medium text-gray-200">${row.model_name||''}</td>
<td>${(row.scored||0).toLocaleString()}</td>
<td class="text-xs">${row.first_seen||''}</td>
<td class="text-xs">${row.last_seen||''}</td>
</tr>`).join('') || '<tr><td colspan="4" class="text-center text-gray-500 py-4">Aucun scoring récent</td></tr>';
// Model metadata cards
const cards = document.getElementById('model-cards');
if (d.models?.length) {
cards.innerHTML = d.models.map(m => `
<div class="bg-gray-800 rounded-lg p-4 border border-gray-700">
<div class="flex items-center gap-3 mb-2">
<span class="text-sm font-semibold text-white">${m.model_name||'?'} v${m.version_id||'?'}</span>
<span class="badge badge-low">${m.algorithm||'?'}</span>
${m.autoencoder ? '<span class="badge badge-medium">+AE</span>' : ''}
</div>
<div class="grid grid-cols-2 md:grid-cols-4 gap-2 text-xs text-gray-400">
<div>Entraîné : <span class="text-gray-300">${m.trained_at||'?'}</span></div>
<div>Échantillons : <span class="text-gray-300">${m.human_samples||'?'}</span></div>
<div>Contamination : <span class="text-gray-300">${m.contamination||'?'}</span></div>
<div>Seuil : <span class="text-gray-300">${m.threshold||'?'}</span></div>
${m.validation ? `<div>Val anomaly rate : <span class="text-gray-300">${(m.validation.val_anomaly_rate*100).toFixed(1)}%</span></div>
<div>Val mean score : <span class="text-gray-300">${m.validation.val_mean_score?.toFixed(4)||'?'}</span></div>
<div>Train size : <span class="text-gray-300">${m.validation.train_size||'?'}</span></div>
<div>Val size : <span class="text-gray-300">${m.validation.val_size||'?'}</span></div>` : ''}
</div>
</div>
`).join('');
} else {
cards.innerHTML = '<span class="text-sm text-gray-500">Aucun fichier de métadonnées trouvé (les modèles sont dans /data/models/)</span>';
}
} catch(e) { console.error(e); }
}
loadModels();
</script>
{% endblock %}