ja4-platform/services/dashboard/backend/templates/models.html

{% extends "base.html" %}
{% block title %}JA4 SOC — Modèles{% endblock %}
{% block page_title %}
    Modèles ML
    <span class="relative inline-block ml-1"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
        <h4>Monitoring des modèles ML</h4>
        <p>Ensemble triple-voix : Extended Isolation Forest (EIF) + Autoencoder (AE) + XGBoost supervisé.</p>
        <p><strong>Cycle :</strong> Toutes les 30 min, le bot-detector ré-entraîne si une dérive est détectée (≥95% features). Les anciens modèles restent en cache.</p>
        <p><strong>Workflow :</strong> Surveillez le volume de scoring, le taux d'anomalie et la santé des modèles. Un taux d'anomalie > 10% peut indiquer une attaque ou un modèle dégradé.</p>
        <p class="doc-source">Source : ml_all_scores (7j), /data/models/*.json</p>
    </div></span>
{% endblock %}
{% block content %}
<div class="space-y-4">
    <!-- KPIs -->
    <div class="grid grid-cols-2 md:grid-cols-3 xl:grid-cols-6 gap-3">
        <div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Sessions scorées (7j)</div><div class="text-xl font-bold text-brand-500" id="kpi-total">—</div></div>
        <div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Modèles actifs</div><div class="text-xl font-bold text-green-400" id="kpi-models">—</div></div>
        <div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Anomalies détectées</div><div class="text-xl font-bold text-red-400" id="kpi-anomalies">—</div></div>
        <div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Taux d'anomalie</div><div class="text-xl font-bold text-orange-400" id="kpi-rate">—</div></div>
        <div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Dernier scoring</div><div class="text-sm font-medium text-gray-200" id="kpi-last">—</div></div>
        <div class="kpi-card"><div class="text-[11px] text-gray-500 mb-1">Dernier entraînement</div><div class="text-sm font-medium text-yellow-400" id="kpi-train">—</div></div>
    </div>

    <!-- Scoring volume timeline + Threat breakdown -->
    <div class="grid grid-cols-1 lg:grid-cols-3 gap-4">
        <div class="section-card lg:col-span-2">
            <div class="section-header"><span class="section-title">Volume de scoring
                <span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
                    <h4>Timeline de scoring</h4>
                    <p>Nombre de sessions scorées par heure et par modèle. La courbe orange montre le score moyen d'anomalie.</p>
                    <p><strong>Interprétation :</strong> Un creux soudain indique un problème de pipeline. Un pic de score moyen = vague d'attaque.</p>
                    <p class="doc-source">Source : ml_all_scores GROUP BY hour, model_name</p>
                </div></span>
            </span></div>
            <div class="section-body"><div id="timeline-chart" style="height:260px"></div></div>
        </div>
        <div class="section-card">
            <div class="section-header"><span class="section-title">Répartition menaces
                <span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
                    <h4>Menaces par modèle</h4>
                    <p>Répartition des niveaux de menace (NORMAL, HIGH, CRITICAL, KNOWN_BOT, LEGITIMATE_BROWSER) par modèle.</p>
                    <p class="doc-source">Source : ml_all_scores GROUP BY model_name, threat_level</p>
                </div></span>
            </span></div>
            <div class="section-body"><div id="threats-chart" style="height:260px"></div></div>
        </div>
    </div>

    <!-- Score avg timeline (anomaly rate) -->
    <div class="section-card">
        <div class="section-header"><span class="section-title">Taux d'anomalie horaire
            <span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
                <h4>Anomaly rate over time</h4>
                <p>Pourcentage de sessions classées HIGH/CRITICAL par heure. Les barres montrent le volume, la ligne le taux.</p>
                <p><strong>Seuil d'alerte :</strong> Un taux > 10% prolongé mérite investigation.</p>
                <p class="doc-source">Source : ml_all_scores</p>
            </div></span>
        </span></div>
        <div class="section-body"><div id="rate-chart" style="height:200px"></div></div>
    </div>

    <!-- Scoring stats table -->
    <div class="section-card overflow-hidden">
        <div class="section-header"><span class="section-title">Statistiques de scoring (7 jours)
            <span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
                <h4>Résumé par modèle</h4>
                <p>Sessions scorées, période active et dernière activité pour chaque modèle. Complet = L3→L7 corrélé, Applicatif = L7 seul.</p>
                <p class="doc-source">Source : ml_all_scores GROUP BY model_name</p>
            </div></span>
        </span></div>
        <div class="overflow-x-auto">
            <table class="data-table"><thead><tr>
                <th>Modèle</th><th>Sessions scorées</th><th>Premier scoring</th><th>Dernier scoring</th>
            </tr></thead><tbody id="scoring-body"></tbody></table>
        </div>
    </div>

    <!-- Model metadata cards -->
    <div class="section-card overflow-hidden">
        <div class="section-header"><span class="section-title">Versions des modèles
            <span class="relative inline-block"><button onclick="docToggle(this)" class="doc-btn">?</button><div class="doc-panel">
                <h4>Métadonnées des modèles</h4>
                <p>Chaque fichier .json dans /data/models/ décrit un modèle entraîné : version, algorithme, paramètres, métriques de validation.</p>
                <p><strong>Gate de validation :</strong> Un modèle n'est utilisé que si val_anomaly_rate &lt; 5% et val_mean_score est raisonnable.</p>
                <p class="doc-source">Source : /data/models/*.json</p>
            </div></span>
        </span></div>
        <div id="model-cards" class="p-5 grid grid-cols-1 lg:grid-cols-2 gap-4">
            <span class="text-sm text-gray-500">Chargement...</span>
        </div>
    </div>
</div>
{% endblock %}
{% block scripts %}
<script>
let mCharts = {};
function initC(id) { const el=document.getElementById(id); if(!el) return null; if(mCharts[id]) mCharts[id].dispose(); mCharts[id]=echarts.init(el); return mCharts[id]; }

async function loadModels() {
    try {
        const [md, tl, th] = await Promise.all([
            fetch('/api/models').then(r=>r.json()),
            fetch('/api/models/timeline').then(r=>r.json()),
            fetch('/api/models/threats').then(r=>r.json()),
        ]);

        // ── KPIs ──
        const stats = md.scoring_stats || [];
        const totalScored = stats.reduce((s,r)=>s+(r.scored||0),0);
        document.getElementById('kpi-total').textContent = fmtNum(totalScored);
        document.getElementById('kpi-models').textContent = stats.length;
        const lastSeen = stats.map(r=>r.last_seen||'').sort().pop() || '—';
        document.getElementById('kpi-last').textContent = lastSeen.substring(0,16);
        if (md.models?.length) {
            const latest = md.models[md.models.length-1];
            document.getElementById('kpi-train').textContent = (latest.trained_at||'').substring(0,16);
        }

        // ── Timeline chart ──
        const tlRows = tl.timeline || [];
        if (tlRows.length) {
            const hours = [...new Set(tlRows.map(r=>r.hour))].sort();
            const models = [...new Set(tlRows.map(r=>r.model_name))];
            const MODEL_COLORS = {'Complet':'#6366f1','Applicatif':'#14b8a6','Complet_24h':'#8b5cf6','Applicatif_24h':'#06b6d4'};
            const byHourModel = {};
            let totalAnom = 0;
            tlRows.forEach(r => { byHourModel[r.hour+'|'+r.model_name] = r; totalAnom += (r.anomalies||0); });
            document.getElementById('kpi-anomalies').textContent = fmtNum(totalAnom);
            document.getElementById('kpi-rate').textContent = totalScored>0 ? (totalAnom/totalScored*100).toFixed(1)+'%' : '—';

            const ch = initC('timeline-chart');
            ch.setOption(ecBase({
                tooltip: ecTooltip({trigger:'axis'}),
                legend: {data:models, bottom:0, textStyle:{color:EC_TEXT,fontSize:10}},
                grid: {left:50,right:20,top:15,bottom:35},
                xAxis: {type:'category', data:hours.map(h=>(h||'').substring(11,16)), axisLine:{lineStyle:{color:EC_GRID}}, axisLabel:{color:EC_TEXT,fontSize:10}},
                yAxis: {type:'value', splitLine:{lineStyle:{color:EC_GRID,type:'dashed'}}, axisLabel:{color:EC_TEXT}},
                series: models.map(m => ({
                    name:m, type:'bar', stack:'vol', barWidth:'70%',
                    data: hours.map(h => (byHourModel[h+'|'+m]||{}).cnt || 0),
                    itemStyle:{color:MODEL_COLORS[m]||'#6b7280'},
                }))
            }));
        }

        // ── Anomaly rate chart ──
        if (tlRows.length) {
            const hours = [...new Set(tlRows.map(r=>r.hour))].sort();
            const hourAgg = {};
            tlRows.forEach(r => {
                if (!hourAgg[r.hour]) hourAgg[r.hour] = {total:0, anom:0};
                hourAgg[r.hour].total += r.cnt||0;
                hourAgg[r.hour].anom += r.anomalies||0;
            });
            const ch = initC('rate-chart');
            ch.setOption(ecBase({
                tooltip: ecTooltip({trigger:'axis'}),
                grid: {left:50,right:50,top:15,bottom:25},
                xAxis: {type:'category', data:hours.map(h=>(h||'').substring(11,16)), axisLine:{lineStyle:{color:EC_GRID}}, axisLabel:{color:EC_TEXT,fontSize:10}},
                yAxis: [
                    {type:'value', name:'Sessions', nameTextStyle:{color:EC_TEXT}, splitLine:{lineStyle:{color:EC_GRID,type:'dashed'}}, axisLabel:{color:EC_TEXT}},
                    {type:'value', name:'Taux %', nameTextStyle:{color:EC_TEXT}, max:100, splitLine:{show:false}, axisLabel:{color:EC_TEXT,formatter:'{value}%'}},
                ],
                series: [
                    {name:'Sessions', type:'bar', yAxisIndex:0, barWidth:'60%',
                     data:hours.map(h=>(hourAgg[h]||{}).total||0),
                     itemStyle:{color:'rgba(99,102,241,0.3)'}},
                    {name:'Taux anomalie', type:'line', yAxisIndex:1, smooth:true,
                     data:hours.map(h=>{const a=hourAgg[h]; return a&&a.total>0 ? +(a.anom/a.total*100).toFixed(1) : 0;}),
                     lineStyle:{color:'#f97316',width:2}, itemStyle:{color:'#f97316'}, symbol:'circle', symbolSize:3,
                     areaStyle:{color:new echarts.graphic.LinearGradient(0,0,0,1,[{offset:0,color:'rgba(249,115,22,0.15)'},{offset:1,color:'rgba(249,115,22,0)'}])}},
                ]
            }));
        }

        // ── Threats chart ──
        const thRows = th.threats || [];
        if (thRows.length) {
            const models = [...new Set(thRows.map(r=>r.model_name))];
            const levels = [...new Set(thRows.map(r=>r.threat_level))];
            const TCOLS = {CRITICAL:'#ef4444',HIGH:'#f97316',MEDIUM:'#eab308',NORMAL:'#6b7280',KNOWN_BOT:'#3b82f6',LEGITIMATE_BROWSER:'#22c55e',ANUBIS_DENY:'#a855f7'};
            const byMT = {};
            thRows.forEach(r => { byMT[r.model_name+'|'+r.threat_level] = r.cnt; });
            const ch = initC('threats-chart');
            ch.setOption(ecBase({
                tooltip: ecTooltip({trigger:'axis'}),
                legend: {data:levels, bottom:0, textStyle:{color:EC_TEXT,fontSize:9}},
                grid: {left:15,right:15,top:15,bottom:50},
                xAxis: {type:'category', data:models, axisLabel:{color:EC_TEXT,fontSize:10}, axisLine:{lineStyle:{color:EC_GRID}}},
                yAxis: {type:'value', splitLine:{lineStyle:{color:EC_GRID,type:'dashed'}}, axisLabel:{color:EC_TEXT}},
                series: levels.map(l => ({
                    name:l, type:'bar', stack:'t', barWidth:'50%',
                    data:models.map(m => byMT[m+'|'+l]||0),
                    itemStyle:{color:TCOLS[l]||'#6b7280'},
                }))
            }));
        }

        // ── Scoring stats table ──
        document.getElementById('scoring-body').innerHTML = stats.map(row => `<tr>
            <td class="font-medium text-gray-200">${escapeHtml(row.model_name||'')}</td>
            <td class="font-mono">${fmtNum(row.scored||0)}</td>
            <td class="text-xs text-gray-400">${(row.first_seen||'').substring(0,16)}</td>
            <td class="text-xs text-gray-400">${(row.last_seen||'').substring(0,16)}</td>
        </tr>`).join('') || '<tr><td colspan="4" class="text-center text-gray-500 py-4">Aucun scoring récent</td></tr>';

        // ── Model cards ──
        const cards = document.getElementById('model-cards');
        if (md.models?.length) {
            cards.innerHTML = md.models.map(m => {
                const valRate = m.validation?.val_anomaly_rate;
                const valBadge = valRate != null
                    ? (valRate > 0.05 ? 'badge-high' : valRate > 0.02 ? 'badge-medium' : 'badge-low')
                    : 'badge-normal';
                return `
                <div class="bg-gray-800 rounded-lg p-4 border border-gray-700 hover:border-gray-600 transition-colors">
                    <div class="flex items-center gap-3 mb-3">
                        <span class="text-sm font-semibold text-white">${escapeHtml(m.model_name||'?')}</span>
                        <span class="badge badge-low text-[10px]">${escapeHtml(m.algorithm||'?')}</span>
                        ${m.autoencoder ? '<span class="badge badge-medium text-[10px]">+AE</span>' : ''}
                        <span class="text-[10px] text-gray-500 ml-auto">v${escapeHtml(m.version_id||'?')}</span>
                    </div>
                    <div class="grid grid-cols-2 md:grid-cols-4 gap-3 text-xs">
                        <div class="bg-gray-900/50 rounded p-2">
                            <div class="text-[9px] text-gray-500 mb-0.5">Entraîné</div>
                            <div class="text-gray-300 font-mono">${(m.trained_at||'?').substring(0,16)}</div>
                        </div>
                        <div class="bg-gray-900/50 rounded p-2">
                            <div class="text-[9px] text-gray-500 mb-0.5">Échantillons</div>
                            <div class="text-gray-300 font-mono">${fmtNum(m.human_samples||0)}</div>
                        </div>
                        <div class="bg-gray-900/50 rounded p-2">
                            <div class="text-[9px] text-gray-500 mb-0.5">Contamination</div>
                            <div class="text-gray-300 font-mono">${m.contamination||'?'}</div>
                        </div>
                        <div class="bg-gray-900/50 rounded p-2">
                            <div class="text-[9px] text-gray-500 mb-0.5">Seuil</div>
                            <div class="text-gray-300 font-mono">${typeof m.threshold==='number'?m.threshold.toFixed(4):m.threshold||'?'}</div>
                        </div>
                    </div>
                    ${m.validation ? `
                    <div class="mt-3 pt-3 border-t border-gray-700">
                        <div class="text-[10px] text-gray-500 mb-2 font-medium">Validation Gate</div>
                        <div class="grid grid-cols-4 gap-2 text-xs">
                            <div><span class="text-gray-500">Taux anomalie :</span> <span class="badge ${valBadge} text-[10px]">${(valRate*100).toFixed(1)}%</span></div>
                            <div><span class="text-gray-500">Score moyen :</span> <span class="text-gray-300">${m.validation.val_mean_score?.toFixed(4)||'?'}</span></div>
                            <div><span class="text-gray-500">Train :</span> <span class="text-gray-300">${fmtNum(m.validation.train_size||0)}</span></div>
                            <div><span class="text-gray-500">Val :</span> <span class="text-gray-300">${fmtNum(m.validation.val_size||0)}</span></div>
                        </div>
                    </div>` : ''}
                    ${m.features_used ? `
                    <div class="mt-2 text-[10px] text-gray-500">${m.features_used} features · ${m.features_pruned||0} élaguées</div>` : ''}
                </div>`;
            }).join('');
        } else {
            cards.innerHTML = '<span class="text-sm text-gray-500">Aucun fichier de métadonnées trouvé</span>';
        }
    } catch(e) { console.error('models load error:', e); }
}

loadModels();
setInterval(loadModels, 60000);
window.addEventListener('resize', () => Object.values(mCharts).forEach(c=>c?.resize()));
</script>
{% endblock %}