ja4-platform/services/dashboard/backend/templates/reflists.html

{% extends "base.html" %}
{% block page_title %}
Listes de référence
<div class="relative inline-block ml-2">
    <button class="doc-btn text-gray-400 hover:text-brand-300 text-sm" onclick="docToggle(this)">ⓘ</button>
    <div class="doc-panel w-96">
        <h3 class="font-semibold text-brand-300 mb-2">Listes de référence</h3>
        <p class="text-xs text-gray-300 leading-relaxed mb-2">
            Visualisation des fichiers CSV et dictionnaires chargés dans ClickHouse.
            Ces listes alimentent les enrichissements en temps réel du pipeline :
            identification de bots connus, résolution ASN/géo, classification navigateur,
            et règles Anubis de filtrage.
        </p>
        <table class="text-[10px] text-gray-400 w-full">
            <tr><td class="pr-2 font-mono text-brand-400">bot_ip</td><td>IP/CIDR de bots connus → nom du bot</td></tr>
            <tr><td class="pr-2 font-mono text-brand-400">bot_ja4</td><td>Fingerprints JA4 de bots → nom du bot</td></tr>
            <tr><td class="pr-2 font-mono text-brand-400">browser_ja4</td><td>Fingerprints JA4 navigateurs → famille, lib TLS</td></tr>
            <tr><td class="pr-2 font-mono text-brand-400">asn_reputation</td><td>ASN → label de réputation (isp, datacenter, cdn…)</td></tr>
            <tr><td class="pr-2 font-mono text-brand-400">iplocate_asn</td><td>IP/CIDR → ASN, pays, nom (géolocalisation)</td></tr>
            <tr><td class="pr-2 font-mono text-brand-400">anubis_*</td><td>Règles Anubis : UA, IP, ASN, pays (filtrage crawlers)</td></tr>
        </table>
    </div>
</div>
{% endblock %}

{% block content %}
<!-- KPI bar -->
<div id="kpi-bar" class="grid grid-cols-2 md:grid-cols-4 lg:grid-cols-5 gap-3 mb-5">
    <div class="bg-gray-900/60 border border-gray-800 rounded-lg p-3">
        <div class="text-[10px] text-gray-500 uppercase tracking-wider">Dictionnaires</div>
        <div id="kpi-dicts" class="text-xl font-bold text-white mt-1">—</div>
    </div>
    <div class="bg-gray-900/60 border border-gray-800 rounded-lg p-3">
        <div class="text-[10px] text-gray-500 uppercase tracking-wider">Bot IPs</div>
        <div id="kpi-botip" class="text-xl font-bold text-red-400 mt-1">—</div>
    </div>
    <div class="bg-gray-900/60 border border-gray-800 rounded-lg p-3">
        <div class="text-[10px] text-gray-500 uppercase tracking-wider">Browser JA4</div>
        <div id="kpi-browser" class="text-xl font-bold text-green-400 mt-1">—</div>
    </div>
    <div class="bg-gray-900/60 border border-gray-800 rounded-lg p-3">
        <div class="text-[10px] text-gray-500 uppercase tracking-wider">ASN Réputation</div>
        <div id="kpi-asn" class="text-xl font-bold text-blue-400 mt-1">—</div>
    </div>
    <div class="bg-gray-900/60 border border-gray-800 rounded-lg p-3">
        <div class="text-[10px] text-gray-500 uppercase tracking-wider">IPLocate ASN</div>
        <div id="kpi-iplocate" class="text-xl font-bold text-purple-400 mt-1">—</div>
    </div>
</div>

<!-- Tab bar -->
<div class="flex flex-wrap gap-1 mb-4 border-b border-gray-800 pb-2">
    <button class="tab-btn active" data-tab="bot_ip">🤖 Bot IP</button>
    <button class="tab-btn" data-tab="bot_ja4">🔑 Bot JA4</button>
    <button class="tab-btn" data-tab="browser_ja4">🌐 Browser JA4</button>
    <button class="tab-btn" data-tab="asn_reputation">🏢 ASN Réputation</button>
    <button class="tab-btn" data-tab="iplocate_asn">🌍 IPLocate</button>
    <button class="tab-btn" data-tab="anubis_ua_rules">🕷 Anubis UA</button>
    <button class="tab-btn" data-tab="anubis_ip_rules">🕷 Anubis IP</button>
    <button class="tab-btn" data-tab="anubis_asn_rules">🕷 Anubis ASN</button>
    <button class="tab-btn" data-tab="anubis_country_rules">🕷 Anubis Pays</button>
</div>

<!-- Controls row -->
<div class="flex flex-wrap items-center gap-3 mb-4">
    <input id="search-input" type="text" placeholder="Rechercher…"
           class="bg-gray-900 border border-gray-700 rounded px-3 py-1.5 text-sm text-gray-200 placeholder-gray-500 w-64 focus:border-brand-500 focus:outline-none">
    <span id="result-count" class="text-xs text-gray-500"></span>
    <div class="flex-1"></div>
    <div class="flex items-center gap-2">
        <button id="btn-prev" onclick="prevPage()" class="px-2 py-1 text-xs bg-gray-800 border border-gray-700 rounded text-gray-400 hover:bg-gray-700 disabled:opacity-30" disabled>← Préc.</button>
        <span id="page-info" class="text-xs text-gray-500">Page 1</span>
        <button id="btn-next" onclick="nextPage()" class="px-2 py-1 text-xs bg-gray-800 border border-gray-700 rounded text-gray-400 hover:bg-gray-700 disabled:opacity-30" disabled>Suiv. →</button>
    </div>
</div>

<!-- Stats panel (chart) -->
<div id="stats-panel" class="bg-gray-900/60 border border-gray-800 rounded-lg p-4 mb-4 hidden">
    <div class="flex items-center gap-2 mb-3">
        <h3 id="stats-title" class="text-sm font-semibold text-gray-200"></h3>
        <div class="relative inline-block">
            <button class="doc-btn text-gray-400 hover:text-brand-300 text-xs" onclick="docToggle(this)">ⓘ</button>
            <div class="doc-panel w-72">
                <p class="text-xs text-gray-300 leading-relaxed">
                    Distribution des entrées par catégorie principale.
                    Cliquer sur une barre pour filtrer le tableau.
                </p>
            </div>
        </div>
    </div>
    <div id="stats-chart" style="height:200px;"></div>
</div>

<!-- Data table -->
<div class="bg-gray-900/60 border border-gray-800 rounded-lg overflow-hidden">
    <div class="overflow-x-auto">
        <table class="w-full text-xs">
            <thead>
                <tr id="table-head" class="border-b border-gray-800 text-gray-400 text-left"></tr>
            </thead>
            <tbody id="table-body" class="divide-y divide-gray-800/50"></tbody>
        </table>
    </div>
</div>
{% endblock %}

{% block scripts %}
<style>
    .tab-btn {
        padding: 6px 14px;
        font-size: 12px;
        font-weight: 500;
        color: #9ca3af;
        background: transparent;
        border: 1px solid transparent;
        border-radius: 6px;
        cursor: pointer;
        transition: all 0.15s;
        white-space: nowrap;
    }
    .tab-btn:hover { color: #e5e7eb; background: rgba(255,255,255,0.04); }
    .tab-btn.active {
        color: #93c5fd;
        background: rgba(59,130,246,0.1);
        border-color: rgba(59,130,246,0.3);
    }
</style>
<script>
const PAGE_SIZE = 200;
let currentTab = 'bot_ip';
let currentOffset = 0;
let currentTotal = 0;
let currentSort = '';
let currentOrder = 'ASC';
let searchTimeout = null;

// Descriptions par liste
const LIST_DOC = {
    bot_ip:     'Plages IP/CIDR de bots et scanners connus. Source : dict_bot_ip (IP_TRIE). Utilisé par la feature is_known_bot dans le scoring ML.',
    bot_ja4:    'Fingerprints JA4 de clients automatisés connus (curl, python-requests, scrapy…). Source : dict_bot_ja4. Utilisé pour le triage pre-ML.',
    browser_ja4:'Fingerprints JA4 de navigateurs légitimes avec famille (Chromium, Firefox, Safari…), librairie TLS et contexte. Source : dict_browser_ja4.',
    asn_reputation:'Réputation de chaque ASN (isp, datacenter, cdn, hosting, education…). Source : dict_asn_reputation. Utilisé pour la baseline ISP humaine.',
    iplocate_asn:'Géolocalisation IP→ASN : réseau, numéro ASN, code pays, nom opérateur. Source : dict_iplocate_asn (IP_TRIE, ~714K entrées).',
    anubis_ua_rules:'Règles Anubis de détection par User-Agent (REGEXP_TREE). Chaque règle associe un pattern regex à un bot_name et une action (ALLOW/DENY/WEIGH).',
    anubis_ip_rules:'Règles Anubis de détection par plage IP (IP_TRIE). Associe des CIDR à des bots connus avec action de filtrage.',
    anubis_asn_rules:'Règles Anubis par ASN : certains ASN sont associés à des botnets ou services automatisés connus.',
    anubis_country_rules:'Règles Anubis par pays : politique de filtrage par code pays (ex: bloquer le trafic de certaines régions).',
};

// Colonnes par liste
const COLUMNS = {
    bot_ip:     [{k:'prefix',f:'IP/CIDR'},{k:'bot_name',f:'Bot'}],
    bot_ja4:    [{k:'ja4',f:'JA4'},{k:'bot_name',f:'Bot'}],
    browser_ja4:[{k:'ja4',f:'JA4'},{k:'browser_family',f:'Famille'},{k:'tls_library',f:'Lib TLS'},{k:'context',f:'Contexte'}],
    asn_reputation:[{k:'src_asn',f:'ASN'},{k:'label',f:'Réputation'}],
    iplocate_asn:[{k:'network',f:'Réseau'},{k:'asn',f:'ASN'},{k:'country_code',f:'Pays'},{k:'name',f:'Nom'}],
    anubis_ua_rules:[{k:'id',f:'ID'},{k:'regexp',f:'Regex'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
    anubis_ip_rules:[{k:'prefix',f:'IP/CIDR'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
    anubis_asn_rules:[{k:'asn',f:'ASN'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
    anubis_country_rules:[{k:'country_code',f:'Pays'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
};

// Tab click
document.querySelectorAll('.tab-btn').forEach(btn => {
    btn.addEventListener('click', () => {
        document.querySelectorAll('.tab-btn').forEach(b => b.classList.remove('active'));
        btn.classList.add('active');
        currentTab = btn.dataset.tab;
        currentOffset = 0;
        currentSort = '';
        currentOrder = 'ASC';
        document.getElementById('search-input').value = '';
        loadStats();
        loadData();
    });
});

// Search
document.getElementById('search-input').addEventListener('input', (e) => {
    clearTimeout(searchTimeout);
    searchTimeout = setTimeout(() => { currentOffset = 0; loadData(); }, 350);
});

function prevPage() { if (currentOffset >= PAGE_SIZE) { currentOffset -= PAGE_SIZE; loadData(); } }
function nextPage() { if (currentOffset + PAGE_SIZE < currentTotal) { currentOffset += PAGE_SIZE; loadData(); } }

function sortBy(col) {
    if (currentSort === col) {
        currentOrder = currentOrder === 'ASC' ? 'DESC' : 'ASC';
    } else {
        currentSort = col;
        currentOrder = 'ASC';
    }
    currentOffset = 0;
    loadData();
}

function renderHead() {
    const cols = COLUMNS[currentTab] || [];
    const thead = document.getElementById('table-head');
    thead.innerHTML = cols.map(c =>
        `<th class="px-3 py-2 text-[11px] cursor-pointer hover:text-brand-300 select-none whitespace-nowrap" onclick="sortBy('${c.k}')">` +
        `${c.f} ${currentSort === c.k ? (currentOrder === 'ASC' ? '↑' : '↓') : '<span class=&quot;text-gray-600&quot;>↕</span>'}</th>`
    ).join('');
}

function cellHtml(col, val) {
    if (val == null || val === '') return '<span class="text-gray-600">—</span>';
    const s = String(val);
    if (col === 'ja4') return fmtJA4Full(s);
    if (col === 'prefix' || col === 'network') return `<span class="font-mono text-gray-300">${esc(s)}</span>`;
    if (col === 'bot_name') return s ? `<span class="text-red-400">${esc(s)}</span>` : '—';
    if (col === 'browser_family') {
        const colors = {Chromium:'text-blue-400',Firefox:'text-orange-400',Safari:'text-cyan-400',Edge:'text-green-400',Opera:'text-red-400'};
        return `<span class="${colors[s]||'text-gray-300'}">${esc(s)}</span>`;
    }
    if (col === 'action') {
        const cls = {DENY:'bg-red-500/20 text-red-300',ALLOW:'bg-green-500/20 text-green-300',WEIGH:'bg-yellow-500/20 text-yellow-300'};
        return `<span class="px-1.5 py-0.5 rounded text-[10px] font-medium ${cls[s]||'text-gray-400'}">${esc(s)||'—'}</span>`;
    }
    if (col === 'label') {
        const cls = {isp:'text-green-400',datacenter:'text-yellow-400',cdn:'text-blue-400',hosting:'text-orange-400',education:'text-purple-400',government:'text-cyan-400',enterprise:'text-indigo-400'};
        return `<span class="${cls[s]||'text-gray-300'}">${esc(s)}</span>`;
    }
    if (col === 'country_code' && s.length === 2) return fmtCountry(s);
    if (col === 'src_asn' || col === 'asn') return `<span class="font-mono text-gray-300">AS${s}</span>`;
    if (col === 'regexp') return `<code class="text-[10px] text-yellow-300 bg-gray-800 px-1 rounded max-w-xs truncate inline-block">${esc(s)}</code>`;
    return `<span class="text-gray-300">${esc(s)}</span>`;
}

function renderBody(rows) {
    const cols = COLUMNS[currentTab] || [];
    const tbody = document.getElementById('table-body');
    if (!rows.length) {
        tbody.innerHTML = `<tr><td colspan="${cols.length}" class="px-4 py-8 text-center text-gray-500">Aucune entrée</td></tr>`;
        return;
    }
    tbody.innerHTML = rows.map(r =>
        '<tr class="hover:bg-gray-800/40 transition-colors">' +
        cols.map(c => `<td class="px-3 py-1.5 whitespace-nowrap">${cellHtml(c.k, r[c.k])}</td>`).join('') +
        '</tr>'
    ).join('');
}

function updatePagination() {
    const pageNum = Math.floor(currentOffset / PAGE_SIZE) + 1;
    const totalPages = Math.max(1, Math.ceil(currentTotal / PAGE_SIZE));
    document.getElementById('page-info').textContent = `Page ${pageNum} / ${totalPages}`;
    document.getElementById('btn-prev').disabled = currentOffset === 0;
    document.getElementById('btn-next').disabled = currentOffset + PAGE_SIZE >= currentTotal;
    document.getElementById('result-count').textContent = `${fmtNum(currentTotal)} entrée${currentTotal > 1 ? 's' : ''}`;
}

async function loadData() {
    const search = document.getElementById('search-input').value.trim();
    renderHead();
    let url = `/api/reflist/${currentTab}?limit=${PAGE_SIZE}&offset=${currentOffset}`;
    if (currentSort) url += `&sort=${currentSort}&order=${currentOrder}`;
    if (search) url += `&search=${encodeURIComponent(search)}`;

    try {
        const resp = await fetch(url);
        if (!resp.ok) throw new Error(resp.statusText);
        const data = await resp.json();
        currentTotal = data.total;
        renderBody(data.rows);
        updatePagination();
    } catch (e) {
        console.error('Reflist load error:', e);
        document.getElementById('table-body').innerHTML =
            `<tr><td colspan="10" class="px-4 py-8 text-center text-red-400">Erreur de chargement : ${esc(e.message)}</td></tr>`;
    }
}

async function loadStats() {
    const panel = document.getElementById('stats-panel');
    try {
        const resp = await fetch(`/api/reflist/${currentTab}/stats`);
        if (!resp.ok) throw new Error(resp.statusText);
        const data = await resp.json();
        if (!data.breakdown || !data.breakdown.length) { panel.classList.add('hidden'); return; }

        panel.classList.remove('hidden');
        document.getElementById('stats-title').textContent =
            `Répartition — ${currentTab.replace(/_/g, ' ')} (${fmtNum(data.total)} entrées)`;

        const labels = data.breakdown.map(r => Object.values(r)[0] || '—');
        const values = data.breakdown.map(r => r.cnt);
        renderBarChart('stats-chart', labels, values);
    } catch (e) {
        panel.classList.add('hidden');
        console.error('Stats load error:', e);
    }
}

async function loadMeta() {
    try {
        const resp = await fetch('/api/dictionaries');
        if (!resp.ok) return;
        const data = await resp.json();
        const dicts = data.dictionaries || [];
        document.getElementById('kpi-dicts').textContent = dicts.length;
        for (const d of dicts) {
            if (d.name === 'dict_bot_ip') document.getElementById('kpi-botip').textContent = fmtNum(d.element_count);
            if (d.name === 'dict_browser_ja4') document.getElementById('kpi-browser').textContent = fmtNum(d.element_count);
            if (d.name === 'dict_asn_reputation') document.getElementById('kpi-asn').textContent = fmtNum(d.element_count);
            if (d.name === 'dict_iplocate_asn') document.getElementById('kpi-iplocate').textContent = fmtNum(d.element_count);
        }
    } catch (e) { console.error('Meta load error:', e); }
}

function renderBarChart(id, labels, values) {
    const el = document.getElementById(id);
    const ch = echarts.getInstanceByDom(el);
    if (ch) ch.dispose();
    const chart = echarts.init(el);
    chart.setOption({
        tooltip: { trigger: 'axis', axisPointer: { type: 'shadow' } },
        grid: { left: 80, right: 20, top: 10, bottom: 30 },
        xAxis: { type: 'value', axisLabel: { color: '#6b7280', fontSize: 10 }, splitLine: { lineStyle: { color: '#1f2937' } } },
        yAxis: { type: 'category', data: labels.slice().reverse(), axisLabel: { color: '#9ca3af', fontSize: 10 }, axisTick: { show: false } },
        series: [{
            type: 'bar',
            data: values.slice().reverse(),
            itemStyle: { color: '#3b82f6', borderRadius: [0, 3, 3, 0] },
            barMaxWidth: 20,
        }],
    });
    window.addEventListener('resize', () => chart.resize());
}

document.addEventListener('DOMContentLoaded', () => {
    loadMeta();
    loadStats();
    loadData();
});
</script>
{% endblock %}