feat: nouvelles techniques de détection et page tactiques SOC

SQL:
- Ajout 5 colonnes d'agrégation (count_xff, count_unusual_ct,
  count_non_std_port, count_login_post, sec_ch_mobile_mismatch)
- Exposition de 5 features calculées dans view_ai_features_1h
- Migration ALTER TABLE pour déploiements existants

Bot-detector:
- 7 nouvelles features ML (has_xff, unusual_content_type_ratio,
  non_standard_port_ratio, login_post_concentration,
  sec_ch_mobile_mismatch, true_window_size, window_mss_ratio)
- Propagation campaign_id vers ml_all_scores (était toujours -1)
- Escalade campagne : HIGH→CRITICAL si cluster ≥5 membres

Dashboard:
- Page Tactiques SOC : brute-force, rotation JA4, récurrence,
  alertes temps réel — 4 KPIs + 4 panneaux + infobulles doc
- Ajout fmtDate() helper global
- Navigation sidebar mise à jour

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

services/bot-detector/bot_detector/config.py

@@ -109,7 +109,8 @@ STRUCTURAL_EXCLUDED_FEATURES: dict[str, list] = {
                    'request_size_variance', 'mss_mobile_mismatch',
                    'ja3_diversity_ratio', 'syn_timing_cv', 'tls12_ratio', 'ip_df_variance',
                    'avg_ttl', 'ttl_std', 'no_window_scale_ratio',
-                   'ja4_drift_ratio'],
+                   'ja4_drift_ratio',
+                   'true_window_size', 'window_mss_ratio'],
 }
 
 # ─── Imports optionnels (bibliothèques lourdes) ────────────────────────────

services/bot-detector/bot_detector/pipeline.py

@@ -306,6 +306,23 @@ def run_semi_supervised_logic(df, features, name, cycle_id, recurrence_map):
         if ENABLE_CLUSTERING:
             anomalies = cluster_anomalies(anomalies, scoring_features, ae_model=ae_model)
 
+            # P2 — Escalade par taille de campagne : les IPs dans un cluster
+            # coordonné de grande taille sont plus menaçantes que des IPs isolées.
+            # Escalader HIGH → CRITICAL si cluster_size ≥ 5.
+            if 'campaign_id' in anomalies.columns:
+                cid_counts = anomalies['campaign_id'].value_counts()
+                for cid, size in cid_counts.items():
+                    if cid < 0:
+                        continue
+                    if size >= 5:
+                        mask = (anomalies['campaign_id'] == cid) & (anomalies['threat_level'] == 'HIGH')
+                        n_escalated = mask.sum()
+                        if n_escalated > 0:
+                            anomalies.loc[mask, 'threat_level'] = 'CRITICAL'
+                            anomalies.loc[mask, 'reason'] = anomalies.loc[mask, 'reason'] + \
+                                f' [Escalade campagne #{cid}, {size} IPs coordonnées]'
+                            log_info(f"[{name}] Escalade campagne #{cid}: {n_escalated} IP(s) HIGH→CRITICAL ({size} membres)")
+
         anomalies['ja4'] = anomalies['ja4'].replace({'': 'HTTP_CLEAR_TEXT'})
         for _, row in anomalies.iterrows():
             log_decision('ANOMALY', cycle_id, name, {
@@ -330,6 +347,14 @@ def run_semi_supervised_logic(df, features, name, cycle_id, recurrence_map):
         anubis_deny    if not anubis_deny.empty    else None,
     ] if df is not None], ignore_index=True)
 
+    # Propager campaign_id des anomalies clusterisées vers all_scored
+    # (all_scored a été capturé avant clustering, ses campaign_id sont tous -1)
+    if not anomalies.empty and 'campaign_id' in anomalies.columns:
+        cid_map = anomalies.set_index(anomalies.index)['campaign_id']
+        matched = all_scored.index.isin(cid_map.index)
+        if matched.any():
+            all_scored.loc[matched, 'campaign_id'] = cid_map
+
     # Inclure anubis_allow dans all_scored pour traçabilité dans ml_all_scores.
     # Ces IPs sont exclues de l'analyse IF mais doivent apparaître dans la table
     # de scores avec threat_level='KNOWN_BOT' et anomaly_score=0.0.

services/bot-detector/bot_detector/preprocessing.py

@@ -41,6 +41,12 @@ FEATURES = [
     'cadence_cv', 'burst_ratio', 'pause_ratio',
     'lag1_autocorrelation', 'benford_deviation',
     'host_diversity', 'host_sweep_speed', 'host_coverage_uniformity',
+    # P0+P1 : features sous-exploitées (SQL existant ou ajouté)
+    'is_fake_navigation',
+    'true_window_size', 'window_mss_ratio',
+    # P1 : nouvelles features de détection
+    'has_xff', 'unusual_content_type_ratio', 'non_standard_port_ratio',
+    'login_post_concentration', 'sec_ch_mobile_mismatch',
 ]
 
 # Features supplémentaires pour le modèle Complet (données TCP/TLS requises)
@@ -100,6 +106,7 @@ def preprocess_df(df: pd.DataFrame) -> pd.DataFrame:
         'has_accept_language', 'has_cookie', 'has_referer', 'ua_ch_mismatch',
         'is_ua_rotating', 'is_alpn_missing', 'sni_host_mismatch', 'alpn_http_mismatch',
         'mss_mobile_mismatch', 'anubis_is_flagged', 'is_rare_ja4',
+        'is_fake_navigation', 'has_xff', 'sec_ch_mobile_mismatch',
     }
     for col in df.columns:
         if col in binary_features:

services/correlator/sql/migrations/02_detection_features.sql

@@ -0,0 +1,17 @@
+-- =============================================================================
+-- 02_detection_features.sql — Ajout des features de détection P0+P1
+-- Colonnes supplémentaires dans agg_host_ip_ja4_1h et agg_header_fingerprint_1h
+-- NOTE : les MVs doivent être recréées (DROP + CREATE) car ALTER VIEW n'existe pas.
+-- Exécuter deploy_schema.sh pour recréer les MVs, ou relancer le schema complet.
+-- =============================================================================
+
+-- agg_host_ip_ja4_1h : nouvelles colonnes de comptage
+ALTER TABLE ja4_processing.agg_host_ip_ja4_1h
+    ADD COLUMN IF NOT EXISTS count_xff SimpleAggregateFunction(sum, UInt64) AFTER count_http_scheme,
+    ADD COLUMN IF NOT EXISTS count_unusual_ct SimpleAggregateFunction(sum, UInt64) AFTER count_xff,
+    ADD COLUMN IF NOT EXISTS count_non_std_port SimpleAggregateFunction(sum, UInt64) AFTER count_unusual_ct,
+    ADD COLUMN IF NOT EXISTS count_login_post SimpleAggregateFunction(sum, UInt64) AFTER count_non_std_port;
+
+-- agg_header_fingerprint_1h : mismatch mobile Sec-CH-UA
+ALTER TABLE ja4_processing.agg_header_fingerprint_1h
+    ADD COLUMN IF NOT EXISTS sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8) AFTER ua_ch_mismatch;

services/dashboard/backend/routes/pages.py

@@ -71,3 +71,8 @@ async def ja4_detail_page(request: Request, fingerprint: str):
 @router.get("/cluster/{cid}")
 async def cluster_detail_page(request: Request, cid: int):
     return templates.TemplateResponse("cluster_detail.html", _ctx(request, "cluster_detail", cid=cid))
+
+
+@router.get("/tactics")
+async def tactics_page(request: Request):
+    return templates.TemplateResponse("tactics.html", _ctx(request, "tactics"))

services/dashboard/backend/templates/base.html

@@ -151,6 +151,10 @@
                 <svg class="w-4 h-4 shrink-0" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M17 20h5v-2a3 3 0 00-5.356-1.857M17 20H7m10 0v-2c0-.656-.126-1.283-.356-1.857M7 20H2v-2a3 3 0 015.356-1.857M7 20v-2c0-.656.126-1.283.356-1.857m0 0a5.002 5.002 0 019.288 0M15 7a3 3 0 11-6 0 3 3 0 016 0z"/></svg>
                 <span class="nav-text">Campagnes</span>
             </a>
+            <a href="/tactics" class="nav-item {% if active_page == 'tactics' %}active{% endif %}" title="Tactiques d'attaque détectées">
+                <svg class="w-4 h-4 shrink-0" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"/></svg>
+                <span class="nav-text">Tactiques</span>
+            </a>
 
             <div class="nav-group-title">Investigation</div>
             <a href="/traffic" class="nav-item {% if active_page == 'traffic' %}active{% endif %}" title="Logs HTTP bruts">
@@ -324,6 +328,10 @@
             if (diff < 86400) return Math.round(diff/3600) + 'h';
             return Math.round(diff/86400) + 'j';
         }
+        function fmtDate(d) {
+            if (!d) return '—';
+            return String(d).substring(0, 16).replace('T', ' ');
+        }
 
         // ── ECharts helpers ──
         const EC_COLORS = ['#6366f1','#22c55e','#f97316','#ef4444','#3b82f6','#eab308','#ec4899','#14b8a6','#8b5cf6','#f43f5e'];

services/dashboard/backend/templates/tactics.html

@@ -0,0 +1,199 @@
+{% extends "base.html" %}
+{% block page_title %}Tactiques d'attaque{% endblock %}
+
+{% block content %}
+<!-- ═══ Tactiques d'attaque — 4 panneaux de détection spécialisés ═══ -->
+
+<!-- ── KPIs ── -->
+<div class="grid grid-cols-2 sm:grid-cols-4 gap-3 mb-4">
+    <div class="kpi-card"><div class="kpi-label">Brute-force IPs</div><div class="kpi-value text-red-400" id="kpi-bf">—</div></div>
+    <div class="kpi-card"><div class="kpi-label">JA4 rotation IPs</div><div class="kpi-value text-orange-400" id="kpi-rot">—</div></div>
+    <div class="kpi-card"><div class="kpi-label">IPs récurrentes</div><div class="kpi-value text-yellow-400" id="kpi-rec">—</div></div>
+    <div class="kpi-card"><div class="kpi-label">Alertes 24h</div><div class="kpi-value text-purple-400" id="kpi-alerts">—</div></div>
+</div>
+
+<!-- ── Brute-force / Credential stuffing ── -->
+<div class="section-card border-red-600/40 mb-4">
+    <div class="section-header">
+        <span class="section-title">
+            <svg class="w-4 h-4 text-red-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 15v2m-6 4h12a2 2 0 002-2v-6a2 2 0 00-2-2H6a2 2 0 00-2 2v6a2 2 0 002 2zm10-10V7a4 4 0 00-8 0v4h8z"/></svg>
+            Brute-force / Credential Stuffing
+        </span>
+        <div class="relative">
+            <button class="doc-btn">ⓘ</button>
+            <div class="doc-panel right-0 w-72">
+                <strong>Détection brute-force</strong><br>
+                IPs envoyant ≥10 requêtes POST/heure sur des endpoints d'authentification.
+                Signale les tentatives de credential stuffing, brute-force de mots de passe,
+                ou abus d'API. Données issues de <code>view_form_bruteforce_detected</code>.
+            </div>
+        </div>
+    </div>
+    <div class="section-body p-0">
+        <div class="overflow-x-auto">
+            <table class="data-table">
+                <thead><tr><th>IP</th><th>Host</th><th class="text-right">POST/h</th><th class="text-right">Paths distincts</th><th>Première vue</th><th>Dernière vue</th></tr></thead>
+                <tbody id="bf-body"><tr><td colspan="6" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
+            </table>
+        </div>
+    </div>
+</div>
+
+<!-- ── JA4 Rotation (évasion) ── -->
+<div class="section-card border-orange-600/40 mb-4">
+    <div class="section-header">
+        <span class="section-title">
+            <svg class="w-4 h-4 text-orange-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"/></svg>
+            Rotation de fingerprint JA4
+        </span>
+        <div class="relative">
+            <button class="doc-btn">ⓘ</button>
+            <div class="doc-panel right-0 w-72">
+                <strong>Détection d'évasion JA4</strong><br>
+                IPs utilisant ≥2 fingerprints JA4 distincts en 24h. Technique d'évasion
+                classique : rotation de la configuration TLS pour contourner les blocages
+                par fingerprint. Données issues de <code>view_host_ip_ja4_rotation</code>.
+            </div>
+        </div>
+    </div>
+    <div class="section-body p-0">
+        <div class="overflow-x-auto">
+            <table class="data-table">
+                <thead><tr><th>IP</th><th>Host</th><th class="text-right">JA4 distincts</th><th>Fingerprints</th><th class="text-right">Hits</th><th>Fenêtre</th></tr></thead>
+                <tbody id="rot-body"><tr><td colspan="6" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
+            </table>
+        </div>
+    </div>
+</div>
+
+<!-- ── Récurrence (menaces persistantes) ── -->
+<div class="section-card border-yellow-600/40 mb-4">
+    <div class="section-header">
+        <span class="section-title">
+            <svg class="w-4 h-4 text-yellow-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 8v4l3 3m6-3a9 9 0 11-18 0 9 9 0 0118 0z"/></svg>
+            Menaces persistantes (récurrence)
+        </span>
+        <div class="relative">
+            <button class="doc-btn">ⓘ</button>
+            <div class="doc-panel right-0 w-72">
+                <strong>IPs récurrentes</strong><br>
+                IPs détectées comme anomalies sur plusieurs fenêtres temporelles distinctes.
+                La récurrence augmente la confiance dans la classification malveillante.
+                Score agravé par <code>log1p(recurrence) × 0.005</code>.
+                Données issues de <code>view_ip_recurrence</code>.
+            </div>
+        </div>
+    </div>
+    <div class="section-body p-0">
+        <div class="overflow-x-auto">
+            <table class="data-table">
+                <thead><tr><th>IP</th><th class="text-right">Récurrence</th><th class="text-right">Pire score</th><th>Pire menace</th><th>Première vue</th><th>Dernière vue</th><th>JA4 top</th><th>Host top</th></tr></thead>
+                <tbody id="rec-body"><tr><td colspan="8" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
+            </table>
+        </div>
+    </div>
+</div>
+
+<!-- ── Alertes temps réel ── -->
+<div class="section-card border-purple-600/40 mb-4">
+    <div class="section-header">
+        <span class="section-title">
+            <svg class="w-4 h-4 text-purple-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 17h5l-1.405-1.405A2.032 2.032 0 0118 14.158V11a6.002 6.002 0 00-4-5.659V5a2 2 0 10-4 0v.341C7.67 6.165 6 8.388 6 11v3.159c0 .538-.214 1.055-.595 1.436L4 17h5m6 0v1a3 3 0 11-6 0v-1m6 0H9"/></svg>
+            Alertes récentes (24h)
+        </span>
+        <div class="relative">
+            <button class="doc-btn">ⓘ</button>
+            <div class="doc-panel right-0 w-72">
+                <strong>Flux d'alertes</strong><br>
+                Dernières détections CRITICAL, HIGH et KNOWN_BOT sur les 24 dernières heures.
+                Chaque alerte inclut le score, la raison SHAP, et le lien vers l'investigation IP.
+            </div>
+        </div>
+    </div>
+    <div class="section-body p-0">
+        <div class="overflow-x-auto">
+            <table class="data-table">
+                <thead><tr><th>Date</th><th>IP</th><th class="text-right">Score</th><th>Menace</th><th>JA4</th><th>Host</th><th>ASN</th><th class="text-right">Hits</th><th>Raison</th></tr></thead>
+                <tbody id="alert-body"><tr><td colspan="9" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
+            </table>
+        </div>
+    </div>
+</div>
+
+<script>
+/* ═══════════════════════════════════════════════════════════════════════
+ *  Tactiques — chargement des 4 sources de données
+ * ═══════════════════════════════════════════════════════════════════════ */
+document.addEventListener('DOMContentLoaded', () => {
+    Promise.all([
+        fetch('/api/brute-force').then(r => r.json()),
+        fetch('/api/ja4-rotation').then(r => r.json()),
+        fetch('/api/recurrence').then(r => r.json()),
+        fetch('/api/alerts?limit=50').then(r => r.json()),
+    ]).then(([bf, rot, rec, alerts]) => {
+        renderBruteForce(bf.data || []);
+        renderRotation(rot.data || []);
+        renderRecurrence(rec.data || []);
+        renderAlerts(alerts.alerts || []);
+    }).catch(err => console.error('Tactics load error:', err));
+});
+
+function renderBruteForce(data) {
+    document.getElementById('kpi-bf').textContent = data.length;
+    const body = document.getElementById('bf-body');
+    if (!data.length) { body.innerHTML = '<tr><td colspan="6" class="text-center text-gray-500 py-8">Aucune tentative de brute-force détectée</td></tr>'; return; }
+    body.innerHTML = data.map(r => `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
+        <td>${fmtIP(r.src_ip)}</td><td class="text-gray-300">${escapeHtml(r.host||'')}</td>
+        <td class="text-right font-mono text-red-400 font-bold">${r.post_count}</td>
+        <td class="text-right">${r.distinct_paths}</td>
+        <td class="text-gray-400 text-xs">${fmtDate(r.first_seen)}</td>
+        <td class="text-gray-400 text-xs">${fmtDate(r.last_seen)}</td></tr>`).join('');
+}
+
+function renderRotation(data) {
+    document.getElementById('kpi-rot').textContent = data.length;
+    const body = document.getElementById('rot-body');
+    if (!data.length) { body.innerHTML = '<tr><td colspan="6" class="text-center text-gray-500 py-8">Aucune rotation JA4 détectée</td></tr>'; return; }
+    body.innerHTML = data.map(r => {
+        const ja4s = (r.ja4_list || []).map(j => fmtJA4(j)).join(', ');
+        return `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
+            <td>${fmtIP(r.src_ip)}</td><td class="text-gray-300">${escapeHtml(r.host||'')}</td>
+            <td class="text-right font-mono text-orange-400 font-bold">${r.distinct_ja4}</td>
+            <td class="max-w-xs truncate">${ja4s}</td>
+            <td class="text-right">${r.total_hits}</td>
+            <td class="text-gray-400 text-xs">${fmtDate(r.window_start)}</td></tr>`;
+    }).join('');
+}
+
+function renderRecurrence(data) {
+    document.getElementById('kpi-rec').textContent = data.length;
+    const body = document.getElementById('rec-body');
+    if (!data.length) { body.innerHTML = '<tr><td colspan="8" class="text-center text-gray-500 py-8">Aucune IP récurrente</td></tr>'; return; }
+    body.innerHTML = data.map(r => `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
+        <td>${fmtIP(r.src_ip)}</td>
+        <td class="text-right font-mono text-yellow-400 font-bold">${r.recurrence}×</td>
+        <td class="text-right font-mono">${fmtScore(r.worst_score)}</td>
+        <td>${threatBadge(r.worst_threat||r.worst_threat_level||'')}</td>
+        <td class="text-gray-400 text-xs">${fmtDate(r.first_seen)}</td>
+        <td class="text-gray-400 text-xs">${fmtDate(r.last_seen)}</td>
+        <td>${r.top_ja4 ? fmtJA4(r.top_ja4) : ''}</td>
+        <td class="text-gray-300">${escapeHtml(r.top_host||'')}</td></tr>`).join('');
+}
+
+function renderAlerts(data) {
+    document.getElementById('kpi-alerts').textContent = data.length;
+    const body = document.getElementById('alert-body');
+    if (!data.length) { body.innerHTML = '<tr><td colspan="9" class="text-center text-gray-500 py-8">Aucune alerte récente</td></tr>'; return; }
+    body.innerHTML = data.map(r => `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
+        <td class="text-gray-400 text-xs whitespace-nowrap">${fmtDate(r.detected_at)}</td>
+        <td>${fmtIP(r.src_ip)}</td>
+        <td class="text-right font-mono">${fmtScore(r.anomaly_score)}</td>
+        <td>${threatBadge(r.threat_level)}</td>
+        <td>${fmtJA4(r.ja4)}</td>
+        <td class="text-gray-300">${escapeHtml(r.host||'')}</td>
+        <td class="text-gray-400 text-xs">${escapeHtml(r.asn_org||'')}</td>
+        <td class="text-right">${r.hits||''}</td>
+        <td class="text-gray-400 text-xs max-w-[200px] truncate" title="${escapeHtml(r.reason||'')}">${escapeHtml((r.reason||'').substring(0,60))}</td></tr>`).join('');
+}
+</script>
+{% endblock %}

shared/clickhouse/05_aggregation_tables.sql

@@ -112,6 +112,11 @@ CREATE TABLE IF NOT EXISTS ja4_processing.agg_host_ip_ja4_1h
     -- HTTP features
     count_no_accept_enc SimpleAggregateFunction(sum, UInt64),
     count_http_scheme SimpleAggregateFunction(sum, UInt64),
+    -- P1 : nouvelles features de détection
+    count_xff SimpleAggregateFunction(sum, UInt64),
+    count_unusual_ct SimpleAggregateFunction(sum, UInt64),
+    count_non_std_port SimpleAggregateFunction(sum, UInt64),
+    count_login_post SimpleAggregateFunction(sum, UInt64),
 
     -- Projection pour les requêtes d'investigation par IP :
     -- ORDER BY actuel (window_start, src_ip, ...) est optimal pour heatmap
@@ -157,7 +162,7 @@ SELECT
     sum(IF(match(src.path, '(?i)\.(png|jpg|jpeg|gif|css|js|ico|woff2|svg|eot)$'), 1, 0)) AS count_assets,
     sum(IF(position(src.client_headers, 'Referer') = 0, 1, 0)) AS count_no_referer,
     uniqState(src.header_user_agent) AS uniq_ua,
-    0 AS max_requests_per_sec,
+    0 AS max_requests_per_sec, -- TODO(P0): calculer via sous-requête par seconde (impossible dans un seul GROUP BY)
     varPopState(toFloat64(length(replaceAll(src.path, '/', '//')) - length(src.path))) AS url_depth_variance,
     sum(IF(src.ip_meta_total_length < 60 OR src.ip_meta_total_length > 1500, 1, 0)) AS count_anomalous_payload,
     uniqState(src.ja3) AS uniq_ja3,
@@ -173,7 +178,13 @@ SELECT
     sum(IF(src.tcp_meta_window_scale = 0 AND src.correlated = 1, 1, 0)) AS count_no_wscale,
     sum(toUInt64(src.correlated)) AS count_correlated,
     sum(IF(length(src.header_accept_encoding) = 0, 1, 0)) AS count_no_accept_enc,
-    sum(IF(src.scheme = 'http', 1, 0)) AS count_http_scheme
+    sum(IF(src.scheme = 'http', 1, 0)) AS count_http_scheme,
+    -- P1 : nouvelles features
+    sum(IF(length(src.header_x_forwarded_for) > 0, 1, 0)) AS count_xff,
+    sum(IF(src.method = 'POST' AND length(src.header_content_type) > 0
+           AND NOT match(src.header_content_type, '(?i)(form-urlencoded|multipart|json|xml|text/plain|grpc|protobuf)'), 1, 0)) AS count_unusual_ct,
+    sum(IF(src.dst_port NOT IN (80, 443, 8080, 8443), 1, 0)) AS count_non_std_port,
+    sum(IF(src.method = 'POST' AND match(src.path, '(?i)(login|signin|auth|token|session|wp-login|connect|oauth)'), 1, 0)) AS count_login_post
 FROM ja4_logs.http_logs AS src
 GROUP BY window_start, src_ip, ja4, host, src_asn;
 
@@ -192,6 +203,7 @@ CREATE TABLE IF NOT EXISTS ja4_processing.agg_header_fingerprint_1h
     has_referer SimpleAggregateFunction(max, UInt8),
     modern_browser_score SimpleAggregateFunction(max, UInt8),
     ua_ch_mismatch SimpleAggregateFunction(max, UInt8),
+    sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8),
     sec_fetch_mode SimpleAggregateFunction(any, String),
     sec_fetch_dest SimpleAggregateFunction(any, String)
 )
@@ -212,6 +224,10 @@ SELECT
     max(toUInt8(if(position(src.client_headers, 'Referer') > 0, 1, 0))) AS has_referer,
     max(toUInt8(if(length(src.header_sec_ch_ua) > 0, 100, if(length(src.header_user_agent) > 0, 50, 0)))) AS modern_browser_score,
     max(toUInt8(if((position(src.header_user_agent, 'Windows') > 0 AND position(src.header_sec_ch_ua_platform, 'Windows') == 0) OR (position(src.header_user_agent, 'iPhone') > 0 AND position(src.header_sec_ch_ua_platform, 'iOS') == 0), 1, 0))) AS ua_ch_mismatch,
+    max(toUInt8(if(
+        (src.header_sec_ch_ua_mobile = '?1' AND position(src.header_user_agent, 'Mobile') == 0 AND position(src.header_user_agent, 'Android') == 0 AND position(src.header_user_agent, 'iPhone') == 0)
+        OR (src.header_sec_ch_ua_mobile = '?0' AND (position(src.header_user_agent, 'iPhone') > 0 OR position(src.header_user_agent, 'Android') > 0)),
+        1, 0))) AS sec_ch_mobile_mismatch,
     any(src.header_sec_fetch_mode) AS sec_fetch_mode,
     any(src.header_sec_fetch_dest) AS sec_fetch_dest
 FROM ja4_logs.http_logs AS src

shared/clickhouse/07_ai_features_view.sql

@@ -127,7 +127,13 @@ WITH base_data AS (
         sqrt(a.ttl_variance_val) AS ttl_std,
         IF(a.count_correlated_val > 0, a.count_no_wscale_val / a.count_correlated_val, 0) AS no_window_scale_ratio,
         a.count_no_accept_enc_val / (a.hits + 1) AS missing_accept_enc_ratio,
-        a.count_http_scheme_val / (a.hits + 1)   AS http_scheme_ratio
+        a.count_http_scheme_val / (a.hits + 1)   AS http_scheme_ratio,
+        -- P1 : nouvelles features de détection
+        IF(a.count_xff_val > 0, 1, 0) AS has_xff,
+        a.count_unusual_ct_val / greatest(a.count_post, 1) AS unusual_content_type_ratio,
+        a.count_non_std_port_val / (a.hits + 1) AS non_standard_port_ratio,
+        a.count_login_post_val / greatest(a.count_post, 1) AS login_post_concentration,
+        h.sec_ch_mobile_mismatch AS sec_ch_mobile_mismatch
     FROM (
         SELECT
             window_start, src_ip, ja4, host, src_asn,
@@ -162,7 +168,12 @@ WITH base_data AS (
             sum(count_no_wscale)   AS count_no_wscale_val,
             sum(count_correlated)  AS count_correlated_val,
             sum(count_no_accept_enc) AS count_no_accept_enc_val,
-            sum(count_http_scheme)   AS count_http_scheme_val
+            sum(count_http_scheme)   AS count_http_scheme_val,
+            -- P1 : nouvelles features de détection
+            sum(count_xff)           AS count_xff_val,
+            sum(count_unusual_ct)    AS count_unusual_ct_val,
+            sum(count_non_std_port)  AS count_non_std_port_val,
+            sum(count_login_post)    AS count_login_post_val
         FROM ja4_processing.agg_host_ip_ja4_1h
         WHERE window_start >= now() - INTERVAL 24 HOUR
         GROUP BY window_start, src_ip, ja4, host, src_asn
@@ -173,6 +184,7 @@ WITH base_data AS (
             max(header_count) AS header_count, max(has_accept_language) AS has_accept_language,
             max(has_cookie) AS has_cookie, max(has_referer) AS has_referer,
             max(modern_browser_score) AS modern_browser_score, max(ua_ch_mismatch) AS ua_ch_mismatch,
+            max(sec_ch_mobile_mismatch) AS sec_ch_mobile_mismatch,
             any(sec_fetch_mode) AS sec_fetch_mode, any(sec_fetch_dest) AS sec_fetch_dest
         FROM ja4_processing.agg_header_fingerprint_1h
         WHERE window_start >= now() - INTERVAL 24 HOUR