feat: nouvelles techniques de détection et page tactiques SOC
SQL: - Ajout 5 colonnes d'agrégation (count_xff, count_unusual_ct, count_non_std_port, count_login_post, sec_ch_mobile_mismatch) - Exposition de 5 features calculées dans view_ai_features_1h - Migration ALTER TABLE pour déploiements existants Bot-detector: - 7 nouvelles features ML (has_xff, unusual_content_type_ratio, non_standard_port_ratio, login_post_concentration, sec_ch_mobile_mismatch, true_window_size, window_mss_ratio) - Propagation campaign_id vers ml_all_scores (était toujours -1) - Escalade campagne : HIGH→CRITICAL si cluster ≥5 membres Dashboard: - Page Tactiques SOC : brute-force, rotation JA4, récurrence, alertes temps réel — 4 KPIs + 4 panneaux + infobulles doc - Ajout fmtDate() helper global - Navigation sidebar mise à jour Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
@ -109,7 +109,8 @@ STRUCTURAL_EXCLUDED_FEATURES: dict[str, list] = {
|
||||
'request_size_variance', 'mss_mobile_mismatch',
|
||||
'ja3_diversity_ratio', 'syn_timing_cv', 'tls12_ratio', 'ip_df_variance',
|
||||
'avg_ttl', 'ttl_std', 'no_window_scale_ratio',
|
||||
'ja4_drift_ratio'],
|
||||
'ja4_drift_ratio',
|
||||
'true_window_size', 'window_mss_ratio'],
|
||||
}
|
||||
|
||||
# ─── Imports optionnels (bibliothèques lourdes) ────────────────────────────
|
||||
|
||||
@ -306,6 +306,23 @@ def run_semi_supervised_logic(df, features, name, cycle_id, recurrence_map):
|
||||
if ENABLE_CLUSTERING:
|
||||
anomalies = cluster_anomalies(anomalies, scoring_features, ae_model=ae_model)
|
||||
|
||||
# P2 — Escalade par taille de campagne : les IPs dans un cluster
|
||||
# coordonné de grande taille sont plus menaçantes que des IPs isolées.
|
||||
# Escalader HIGH → CRITICAL si cluster_size ≥ 5.
|
||||
if 'campaign_id' in anomalies.columns:
|
||||
cid_counts = anomalies['campaign_id'].value_counts()
|
||||
for cid, size in cid_counts.items():
|
||||
if cid < 0:
|
||||
continue
|
||||
if size >= 5:
|
||||
mask = (anomalies['campaign_id'] == cid) & (anomalies['threat_level'] == 'HIGH')
|
||||
n_escalated = mask.sum()
|
||||
if n_escalated > 0:
|
||||
anomalies.loc[mask, 'threat_level'] = 'CRITICAL'
|
||||
anomalies.loc[mask, 'reason'] = anomalies.loc[mask, 'reason'] + \
|
||||
f' [Escalade campagne #{cid}, {size} IPs coordonnées]'
|
||||
log_info(f"[{name}] Escalade campagne #{cid}: {n_escalated} IP(s) HIGH→CRITICAL ({size} membres)")
|
||||
|
||||
anomalies['ja4'] = anomalies['ja4'].replace({'': 'HTTP_CLEAR_TEXT'})
|
||||
for _, row in anomalies.iterrows():
|
||||
log_decision('ANOMALY', cycle_id, name, {
|
||||
@ -330,6 +347,14 @@ def run_semi_supervised_logic(df, features, name, cycle_id, recurrence_map):
|
||||
anubis_deny if not anubis_deny.empty else None,
|
||||
] if df is not None], ignore_index=True)
|
||||
|
||||
# Propager campaign_id des anomalies clusterisées vers all_scored
|
||||
# (all_scored a été capturé avant clustering, ses campaign_id sont tous -1)
|
||||
if not anomalies.empty and 'campaign_id' in anomalies.columns:
|
||||
cid_map = anomalies.set_index(anomalies.index)['campaign_id']
|
||||
matched = all_scored.index.isin(cid_map.index)
|
||||
if matched.any():
|
||||
all_scored.loc[matched, 'campaign_id'] = cid_map
|
||||
|
||||
# Inclure anubis_allow dans all_scored pour traçabilité dans ml_all_scores.
|
||||
# Ces IPs sont exclues de l'analyse IF mais doivent apparaître dans la table
|
||||
# de scores avec threat_level='KNOWN_BOT' et anomaly_score=0.0.
|
||||
|
||||
@ -41,6 +41,12 @@ FEATURES = [
|
||||
'cadence_cv', 'burst_ratio', 'pause_ratio',
|
||||
'lag1_autocorrelation', 'benford_deviation',
|
||||
'host_diversity', 'host_sweep_speed', 'host_coverage_uniformity',
|
||||
# P0+P1 : features sous-exploitées (SQL existant ou ajouté)
|
||||
'is_fake_navigation',
|
||||
'true_window_size', 'window_mss_ratio',
|
||||
# P1 : nouvelles features de détection
|
||||
'has_xff', 'unusual_content_type_ratio', 'non_standard_port_ratio',
|
||||
'login_post_concentration', 'sec_ch_mobile_mismatch',
|
||||
]
|
||||
|
||||
# Features supplémentaires pour le modèle Complet (données TCP/TLS requises)
|
||||
@ -100,6 +106,7 @@ def preprocess_df(df: pd.DataFrame) -> pd.DataFrame:
|
||||
'has_accept_language', 'has_cookie', 'has_referer', 'ua_ch_mismatch',
|
||||
'is_ua_rotating', 'is_alpn_missing', 'sni_host_mismatch', 'alpn_http_mismatch',
|
||||
'mss_mobile_mismatch', 'anubis_is_flagged', 'is_rare_ja4',
|
||||
'is_fake_navigation', 'has_xff', 'sec_ch_mobile_mismatch',
|
||||
}
|
||||
for col in df.columns:
|
||||
if col in binary_features:
|
||||
|
||||
17
services/correlator/sql/migrations/02_detection_features.sql
Normal file
17
services/correlator/sql/migrations/02_detection_features.sql
Normal file
@ -0,0 +1,17 @@
|
||||
-- =============================================================================
|
||||
-- 02_detection_features.sql — Ajout des features de détection P0+P1
|
||||
-- Colonnes supplémentaires dans agg_host_ip_ja4_1h et agg_header_fingerprint_1h
|
||||
-- NOTE : les MVs doivent être recréées (DROP + CREATE) car ALTER VIEW n'existe pas.
|
||||
-- Exécuter deploy_schema.sh pour recréer les MVs, ou relancer le schema complet.
|
||||
-- =============================================================================
|
||||
|
||||
-- agg_host_ip_ja4_1h : nouvelles colonnes de comptage
|
||||
ALTER TABLE ja4_processing.agg_host_ip_ja4_1h
|
||||
ADD COLUMN IF NOT EXISTS count_xff SimpleAggregateFunction(sum, UInt64) AFTER count_http_scheme,
|
||||
ADD COLUMN IF NOT EXISTS count_unusual_ct SimpleAggregateFunction(sum, UInt64) AFTER count_xff,
|
||||
ADD COLUMN IF NOT EXISTS count_non_std_port SimpleAggregateFunction(sum, UInt64) AFTER count_unusual_ct,
|
||||
ADD COLUMN IF NOT EXISTS count_login_post SimpleAggregateFunction(sum, UInt64) AFTER count_non_std_port;
|
||||
|
||||
-- agg_header_fingerprint_1h : mismatch mobile Sec-CH-UA
|
||||
ALTER TABLE ja4_processing.agg_header_fingerprint_1h
|
||||
ADD COLUMN IF NOT EXISTS sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8) AFTER ua_ch_mismatch;
|
||||
@ -71,3 +71,8 @@ async def ja4_detail_page(request: Request, fingerprint: str):
|
||||
@router.get("/cluster/{cid}")
|
||||
async def cluster_detail_page(request: Request, cid: int):
|
||||
return templates.TemplateResponse("cluster_detail.html", _ctx(request, "cluster_detail", cid=cid))
|
||||
|
||||
|
||||
@router.get("/tactics")
|
||||
async def tactics_page(request: Request):
|
||||
return templates.TemplateResponse("tactics.html", _ctx(request, "tactics"))
|
||||
|
||||
@ -151,6 +151,10 @@
|
||||
<svg class="w-4 h-4 shrink-0" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M17 20h5v-2a3 3 0 00-5.356-1.857M17 20H7m10 0v-2c0-.656-.126-1.283-.356-1.857M7 20H2v-2a3 3 0 015.356-1.857M7 20v-2c0-.656.126-1.283.356-1.857m0 0a5.002 5.002 0 019.288 0M15 7a3 3 0 11-6 0 3 3 0 016 0z"/></svg>
|
||||
<span class="nav-text">Campagnes</span>
|
||||
</a>
|
||||
<a href="/tactics" class="nav-item {% if active_page == 'tactics' %}active{% endif %}" title="Tactiques d'attaque détectées">
|
||||
<svg class="w-4 h-4 shrink-0" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M13 10V3L4 14h7v7l9-11h-7z"/></svg>
|
||||
<span class="nav-text">Tactiques</span>
|
||||
</a>
|
||||
|
||||
<div class="nav-group-title">Investigation</div>
|
||||
<a href="/traffic" class="nav-item {% if active_page == 'traffic' %}active{% endif %}" title="Logs HTTP bruts">
|
||||
@ -324,6 +328,10 @@
|
||||
if (diff < 86400) return Math.round(diff/3600) + 'h';
|
||||
return Math.round(diff/86400) + 'j';
|
||||
}
|
||||
function fmtDate(d) {
|
||||
if (!d) return '—';
|
||||
return String(d).substring(0, 16).replace('T', ' ');
|
||||
}
|
||||
|
||||
// ── ECharts helpers ──
|
||||
const EC_COLORS = ['#6366f1','#22c55e','#f97316','#ef4444','#3b82f6','#eab308','#ec4899','#14b8a6','#8b5cf6','#f43f5e'];
|
||||
|
||||
199
services/dashboard/backend/templates/tactics.html
Normal file
199
services/dashboard/backend/templates/tactics.html
Normal file
@ -0,0 +1,199 @@
|
||||
{% extends "base.html" %}
|
||||
{% block page_title %}Tactiques d'attaque{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<!-- ═══ Tactiques d'attaque — 4 panneaux de détection spécialisés ═══ -->
|
||||
|
||||
<!-- ── KPIs ── -->
|
||||
<div class="grid grid-cols-2 sm:grid-cols-4 gap-3 mb-4">
|
||||
<div class="kpi-card"><div class="kpi-label">Brute-force IPs</div><div class="kpi-value text-red-400" id="kpi-bf">—</div></div>
|
||||
<div class="kpi-card"><div class="kpi-label">JA4 rotation IPs</div><div class="kpi-value text-orange-400" id="kpi-rot">—</div></div>
|
||||
<div class="kpi-card"><div class="kpi-label">IPs récurrentes</div><div class="kpi-value text-yellow-400" id="kpi-rec">—</div></div>
|
||||
<div class="kpi-card"><div class="kpi-label">Alertes 24h</div><div class="kpi-value text-purple-400" id="kpi-alerts">—</div></div>
|
||||
</div>
|
||||
|
||||
<!-- ── Brute-force / Credential stuffing ── -->
|
||||
<div class="section-card border-red-600/40 mb-4">
|
||||
<div class="section-header">
|
||||
<span class="section-title">
|
||||
<svg class="w-4 h-4 text-red-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 15v2m-6 4h12a2 2 0 002-2v-6a2 2 0 00-2-2H6a2 2 0 00-2 2v6a2 2 0 002 2zm10-10V7a4 4 0 00-8 0v4h8z"/></svg>
|
||||
Brute-force / Credential Stuffing
|
||||
</span>
|
||||
<div class="relative">
|
||||
<button class="doc-btn">ⓘ</button>
|
||||
<div class="doc-panel right-0 w-72">
|
||||
<strong>Détection brute-force</strong><br>
|
||||
IPs envoyant ≥10 requêtes POST/heure sur des endpoints d'authentification.
|
||||
Signale les tentatives de credential stuffing, brute-force de mots de passe,
|
||||
ou abus d'API. Données issues de <code>view_form_bruteforce_detected</code>.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="section-body p-0">
|
||||
<div class="overflow-x-auto">
|
||||
<table class="data-table">
|
||||
<thead><tr><th>IP</th><th>Host</th><th class="text-right">POST/h</th><th class="text-right">Paths distincts</th><th>Première vue</th><th>Dernière vue</th></tr></thead>
|
||||
<tbody id="bf-body"><tr><td colspan="6" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── JA4 Rotation (évasion) ── -->
|
||||
<div class="section-card border-orange-600/40 mb-4">
|
||||
<div class="section-header">
|
||||
<span class="section-title">
|
||||
<svg class="w-4 h-4 text-orange-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M4 4v5h.582m15.356 2A8.001 8.001 0 004.582 9m0 0H9m11 11v-5h-.581m0 0a8.003 8.003 0 01-15.357-2m15.357 2H15"/></svg>
|
||||
Rotation de fingerprint JA4
|
||||
</span>
|
||||
<div class="relative">
|
||||
<button class="doc-btn">ⓘ</button>
|
||||
<div class="doc-panel right-0 w-72">
|
||||
<strong>Détection d'évasion JA4</strong><br>
|
||||
IPs utilisant ≥2 fingerprints JA4 distincts en 24h. Technique d'évasion
|
||||
classique : rotation de la configuration TLS pour contourner les blocages
|
||||
par fingerprint. Données issues de <code>view_host_ip_ja4_rotation</code>.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="section-body p-0">
|
||||
<div class="overflow-x-auto">
|
||||
<table class="data-table">
|
||||
<thead><tr><th>IP</th><th>Host</th><th class="text-right">JA4 distincts</th><th>Fingerprints</th><th class="text-right">Hits</th><th>Fenêtre</th></tr></thead>
|
||||
<tbody id="rot-body"><tr><td colspan="6" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Récurrence (menaces persistantes) ── -->
|
||||
<div class="section-card border-yellow-600/40 mb-4">
|
||||
<div class="section-header">
|
||||
<span class="section-title">
|
||||
<svg class="w-4 h-4 text-yellow-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M12 8v4l3 3m6-3a9 9 0 11-18 0 9 9 0 0118 0z"/></svg>
|
||||
Menaces persistantes (récurrence)
|
||||
</span>
|
||||
<div class="relative">
|
||||
<button class="doc-btn">ⓘ</button>
|
||||
<div class="doc-panel right-0 w-72">
|
||||
<strong>IPs récurrentes</strong><br>
|
||||
IPs détectées comme anomalies sur plusieurs fenêtres temporelles distinctes.
|
||||
La récurrence augmente la confiance dans la classification malveillante.
|
||||
Score agravé par <code>log1p(recurrence) × 0.005</code>.
|
||||
Données issues de <code>view_ip_recurrence</code>.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="section-body p-0">
|
||||
<div class="overflow-x-auto">
|
||||
<table class="data-table">
|
||||
<thead><tr><th>IP</th><th class="text-right">Récurrence</th><th class="text-right">Pire score</th><th>Pire menace</th><th>Première vue</th><th>Dernière vue</th><th>JA4 top</th><th>Host top</th></tr></thead>
|
||||
<tbody id="rec-body"><tr><td colspan="8" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Alertes temps réel ── -->
|
||||
<div class="section-card border-purple-600/40 mb-4">
|
||||
<div class="section-header">
|
||||
<span class="section-title">
|
||||
<svg class="w-4 h-4 text-purple-400" fill="none" stroke="currentColor" viewBox="0 0 24 24"><path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M15 17h5l-1.405-1.405A2.032 2.032 0 0118 14.158V11a6.002 6.002 0 00-4-5.659V5a2 2 0 10-4 0v.341C7.67 6.165 6 8.388 6 11v3.159c0 .538-.214 1.055-.595 1.436L4 17h5m6 0v1a3 3 0 11-6 0v-1m6 0H9"/></svg>
|
||||
Alertes récentes (24h)
|
||||
</span>
|
||||
<div class="relative">
|
||||
<button class="doc-btn">ⓘ</button>
|
||||
<div class="doc-panel right-0 w-72">
|
||||
<strong>Flux d'alertes</strong><br>
|
||||
Dernières détections CRITICAL, HIGH et KNOWN_BOT sur les 24 dernières heures.
|
||||
Chaque alerte inclut le score, la raison SHAP, et le lien vers l'investigation IP.
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="section-body p-0">
|
||||
<div class="overflow-x-auto">
|
||||
<table class="data-table">
|
||||
<thead><tr><th>Date</th><th>IP</th><th class="text-right">Score</th><th>Menace</th><th>JA4</th><th>Host</th><th>ASN</th><th class="text-right">Hits</th><th>Raison</th></tr></thead>
|
||||
<tbody id="alert-body"><tr><td colspan="9" class="text-center text-gray-500 py-8">Chargement…</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
/* ═══════════════════════════════════════════════════════════════════════
|
||||
* Tactiques — chargement des 4 sources de données
|
||||
* ═══════════════════════════════════════════════════════════════════════ */
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
Promise.all([
|
||||
fetch('/api/brute-force').then(r => r.json()),
|
||||
fetch('/api/ja4-rotation').then(r => r.json()),
|
||||
fetch('/api/recurrence').then(r => r.json()),
|
||||
fetch('/api/alerts?limit=50').then(r => r.json()),
|
||||
]).then(([bf, rot, rec, alerts]) => {
|
||||
renderBruteForce(bf.data || []);
|
||||
renderRotation(rot.data || []);
|
||||
renderRecurrence(rec.data || []);
|
||||
renderAlerts(alerts.alerts || []);
|
||||
}).catch(err => console.error('Tactics load error:', err));
|
||||
});
|
||||
|
||||
function renderBruteForce(data) {
|
||||
document.getElementById('kpi-bf').textContent = data.length;
|
||||
const body = document.getElementById('bf-body');
|
||||
if (!data.length) { body.innerHTML = '<tr><td colspan="6" class="text-center text-gray-500 py-8">Aucune tentative de brute-force détectée</td></tr>'; return; }
|
||||
body.innerHTML = data.map(r => `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
|
||||
<td>${fmtIP(r.src_ip)}</td><td class="text-gray-300">${escapeHtml(r.host||'')}</td>
|
||||
<td class="text-right font-mono text-red-400 font-bold">${r.post_count}</td>
|
||||
<td class="text-right">${r.distinct_paths}</td>
|
||||
<td class="text-gray-400 text-xs">${fmtDate(r.first_seen)}</td>
|
||||
<td class="text-gray-400 text-xs">${fmtDate(r.last_seen)}</td></tr>`).join('');
|
||||
}
|
||||
|
||||
function renderRotation(data) {
|
||||
document.getElementById('kpi-rot').textContent = data.length;
|
||||
const body = document.getElementById('rot-body');
|
||||
if (!data.length) { body.innerHTML = '<tr><td colspan="6" class="text-center text-gray-500 py-8">Aucune rotation JA4 détectée</td></tr>'; return; }
|
||||
body.innerHTML = data.map(r => {
|
||||
const ja4s = (r.ja4_list || []).map(j => fmtJA4(j)).join(', ');
|
||||
return `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
|
||||
<td>${fmtIP(r.src_ip)}</td><td class="text-gray-300">${escapeHtml(r.host||'')}</td>
|
||||
<td class="text-right font-mono text-orange-400 font-bold">${r.distinct_ja4}</td>
|
||||
<td class="max-w-xs truncate">${ja4s}</td>
|
||||
<td class="text-right">${r.total_hits}</td>
|
||||
<td class="text-gray-400 text-xs">${fmtDate(r.window_start)}</td></tr>`;
|
||||
}).join('');
|
||||
}
|
||||
|
||||
function renderRecurrence(data) {
|
||||
document.getElementById('kpi-rec').textContent = data.length;
|
||||
const body = document.getElementById('rec-body');
|
||||
if (!data.length) { body.innerHTML = '<tr><td colspan="8" class="text-center text-gray-500 py-8">Aucune IP récurrente</td></tr>'; return; }
|
||||
body.innerHTML = data.map(r => `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
|
||||
<td>${fmtIP(r.src_ip)}</td>
|
||||
<td class="text-right font-mono text-yellow-400 font-bold">${r.recurrence}×</td>
|
||||
<td class="text-right font-mono">${fmtScore(r.worst_score)}</td>
|
||||
<td>${threatBadge(r.worst_threat||r.worst_threat_level||'')}</td>
|
||||
<td class="text-gray-400 text-xs">${fmtDate(r.first_seen)}</td>
|
||||
<td class="text-gray-400 text-xs">${fmtDate(r.last_seen)}</td>
|
||||
<td>${r.top_ja4 ? fmtJA4(r.top_ja4) : ''}</td>
|
||||
<td class="text-gray-300">${escapeHtml(r.top_host||'')}</td></tr>`).join('');
|
||||
}
|
||||
|
||||
function renderAlerts(data) {
|
||||
document.getElementById('kpi-alerts').textContent = data.length;
|
||||
const body = document.getElementById('alert-body');
|
||||
if (!data.length) { body.innerHTML = '<tr><td colspan="9" class="text-center text-gray-500 py-8">Aucune alerte récente</td></tr>'; return; }
|
||||
body.innerHTML = data.map(r => `<tr class="cursor-pointer hover:bg-gray-800/60" onclick="location.href='/ip/${r.src_ip}'">
|
||||
<td class="text-gray-400 text-xs whitespace-nowrap">${fmtDate(r.detected_at)}</td>
|
||||
<td>${fmtIP(r.src_ip)}</td>
|
||||
<td class="text-right font-mono">${fmtScore(r.anomaly_score)}</td>
|
||||
<td>${threatBadge(r.threat_level)}</td>
|
||||
<td>${fmtJA4(r.ja4)}</td>
|
||||
<td class="text-gray-300">${escapeHtml(r.host||'')}</td>
|
||||
<td class="text-gray-400 text-xs">${escapeHtml(r.asn_org||'')}</td>
|
||||
<td class="text-right">${r.hits||''}</td>
|
||||
<td class="text-gray-400 text-xs max-w-[200px] truncate" title="${escapeHtml(r.reason||'')}">${escapeHtml((r.reason||'').substring(0,60))}</td></tr>`).join('');
|
||||
}
|
||||
</script>
|
||||
{% endblock %}
|
||||
@ -112,6 +112,11 @@ CREATE TABLE IF NOT EXISTS ja4_processing.agg_host_ip_ja4_1h
|
||||
-- HTTP features
|
||||
count_no_accept_enc SimpleAggregateFunction(sum, UInt64),
|
||||
count_http_scheme SimpleAggregateFunction(sum, UInt64),
|
||||
-- P1 : nouvelles features de détection
|
||||
count_xff SimpleAggregateFunction(sum, UInt64),
|
||||
count_unusual_ct SimpleAggregateFunction(sum, UInt64),
|
||||
count_non_std_port SimpleAggregateFunction(sum, UInt64),
|
||||
count_login_post SimpleAggregateFunction(sum, UInt64),
|
||||
|
||||
-- Projection pour les requêtes d'investigation par IP :
|
||||
-- ORDER BY actuel (window_start, src_ip, ...) est optimal pour heatmap
|
||||
@ -157,7 +162,7 @@ SELECT
|
||||
sum(IF(match(src.path, '(?i)\.(png|jpg|jpeg|gif|css|js|ico|woff2|svg|eot)$'), 1, 0)) AS count_assets,
|
||||
sum(IF(position(src.client_headers, 'Referer') = 0, 1, 0)) AS count_no_referer,
|
||||
uniqState(src.header_user_agent) AS uniq_ua,
|
||||
0 AS max_requests_per_sec,
|
||||
0 AS max_requests_per_sec, -- TODO(P0): calculer via sous-requête par seconde (impossible dans un seul GROUP BY)
|
||||
varPopState(toFloat64(length(replaceAll(src.path, '/', '//')) - length(src.path))) AS url_depth_variance,
|
||||
sum(IF(src.ip_meta_total_length < 60 OR src.ip_meta_total_length > 1500, 1, 0)) AS count_anomalous_payload,
|
||||
uniqState(src.ja3) AS uniq_ja3,
|
||||
@ -173,7 +178,13 @@ SELECT
|
||||
sum(IF(src.tcp_meta_window_scale = 0 AND src.correlated = 1, 1, 0)) AS count_no_wscale,
|
||||
sum(toUInt64(src.correlated)) AS count_correlated,
|
||||
sum(IF(length(src.header_accept_encoding) = 0, 1, 0)) AS count_no_accept_enc,
|
||||
sum(IF(src.scheme = 'http', 1, 0)) AS count_http_scheme
|
||||
sum(IF(src.scheme = 'http', 1, 0)) AS count_http_scheme,
|
||||
-- P1 : nouvelles features
|
||||
sum(IF(length(src.header_x_forwarded_for) > 0, 1, 0)) AS count_xff,
|
||||
sum(IF(src.method = 'POST' AND length(src.header_content_type) > 0
|
||||
AND NOT match(src.header_content_type, '(?i)(form-urlencoded|multipart|json|xml|text/plain|grpc|protobuf)'), 1, 0)) AS count_unusual_ct,
|
||||
sum(IF(src.dst_port NOT IN (80, 443, 8080, 8443), 1, 0)) AS count_non_std_port,
|
||||
sum(IF(src.method = 'POST' AND match(src.path, '(?i)(login|signin|auth|token|session|wp-login|connect|oauth)'), 1, 0)) AS count_login_post
|
||||
FROM ja4_logs.http_logs AS src
|
||||
GROUP BY window_start, src_ip, ja4, host, src_asn;
|
||||
|
||||
@ -192,6 +203,7 @@ CREATE TABLE IF NOT EXISTS ja4_processing.agg_header_fingerprint_1h
|
||||
has_referer SimpleAggregateFunction(max, UInt8),
|
||||
modern_browser_score SimpleAggregateFunction(max, UInt8),
|
||||
ua_ch_mismatch SimpleAggregateFunction(max, UInt8),
|
||||
sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8),
|
||||
sec_fetch_mode SimpleAggregateFunction(any, String),
|
||||
sec_fetch_dest SimpleAggregateFunction(any, String)
|
||||
)
|
||||
@ -212,6 +224,10 @@ SELECT
|
||||
max(toUInt8(if(position(src.client_headers, 'Referer') > 0, 1, 0))) AS has_referer,
|
||||
max(toUInt8(if(length(src.header_sec_ch_ua) > 0, 100, if(length(src.header_user_agent) > 0, 50, 0)))) AS modern_browser_score,
|
||||
max(toUInt8(if((position(src.header_user_agent, 'Windows') > 0 AND position(src.header_sec_ch_ua_platform, 'Windows') == 0) OR (position(src.header_user_agent, 'iPhone') > 0 AND position(src.header_sec_ch_ua_platform, 'iOS') == 0), 1, 0))) AS ua_ch_mismatch,
|
||||
max(toUInt8(if(
|
||||
(src.header_sec_ch_ua_mobile = '?1' AND position(src.header_user_agent, 'Mobile') == 0 AND position(src.header_user_agent, 'Android') == 0 AND position(src.header_user_agent, 'iPhone') == 0)
|
||||
OR (src.header_sec_ch_ua_mobile = '?0' AND (position(src.header_user_agent, 'iPhone') > 0 OR position(src.header_user_agent, 'Android') > 0)),
|
||||
1, 0))) AS sec_ch_mobile_mismatch,
|
||||
any(src.header_sec_fetch_mode) AS sec_fetch_mode,
|
||||
any(src.header_sec_fetch_dest) AS sec_fetch_dest
|
||||
FROM ja4_logs.http_logs AS src
|
||||
|
||||
@ -127,7 +127,13 @@ WITH base_data AS (
|
||||
sqrt(a.ttl_variance_val) AS ttl_std,
|
||||
IF(a.count_correlated_val > 0, a.count_no_wscale_val / a.count_correlated_val, 0) AS no_window_scale_ratio,
|
||||
a.count_no_accept_enc_val / (a.hits + 1) AS missing_accept_enc_ratio,
|
||||
a.count_http_scheme_val / (a.hits + 1) AS http_scheme_ratio
|
||||
a.count_http_scheme_val / (a.hits + 1) AS http_scheme_ratio,
|
||||
-- P1 : nouvelles features de détection
|
||||
IF(a.count_xff_val > 0, 1, 0) AS has_xff,
|
||||
a.count_unusual_ct_val / greatest(a.count_post, 1) AS unusual_content_type_ratio,
|
||||
a.count_non_std_port_val / (a.hits + 1) AS non_standard_port_ratio,
|
||||
a.count_login_post_val / greatest(a.count_post, 1) AS login_post_concentration,
|
||||
h.sec_ch_mobile_mismatch AS sec_ch_mobile_mismatch
|
||||
FROM (
|
||||
SELECT
|
||||
window_start, src_ip, ja4, host, src_asn,
|
||||
@ -162,7 +168,12 @@ WITH base_data AS (
|
||||
sum(count_no_wscale) AS count_no_wscale_val,
|
||||
sum(count_correlated) AS count_correlated_val,
|
||||
sum(count_no_accept_enc) AS count_no_accept_enc_val,
|
||||
sum(count_http_scheme) AS count_http_scheme_val
|
||||
sum(count_http_scheme) AS count_http_scheme_val,
|
||||
-- P1 : nouvelles features de détection
|
||||
sum(count_xff) AS count_xff_val,
|
||||
sum(count_unusual_ct) AS count_unusual_ct_val,
|
||||
sum(count_non_std_port) AS count_non_std_port_val,
|
||||
sum(count_login_post) AS count_login_post_val
|
||||
FROM ja4_processing.agg_host_ip_ja4_1h
|
||||
WHERE window_start >= now() - INTERVAL 24 HOUR
|
||||
GROUP BY window_start, src_ip, ja4, host, src_asn
|
||||
@ -173,6 +184,7 @@ WITH base_data AS (
|
||||
max(header_count) AS header_count, max(has_accept_language) AS has_accept_language,
|
||||
max(has_cookie) AS has_cookie, max(has_referer) AS has_referer,
|
||||
max(modern_browser_score) AS modern_browser_score, max(ua_ch_mismatch) AS ua_ch_mismatch,
|
||||
max(sec_ch_mobile_mismatch) AS sec_ch_mobile_mismatch,
|
||||
any(sec_fetch_mode) AS sec_fetch_mode, any(sec_fetch_dest) AS sec_fetch_dest
|
||||
FROM ja4_processing.agg_header_fingerprint_1h
|
||||
WHERE window_start >= now() - INTERVAL 24 HOUR
|
||||
|
||||
Reference in New Issue
Block a user