85d3b95b7b
Complete implementation of HTTP/2 passive fingerprinting per thesis §2.5.3: mod-reqin-log (C module): - Replace connection-level filter with ap_hook_process_connection (APR_HOOK_FIRST) to capture H2 preface before mod_http2 takes over the connection - AP_MODE_SPECULATIVE read of 512 bytes from c->input_filters - Parse SETTINGS, WINDOW_UPDATE, PRIORITY flags, pseudo-header order - Output individual SETTINGS params as separate JSON fields (IDs 1-6, 8) - Read H2 notes from c1 (master connection) for mod_http2 secondary conns - Fix header_order_signature JSON length bug (26→strlen) ClickHouse schema: - Add 8 new columns to http_logs: h2_has_priority, h2_header_table_size, h2_enable_push, h2_max_concurrent_streams, h2_initial_window_size, h2_max_frame_size, h2_max_header_list_size, h2_enable_connect_protocol - Use Int32/Int64 with DEFAULT -1 to distinguish absent vs zero - Update mv_http_logs to extract individual fields via JSONHas/JSONExtractInt - Migration 04_http2_fields.sql updated for existing deployments Correlator: - Accept both timestamp_ns and timestamp field names (backward compat) Integration: - Enable HTTP/2 in Apache: Protocols h2 http/1.1 in httpd-integration.conf Validated end-to-end via Playwright: H2 curl traffic → mod-reqin-log → correlator → ClickHouse with all 12 H2 columns populated correctly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
486 lines
24 KiB
Python
486 lines
24 KiB
Python
"""Gestion des modèles : chargement, entraînement, versionnement.
|
|
|
|
IsolationForest (EIF), Autoencoder (PyTorch), XGBoost supervisé.
|
|
"""
|
|
import os
|
|
import json
|
|
import glob
|
|
import joblib
|
|
import numpy as np
|
|
import pandas as pd
|
|
from datetime import datetime
|
|
|
|
from .config import (
|
|
MODEL_DIR, MODEL_HISTORY_COUNT, RETRAIN_INTERVAL_H, DRIFT_THRESHOLD,
|
|
N_ESTIMATORS, CONTAMINATION, ANOMALY_THRESHOLD, AE_WEIGHT, AE_EPOCHS, AE_LATENT_DIM,
|
|
AE_LEARNING_RATE, XGB_WEIGHT, XGB_MIN_LABELS, XGB_RETRAIN_INTERVAL_H,
|
|
EIF_AVAILABLE, TORCH_AVAILABLE, XGB_AVAILABLE, DB,
|
|
IsolationForest, StandardScaler,
|
|
)
|
|
from .log import log_info, log_decision, append_training_history
|
|
from .scoring import compute_drift_score
|
|
|
|
# Imports conditionnels depuis config (déjà importés une seule fois)
|
|
if EIF_AVAILABLE:
|
|
from .config import ExtendedIsolationForest
|
|
|
|
if TORCH_AVAILABLE:
|
|
from .config import torch, nn
|
|
|
|
if XGB_AVAILABLE:
|
|
import xgboost as xgb
|
|
|
|
|
|
# ─── Caches de modèles ─────────────────────────────────────────────────────
|
|
_model_cache: dict = {}
|
|
_xgb_cache: dict = {}
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# GESTION DES MODÈLES
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
def _current_pointer_path(name: str) -> str:
|
|
"""Retourne le chemin du fichier pointeur vers la version courante du modèle ``name``."""
|
|
return os.path.join(MODEL_DIR, f'model_{name}.current')
|
|
|
|
def _get_current_version(name: str):
|
|
"""Lit le fichier pointeur et retourne (chemin_modèle, métadonnées) ou (None, None) si absent."""
|
|
pointer = _current_pointer_path(name)
|
|
if not os.path.exists(pointer): return None, None
|
|
with open(pointer) as f: version_id = f.read().strip()
|
|
model_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.joblib')
|
|
meta_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.meta.json')
|
|
if not os.path.exists(model_path) or not os.path.exists(meta_path): return None, None
|
|
with open(meta_path) as f: meta = json.load(f)
|
|
return model_path, meta
|
|
|
|
def _purge_old_versions(name: str):
|
|
"""Supprime les versions excédentaires du modèle ``name`` en ne conservant que MODEL_HISTORY_COUNT fichiers."""
|
|
pattern = os.path.join(MODEL_DIR, f'model_{name}_*.joblib')
|
|
versions = sorted(glob.glob(pattern))
|
|
to_delete = versions[:-MODEL_HISTORY_COUNT] if len(versions) > MODEL_HISTORY_COUNT else []
|
|
for joblib_path in to_delete:
|
|
version_id = os.path.basename(joblib_path).replace(f'model_{name}_', '').replace('.joblib', '')
|
|
meta_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.meta.json')
|
|
os.remove(joblib_path)
|
|
if os.path.exists(meta_path): os.remove(meta_path)
|
|
log_info(f"[{name}] Version purgée : {version_id} (limite={MODEL_HISTORY_COUNT})")
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# AUTOENCODER — Second scorer parallèle (détection d'anomalies par reconstruction)
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
class TrafficAutoEncoder:
|
|
"""Autoencoder symétrique pour détection d'anomalies par erreur de reconstruction.
|
|
|
|
Architecture : encoder (n→64→32→latent_dim) — decoder (latent_dim→32→64→n)
|
|
Activation : ReLU + BatchNorm (encoder/decoder), sigmoid (sortie — données normalisées [0,1])
|
|
Score = MSE(input, reconstruction) par échantillon.
|
|
L'espace latent (16-dim par défaut) peut servir de features compressées pour HDBSCAN.
|
|
"""
|
|
|
|
def __init__(self, n_features: int, latent_dim: int = AE_LATENT_DIM):
|
|
if not TORCH_AVAILABLE:
|
|
raise RuntimeError("PyTorch non disponible — autoencoder désactivé.")
|
|
self.n_features = n_features
|
|
self.latent_dim = latent_dim
|
|
self.device = torch.device('cpu')
|
|
self._build_model()
|
|
self._scaler_min = None
|
|
self._scaler_range = None
|
|
|
|
def _build_model(self):
|
|
# Architecture fixe n→64→32→16→32→64→n (§2.4.3 thèse)
|
|
dim1 = 64
|
|
dim2 = 32
|
|
self.encoder = nn.Sequential(
|
|
nn.Linear(self.n_features, dim1), nn.BatchNorm1d(dim1), nn.ReLU(),
|
|
nn.Linear(dim1, dim2), nn.BatchNorm1d(dim2), nn.ReLU(),
|
|
nn.Linear(dim2, self.latent_dim),
|
|
).to(self.device)
|
|
self.decoder = nn.Sequential(
|
|
nn.Linear(self.latent_dim, dim2), nn.BatchNorm1d(dim2), nn.ReLU(),
|
|
nn.Linear(dim2, dim1), nn.BatchNorm1d(dim1), nn.ReLU(),
|
|
nn.Linear(dim1, self.n_features), nn.Sigmoid(),
|
|
).to(self.device)
|
|
self._all_params = list(self.encoder.parameters()) + list(self.decoder.parameters())
|
|
|
|
def _to_tensor(self, X: np.ndarray) -> 'torch.Tensor':
|
|
"""Normalise [0,1] via min-max puis convertit en Tensor."""
|
|
if self._scaler_min is not None:
|
|
X_norm = (X - self._scaler_min) / (self._scaler_range + 1e-9)
|
|
else:
|
|
X_norm = X
|
|
return torch.tensor(np.clip(X_norm, 0, 1), dtype=torch.float32, device=self.device)
|
|
|
|
def fit(self, X: np.ndarray, epochs: int = AE_EPOCHS, lr: float = AE_LEARNING_RATE,
|
|
batch_size: int = 256) -> dict:
|
|
"""Entraîne l'autoencoder sur la baseline humaine (données normales)."""
|
|
self._scaler_min = X.min(axis=0)
|
|
self._scaler_range = X.max(axis=0) - self._scaler_min
|
|
X_t = self._to_tensor(X)
|
|
|
|
dataset = torch.utils.data.TensorDataset(X_t)
|
|
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)
|
|
optimizer = torch.optim.Adam(self._all_params, lr=lr, weight_decay=1e-5)
|
|
criterion = nn.MSELoss()
|
|
|
|
self.encoder.train()
|
|
self.decoder.train()
|
|
losses = []
|
|
for epoch in range(epochs):
|
|
epoch_loss = 0.0
|
|
for (batch,) in loader:
|
|
latent = self.encoder(batch)
|
|
recon = self.decoder(latent)
|
|
loss = criterion(recon, batch)
|
|
optimizer.zero_grad()
|
|
loss.backward()
|
|
optimizer.step()
|
|
epoch_loss += loss.item() * len(batch)
|
|
losses.append(epoch_loss / len(X_t))
|
|
return {'final_loss': losses[-1], 'epochs': epochs, 'n_samples': len(X)}
|
|
|
|
def score_samples(self, X: np.ndarray) -> np.ndarray:
|
|
"""Retourne l'erreur de reconstruction MSE par échantillon (plus élevé = plus anomal)."""
|
|
self.encoder.eval()
|
|
self.decoder.eval()
|
|
X_t = self._to_tensor(X)
|
|
with torch.no_grad():
|
|
latent = self.encoder(X_t)
|
|
recon = self.decoder(latent)
|
|
mse = ((recon - X_t) ** 2).mean(dim=1).numpy()
|
|
return mse
|
|
|
|
def encode(self, X: np.ndarray) -> np.ndarray:
|
|
"""Retourne l'espace latent (pour HDBSCAN clustering)."""
|
|
self.encoder.eval()
|
|
X_t = self._to_tensor(X)
|
|
with torch.no_grad():
|
|
return self.encoder(X_t).numpy()
|
|
|
|
def state_dict(self) -> dict:
|
|
return {
|
|
'encoder': self.encoder.state_dict(),
|
|
'decoder': self.decoder.state_dict(),
|
|
'scaler_min': self._scaler_min,
|
|
'scaler_range': self._scaler_range,
|
|
'n_features': self.n_features,
|
|
'latent_dim': self.latent_dim,
|
|
}
|
|
|
|
@classmethod
|
|
def load_state_dict(cls, state: dict) -> 'TrafficAutoEncoder':
|
|
ae = cls(state['n_features'], state['latent_dim'])
|
|
ae._scaler_min = state['scaler_min']
|
|
ae._scaler_range = state['scaler_range']
|
|
ae.encoder.load_state_dict(state['encoder'])
|
|
ae.decoder.load_state_dict(state['decoder'])
|
|
return ae
|
|
|
|
|
|
def _ae_model_path(name: str, version_id: str) -> str:
|
|
return os.path.join(MODEL_DIR, f'ae_{name}_{version_id}.pt')
|
|
|
|
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
# XGBOOST — Troisième voix supervisée (labels historiques + feedback SOC)
|
|
# ═══════════════════════════════════════════════════════════════════════════════
|
|
def _xgb_model_path(name: str) -> str:
|
|
return os.path.join(MODEL_DIR, f'xgb_{name}.json')
|
|
|
|
|
|
def _xgb_meta_path(name: str) -> str:
|
|
return os.path.join(MODEL_DIR, f'xgb_{name}.meta.json')
|
|
|
|
|
|
def _load_xgb_labels(client, features: list, min_labels: int = XGB_MIN_LABELS) -> tuple:
|
|
"""Charge les labels historiques depuis ml_all_scores + view_ai_features_1h.
|
|
|
|
Les labels (threat_level) viennent de ml_all_scores, les features de
|
|
view_ai_features_1h via une jointure sur (src_ip, ja4, host).
|
|
Les features absentes de la vue (ex: thesis §5 features) sont ignorées.
|
|
|
|
Positifs : threat_level IN ('HIGH', 'CRITICAL', 'ANUBIS_DENY', 'KNOWN_BOT') → label=1
|
|
Négatifs : threat_level IN ('NORMAL', 'LEGITIMATE_BROWSER') → label=0
|
|
Retourne (X, y, usable_features) ou (None, None, None) si insuffisant.
|
|
"""
|
|
try:
|
|
# Découvrir les colonnes disponibles dans la vue
|
|
cols_result = client.query(
|
|
f"SELECT name FROM system.columns "
|
|
f"WHERE database = '{DB}' AND table = 'view_ai_features_1h'"
|
|
)
|
|
available_cols = {row[0] for row in cols_result.result_rows} if cols_result.result_rows else set()
|
|
usable_features = [f for f in features if f in available_cols]
|
|
if len(usable_features) < 10:
|
|
log_info(f"[XGB] Seulement {len(usable_features)} features disponibles dans view_ai_features_1h — insuffisant.")
|
|
return None, None, None
|
|
|
|
feature_cols = ', '.join(f'f.{c}' for c in usable_features)
|
|
result = client.query(
|
|
f"SELECT {feature_cols}, s.threat_level "
|
|
f"FROM {DB}.ml_all_scores AS s "
|
|
f"INNER JOIN {DB}.view_ai_features_1h AS f "
|
|
f" ON s.src_ip = f.src_ip AND s.ja4 = f.ja4 AND s.host = f.host "
|
|
f"WHERE s.threat_level IN ('NORMAL', 'LEGITIMATE_BROWSER', 'HIGH', 'CRITICAL', 'ANUBIS_DENY', 'KNOWN_BOT') "
|
|
f"AND s.window_start >= now() - INTERVAL 7 DAY "
|
|
f"ORDER BY rand() LIMIT 50000"
|
|
)
|
|
if not result.result_rows:
|
|
return None, None, None
|
|
cols = usable_features + ['threat_level']
|
|
df = pd.DataFrame(result.result_rows, columns=cols)
|
|
df[usable_features] = df[usable_features].apply(pd.to_numeric, errors='coerce')
|
|
df = df.replace([np.inf, -np.inf], np.nan).dropna(subset=usable_features)
|
|
y = (~df['threat_level'].isin(['NORMAL', 'LEGITIMATE_BROWSER'])).astype(int)
|
|
if y.sum() < 10 or len(y) < min_labels:
|
|
return None, None, None
|
|
X = df[usable_features].values
|
|
return X, y.values, usable_features
|
|
except Exception as exc:
|
|
log_info(f"[XGB] Erreur chargement labels : {exc}")
|
|
return None, None, None
|
|
|
|
|
|
def load_or_train_xgb(name: str, client, features: list, cycle_id: str):
|
|
"""Charge ou entraîne le modèle XGBoost supervisé.
|
|
|
|
Retourne (XGBClassifier, list[str] features) ou (None, None) si indisponible.
|
|
"""
|
|
if not XGB_AVAILABLE or XGB_WEIGHT <= 0:
|
|
return None, None
|
|
|
|
model_path = _xgb_model_path(name)
|
|
meta_path = _xgb_meta_path(name)
|
|
|
|
# Charger le modèle existant si récent
|
|
if os.path.exists(model_path) and os.path.exists(meta_path):
|
|
try:
|
|
with open(meta_path) as f:
|
|
meta = json.load(f)
|
|
trained_at = datetime.fromisoformat(meta['trained_at'])
|
|
age_h = (datetime.now() - trained_at).total_seconds() / 3600
|
|
if age_h < XGB_RETRAIN_INTERVAL_H:
|
|
model = xgb.XGBClassifier()
|
|
model.load_model(model_path)
|
|
log_info(f"[XGB][{name}] Modèle rechargé ({age_h:.1f}h / {XGB_RETRAIN_INTERVAL_H}h, {meta.get('n_labels', '?')} labels).")
|
|
return model, meta.get('features', features)
|
|
except Exception as exc:
|
|
log_info(f"[XGB][{name}] Erreur chargement : {exc}")
|
|
|
|
# Entraîner un nouveau modèle
|
|
X, y, xgb_features = _load_xgb_labels(client, features)
|
|
if X is None:
|
|
log_info(f"[XGB][{name}] Labels insuffisants (< {XGB_MIN_LABELS}) — XGBoost désactivé ce cycle.")
|
|
# Tenter de réutiliser un modèle ancien
|
|
if os.path.exists(model_path) and os.path.exists(meta_path):
|
|
try:
|
|
model = xgb.XGBClassifier()
|
|
model.load_model(model_path)
|
|
with open(meta_path) as f:
|
|
meta = json.load(f)
|
|
return model, meta.get('features', features)
|
|
except Exception:
|
|
pass
|
|
return None, None
|
|
|
|
scale_pos = max(1, int((y == 0).sum() / max((y == 1).sum(), 1)))
|
|
model = xgb.XGBClassifier(
|
|
n_estimators=200, max_depth=6, learning_rate=0.1,
|
|
scale_pos_weight=scale_pos, eval_metric='logloss',
|
|
random_state=42, n_jobs=-1,
|
|
tree_method='hist',
|
|
)
|
|
model.fit(X, y, verbose=False)
|
|
|
|
model.save_model(model_path)
|
|
meta = {
|
|
'trained_at': datetime.now().isoformat(),
|
|
'n_labels': len(y), 'n_positive': int(y.sum()),
|
|
'n_negative': int((y == 0).sum()), 'n_features': len(xgb_features),
|
|
'features': xgb_features,
|
|
'scale_pos_weight': scale_pos, 'model_name': name,
|
|
}
|
|
with open(meta_path, 'w') as f:
|
|
json.dump(meta, f, indent=2)
|
|
|
|
log_info(f"[XGB][{name}] Modèle entraîné : {len(y)} labels ({y.sum()} positifs), scale_pos_weight={scale_pos}")
|
|
log_decision('XGB_TRAINED', cycle_id, name, meta)
|
|
return model, xgb_features
|
|
|
|
|
|
def load_or_train_model(name: str, human_baseline: pd.DataFrame, features: list, cycle_id: str):
|
|
"""Charge le modèle IsolationForest existant ou en entraîne un nouveau si nécessaire.
|
|
|
|
Réutilise le modèle si son âge est inférieur à RETRAIN_INTERVAL_H et si aucune
|
|
dérive conceptuelle significative n'est détectée (A1). En cas d'expiration ou de
|
|
dérive, entraîne un nouveau modèle sur ``human_baseline``, le sérialise sur disque,
|
|
met à jour le fichier pointeur et purge les anciennes versions.
|
|
|
|
Retourne (IsolationForest, TrafficAutoEncoder|None, list[str] features).
|
|
"""
|
|
model_path, meta = _get_current_version(name)
|
|
if model_path and meta:
|
|
trained_at = datetime.fromisoformat(meta['trained_at'])
|
|
age_h = (datetime.now() - trained_at).total_seconds() / 3600
|
|
age_ok = age_h < RETRAIN_INTERVAL_H
|
|
|
|
# A1 — Dérive conceptuelle : comparer la distribution actuelle avec celle de l'entraînement
|
|
drift_score = 0.0
|
|
drift_forced = False
|
|
if age_ok and 'baseline_stats' in meta:
|
|
drift_score = compute_drift_score(meta['baseline_stats'], human_baseline, features,
|
|
name=name, cycle_id=cycle_id)
|
|
if drift_score >= DRIFT_THRESHOLD:
|
|
drift_forced = True
|
|
log_info(f"[{name}] Dérive détectée ({drift_score:.0%} features) — retraining forcé.")
|
|
log_decision('DRIFT_DETECTED', cycle_id, name, {
|
|
'version_id': meta['version_id'], 'drift_score': round(drift_score, 3),
|
|
'drift_threshold': DRIFT_THRESHOLD, 'model_age_hours': round(age_h, 2)
|
|
})
|
|
|
|
if age_ok and not drift_forced:
|
|
log_info(f"[{name}] Modèle v{meta['version_id']} valide ({age_h:.1f}h / {RETRAIN_INTERVAL_H}h, drift={drift_score:.0%}) — réutilisation.")
|
|
log_decision('MODEL_LOADED', cycle_id, name, {
|
|
'version_id': meta['version_id'], 'model_age_hours': round(age_h, 2),
|
|
'trained_at': meta['trained_at'], 'human_samples': meta.get('human_samples', '?'),
|
|
'retrain_in_hours': round(RETRAIN_INTERVAL_H - age_h, 1), 'drift_score': round(drift_score, 3)
|
|
})
|
|
ae_loaded = None
|
|
if TORCH_AVAILABLE and AE_WEIGHT > 0:
|
|
ae_path = _ae_model_path(name, meta['version_id'])
|
|
if os.path.exists(ae_path):
|
|
try:
|
|
ae_loaded = TrafficAutoEncoder.load_state_dict(torch.load(ae_path, weights_only=False))
|
|
log_info(f"[{name}] Autoencoder v{meta['version_id']} rechargé.")
|
|
except Exception as exc:
|
|
log_info(f"[{name}] Erreur chargement AE : {exc} — AE désactivé ce cycle.")
|
|
return joblib.load(model_path), ae_loaded, meta.get('features', features)
|
|
elif not drift_forced:
|
|
log_info(f"[{name}] Modèle v{meta['version_id']} expiré ({age_h:.1f}h ≥ {RETRAIN_INTERVAL_H}h) — retraining.")
|
|
|
|
version_id = datetime.now().strftime('%Y%m%d_%H%M%S')
|
|
log_info(f"[{name}] Entraînement EIF v{version_id} — {len(human_baseline)} sessions ISP, {len(features)} features, contamination={CONTAMINATION}")
|
|
|
|
X = human_baseline[features].replace([np.inf, -np.inf], np.nan)
|
|
X = X.fillna(X.median())
|
|
|
|
# Feature pruning : retirer les features à variance quasi-nulle (inutiles pour les arbres)
|
|
PRUNE_VARIANCE_THRESHOLD = float(os.getenv('PRUNE_VARIANCE_THRESHOLD', '1e-6'))
|
|
feature_variances = X.var()
|
|
low_var_features = feature_variances[feature_variances < PRUNE_VARIANCE_THRESHOLD].index.tolist()
|
|
if low_var_features:
|
|
log_info(f"[{name}] Élagage : {len(low_var_features)} feature(s) à variance < {PRUNE_VARIANCE_THRESHOLD} retirées : {low_var_features}")
|
|
X = X.drop(columns=low_var_features)
|
|
features = [f for f in features if f not in low_var_features]
|
|
log_decision('FEATURE_PRUNED', cycle_id, name, {'pruned': low_var_features, 'remaining': len(features)})
|
|
|
|
# Validation split : réserver 20% pour évaluation offline
|
|
val_size = max(1, int(len(X) * 0.2))
|
|
X_train = X.iloc[:-val_size]
|
|
X_val = X.iloc[-val_size:]
|
|
|
|
if EIF_AVAILABLE:
|
|
model = ExtendedIsolationForest(
|
|
ntrees=300, ndim=min(3, len(features)),
|
|
sample_size='auto', missing_action='impute',
|
|
random_seed=42, nthreads=-1
|
|
)
|
|
else:
|
|
model = IsolationForest(n_estimators=300, contamination=CONTAMINATION, random_state=42, n_jobs=-1)
|
|
model.fit(X_train)
|
|
|
|
# Évaluation offline : score moyen sur la validation (devrait être > 0 pour du trafic humain sklearn)
|
|
val_scores = model.decision_function(X_val)
|
|
# Unifier la convention : négatif = anomal (isotree: 0.5 - score)
|
|
if EIF_AVAILABLE:
|
|
val_scores = 0.5 - val_scores
|
|
val_mean_score = float(np.mean(val_scores))
|
|
val_anomaly_rate = float(np.mean(val_scores < 0))
|
|
log_info(f"[{name}] Validation : score moyen={val_mean_score:.4f}, taux anomalie={val_anomaly_rate:.2%} ({len(X_val)} échantillons)")
|
|
|
|
# GATE CONDITION : rejeter le modèle si la baseline semble contaminée
|
|
VAL_ANOMALY_GATE = float(os.getenv('VAL_ANOMALY_GATE', '0.20'))
|
|
if val_anomaly_rate > VAL_ANOMALY_GATE:
|
|
log_info(f"[{name}] ⚠ REJET : val_anomaly_rate={val_anomaly_rate:.2%} > gate={VAL_ANOMALY_GATE:.0%} — baseline probablement contaminée.")
|
|
log_decision('MODEL_REJECTED', cycle_id, name, {
|
|
'val_anomaly_rate': round(val_anomaly_rate, 4), 'gate': VAL_ANOMALY_GATE,
|
|
'val_mean_score': round(val_mean_score, 4), 'version_id': version_id,
|
|
})
|
|
# Tenter de réutiliser le modèle précédent
|
|
if model_path and os.path.exists(model_path):
|
|
log_info(f"[{name}] Conservation du modèle précédent v{meta.get('version_id', '?')}.")
|
|
ae_prev = None
|
|
if TORCH_AVAILABLE and AE_WEIGHT > 0:
|
|
ae_prev_path = _ae_model_path(name, meta.get('version_id', ''))
|
|
if os.path.exists(ae_prev_path):
|
|
try:
|
|
ae_prev = TrafficAutoEncoder.load_state_dict(torch.load(ae_prev_path, weights_only=False))
|
|
except Exception:
|
|
pass
|
|
return joblib.load(model_path), ae_prev, meta.get('features', features)
|
|
log_info(f"[{name}] Aucun modèle précédent — utilisation du modèle rejeté par défaut.")
|
|
|
|
# A1/§4 — Sauvegarder les statistiques de distribution avec quantile digest 9 points
|
|
# (p5…p95) pour une meilleure fidélité de la détection de dérive KS+KL
|
|
baseline_stats = {
|
|
f: {
|
|
'mean': float(X[f].mean()), 'std': float(X[f].std()),
|
|
'p5': float(X[f].quantile(0.05)),
|
|
'p10': float(X[f].quantile(0.10)),
|
|
'p25': float(X[f].quantile(0.25)),
|
|
'p50': float(X[f].quantile(0.50)),
|
|
'p75': float(X[f].quantile(0.75)),
|
|
'p90': float(X[f].quantile(0.90)),
|
|
'p95': float(X[f].quantile(0.95)),
|
|
}
|
|
for f in features
|
|
}
|
|
|
|
new_model_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.joblib')
|
|
new_meta_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.meta.json')
|
|
joblib.dump(model, new_model_path)
|
|
|
|
# Entraînement de l'Autoencoder en parallèle (si PyTorch disponible et AE_WEIGHT > 0)
|
|
ae_model = None
|
|
if TORCH_AVAILABLE and AE_WEIGHT > 0:
|
|
try:
|
|
ae_model = TrafficAutoEncoder(n_features=len(features))
|
|
ae_stats = ae_model.fit(X_train.values)
|
|
ae_path = _ae_model_path(name, version_id)
|
|
torch.save(ae_model.state_dict(), ae_path)
|
|
log_info(f"[{name}] Autoencoder entraîné : loss={ae_stats['final_loss']:.6f}, epochs={ae_stats['epochs']}")
|
|
except Exception as exc:
|
|
log_info(f"[{name}] Autoencoder training échoué : {exc} — AE désactivé.")
|
|
ae_model = None
|
|
|
|
previous_version = meta.get('version_id', None) if meta else None
|
|
new_meta = {
|
|
'version_id': version_id, 'trained_at': datetime.now().isoformat(),
|
|
'human_samples': len(human_baseline), 'contamination': CONTAMINATION,
|
|
'threshold': ANOMALY_THRESHOLD, 'features': features,
|
|
'model_name': name, 'previous_version': previous_version,
|
|
'retrain_interval': RETRAIN_INTERVAL_H, 'baseline_stats': baseline_stats,
|
|
'algorithm': 'ExtendedIsolationForest' if EIF_AVAILABLE else 'IsolationForest',
|
|
'autoencoder': ae_model is not None,
|
|
'ae_weight': AE_WEIGHT if ae_model else 0.0,
|
|
'validation': {
|
|
'val_size': len(X_val), 'train_size': len(X_train),
|
|
'val_mean_score': round(val_mean_score, 4),
|
|
'val_anomaly_rate': round(val_anomaly_rate, 4),
|
|
}
|
|
}
|
|
with open(new_meta_path, 'w') as f: json.dump(new_meta, f, indent=2)
|
|
with open(_current_pointer_path(name), 'w') as f: f.write(version_id)
|
|
|
|
append_training_history({k: v for k, v in new_meta.items() if k != 'baseline_stats'})
|
|
_purge_old_versions(name)
|
|
|
|
log_info(f"[{name}] Modèle v{version_id} sauvegardé → {new_model_path} (AE={'oui' if ae_model is not None else 'non'})")
|
|
log_decision('MODEL_TRAINED', cycle_id, name, {
|
|
'version_id': version_id, 'previous_version': previous_version,
|
|
'human_samples': len(human_baseline), 'next_retrain_in_h': RETRAIN_INTERVAL_H,
|
|
'history_kept': MODEL_HISTORY_COUNT
|
|
})
|
|
return model, ae_model, features
|