refactor(bot-detector): extract monolith into modular package
Split bot_detector.py (~1982 lines) into 10 focused modules: - config.py: all configuration constants and optional imports - log.py: logging utilities (log_info, log_decision, append_training_history) - infra.py: ClickHouse client, health check HTTP server, shutdown - browser.py: multifactorial browser identification (5 axes) - scoring.py: drift detection, feature validation, SHAP, clustering - models.py: EIF, Autoencoder, XGBoost model management - preprocessing.py: data preprocessing and feature list definitions - pipeline.py: core semi-supervised scoring loop - cycle.py: main analysis cycle orchestration - __main__.py: entry point with startup banner Update Dockerfile to copy package directory and use python -m bot_detector. All 36 existing tests pass unchanged. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
478
services/bot-detector/bot_detector/models.py
Normal file
478
services/bot-detector/bot_detector/models.py
Normal file
@ -0,0 +1,478 @@
|
||||
"""Gestion des modèles : chargement, entraînement, versionnement.
|
||||
|
||||
IsolationForest (EIF), Autoencoder (PyTorch), XGBoost supervisé.
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import glob
|
||||
import joblib
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from datetime import datetime
|
||||
|
||||
from .config import (
|
||||
MODEL_DIR, MODEL_HISTORY_COUNT, RETRAIN_INTERVAL_H, DRIFT_THRESHOLD,
|
||||
N_ESTIMATORS, CONTAMINATION, ANOMALY_THRESHOLD, AE_WEIGHT, AE_EPOCHS, AE_LATENT_DIM,
|
||||
AE_LEARNING_RATE, XGB_WEIGHT, XGB_MIN_LABELS, XGB_RETRAIN_INTERVAL_H,
|
||||
EIF_AVAILABLE, TORCH_AVAILABLE, XGB_AVAILABLE, DB,
|
||||
IsolationForest, StandardScaler,
|
||||
)
|
||||
from .log import log_info, log_decision, append_training_history
|
||||
from .scoring import compute_drift_score
|
||||
|
||||
# Imports conditionnels depuis config (déjà importés une seule fois)
|
||||
if EIF_AVAILABLE:
|
||||
from .config import ExtendedIsolationForest
|
||||
|
||||
if TORCH_AVAILABLE:
|
||||
from .config import torch, nn
|
||||
|
||||
if XGB_AVAILABLE:
|
||||
import xgboost as xgb
|
||||
|
||||
|
||||
# ─── Caches de modèles ─────────────────────────────────────────────────────
|
||||
_model_cache: dict = {}
|
||||
_xgb_cache: dict = {}
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# GESTION DES MODÈLES
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
def _current_pointer_path(name: str) -> str:
|
||||
"""Retourne le chemin du fichier pointeur vers la version courante du modèle ``name``."""
|
||||
return os.path.join(MODEL_DIR, f'model_{name}.current')
|
||||
|
||||
def _get_current_version(name: str):
|
||||
"""Lit le fichier pointeur et retourne (chemin_modèle, métadonnées) ou (None, None) si absent."""
|
||||
pointer = _current_pointer_path(name)
|
||||
if not os.path.exists(pointer): return None, None
|
||||
with open(pointer) as f: version_id = f.read().strip()
|
||||
model_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.joblib')
|
||||
meta_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.meta.json')
|
||||
if not os.path.exists(model_path) or not os.path.exists(meta_path): return None, None
|
||||
with open(meta_path) as f: meta = json.load(f)
|
||||
return model_path, meta
|
||||
|
||||
def _purge_old_versions(name: str):
|
||||
"""Supprime les versions excédentaires du modèle ``name`` en ne conservant que MODEL_HISTORY_COUNT fichiers."""
|
||||
pattern = os.path.join(MODEL_DIR, f'model_{name}_*.joblib')
|
||||
versions = sorted(glob.glob(pattern))
|
||||
to_delete = versions[:-MODEL_HISTORY_COUNT] if len(versions) > MODEL_HISTORY_COUNT else []
|
||||
for joblib_path in to_delete:
|
||||
version_id = os.path.basename(joblib_path).replace(f'model_{name}_', '').replace('.joblib', '')
|
||||
meta_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.meta.json')
|
||||
os.remove(joblib_path)
|
||||
if os.path.exists(meta_path): os.remove(meta_path)
|
||||
log_info(f"[{name}] Version purgée : {version_id} (limite={MODEL_HISTORY_COUNT})")
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# AUTOENCODER — Second scorer parallèle (détection d'anomalies par reconstruction)
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
class TrafficAutoEncoder:
|
||||
"""Autoencoder symétrique pour détection d'anomalies par erreur de reconstruction.
|
||||
|
||||
Architecture : encoder (n→64→32→latent_dim) — decoder (latent_dim→32→64→n)
|
||||
Activation : ReLU + BatchNorm (encoder/decoder), sigmoid (sortie — données normalisées [0,1])
|
||||
Score = MSE(input, reconstruction) par échantillon.
|
||||
L'espace latent (16-dim par défaut) peut servir de features compressées pour HDBSCAN.
|
||||
"""
|
||||
|
||||
def __init__(self, n_features: int, latent_dim: int = AE_LATENT_DIM):
|
||||
if not TORCH_AVAILABLE:
|
||||
raise RuntimeError("PyTorch non disponible — autoencoder désactivé.")
|
||||
self.n_features = n_features
|
||||
self.latent_dim = latent_dim
|
||||
self.device = torch.device('cpu')
|
||||
self._build_model()
|
||||
self._scaler_min = None
|
||||
self._scaler_range = None
|
||||
|
||||
def _build_model(self):
|
||||
dim1 = min(64, max(self.n_features, self.latent_dim + 4))
|
||||
dim2 = min(32, max(dim1 // 2, self.latent_dim + 2))
|
||||
self.encoder = nn.Sequential(
|
||||
nn.Linear(self.n_features, dim1), nn.BatchNorm1d(dim1), nn.ReLU(),
|
||||
nn.Linear(dim1, dim2), nn.BatchNorm1d(dim2), nn.ReLU(),
|
||||
nn.Linear(dim2, self.latent_dim),
|
||||
).to(self.device)
|
||||
self.decoder = nn.Sequential(
|
||||
nn.Linear(self.latent_dim, dim2), nn.BatchNorm1d(dim2), nn.ReLU(),
|
||||
nn.Linear(dim2, dim1), nn.BatchNorm1d(dim1), nn.ReLU(),
|
||||
nn.Linear(dim1, self.n_features), nn.Sigmoid(),
|
||||
).to(self.device)
|
||||
self._all_params = list(self.encoder.parameters()) + list(self.decoder.parameters())
|
||||
|
||||
def _to_tensor(self, X: np.ndarray) -> 'torch.Tensor':
|
||||
"""Normalise [0,1] via min-max puis convertit en Tensor."""
|
||||
if self._scaler_min is not None:
|
||||
X_norm = (X - self._scaler_min) / (self._scaler_range + 1e-9)
|
||||
else:
|
||||
X_norm = X
|
||||
return torch.tensor(np.clip(X_norm, 0, 1), dtype=torch.float32, device=self.device)
|
||||
|
||||
def fit(self, X: np.ndarray, epochs: int = AE_EPOCHS, lr: float = AE_LEARNING_RATE,
|
||||
batch_size: int = 256) -> dict:
|
||||
"""Entraîne l'autoencoder sur la baseline humaine (données normales)."""
|
||||
self._scaler_min = X.min(axis=0)
|
||||
self._scaler_range = X.max(axis=0) - self._scaler_min
|
||||
X_t = self._to_tensor(X)
|
||||
|
||||
dataset = torch.utils.data.TensorDataset(X_t)
|
||||
loader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=True)
|
||||
optimizer = torch.optim.Adam(self._all_params, lr=lr, weight_decay=1e-5)
|
||||
criterion = nn.MSELoss()
|
||||
|
||||
self.encoder.train()
|
||||
self.decoder.train()
|
||||
losses = []
|
||||
for epoch in range(epochs):
|
||||
epoch_loss = 0.0
|
||||
for (batch,) in loader:
|
||||
latent = self.encoder(batch)
|
||||
recon = self.decoder(latent)
|
||||
loss = criterion(recon, batch)
|
||||
optimizer.zero_grad()
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
epoch_loss += loss.item() * len(batch)
|
||||
losses.append(epoch_loss / len(X_t))
|
||||
return {'final_loss': losses[-1], 'epochs': epochs, 'n_samples': len(X)}
|
||||
|
||||
def score_samples(self, X: np.ndarray) -> np.ndarray:
|
||||
"""Retourne l'erreur de reconstruction MSE par échantillon (plus élevé = plus anomal)."""
|
||||
self.encoder.eval()
|
||||
self.decoder.eval()
|
||||
X_t = self._to_tensor(X)
|
||||
with torch.no_grad():
|
||||
latent = self.encoder(X_t)
|
||||
recon = self.decoder(latent)
|
||||
mse = ((recon - X_t) ** 2).mean(dim=1).numpy()
|
||||
return mse
|
||||
|
||||
def encode(self, X: np.ndarray) -> np.ndarray:
|
||||
"""Retourne l'espace latent (pour HDBSCAN clustering)."""
|
||||
self.encoder.eval()
|
||||
X_t = self._to_tensor(X)
|
||||
with torch.no_grad():
|
||||
return self.encoder(X_t).numpy()
|
||||
|
||||
def state_dict(self) -> dict:
|
||||
return {
|
||||
'encoder': self.encoder.state_dict(),
|
||||
'decoder': self.decoder.state_dict(),
|
||||
'scaler_min': self._scaler_min,
|
||||
'scaler_range': self._scaler_range,
|
||||
'n_features': self.n_features,
|
||||
'latent_dim': self.latent_dim,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def load_state_dict(cls, state: dict) -> 'TrafficAutoEncoder':
|
||||
ae = cls(state['n_features'], state['latent_dim'])
|
||||
ae._scaler_min = state['scaler_min']
|
||||
ae._scaler_range = state['scaler_range']
|
||||
ae.encoder.load_state_dict(state['encoder'])
|
||||
ae.decoder.load_state_dict(state['decoder'])
|
||||
return ae
|
||||
|
||||
|
||||
def _ae_model_path(name: str, version_id: str) -> str:
|
||||
return os.path.join(MODEL_DIR, f'ae_{name}_{version_id}.pt')
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
# XGBOOST — Troisième voix supervisée (labels historiques + feedback SOC)
|
||||
# ═══════════════════════════════════════════════════════════════════════════════
|
||||
def _xgb_model_path(name: str) -> str:
|
||||
return os.path.join(MODEL_DIR, f'xgb_{name}.json')
|
||||
|
||||
|
||||
def _xgb_meta_path(name: str) -> str:
|
||||
return os.path.join(MODEL_DIR, f'xgb_{name}.meta.json')
|
||||
|
||||
|
||||
def _load_xgb_labels(client, features: list, min_labels: int = XGB_MIN_LABELS) -> tuple:
|
||||
"""Charge les labels historiques depuis ml_all_scores + view_ai_features_1h.
|
||||
|
||||
Les labels (threat_level) viennent de ml_all_scores, les features de
|
||||
view_ai_features_1h via une jointure sur (src_ip, ja4, host).
|
||||
Les features absentes de la vue (ex: thesis §5 features) sont ignorées.
|
||||
|
||||
Positifs : threat_level IN ('HIGH', 'CRITICAL', 'ANUBIS_DENY', 'KNOWN_BOT') → label=1
|
||||
Négatifs : threat_level IN ('NORMAL', 'LEGITIMATE_BROWSER') → label=0
|
||||
Retourne (X, y, usable_features) ou (None, None, None) si insuffisant.
|
||||
"""
|
||||
try:
|
||||
# Découvrir les colonnes disponibles dans la vue
|
||||
cols_result = client.query(
|
||||
f"SELECT name FROM system.columns "
|
||||
f"WHERE database = '{DB}' AND table = 'view_ai_features_1h'"
|
||||
)
|
||||
available_cols = {row[0] for row in cols_result.result_rows} if cols_result.result_rows else set()
|
||||
usable_features = [f for f in features if f in available_cols]
|
||||
if len(usable_features) < 10:
|
||||
log_info(f"[XGB] Seulement {len(usable_features)} features disponibles dans view_ai_features_1h — insuffisant.")
|
||||
return None, None, None
|
||||
|
||||
feature_cols = ', '.join(f'f.{c}' for c in usable_features)
|
||||
result = client.query(
|
||||
f"SELECT {feature_cols}, s.threat_level "
|
||||
f"FROM {DB}.ml_all_scores AS s "
|
||||
f"INNER JOIN {DB}.view_ai_features_1h AS f "
|
||||
f" ON s.src_ip = f.src_ip AND s.ja4 = f.ja4 AND s.host = f.host "
|
||||
f"WHERE s.threat_level IN ('NORMAL', 'LEGITIMATE_BROWSER', 'HIGH', 'CRITICAL', 'ANUBIS_DENY', 'KNOWN_BOT') "
|
||||
f"AND s.window_start >= now() - INTERVAL 7 DAY "
|
||||
f"ORDER BY rand() LIMIT 50000"
|
||||
)
|
||||
if not result.result_rows:
|
||||
return None, None, None
|
||||
cols = usable_features + ['threat_level']
|
||||
df = pd.DataFrame(result.result_rows, columns=cols)
|
||||
df[usable_features] = df[usable_features].apply(pd.to_numeric, errors='coerce')
|
||||
df = df.replace([np.inf, -np.inf], np.nan).dropna(subset=usable_features)
|
||||
y = (~df['threat_level'].isin(['NORMAL', 'LEGITIMATE_BROWSER'])).astype(int)
|
||||
if y.sum() < 10 or len(y) < min_labels:
|
||||
return None, None, None
|
||||
X = df[usable_features].values
|
||||
return X, y.values, usable_features
|
||||
except Exception as exc:
|
||||
log_info(f"[XGB] Erreur chargement labels : {exc}")
|
||||
return None, None, None
|
||||
|
||||
|
||||
def load_or_train_xgb(name: str, client, features: list, cycle_id: str):
|
||||
"""Charge ou entraîne le modèle XGBoost supervisé.
|
||||
|
||||
Retourne (XGBClassifier, list[str] features) ou (None, None) si indisponible.
|
||||
"""
|
||||
if not XGB_AVAILABLE or XGB_WEIGHT <= 0:
|
||||
return None, None
|
||||
|
||||
model_path = _xgb_model_path(name)
|
||||
meta_path = _xgb_meta_path(name)
|
||||
|
||||
# Charger le modèle existant si récent
|
||||
if os.path.exists(model_path) and os.path.exists(meta_path):
|
||||
try:
|
||||
with open(meta_path) as f:
|
||||
meta = json.load(f)
|
||||
trained_at = datetime.fromisoformat(meta['trained_at'])
|
||||
age_h = (datetime.now() - trained_at).total_seconds() / 3600
|
||||
if age_h < XGB_RETRAIN_INTERVAL_H:
|
||||
model = xgb.XGBClassifier()
|
||||
model.load_model(model_path)
|
||||
log_info(f"[XGB][{name}] Modèle rechargé ({age_h:.1f}h / {XGB_RETRAIN_INTERVAL_H}h, {meta.get('n_labels', '?')} labels).")
|
||||
return model, meta.get('features', features)
|
||||
except Exception as exc:
|
||||
log_info(f"[XGB][{name}] Erreur chargement : {exc}")
|
||||
|
||||
# Entraîner un nouveau modèle
|
||||
X, y, xgb_features = _load_xgb_labels(client, features)
|
||||
if X is None:
|
||||
log_info(f"[XGB][{name}] Labels insuffisants (< {XGB_MIN_LABELS}) — XGBoost désactivé ce cycle.")
|
||||
# Tenter de réutiliser un modèle ancien
|
||||
if os.path.exists(model_path) and os.path.exists(meta_path):
|
||||
try:
|
||||
model = xgb.XGBClassifier()
|
||||
model.load_model(model_path)
|
||||
with open(meta_path) as f:
|
||||
meta = json.load(f)
|
||||
return model, meta.get('features', features)
|
||||
except Exception:
|
||||
pass
|
||||
return None, None
|
||||
|
||||
scale_pos = max(1, int((y == 0).sum() / max((y == 1).sum(), 1)))
|
||||
model = xgb.XGBClassifier(
|
||||
n_estimators=200, max_depth=6, learning_rate=0.1,
|
||||
scale_pos_weight=scale_pos, eval_metric='logloss',
|
||||
random_state=42, n_jobs=-1,
|
||||
tree_method='hist',
|
||||
)
|
||||
model.fit(X, y, verbose=False)
|
||||
|
||||
model.save_model(model_path)
|
||||
meta = {
|
||||
'trained_at': datetime.now().isoformat(),
|
||||
'n_labels': len(y), 'n_positive': int(y.sum()),
|
||||
'n_negative': int((y == 0).sum()), 'n_features': len(xgb_features),
|
||||
'features': xgb_features,
|
||||
'scale_pos_weight': scale_pos, 'model_name': name,
|
||||
}
|
||||
with open(meta_path, 'w') as f:
|
||||
json.dump(meta, f, indent=2)
|
||||
|
||||
log_info(f"[XGB][{name}] Modèle entraîné : {len(y)} labels ({y.sum()} positifs), scale_pos_weight={scale_pos}")
|
||||
log_decision('XGB_TRAINED', cycle_id, name, meta)
|
||||
return model, xgb_features
|
||||
|
||||
|
||||
def load_or_train_model(name: str, human_baseline: pd.DataFrame, features: list, cycle_id: str):
|
||||
"""Charge le modèle IsolationForest existant ou en entraîne un nouveau si nécessaire.
|
||||
|
||||
Réutilise le modèle si son âge est inférieur à RETRAIN_INTERVAL_H et si aucune
|
||||
dérive conceptuelle significative n'est détectée (A1). En cas d'expiration ou de
|
||||
dérive, entraîne un nouveau modèle sur ``human_baseline``, le sérialise sur disque,
|
||||
met à jour le fichier pointeur et purge les anciennes versions.
|
||||
|
||||
Retourne (IsolationForest, TrafficAutoEncoder|None, list[str] features).
|
||||
"""
|
||||
model_path, meta = _get_current_version(name)
|
||||
if model_path and meta:
|
||||
trained_at = datetime.fromisoformat(meta['trained_at'])
|
||||
age_h = (datetime.now() - trained_at).total_seconds() / 3600
|
||||
age_ok = age_h < RETRAIN_INTERVAL_H
|
||||
|
||||
# A1 — Dérive conceptuelle : comparer la distribution actuelle avec celle de l'entraînement
|
||||
drift_score = 0.0
|
||||
drift_forced = False
|
||||
if age_ok and 'baseline_stats' in meta:
|
||||
drift_score = compute_drift_score(meta['baseline_stats'], human_baseline, features)
|
||||
if drift_score >= DRIFT_THRESHOLD:
|
||||
drift_forced = True
|
||||
log_info(f"[{name}] Dérive détectée ({drift_score:.0%} features) — retraining forcé.")
|
||||
log_decision('DRIFT_DETECTED', cycle_id, name, {
|
||||
'version_id': meta['version_id'], 'drift_score': round(drift_score, 3),
|
||||
'drift_threshold': DRIFT_THRESHOLD, 'model_age_hours': round(age_h, 2)
|
||||
})
|
||||
|
||||
if age_ok and not drift_forced:
|
||||
log_info(f"[{name}] Modèle v{meta['version_id']} valide ({age_h:.1f}h / {RETRAIN_INTERVAL_H}h, drift={drift_score:.0%}) — réutilisation.")
|
||||
log_decision('MODEL_LOADED', cycle_id, name, {
|
||||
'version_id': meta['version_id'], 'model_age_hours': round(age_h, 2),
|
||||
'trained_at': meta['trained_at'], 'human_samples': meta.get('human_samples', '?'),
|
||||
'retrain_in_hours': round(RETRAIN_INTERVAL_H - age_h, 1), 'drift_score': round(drift_score, 3)
|
||||
})
|
||||
ae_loaded = None
|
||||
if TORCH_AVAILABLE and AE_WEIGHT > 0:
|
||||
ae_path = _ae_model_path(name, meta['version_id'])
|
||||
if os.path.exists(ae_path):
|
||||
try:
|
||||
ae_loaded = TrafficAutoEncoder.load_state_dict(torch.load(ae_path, weights_only=False))
|
||||
log_info(f"[{name}] Autoencoder v{meta['version_id']} rechargé.")
|
||||
except Exception as exc:
|
||||
log_info(f"[{name}] Erreur chargement AE : {exc} — AE désactivé ce cycle.")
|
||||
return joblib.load(model_path), ae_loaded, meta.get('features', features)
|
||||
elif not drift_forced:
|
||||
log_info(f"[{name}] Modèle v{meta['version_id']} expiré ({age_h:.1f}h ≥ {RETRAIN_INTERVAL_H}h) — retraining.")
|
||||
|
||||
version_id = datetime.now().strftime('%Y%m%d_%H%M%S')
|
||||
log_info(f"[{name}] Entraînement EIF v{version_id} — {len(human_baseline)} sessions ISP, {len(features)} features, contamination={CONTAMINATION}")
|
||||
|
||||
X = human_baseline[features].replace([np.inf, -np.inf], np.nan)
|
||||
X = X.fillna(X.median())
|
||||
|
||||
# Feature pruning : retirer les features à variance quasi-nulle (inutiles pour les arbres)
|
||||
PRUNE_VARIANCE_THRESHOLD = float(os.getenv('PRUNE_VARIANCE_THRESHOLD', '1e-6'))
|
||||
feature_variances = X.var()
|
||||
low_var_features = feature_variances[feature_variances < PRUNE_VARIANCE_THRESHOLD].index.tolist()
|
||||
if low_var_features:
|
||||
log_info(f"[{name}] Élagage : {len(low_var_features)} feature(s) à variance < {PRUNE_VARIANCE_THRESHOLD} retirées : {low_var_features}")
|
||||
X = X.drop(columns=low_var_features)
|
||||
features = [f for f in features if f not in low_var_features]
|
||||
log_decision('FEATURE_PRUNED', cycle_id, name, {'pruned': low_var_features, 'remaining': len(features)})
|
||||
|
||||
# Validation split : réserver 20% pour évaluation offline
|
||||
val_size = max(1, int(len(X) * 0.2))
|
||||
X_train = X.iloc[:-val_size]
|
||||
X_val = X.iloc[-val_size:]
|
||||
|
||||
if EIF_AVAILABLE:
|
||||
model = ExtendedIsolationForest(
|
||||
ntrees=300, ndim=min(3, len(features)),
|
||||
sample_size='auto', missing_action='impute',
|
||||
random_seed=42, nthreads=-1
|
||||
)
|
||||
else:
|
||||
model = IsolationForest(n_estimators=300, contamination=CONTAMINATION, random_state=42, n_jobs=-1)
|
||||
model.fit(X_train)
|
||||
|
||||
# Évaluation offline : score moyen sur la validation (devrait être > 0 pour du trafic humain sklearn)
|
||||
val_scores = model.decision_function(X_val)
|
||||
# Unifier la convention : négatif = anomal (isotree: 0.5 - score)
|
||||
if EIF_AVAILABLE:
|
||||
val_scores = 0.5 - val_scores
|
||||
val_mean_score = float(np.mean(val_scores))
|
||||
val_anomaly_rate = float(np.mean(val_scores < 0))
|
||||
log_info(f"[{name}] Validation : score moyen={val_mean_score:.4f}, taux anomalie={val_anomaly_rate:.2%} ({len(X_val)} échantillons)")
|
||||
|
||||
# GATE CONDITION : rejeter le modèle si la baseline semble contaminée
|
||||
VAL_ANOMALY_GATE = float(os.getenv('VAL_ANOMALY_GATE', '0.20'))
|
||||
if val_anomaly_rate > VAL_ANOMALY_GATE:
|
||||
log_info(f"[{name}] ⚠ REJET : val_anomaly_rate={val_anomaly_rate:.2%} > gate={VAL_ANOMALY_GATE:.0%} — baseline probablement contaminée.")
|
||||
log_decision('MODEL_REJECTED', cycle_id, name, {
|
||||
'val_anomaly_rate': round(val_anomaly_rate, 4), 'gate': VAL_ANOMALY_GATE,
|
||||
'val_mean_score': round(val_mean_score, 4), 'version_id': version_id,
|
||||
})
|
||||
# Tenter de réutiliser le modèle précédent
|
||||
if model_path and os.path.exists(model_path):
|
||||
log_info(f"[{name}] Conservation du modèle précédent v{meta.get('version_id', '?')}.")
|
||||
ae_prev = None
|
||||
if TORCH_AVAILABLE and AE_WEIGHT > 0:
|
||||
ae_prev_path = _ae_model_path(name, meta.get('version_id', ''))
|
||||
if os.path.exists(ae_prev_path):
|
||||
try:
|
||||
ae_prev = TrafficAutoEncoder.load_state_dict(torch.load(ae_prev_path, weights_only=False))
|
||||
except Exception:
|
||||
pass
|
||||
return joblib.load(model_path), ae_prev, meta.get('features', features)
|
||||
log_info(f"[{name}] Aucun modèle précédent — utilisation du modèle rejeté par défaut.")
|
||||
|
||||
# A1 — Sauvegarder les statistiques de distribution avec quantile digest pour drift detection
|
||||
baseline_stats = {
|
||||
f: {
|
||||
'mean': float(X[f].mean()), 'std': float(X[f].std()),
|
||||
'p10': float(X[f].quantile(0.10)), 'p25': float(X[f].quantile(0.25)),
|
||||
'p50': float(X[f].quantile(0.50)), 'p75': float(X[f].quantile(0.75)),
|
||||
'p90': float(X[f].quantile(0.90)),
|
||||
}
|
||||
for f in features
|
||||
}
|
||||
|
||||
new_model_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.joblib')
|
||||
new_meta_path = os.path.join(MODEL_DIR, f'model_{name}_{version_id}.meta.json')
|
||||
joblib.dump(model, new_model_path)
|
||||
|
||||
# Entraînement de l'Autoencoder en parallèle (si PyTorch disponible et AE_WEIGHT > 0)
|
||||
ae_model = None
|
||||
if TORCH_AVAILABLE and AE_WEIGHT > 0:
|
||||
try:
|
||||
ae_model = TrafficAutoEncoder(n_features=len(features))
|
||||
ae_stats = ae_model.fit(X_train.values)
|
||||
ae_path = _ae_model_path(name, version_id)
|
||||
torch.save(ae_model.state_dict(), ae_path)
|
||||
log_info(f"[{name}] Autoencoder entraîné : loss={ae_stats['final_loss']:.6f}, epochs={ae_stats['epochs']}")
|
||||
except Exception as exc:
|
||||
log_info(f"[{name}] Autoencoder training échoué : {exc} — AE désactivé.")
|
||||
ae_model = None
|
||||
|
||||
previous_version = meta.get('version_id', None) if meta else None
|
||||
new_meta = {
|
||||
'version_id': version_id, 'trained_at': datetime.now().isoformat(),
|
||||
'human_samples': len(human_baseline), 'contamination': CONTAMINATION,
|
||||
'threshold': ANOMALY_THRESHOLD, 'features': features,
|
||||
'model_name': name, 'previous_version': previous_version,
|
||||
'retrain_interval': RETRAIN_INTERVAL_H, 'baseline_stats': baseline_stats,
|
||||
'algorithm': 'ExtendedIsolationForest' if EIF_AVAILABLE else 'IsolationForest',
|
||||
'autoencoder': ae_model is not None,
|
||||
'ae_weight': AE_WEIGHT if ae_model else 0.0,
|
||||
'validation': {
|
||||
'val_size': len(X_val), 'train_size': len(X_train),
|
||||
'val_mean_score': round(val_mean_score, 4),
|
||||
'val_anomaly_rate': round(val_anomaly_rate, 4),
|
||||
}
|
||||
}
|
||||
with open(new_meta_path, 'w') as f: json.dump(new_meta, f, indent=2)
|
||||
with open(_current_pointer_path(name), 'w') as f: f.write(version_id)
|
||||
|
||||
append_training_history({k: v for k, v in new_meta.items() if k != 'baseline_stats'})
|
||||
_purge_old_versions(name)
|
||||
|
||||
log_info(f"[{name}] Modèle v{version_id} sauvegardé → {new_model_path} (AE={'oui' if ae_model is not None else 'non'})")
|
||||
log_decision('MODEL_TRAINED', cycle_id, name, {
|
||||
'version_id': version_id, 'previous_version': previous_version,
|
||||
'human_samples': len(human_baseline), 'next_retrain_in_h': RETRAIN_INTERVAL_H,
|
||||
'history_kept': MODEL_HISTORY_COUNT
|
||||
})
|
||||
return model, ae_model, features
|
||||
Reference in New Issue
Block a user