feat(dashboard): rebuild SOC dashboard + fix ClickHouse SQL
Complete rewrite of the SOC dashboard using FastAPI + Jinja2 + htmx + Chart.js + Tailwind CSS. Replaces the old React/Vite frontend with server-rendered templates. Dashboard pages: - Overview: KPIs, timeline chart, threat distribution, top IPs - Detections: paginated/filterable anomaly table - Scores: ml_all_scores with AE error & XGB prob columns - Traffic: HTTP logs with method/host filters - IP Investigation: full deep-dive (scores, features, HTTP logs, classify) - Classification: SOC feedback form + history - Features: AI + thesis feature stats - Models: scoring stats + model metadata API: 9 JSON endpoints with parameterized queries, sort whitelists SQL fixes: - 05_aggregation_tables: add deduplicate_merge_projection_mode - 11_views: fix nested aggregate (argMax inside sum) - 12_thesis_features: remove invalid 'let' bindings, fix groupArrayIf type Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
@ -1,237 +1,37 @@
|
||||
"""
|
||||
Bot Detector Dashboard - API Backend
|
||||
FastAPI application pour servir le dashboard web
|
||||
"""
|
||||
"""JA4 SOC Dashboard — FastAPI application."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import FastAPI, HTTPException
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from fastapi.responses import FileResponse
|
||||
import os
|
||||
|
||||
from .config import settings
|
||||
from .database import db
|
||||
from .routes import metrics, detections, variability, attributes, analysis, entities, incidents, audit, reputation, fingerprints
|
||||
from .routes import bruteforce, tcp_spoofing, header_fingerprint, heatmap, botnets, rotation, ml_features, investigation_summary, search, clustering
|
||||
from backend.routes.api import router as api_router
|
||||
from backend.routes.pages import router as pages_router
|
||||
|
||||
# Configuration logging
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
||||
|
||||
app = FastAPI(title="JA4 SOC Dashboard", version="1.0.0")
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Gestion du cycle de vie de l'application"""
|
||||
# Startup
|
||||
logger.info("Démarrage du Bot Detector Dashboard API...")
|
||||
logger.info(f"ClickHouse: {settings.CLICKHOUSE_HOST}:{settings.CLICKHOUSE_PORT}")
|
||||
logger.info(f"Database: {settings.CLICKHOUSE_DB}")
|
||||
|
||||
# Tester la connexion ClickHouse
|
||||
try:
|
||||
client = db.connect()
|
||||
client.ping()
|
||||
logger.info("Connexion ClickHouse établie avec succès")
|
||||
except Exception as e:
|
||||
logger.error(f"Échec de connexion ClickHouse: {e}")
|
||||
raise
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
logger.info("Arrêt du Bot Detector Dashboard API...")
|
||||
db.close()
|
||||
|
||||
|
||||
# Création de l'application FastAPI
|
||||
OPENAPI_TAGS = [
|
||||
{
|
||||
"name": "Metrics",
|
||||
"description": "Métriques globales : comptages, niveaux de menace, baseline et distribution des scores ML.",
|
||||
},
|
||||
{
|
||||
"name": "Detections",
|
||||
"description": "Liste paginée et filtrée des anomalies détectées par le modèle ML. Supporte tri, recherche texte et regroupement par IP.",
|
||||
},
|
||||
{
|
||||
"name": "investigation",
|
||||
"description": (
|
||||
"**Point d'entrée principal pour l'analyse d'une IP.** "
|
||||
"Agrège en un seul appel : score ML, brute-force, spoofing TCP, rotation JA4, persistance et timeline 24h. "
|
||||
"Retourne un `risk_score` heuristique de 0 à 100."
|
||||
),
|
||||
},
|
||||
{
|
||||
"name": "Reputation",
|
||||
"description": "Réputation externe d'une IP via IP-API.com et IPinfo.io (sans clé API). Détecte proxies, VPN, Tor, hébergeurs.",
|
||||
},
|
||||
{
|
||||
"name": "Analysis",
|
||||
"description": "Analyses approfondies par IP : subnet, pays, empreintes JA4, user-agents, recommandation SOC et gestion des classifications.",
|
||||
},
|
||||
{
|
||||
"name": "Entities",
|
||||
"description": "Investigation par entité (IP, JA4, subnet, user-agent, host). Retourne détections associées, user-agents, chemins, paramètres et entités liées.",
|
||||
},
|
||||
{
|
||||
"name": "Incidents",
|
||||
"description": "Clusters d'incidents actifs regroupés par similarité comportementale. Permet la classification et le suivi des incidents.",
|
||||
},
|
||||
{
|
||||
"name": "Fingerprints",
|
||||
"description": "Analyse des empreintes JA4/TLS : spoofing, matrice JA4↔UA, user-agents suspects, cohérence par IP, JA4 légitimes et corrélation ASN.",
|
||||
},
|
||||
{
|
||||
"name": "Bruteforce",
|
||||
"description": "Détection des attaques brute-force : cibles, attaquants, timeline et détail par host.",
|
||||
},
|
||||
{
|
||||
"name": "TCP Spoofing",
|
||||
"description": "Détection du spoofing TCP/OS fingerprinting : vue d'ensemble, liste et matrice TTL×MSS.",
|
||||
},
|
||||
{
|
||||
"name": "Header Fingerprint",
|
||||
"description": "Clusters de fingerprints d'en-têtes HTTP suspects et IPs associées.",
|
||||
},
|
||||
{
|
||||
"name": "Heatmap",
|
||||
"description": "Heatmap horaire du trafic, top hosts et matrice activité/heure.",
|
||||
},
|
||||
{
|
||||
"name": "Botnets",
|
||||
"description": "Détection de botnets : spread JA4, distribution géographique par JA4, résumé global.",
|
||||
},
|
||||
{
|
||||
"name": "Rotation",
|
||||
"description": "Détection de la rotation JA4 (évasion de détection), menaces persistantes, historique JA4 par IP et score de sophistication.",
|
||||
},
|
||||
{
|
||||
"name": "ML Features",
|
||||
"description": "Données brutes du modèle ML : top anomalies, radar par IP, distribution des scores, tendances, features B et scatter plot.",
|
||||
},
|
||||
{
|
||||
"name": "Attributes",
|
||||
"description": "Listes des valeurs distinctes d'attributs (JA4, user-agents, ASN, pays…) avec comptages.",
|
||||
},
|
||||
{
|
||||
"name": "Variability",
|
||||
"description": "Variabilité comportementale : IPs par attribut, attributs par valeur, analyse des user-agents.",
|
||||
},
|
||||
{
|
||||
"name": "Clustering",
|
||||
"description": "Clustering K-Means des IPs sur les features ML. Statut du cache, clusters, points et IPs par cluster.",
|
||||
},
|
||||
{
|
||||
"name": "Search",
|
||||
"description": "Recherche rapide cross-entités (IP, JA4, host, user-agent, pays, ASN).",
|
||||
},
|
||||
{
|
||||
"name": "Audit",
|
||||
"description": "Journal d'audit SOC : création de logs, consultation filtrée, statistiques et activité par utilisateur.",
|
||||
},
|
||||
]
|
||||
|
||||
app = FastAPI(
|
||||
title="Bot Detector Dashboard API",
|
||||
description=(
|
||||
"API REST du **Bot Detector SOC Dashboard**.\n\n"
|
||||
"Permet d'interroger les bases ClickHouse (`ja4_logs` / `ja4_processing`) pour visualiser et analyser "
|
||||
"les détections de bots générées par le service `bot_detector_ai`.\n\n"
|
||||
"**Endpoint clé :** `GET /api/investigation/{ip}/summary` — synthèse complète en un appel.\n\n"
|
||||
"Documentation interactive : `/docs` (Swagger UI) · `/redoc` (ReDoc)"
|
||||
),
|
||||
version="1.0.0",
|
||||
openapi_tags=OPENAPI_TAGS,
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# Configuration CORS
|
||||
# CORS — allow all origins for dashboard access
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=settings.CORS_ORIGINS,
|
||||
allow_origins=["*"],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# Enregistrement des routes
|
||||
app.include_router(metrics.router)
|
||||
app.include_router(detections.router)
|
||||
app.include_router(variability.router)
|
||||
app.include_router(attributes.router)
|
||||
app.include_router(analysis.router)
|
||||
app.include_router(entities.router)
|
||||
app.include_router(incidents.router)
|
||||
app.include_router(audit.router)
|
||||
app.include_router(reputation.router)
|
||||
app.include_router(fingerprints.router)
|
||||
app.include_router(bruteforce.router)
|
||||
app.include_router(tcp_spoofing.router)
|
||||
app.include_router(header_fingerprint.router)
|
||||
app.include_router(heatmap.router)
|
||||
app.include_router(botnets.router)
|
||||
app.include_router(rotation.router)
|
||||
app.include_router(ml_features.router)
|
||||
app.include_router(investigation_summary.router)
|
||||
app.include_router(search.router)
|
||||
app.include_router(clustering.router)
|
||||
# Static assets
|
||||
app.mount("/static", StaticFiles(directory="backend/static"), name="static")
|
||||
|
||||
# Routers — API first so /api/* paths match before page catch-all
|
||||
app.include_router(api_router)
|
||||
app.include_router(pages_router)
|
||||
|
||||
|
||||
# Chemin vers le fichier index.html du frontend (utilisé par serve_frontend et serve_spa)
|
||||
_FRONTEND_INDEX = os.path.join(os.path.dirname(__file__), "..", "frontend", "dist", "index.html")
|
||||
|
||||
# Route pour servir le frontend
|
||||
@app.get("/")
|
||||
async def serve_frontend():
|
||||
"""Sert l'application React"""
|
||||
if os.path.exists(_FRONTEND_INDEX):
|
||||
return FileResponse(_FRONTEND_INDEX)
|
||||
return {"message": "Dashboard API - Frontend non construit. Voir /docs pour l'API."}
|
||||
|
||||
|
||||
# Servir les assets statiques
|
||||
_assets_path = os.path.join(os.path.dirname(__file__), "..", "frontend", "dist", "assets")
|
||||
if os.path.exists(_assets_path):
|
||||
try:
|
||||
app.mount("/assets", StaticFiles(directory=_assets_path), name="assets")
|
||||
except Exception as _e:
|
||||
logger.warning(f"Impossible de monter les assets statiques : {_e}")
|
||||
|
||||
|
||||
# Health check
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
"""Endpoint de santé pour le health check Docker"""
|
||||
try:
|
||||
db.connect().ping()
|
||||
return {"status": "healthy", "clickhouse": "connected"}
|
||||
except Exception as e:
|
||||
return {"status": "unhealthy", "clickhouse": "disconnected", "error": str(e)}
|
||||
|
||||
|
||||
# Route catch-all pour le routing SPA (React Router) - DOIT ÊTRE EN DERNIER
|
||||
# Sauf pour /api/* qui doit être géré par les routers
|
||||
@app.get("/{full_path:path}")
|
||||
async def serve_spa(full_path: str):
|
||||
"""Redirige toutes les routes vers index.html pour le routing React"""
|
||||
# Ne pas intercepter les routes API
|
||||
if full_path.startswith("api/"):
|
||||
raise HTTPException(status_code=404, detail="API endpoint not found")
|
||||
|
||||
if os.path.exists(_FRONTEND_INDEX):
|
||||
return FileResponse(_FRONTEND_INDEX)
|
||||
return {"message": "Dashboard API - Frontend non construit"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
host=settings.API_HOST,
|
||||
port=settings.API_PORT,
|
||||
reload=True
|
||||
)
|
||||
async def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
Reference in New Issue
Block a user