Files
ja4-platform/services/dashboard/backend/routes/heatmap.py
toto d469e39da7 feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized
Services:
- ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap)
- logcorrelator: JA4 log correlation engine (Go, ClickHouse)
- mod_reqin_log: Apache module (C, JSON request logging)
- bot_detector: ML bot detection pipeline (Python)
- dashboard: FastAPI/Streamlit analytics UI (Python)

Shared libraries:
- shared/go/ja4common: logger, config, shutdown, ipfilter (Go module)
- shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package)
- shared/clickhouse/: canonical SQL migrations (10 files)

Build & packaging:
- Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10)
- go.work workspace linking sentinel, correlator, ja4common
- Makefile with test-all, build-all, rpm-* targets

Fixes applied:
- go.work: 1.21 → 1.24.6 (required by sentinel)
- correlator Dockerfiles: golang:1.21 → golang:1.24
- replace directives in go.mod for ja4common local path
- pyproject.toml: setuptools.backends → setuptools.build_meta
- Removed static libpcap linking (unavailable on Rocky 9)
- Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32)
- Rewrote corrupted test files (logger_test.go × 2)

Test coverage:
- correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%)
- sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse)

Documentation:
- README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-07 16:42:59 +02:00

145 lines
4.8 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
Endpoints pour la heatmap temporelle (hits par heure / hôte)
"""
from collections import defaultdict
from fastapi import APIRouter, HTTPException, Query
from ..database import db
router = APIRouter(prefix="/api/heatmap", tags=["heatmap"])
@router.get("/hourly")
async def get_heatmap_hourly():
"""Hits agrégés par heure sur les 72 dernières heures."""
try:
sql = """
SELECT
toHour(window_start) AS hour,
sum(hits) AS hits,
uniq(replaceRegexpAll(toString(src_ip), '^::ffff:', '')) AS unique_ips,
max(max_requests_per_sec) AS max_rps
FROM mabase_prod.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 72 HOUR
GROUP BY hour
ORDER BY hour ASC
"""
result = db.query(sql)
hours = [
{
"hour": int(row[0]),
"hits": int(row[1]),
"unique_ips": int(row[2]),
"max_rps": int(row[3]),
}
for row in result.result_rows
]
return {"hours": hours}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/top-hosts")
async def get_heatmap_top_hosts(limit: int = Query(20, ge=1, le=100)):
"""Hôtes les plus ciblés avec répartition horaire sur 24h."""
try:
# Aggregate overall stats per host
agg_sql = """
SELECT
host,
sum(hits) AS total_hits,
uniq(replaceRegexpAll(toString(src_ip), '^::ffff:', '')) AS unique_ips,
uniq(ja4) AS unique_ja4s
FROM mabase_prod.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 72 HOUR
GROUP BY host
ORDER BY total_hits DESC
LIMIT %(limit)s
"""
agg_res = db.query(agg_sql, {"limit": limit})
top_hosts = [str(r[0]) for r in agg_res.result_rows]
host_stats = {
str(r[0]): {
"host": str(r[0]),
"total_hits": int(r[1]),
"unique_ips": int(r[2]),
"unique_ja4s":int(r[3]),
}
for r in agg_res.result_rows
}
if not top_hosts:
return {"items": []}
# Hourly breakdown per host
hourly_sql = """
SELECT
host,
toHour(window_start) AS hour,
sum(hits) AS hits
FROM mabase_prod.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 72 HOUR
AND host IN %(hosts)s
GROUP BY host, hour
"""
hourly_res = db.query(hourly_sql, {"hosts": top_hosts})
hourly_map: dict = defaultdict(lambda: [0] * 24)
for row in hourly_res.result_rows:
h = str(row[0])
hour = int(row[1])
hits = int(row[2])
hourly_map[h][hour] += hits
items = []
for host in top_hosts:
entry = dict(host_stats[host])
entry["hourly_hits"] = hourly_map[host]
items.append(entry)
return {"items": items}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@router.get("/matrix")
async def get_heatmap_matrix():
"""Matrice top-15 hôtes × 24 heures (sum hits) sur les 72 dernières heures."""
try:
top_sql = """
SELECT host, sum(hits) AS total_hits
FROM mabase_prod.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 72 HOUR
GROUP BY host
ORDER BY total_hits DESC
"""
top_res = db.query(top_sql)
top_hosts = [str(r[0]) for r in top_res.result_rows]
if not top_hosts:
return {"hosts": [], "matrix": []}
cell_sql = """
SELECT
host,
toHour(window_start) AS hour,
sum(hits) AS hits
FROM mabase_prod.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 72 HOUR
AND host IN %(hosts)s
GROUP BY host, hour
"""
cell_res = db.query(cell_sql, {"hosts": top_hosts})
matrix_map: dict = defaultdict(lambda: [0] * 24)
for row in cell_res.result_rows:
h = str(row[0])
hour = int(row[1])
hits = int(row[2])
matrix_map[h][hour] += hits
matrix = [matrix_map[h] for h in top_hosts]
return {"hosts": top_hosts, "matrix": matrix}
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))