refactor: replace hardcoded mabase_prod DB prefix with configurable settings

Replace all hardcoded 'mabase_prod.' table prefixes in dashboard route
SQL queries with configurable database names from settings:

- http_logs, http_logs_raw → settings.CLICKHOUSE_DB_LOGS
- All other tables → settings.CLICKHOUSE_DB_PROCESSING

Also qualify previously unqualified table references (bare FROM/JOIN
table_name) with the appropriate database prefix for consistency.

Each route file now imports 'from ..config import settings' and uses
f-strings with {settings.CLICKHOUSE_DB_PROCESSING} or
{settings.CLICKHOUSE_DB_LOGS} for database-qualified table names.

Files updated: analysis, attributes, audit, botnets, bruteforce,
clustering, detections, entities, fingerprints, header_fingerprint,
heatmap, incidents, investigation_summary, metrics, ml_features,
rotation, search, tcp_spoofing, variability (19 files).

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 19:03:05 +02:00
parent dba2676fa7
commit b6391afbeb
19 changed files with 225 additions and 206 deletions

View File

@ -8,6 +8,7 @@ from fastapi import APIRouter, HTTPException
from ..database import db
from ..services.tcp_fingerprint import fingerprint_os, detect_spoof, declared_os_from_ua
from ..config import settings
router = APIRouter(prefix="/api/investigation", tags=["investigation"])
@ -25,7 +26,7 @@ async def get_ip_full_summary(ip: str):
clean_ip = ip.replace("::ffff:", "").strip()
try:
# ── 1. Score ML / features ─────────────────────────────────────────────
ml_sql = """
ml_sql = f"""
SELECT
max(abs(anomaly_score)) AS max_score,
any(threat_level) AS threat_level,
@ -33,7 +34,7 @@ async def get_ip_full_summary(ip: str):
count() AS total_detections,
uniq(host) AS distinct_hosts,
uniq(ja4) AS distinct_ja4
FROM mabase_prod.ml_detected_anomalies
FROM {settings.CLICKHOUSE_DB_PROCESSING}.ml_detected_anomalies
WHERE replaceRegexpAll(toString(src_ip), '^::ffff:', '') = %(ip)s
"""
ml_res = db.query(ml_sql, {"ip": clean_ip})
@ -48,13 +49,13 @@ async def get_ip_full_summary(ip: str):
}
# ── 2. Brute force ─────────────────────────────────────────────────────
bf_sql = """
bf_sql = f"""
SELECT
uniq(host) AS hosts_attacked,
sum(hits) AS total_hits,
sum(query_params_count) AS total_params,
groupArray(3)(host) AS top_hosts
FROM mabase_prod.view_form_bruteforce_detected
FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_form_bruteforce_detected
WHERE replaceRegexpAll(toString(src_ip), '^::ffff:', '') = %(ip)s
"""
bf_res = db.query(bf_sql, {"ip": clean_ip})
@ -68,14 +69,14 @@ async def get_ip_full_summary(ip: str):
}
# ── 3. TCP spoofing — fingerprinting multi-signal ─────────────────────
tcp_sql = """
tcp_sql = f"""
SELECT
any(tcp_ttl_raw) AS ttl,
any(tcp_win_raw) AS win,
any(tcp_scale_raw) AS scale,
any(tcp_mss_raw) AS mss,
any(first_ua) AS ua
FROM mabase_prod.agg_host_ip_ja4_1h
FROM {settings.CLICKHOUSE_DB_PROCESSING}.agg_host_ip_ja4_1h
WHERE replaceRegexpAll(toString(src_ip), '^::ffff:', '') = %(ip)s
AND window_start >= now() - INTERVAL 24 HOUR
AND tcp_ttl_raw > 0
@ -109,9 +110,9 @@ async def get_ip_full_summary(ip: str):
}
# ── 4. JA4 rotation ────────────────────────────────────────────────────
rot_sql = """
rot_sql = f"""
SELECT distinct_ja4_count, total_hits
FROM mabase_prod.view_host_ip_ja4_rotation
FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_host_ip_ja4_rotation
WHERE replaceRegexpAll(toString(src_ip), '^::ffff:', '') = %(ip)s
LIMIT 1
"""
@ -123,9 +124,9 @@ async def get_ip_full_summary(ip: str):
rot_data = {"rotating": cnt > 1, "distinct_ja4_count": cnt, "total_hits": int(row[1] or 0)}
# ── 5. Persistance ─────────────────────────────────────────────────────
pers_sql = """
pers_sql = f"""
SELECT recurrence, worst_score, worst_threat_level, first_seen, last_seen
FROM mabase_prod.view_ip_recurrence
FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_ip_recurrence
WHERE replaceRegexpAll(toString(src_ip), '^::ffff:', '') = %(ip)s
LIMIT 1
"""
@ -143,12 +144,12 @@ async def get_ip_full_summary(ip: str):
}
# ── 6. Timeline 24h ────────────────────────────────────────────────────
tl_sql = """
tl_sql = f"""
SELECT
toHour(window_start) AS hour,
sum(hits) AS hits,
groupUniqArray(3)(ja4) AS ja4s
FROM mabase_prod.agg_host_ip_ja4_1h
FROM {settings.CLICKHOUSE_DB_PROCESSING}.agg_host_ip_ja4_1h
WHERE replaceRegexpAll(toString(src_ip), '^::ffff:', '') = %(ip)s
AND window_start >= now() - INTERVAL 24 HOUR
GROUP BY hour