refactor: replace hardcoded mabase_prod DB prefix with configurable settings
Replace all hardcoded 'mabase_prod.' table prefixes in dashboard route
SQL queries with configurable database names from settings:
- http_logs, http_logs_raw → settings.CLICKHOUSE_DB_LOGS
- All other tables → settings.CLICKHOUSE_DB_PROCESSING
Also qualify previously unqualified table references (bare FROM/JOIN
table_name) with the appropriate database prefix for consistency.
Each route file now imports 'from ..config import settings' and uses
f-strings with {settings.CLICKHOUSE_DB_PROCESSING} or
{settings.CLICKHOUSE_DB_LOGS} for database-qualified table names.
Files updated: analysis, attributes, audit, botnets, bruteforce,
clustering, detections, entities, fingerprints, header_fingerprint,
heatmap, incidents, investigation_summary, metrics, ml_features,
rotation, search, tcp_spoofing, variability (19 files).
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
@ -11,6 +11,7 @@ from ..models import (
|
||||
EntityRelatedAttributes,
|
||||
EntityAttributeValue
|
||||
)
|
||||
from ..config import settings
|
||||
|
||||
router = APIRouter(prefix="/api/entities", tags=["Entities"])
|
||||
|
||||
@ -24,7 +25,7 @@ def get_entity_stats(entity_type: str, entity_value: str, hours: int = 24) -> Op
|
||||
"""
|
||||
Récupère les statistiques pour une entité donnée
|
||||
"""
|
||||
query = """
|
||||
query = f"""
|
||||
SELECT
|
||||
entity_type,
|
||||
entity_value,
|
||||
@ -32,7 +33,7 @@ def get_entity_stats(entity_type: str, entity_value: str, hours: int = 24) -> Op
|
||||
sum(unique_ips) as unique_ips,
|
||||
min(log_date) as first_seen,
|
||||
max(log_date) as last_seen
|
||||
FROM mabase_prod.view_dashboard_entities
|
||||
FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities
|
||||
WHERE entity_type = %(entity_type)s
|
||||
AND entity_value = %(entity_value)s
|
||||
AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR)
|
||||
@ -64,13 +65,13 @@ def get_related_attributes(entity_type: str, entity_value: str, hours: int = 24)
|
||||
Récupère les attributs associés à une entité
|
||||
"""
|
||||
# Requête pour agréger tous les attributs associés
|
||||
query = """
|
||||
query = f"""
|
||||
SELECT
|
||||
(SELECT groupUniqArray(toString(src_ip)) FROM mabase_prod.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR)) as ips,
|
||||
(SELECT groupUniqArray(ja4) FROM mabase_prod.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND ja4 != '') as ja4s,
|
||||
(SELECT groupUniqArray(host) FROM mabase_prod.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND host != '') as hosts,
|
||||
(SELECT groupUniqArrayArray(asns) FROM mabase_prod.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND notEmpty(asns)) as asns,
|
||||
(SELECT groupUniqArrayArray(countries) FROM mabase_prod.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND notEmpty(countries)) as countries
|
||||
(SELECT groupUniqArray(toString(src_ip)) FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR)) as ips,
|
||||
(SELECT groupUniqArray(ja4) FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND ja4 != '') as ja4s,
|
||||
(SELECT groupUniqArray(host) FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND host != '') as hosts,
|
||||
(SELECT groupUniqArrayArray(asns) FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND notEmpty(asns)) as asns,
|
||||
(SELECT groupUniqArrayArray(countries) FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities WHERE entity_type = %(entity_type)s AND entity_value = %(entity_value)s AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR) AND notEmpty(countries)) as countries
|
||||
"""
|
||||
|
||||
result = db.query(query, {
|
||||
@ -110,7 +111,7 @@ def get_array_values(entity_type: str, entity_value: str, array_field: str, hour
|
||||
FROM (
|
||||
SELECT
|
||||
arrayJoin({array_field}) as value
|
||||
FROM mabase_prod.view_dashboard_entities
|
||||
FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities
|
||||
WHERE entity_type = %(entity_type)s
|
||||
AND entity_value = %(entity_value)s
|
||||
AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR)
|
||||
@ -156,7 +157,7 @@ async def get_subnet_investigation(
|
||||
subnet_third = subnet_parts[2]
|
||||
|
||||
# Stats globales du subnet - utilise ml_detected_anomalies + view_dashboard_entities pour UA
|
||||
stats_query = """
|
||||
stats_query = f"""
|
||||
WITH cleaned_ips AS (
|
||||
SELECT
|
||||
replaceRegexpAll(toString(src_ip), '^::ffff:', '') AS clean_ip,
|
||||
@ -165,7 +166,7 @@ async def get_subnet_investigation(
|
||||
host,
|
||||
country_code,
|
||||
asn_number
|
||||
FROM ml_detected_anomalies
|
||||
FROM {settings.CLICKHOUSE_DB_PROCESSING}.ml_detected_anomalies
|
||||
WHERE detected_at >= now() - INTERVAL %(hours)s HOUR
|
||||
),
|
||||
subnet_filter AS (
|
||||
@ -180,7 +181,7 @@ async def get_subnet_investigation(
|
||||
SELECT
|
||||
entity_value AS ip,
|
||||
arrayJoin(user_agents) AS user_agent
|
||||
FROM view_dashboard_entities
|
||||
FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities
|
||||
WHERE entity_type = 'ip'
|
||||
AND log_date >= toDate(now() - INTERVAL %(hours)s HOUR)
|
||||
AND splitByChar('.', entity_value)[1] = %(subnet_prefix)s
|
||||
@ -227,7 +228,7 @@ async def get_subnet_investigation(
|
||||
}
|
||||
|
||||
# Liste des IPs avec détails - 2 requêtes séparées + fusion en Python
|
||||
ips_query = """
|
||||
ips_query = f"""
|
||||
WITH cleaned_ips AS (
|
||||
SELECT
|
||||
replaceRegexpAll(toString(src_ip), '^::ffff:', '') AS clean_ip,
|
||||
@ -237,7 +238,7 @@ async def get_subnet_investigation(
|
||||
asn_number,
|
||||
threat_level,
|
||||
anomaly_score
|
||||
FROM ml_detected_anomalies
|
||||
FROM {settings.CLICKHOUSE_DB_PROCESSING}.ml_detected_anomalies
|
||||
WHERE detected_at >= now() - INTERVAL %(hours)s HOUR
|
||||
),
|
||||
subnet_filter AS (
|
||||
@ -282,7 +283,7 @@ async def get_subnet_investigation(
|
||||
SELECT
|
||||
entity_value AS ip,
|
||||
uniq(arrayJoin(user_agents)) AS unique_ua
|
||||
FROM view_dashboard_entities
|
||||
FROM {settings.CLICKHOUSE_DB_PROCESSING}.view_dashboard_entities
|
||||
PREWHERE entity_type = 'ip'
|
||||
WHERE entity_value IN ({ip_values})
|
||||
AND log_date >= today() - INTERVAL 30 DAY
|
||||
|
||||
Reference in New Issue
Block a user