feat: split ClickHouse into dual configurable databases (ja4_logs / ja4_processing)
Architecture: - ja4_logs: raw log ingestion (http_logs_raw, http_logs, mv_http_logs) - ja4_processing: analytics, aggregation, ML, dictionaries, audit Configuration (env vars): - CLICKHOUSE_DB_LOGS (default: ja4_logs) - CLICKHOUSE_DB_PROCESSING (default: ja4_processing) Changes: - SQL migrations (10 files): all mabase_prod refs → ja4_logs or ja4_processing with correct cross-database references (MVs, views, dicts) - deploy_schema.sh: substitutes DB names from env vars at deploy time - Python shared settings: added CLICKHOUSE_DB_LOGS + CLICKHOUSE_DB_PROCESSING - Dashboard routes (19 files): replaced ~80 hardcoded mabase_prod refs with settings.CLICKHOUSE_DB_LOGS / settings.CLICKHOUSE_DB_PROCESSING - Bot-detector: DB → CLICKHOUSE_DB_PROCESSING, fetch_rules.py configurable - Correlator: DSN example updated to ja4_logs - Docker-compose + .env files: new env vars with defaults - All documentation updated (14 markdown files) All tests pass: sentinel 10/10, correlator 67.1%, bot-detector 11, dashboard 20, ja4_common 18 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
@ -1,6 +1,6 @@
|
||||
DROP TABLE IF EXISTS mabase_prod.ref_bot_networks;
|
||||
DROP TABLE IF EXISTS ja4_processing.ref_bot_networks;
|
||||
|
||||
CREATE TABLE mabase_prod.ref_bot_networks (
|
||||
CREATE TABLE ja4_processing.ref_bot_networks (
|
||||
-- On utilise IPv6CIDR car il accepte aussi les IPv4 au format ::ffff:1.2.3.4/120
|
||||
network IPv6CIDR,
|
||||
bot_name LowCardinality(String),
|
||||
@ -11,11 +11,11 @@ ORDER BY (network, bot_name);
|
||||
|
||||
|
||||
-- Création de la table lisant le fichier des IPs
|
||||
CREATE TABLE mabase_prod.bot_ip (
|
||||
CREATE TABLE ja4_processing.bot_ip (
|
||||
ip String
|
||||
) ENGINE = File(CSV, 'bot_ip.csv');
|
||||
|
||||
-- Création de la table lisant le fichier des signatures JA4
|
||||
CREATE TABLE mabase_prod.bot_ja4 (
|
||||
CREATE TABLE ja4_processing.bot_ja4 (
|
||||
ja4 String
|
||||
) ENGINE = File(CSV, 'bot_ja4.csv');
|
||||
|
||||
@ -11,13 +11,14 @@
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Base de données
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE DATABASE IF NOT EXISTS mabase_prod;
|
||||
CREATE DATABASE IF NOT EXISTS ja4_logs;
|
||||
CREATE DATABASE IF NOT EXISTS ja4_processing;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Table brute : cible directe des inserts du service
|
||||
-- Le service n'insère que dans cette table (colonne raw_json).
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS mabase_prod.http_logs_raw
|
||||
CREATE TABLE IF NOT EXISTS ja4_logs.http_logs_raw
|
||||
(
|
||||
`raw_json` String CODEC(ZSTD(3)),
|
||||
`ingest_time` DateTime DEFAULT now()
|
||||
@ -34,7 +35,7 @@ SETTINGS
|
||||
-- Table parsée : alimentée automatiquement par la vue matérialisée
|
||||
-- -----------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE mabase_prod.http_logs
|
||||
CREATE TABLE ja4_logs.http_logs
|
||||
(
|
||||
-- Temporel
|
||||
`time` DateTime,
|
||||
@ -118,10 +119,10 @@ SETTINGS
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vue matérialisée : parse le JSON de http_logs_raw vers http_logs
|
||||
-- -----------------------------------------------------------------------------
|
||||
DROP VIEW IF EXISTS mabase_prod.mv_http_logs;
|
||||
DROP VIEW IF EXISTS ja4_logs.mv_http_logs;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS mabase_prod.mv_http_logs
|
||||
TO mabase_prod.http_logs
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS ja4_logs.mv_http_logs
|
||||
TO ja4_logs.http_logs
|
||||
AS
|
||||
SELECT
|
||||
parseDateTimeBestEffort(coalesce(JSONExtractString(raw_json, 'time'), '1970-01-01T00:00:00Z')) AS time,
|
||||
@ -133,31 +134,31 @@ SELECT
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'dst_port'), 0)) AS dst_port,
|
||||
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'asn',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
toUInt32(0)
|
||||
) AS src_asn,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'country_code',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_country_code,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'name',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_as_name,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'org',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_org,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'domain',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
@ -211,7 +212,7 @@ SELECT
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Mode'), '') AS header_sec_fetch_mode,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Site'), '') AS header_sec_fetch_site
|
||||
|
||||
FROM mabase_prod.http_logs_raw;
|
||||
FROM ja4_logs.http_logs_raw;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Utilisateurs et permissions
|
||||
@ -220,15 +221,15 @@ CREATE USER IF NOT EXISTS data_writer IDENTIFIED WITH plaintext_password BY 'Cha
|
||||
CREATE USER IF NOT EXISTS analyst IDENTIFIED WITH plaintext_password BY 'ChangeMe';
|
||||
|
||||
-- data_writer : INSERT uniquement sur la table brute
|
||||
GRANT INSERT ON mabase_prod.http_logs_raw TO data_writer;
|
||||
GRANT SELECT ON mabase_prod.http_logs_raw TO data_writer;
|
||||
GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;
|
||||
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
|
||||
|
||||
-- analyst : lecture sur la table parsée
|
||||
GRANT SELECT ON mabase_prod.http_logs TO analyst;
|
||||
GRANT SELECT ON ja4_logs.http_logs TO analyst;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vérifications post-installation
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- SELECT count(*), min(ingest_time), max(ingest_time) FROM mabase_prod.http_logs_raw;
|
||||
-- SELECT count(*), min(time), max(time) FROM mabase_prod.http_logs;
|
||||
-- SELECT time, src_ip, dst_ip, method, host, path, ja4 FROM mabase_prod.http_logs ORDER BY time DESC LIMIT 10;
|
||||
-- SELECT count(*), min(ingest_time), max(ingest_time) FROM ja4_logs.http_logs_raw;
|
||||
-- SELECT count(*), min(time), max(time) FROM ja4_logs.http_logs;
|
||||
-- SELECT time, src_ip, dst_ip, method, host, path, ja4 FROM ja4_logs.http_logs ORDER BY time DESC LIMIT 10;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
DROP DICTIONARY IF EXISTS mabase_prod.dict_iplocate_asn;
|
||||
DROP DICTIONARY IF EXISTS ja4_processing.dict_iplocate_asn;
|
||||
|
||||
CREATE DICTIONARY IF NOT EXISTS mabase_prod.dict_iplocate_asn
|
||||
CREATE DICTIONARY IF NOT EXISTS ja4_processing.dict_iplocate_asn
|
||||
(
|
||||
network String,
|
||||
asn UInt32,
|
||||
@ -17,10 +17,10 @@ LIFETIME(MIN 3600 MAX 7200);
|
||||
|
||||
|
||||
-- Suppression si existe pour reconfiguration
|
||||
DROP TABLE IF EXISTS mabase_prod.ref_bot_networks;
|
||||
DROP TABLE IF EXISTS ja4_processing.ref_bot_networks;
|
||||
|
||||
-- Table optimisée pour le filtrage binaire de CIDR
|
||||
CREATE TABLE mabase_prod.ref_bot_networks (
|
||||
CREATE TABLE ja4_processing.ref_bot_networks (
|
||||
network IPv6CIDR, -- Gère nativement '1.2.3.0/24' et '2001:db8::/32'
|
||||
bot_name LowCardinality(String),
|
||||
is_legitimate UInt8, -- 1 = Whitelist, 0 = Blacklist
|
||||
|
||||
Reference in New Issue
Block a user