code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: split ClickHouse into dual configurable databases (ja4_logs / ja4_processing)

Browse Source 
Architecture:
- ja4_logs: raw log ingestion (http_logs_raw, http_logs, mv_http_logs)
- ja4_processing: analytics, aggregation, ML, dictionaries, audit

Configuration (env vars):
- CLICKHOUSE_DB_LOGS (default: ja4_logs)
- CLICKHOUSE_DB_PROCESSING (default: ja4_processing)

Changes:
- SQL migrations (10 files): all mabase_prod refs → ja4_logs or ja4_processing
  with correct cross-database references (MVs, views, dicts)
- deploy_schema.sh: substitutes DB names from env vars at deploy time
- Python shared settings: added CLICKHOUSE_DB_LOGS + CLICKHOUSE_DB_PROCESSING
- Dashboard routes (19 files): replaced ~80 hardcoded mabase_prod refs
  with settings.CLICKHOUSE_DB_LOGS / settings.CLICKHOUSE_DB_PROCESSING
- Bot-detector: DB → CLICKHOUSE_DB_PROCESSING, fetch_rules.py configurable
- Correlator: DSN example updated to ja4_logs
- Docker-compose + .env files: new env vars with defaults
- All documentation updated (14 markdown files)

All tests pass: sentinel 10/10, correlator 67.1%, bot-detector 11, dashboard 20, ja4_common 18

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 19:10:35 +02:00
parent b6391afbeb
commit 9f3e0621e5
46 changed files with 638 additions and 549 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
shared/clickhouse/00_database.sql
View File

@ -1,4 +1,5 @@
-- =============================================================================
-- 00_database.sql — Create the production database
-- 00_database.sql — Create the production databases
-- =============================================================================
CREATE DATABASE IF NOT EXISTS mabase_prod;
CREATE DATABASE IF NOT EXISTS ja4_logs;
CREATE DATABASE IF NOT EXISTS ja4_processing;

2
shared/clickhouse/01_raw_tables.sql
View File

@ -2,7 +2,7 @@
-- 01_raw_tables.sql — Raw ingest table (direct target for logcorrelator inserts)
-- =============================================================================
CREATE TABLE IF NOT EXISTS mabase_prod.http_logs_raw
CREATE TABLE IF NOT EXISTS ja4_logs.http_logs_raw
(
`raw_json` String CODEC(ZSTD(3)),
`ingest_time` DateTime DEFAULT now()

10
shared/clickhouse/02_dictionaries.sql
View File

@ -6,9 +6,9 @@
-- IPLocate ASN geo-location dictionary
-- CSV file must be placed at: /var/lib/clickhouse/user_files/iplocate-ip-to-asn.csv
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS mabase_prod.dict_iplocate_asn;
DROP DICTIONARY IF EXISTS ja4_processing.dict_iplocate_asn;
CREATE DICTIONARY IF NOT EXISTS mabase_prod.dict_iplocate_asn
CREATE DICTIONARY IF NOT EXISTS ja4_processing.dict_iplocate_asn
(
network String,
asn UInt32,
@ -26,7 +26,7 @@ LIFETIME(MIN 3600 MAX 7200);
-- -----------------------------------------------------------------------------
-- Bot network CIDR reference table (ReplacingMergeTree for upserts)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.ref_bot_networks
CREATE TABLE IF NOT EXISTS ja4_processing.ref_bot_networks
(
network IPv6CIDR,
bot_name LowCardinality(String),
@ -41,7 +41,7 @@ ORDER BY (network, bot_name);
-- Bot IP flat table (CSV file engine)
-- CSV file must be placed at: /var/lib/clickhouse/user_files/bot_ip.csv
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.bot_ip
CREATE TABLE IF NOT EXISTS ja4_processing.bot_ip
(
ip String
)
@ -52,7 +52,7 @@ ENGINE = File(CSV, 'bot_ip.csv');
-- Bot JA4 flat table (CSV file engine)
-- CSV file must be placed at: /var/lib/clickhouse/user_files/bot_ja4.csv
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.bot_ja4
CREATE TABLE IF NOT EXISTS ja4_processing.bot_ja4
(
ja4 String
)

40
shared/clickhouse/03_anubis_tables.sql
View File

@ -6,7 +6,7 @@
-- -----------------------------------------------------------------------------
-- 1. TABLE SOURCE — User-Agent rules (for REGEXP_TREE dictionary)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.anubis_ua_rules
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_ua_rules
(
id UInt64,
parent_id UInt64,
@ -21,7 +21,7 @@ ORDER BY id;
-- -----------------------------------------------------------------------------
-- 2. TABLE SOURCE — IP/CIDR rules (for IP_TRIE dictionary)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.anubis_ip_rules
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_ip_rules
(
prefix String,
bot_name LowCardinality(String),
@ -36,29 +36,29 @@ ORDER BY prefix;
-- -----------------------------------------------------------------------------
-- 3. DICTIONARY — UA REGEXP_TREE
-- dictGet('mabase_prod.dict_anubis_ua', 'bot_name', header_user_agent)
-- dictGet('ja4_processing.dict_anubis_ua', 'bot_name', header_user_agent)
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS mabase_prod.dict_anubis_ua;
CREATE DICTIONARY mabase_prod.dict_anubis_ua
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_ua;
CREATE DICTIONARY ja4_processing.dict_anubis_ua
(
regexp String,
bot_name String,
action String
)
PRIMARY KEY regexp
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'mabase_prod' TABLE 'anubis_ua_rules'))
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'ja4_processing' TABLE 'anubis_ua_rules'))
LAYOUT(REGEXP_TREE)
LIFETIME(MIN 300 MAX 600);
-- -----------------------------------------------------------------------------
-- 4. DICTIONARY — IP IP_TRIE
-- dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', toIPv6(src_ip), '')
-- dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', toIPv6(src_ip), '')
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS mabase_prod.dict_anubis_ip;
CREATE DICTIONARY mabase_prod.dict_anubis_ip
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_ip;
CREATE DICTIONARY ja4_processing.dict_anubis_ip
(
prefix String,
bot_name String,
@ -68,7 +68,7 @@ CREATE DICTIONARY mabase_prod.dict_anubis_ip
category String
)
PRIMARY KEY prefix
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'mabase_prod' TABLE 'anubis_ip_rules'))
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'ja4_processing' TABLE 'anubis_ip_rules'))
LAYOUT(IP_TRIE())
LIFETIME(MIN 300 MAX 600);
@ -76,7 +76,7 @@ LIFETIME(MIN 300 MAX 600);
-- -----------------------------------------------------------------------------
-- 5. TABLE SOURCE — ASN rules (for Flat dictionary)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.anubis_asn_rules
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_asn_rules
(
asn UInt32,
bot_name LowCardinality(String),
@ -90,7 +90,7 @@ ORDER BY asn;
-- -----------------------------------------------------------------------------
-- 6. TABLE SOURCE — Country rules (for Flat dictionary)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.anubis_country_rules
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_country_rules
(
country_code LowCardinality(String),
bot_name LowCardinality(String),
@ -103,11 +103,11 @@ ORDER BY country_code;
-- -----------------------------------------------------------------------------
-- 7. DICTIONARY — ASN Flat
-- dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', src_asn, '')
-- dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', src_asn, '')
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS mabase_prod.dict_anubis_asn;
CREATE DICTIONARY mabase_prod.dict_anubis_asn
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_asn;
CREATE DICTIONARY ja4_processing.dict_anubis_asn
(
asn UInt32,
bot_name String,
@ -115,18 +115,18 @@ CREATE DICTIONARY mabase_prod.dict_anubis_asn
category String
)
PRIMARY KEY asn
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'mabase_prod' TABLE 'anubis_asn_rules'))
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'ja4_processing' TABLE 'anubis_asn_rules'))
LAYOUT(FLAT())
LIFETIME(MIN 300 MAX 600);
-- -----------------------------------------------------------------------------
-- 8. DICTIONARY — Country Flat
-- dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', src_country_code, '')
-- dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', src_country_code, '')
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS mabase_prod.dict_anubis_country;
CREATE DICTIONARY mabase_prod.dict_anubis_country
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_country;
CREATE DICTIONARY ja4_processing.dict_anubis_country
(
country_code String,
bot_name String,
@ -134,6 +134,6 @@ CREATE DICTIONARY mabase_prod.dict_anubis_country
category String
)
PRIMARY KEY country_code
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'mabase_prod' TABLE 'anubis_country_rules'))
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'ja4_processing' TABLE 'anubis_country_rules'))
LAYOUT(FLAT())
LIFETIME(MIN 300 MAX 600);

116
shared/clickhouse/04_mv_http_logs.sql
View File

@ -11,7 +11,7 @@
-- http_logs — parsed destination table (populated by mv_http_logs)
-- Includes Anubis enrichment columns added by deploy_schema.sql items 9+
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.http_logs
CREATE TABLE IF NOT EXISTS ja4_logs.http_logs
(
-- Temporal
`time` DateTime,
@ -104,16 +104,16 @@ SETTINGS
-- > (4) ASN > (5) Country
-- Uses WITH clause CTEs for clean intermediate values.
-- -----------------------------------------------------------------------------
DROP VIEW IF EXISTS mabase_prod.mv_http_logs;
DROP VIEW IF EXISTS ja4_logs.mv_http_logs;
CREATE MATERIALIZED VIEW mabase_prod.mv_http_logs
TO mabase_prod.http_logs
CREATE MATERIALIZED VIEW ja4_logs.mv_http_logs
TO ja4_logs.http_logs
AS
WITH
coalesce(JSONExtractString(raw_json, 'header_User-Agent'), '') AS _ua,
toIPv6(toIPv4(coalesce(JSONExtractString(raw_json, 'src_ip'), '0.0.0.0'))) AS _ip,
toUInt32(dictGetOrDefault('mabase_prod.dict_iplocate_asn', 'asn', _ip, toUInt32(0))) AS _asn,
dictGetOrDefault('mabase_prod.dict_iplocate_asn', 'country_code', _ip, '') AS _cc
toUInt32(dictGetOrDefault('ja4_processing.dict_iplocate_asn', 'asn', _ip, toUInt32(0))) AS _asn,
dictGetOrDefault('ja4_processing.dict_iplocate_asn', 'country_code', _ip, '') AS _cc
SELECT
parseDateTimeBestEffort(coalesce(JSONExtractString(raw_json, 'time'), '1970-01-01T00:00:00Z')) AS time,
toDate(time) AS log_date,
@ -123,9 +123,9 @@ SELECT
_cc AS src_country_code,
toIPv4(coalesce(JSONExtractString(raw_json, 'dst_ip'), '0.0.0.0')) AS dst_ip,
toUInt16(coalesce(JSONExtractUInt(raw_json, 'dst_port'), 0)) AS dst_port,
dictGetOrDefault('mabase_prod.dict_iplocate_asn', 'name', _ip, '') AS src_as_name,
dictGetOrDefault('mabase_prod.dict_iplocate_asn', 'org', _ip, '') AS src_org,
dictGetOrDefault('mabase_prod.dict_iplocate_asn', 'domain', _ip, '') AS src_domain,
dictGetOrDefault('ja4_processing.dict_iplocate_asn', 'name', _ip, '') AS src_as_name,
dictGetOrDefault('ja4_processing.dict_iplocate_asn', 'org', _ip, '') AS src_org,
dictGetOrDefault('ja4_processing.dict_iplocate_asn', 'domain', _ip, '') AS src_domain,
coalesce(JSONExtractString(raw_json, 'method'), '') AS method,
coalesce(JSONExtractString(raw_json, 'scheme'), '') AS scheme,
coalesce(JSONExtractString(raw_json, 'host'), '') AS host,
@ -172,63 +172,63 @@ SELECT
-- Anubis enrichment: combined UA+IP priority logic
-- Priority: (1) UA+IP [same rule_id] > (2) UA only > (3) IP only > (4) ASN > (5) Country
CASE
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('mabase_prod.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('mabase_prod.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua)
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua)
WHEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', _ip, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', _asn, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', _cc, '')
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '')
ELSE ''
END AS anubis_bot_name,
CASE
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('mabase_prod.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('mabase_prod.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('mabase_prod.dict_anubis_ua', 'action', _ua)
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('mabase_prod.dict_anubis_ua', 'action', _ua)
WHEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'action', _ip, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'action', _asn, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'action', _cc, '')
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', _ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', _ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'action', _ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'action', _asn, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'action', _cc, '')
ELSE ''
END AS anubis_bot_action,
CASE
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('mabase_prod.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('mabase_prod.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('mabase_prod.dict_anubis_ua', 'category', _ua)
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('mabase_prod.dict_anubis_ua', 'category', _ua)
WHEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'category', _ip, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'category', _asn, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'category', _cc, '')
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', _ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', _ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'category', _ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'category', _asn, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'category', _cc, '')
ELSE ''
END AS anubis_bot_category
FROM mabase_prod.http_logs_raw;
FROM ja4_logs.http_logs_raw;

32
shared/clickhouse/05_aggregation_tables.sql
View File

@ -7,8 +7,8 @@
-- Bot reputation dictionaries (in-RAM for fast lookup)
-- CSV files must be placed at: /var/lib/clickhouse/user_files/
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS mabase_prod.dict_bot_ip;
CREATE DICTIONARY mabase_prod.dict_bot_ip
DROP DICTIONARY IF EXISTS ja4_processing.dict_bot_ip;
CREATE DICTIONARY ja4_processing.dict_bot_ip
(
prefix String,
bot_name String
@ -18,8 +18,8 @@ SOURCE(FILE(path '/var/lib/clickhouse/user_files/bot_ip.csv' format 'CSV'))
LAYOUT(IP_TRIE())
LIFETIME(MIN 300 MAX 300);
DROP DICTIONARY IF EXISTS mabase_prod.dict_bot_ja4;
CREATE DICTIONARY mabase_prod.dict_bot_ja4
DROP DICTIONARY IF EXISTS ja4_processing.dict_bot_ja4;
CREATE DICTIONARY ja4_processing.dict_bot_ja4
(
ja4 String,
bot_name String
@ -29,8 +29,8 @@ SOURCE(FILE(path '/var/lib/clickhouse/user_files/bot_ja4.csv' format 'CSV'))
LAYOUT(COMPLEX_KEY_HASHED())
LIFETIME(MIN 300 MAX 300);
DROP DICTIONARY IF EXISTS mabase_prod.dict_asn_reputation;
CREATE DICTIONARY mabase_prod.dict_asn_reputation
DROP DICTIONARY IF EXISTS ja4_processing.dict_asn_reputation;
CREATE DICTIONARY ja4_processing.dict_asn_reputation
(
src_asn UInt64,
label String
@ -44,7 +44,7 @@ LIFETIME(MIN 300 MAX 300);
-- -----------------------------------------------------------------------------
-- agg_host_ip_ja4_1h — behavioral aggregation (L4/L5/L7)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.agg_host_ip_ja4_1h
CREATE TABLE IF NOT EXISTS ja4_processing.agg_host_ip_ja4_1h
(
window_start DateTime,
src_ip IPv6, ja4 String, host String, src_asn UInt32,
@ -107,10 +107,10 @@ ORDER BY (window_start, src_ip, ja4, host);
-- -----------------------------------------------------------------------------
-- mv_agg_host_ip_ja4_1h — feeds agg_host_ip_ja4_1h from http_logs
-- -----------------------------------------------------------------------------
DROP VIEW IF EXISTS mabase_prod.mv_agg_host_ip_ja4_1h;
DROP VIEW IF EXISTS ja4_processing.mv_agg_host_ip_ja4_1h;
CREATE MATERIALIZED VIEW mabase_prod.mv_agg_host_ip_ja4_1h
TO mabase_prod.agg_host_ip_ja4_1h AS
CREATE MATERIALIZED VIEW ja4_processing.mv_agg_host_ip_ja4_1h
TO ja4_processing.agg_host_ip_ja4_1h AS
SELECT
toStartOfHour(src.time) AS window_start,
toIPv6(src.src_ip) AS src_ip, src.ja4, src.host, src.src_asn,
@ -151,14 +151,14 @@ SELECT
sum(toUInt64(src.correlated)) AS count_correlated,
sum(IF(length(src.header_accept_encoding) = 0, 1, 0)) AS count_no_accept_enc,
sum(IF(src.scheme = 'http', 1, 0)) AS count_http_scheme
FROM mabase_prod.http_logs AS src
FROM ja4_logs.http_logs AS src
GROUP BY window_start, src_ip, ja4, host, src_asn;
-- -----------------------------------------------------------------------------
-- agg_header_fingerprint_1h — header fingerprint aggregation (L7)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.agg_header_fingerprint_1h
CREATE TABLE IF NOT EXISTS ja4_processing.agg_header_fingerprint_1h
(
window_start DateTime,
src_ip IPv6,
@ -175,10 +175,10 @@ CREATE TABLE IF NOT EXISTS mabase_prod.agg_header_fingerprint_1h
ENGINE = AggregatingMergeTree()
ORDER BY (window_start, src_ip);
DROP VIEW IF EXISTS mabase_prod.mv_agg_header_fingerprint_1h;
DROP VIEW IF EXISTS ja4_processing.mv_agg_header_fingerprint_1h;
CREATE MATERIALIZED VIEW mabase_prod.mv_agg_header_fingerprint_1h
TO mabase_prod.agg_header_fingerprint_1h AS
CREATE MATERIALIZED VIEW ja4_processing.mv_agg_header_fingerprint_1h
TO ja4_processing.agg_header_fingerprint_1h AS
SELECT
toStartOfHour(src.time) AS window_start,
toIPv6(src.src_ip) AS src_ip,
@ -191,5 +191,5 @@ SELECT
max(toUInt8(if((position(src.header_user_agent, 'Windows') > 0 AND position(src.header_sec_ch_ua_platform, 'Windows') == 0) OR (position(src.header_user_agent, 'iPhone') > 0 AND position(src.header_sec_ch_ua_platform, 'iOS') == 0), 1, 0))) AS ua_ch_mismatch,
any(src.header_sec_fetch_mode) AS sec_fetch_mode,
any(src.header_sec_fetch_dest) AS sec_fetch_dest
FROM mabase_prod.http_logs AS src
FROM ja4_logs.http_logs AS src
GROUP BY window_start, src.src_ip;

8
shared/clickhouse/06_ml_tables.sql
View File

@ -6,7 +6,7 @@
-- -----------------------------------------------------------------------------
-- ml_detected_anomalies — anomaly detections above threat threshold
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.ml_detected_anomalies
CREATE TABLE IF NOT EXISTS ja4_processing.ml_detected_anomalies
(
detected_at DateTime, src_ip IPv6, ja4 String, host String, bot_name String,
anomaly_score Float32, threat_level String, model_name String, recurrence UInt32,
@ -42,7 +42,7 @@ TTL detected_at + INTERVAL 30 DAY;
-- -----------------------------------------------------------------------------
-- ml_all_scores — all classifications (no threshold, for observability)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS mabase_prod.ml_all_scores
CREATE TABLE IF NOT EXISTS ja4_processing.ml_all_scores
(
detected_at DateTime,
window_start DateTime,
@ -78,7 +78,7 @@ SETTINGS index_granularity = 8192;
-- -----------------------------------------------------------------------------
-- view_ip_recurrence — recurrence aggregation over ml_detected_anomalies
-- -----------------------------------------------------------------------------
CREATE OR REPLACE VIEW mabase_prod.view_ip_recurrence AS
CREATE OR REPLACE VIEW ja4_processing.view_ip_recurrence AS
SELECT
src_ip,
count() AS recurrence,
@ -86,5 +86,5 @@ SELECT
max(detected_at) AS last_seen,
min(anomaly_score) AS worst_score,
argMin(threat_level, anomaly_score) AS worst_threat_level
FROM mabase_prod.ml_detected_anomalies
FROM ja4_processing.ml_detected_anomalies
GROUP BY src_ip;

108
shared/clickhouse/07_ai_features_view.sql
View File

@ -4,75 +4,75 @@
-- Includes combined UA+IP priority logic and Anubis bot_name/action/category.
-- =============================================================================
CREATE OR REPLACE VIEW mabase_prod.view_ai_features_1h AS
CREATE OR REPLACE VIEW ja4_processing.view_ai_features_1h AS
WITH base_data AS (
SELECT
a.window_start, a.src_ip, a.ja4, a.host,
toString(a.src_asn) AS asn_number,
a.src_as_name AS asn_org, a.src_org AS asn_detail, a.src_domain AS asn_domain,
a.src_country_code AS country_code,
dictGetOrDefault('mabase_prod.dict_asn_reputation', 'label', toUInt64(a.src_asn), 'unknown') AS asn_label,
dictGetOrDefault('ja4_processing.dict_asn_reputation', 'label', toUInt64(a.src_asn), 'unknown') AS asn_label,
COALESCE(
nullIf(dictGetOrDefault('mabase_prod.dict_bot_ip', 'bot_name', a.src_ip, ''), ''),
nullIf(dictGetOrDefault('mabase_prod.dict_bot_ja4', 'bot_name', tuple(a.ja4), ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_bot_ip', 'bot_name', a.src_ip, ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_bot_ja4', 'bot_name', tuple(a.ja4), ''), ''),
''
) AS bot_name,
-- Anubis: combined UA+IP priority logic > UA only > IP only > ASN > Country
CASE
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('mabase_prod.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('mabase_prod.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua)
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua)
WHEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', a.src_ip, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', a.src_country_code, '')
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '')
ELSE ''
END AS anubis_bot_name,
CASE
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('mabase_prod.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('mabase_prod.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('mabase_prod.dict_anubis_ua', 'action', a.first_ua)
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('mabase_prod.dict_anubis_ua', 'action', a.first_ua)
WHEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'action', a.src_ip, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'action', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'action', a.src_country_code, '')
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', a.first_ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', a.first_ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'action', a.src_ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'action', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'action', a.src_country_code, '')
ELSE ''
END AS anubis_bot_action,
CASE
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('mabase_prod.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('mabase_prod.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('mabase_prod.dict_anubis_ua', 'category', a.first_ua)
WHEN dictGet('mabase_prod.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('mabase_prod.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('mabase_prod.dict_anubis_ua', 'category', a.first_ua)
WHEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('mabase_prod.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_ip', 'category', a.src_ip, '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_asn', 'category', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('mabase_prod.dict_anubis_country', 'category', a.src_country_code, '')
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', a.first_ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', a.first_ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'category', a.src_ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'category', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'category', a.src_country_code, '')
ELSE ''
END AS anubis_bot_category,
a.hits AS hits,
@ -162,7 +162,7 @@ WITH base_data AS (
sum(count_correlated) AS count_correlated_val,
sum(count_no_accept_enc) AS count_no_accept_enc_val,
sum(count_http_scheme) AS count_http_scheme_val
FROM mabase_prod.agg_host_ip_ja4_1h
FROM ja4_processing.agg_host_ip_ja4_1h
WHERE window_start >= now() - INTERVAL 24 HOUR
GROUP BY window_start, src_ip, ja4, host, src_asn
) a
@ -173,7 +173,7 @@ WITH base_data AS (
max(has_cookie) AS has_cookie, max(has_referer) AS has_referer,
max(modern_browser_score) AS modern_browser_score, max(ua_ch_mismatch) AS ua_ch_mismatch,
any(sec_fetch_mode) AS sec_fetch_mode, any(sec_fetch_dest) AS sec_fetch_dest
FROM mabase_prod.agg_header_fingerprint_1h
FROM ja4_processing.agg_header_fingerprint_1h
WHERE window_start >= now() - INTERVAL 24 HOUR
GROUP BY window_start, src_ip
) h ON a.src_ip = h.src_ip AND a.window_start = h.window_start

22
shared/clickhouse/08_users.sql
View File

@ -7,14 +7,16 @@
CREATE USER IF NOT EXISTS data_writer IDENTIFIED WITH plaintext_password BY 'ChangeMe';
CREATE USER IF NOT EXISTS analyst IDENTIFIED WITH plaintext_password BY 'ChangeMe';
-- data_writer: INSERT on raw table only (fed by correlator service)
GRANT INSERT ON mabase_prod.http_logs_raw TO data_writer;
GRANT SELECT ON mabase_prod.http_logs_raw TO data_writer;
-- data_writer: INSERT on raw table in ja4_logs (fed by correlator service)
GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
-- analyst: read access on parsed and derived tables
GRANT SELECT ON mabase_prod.http_logs TO analyst;
GRANT SELECT ON mabase_prod.ml_detected_anomalies TO analyst;
GRANT SELECT ON mabase_prod.ml_all_scores TO analyst;
GRANT SELECT ON mabase_prod.view_ai_features_1h TO analyst;
GRANT SELECT ON mabase_prod.view_ip_recurrence TO analyst;
GRANT SELECT ON mabase_prod.audit_logs TO analyst;
-- analyst: read access on ja4_logs (parsed logs)
GRANT SELECT ON ja4_logs.http_logs TO analyst;
-- analyst: read access on ja4_processing (analytics, ML, views, audit)
GRANT SELECT ON ja4_processing.ml_detected_anomalies TO analyst;
GRANT SELECT ON ja4_processing.ml_all_scores TO analyst;
GRANT SELECT ON ja4_processing.view_ai_features_1h TO analyst;
GRANT SELECT ON ja4_processing.view_ip_recurrence TO analyst;
GRANT SELECT ON ja4_processing.audit_logs TO analyst;

4
shared/clickhouse/09_audit_table.sql
View File

@ -1,9 +1,9 @@
-- =============================================================================
-- 09_audit_table.sql — SOC audit log table for dashboard activity tracking
-- Referenced as mabase_prod.audit_logs in dashboard/backend/routes/audit.py
-- Referenced as ja4_processing.audit_logs in dashboard/backend/routes/audit.py
-- =============================================================================
CREATE TABLE IF NOT EXISTS mabase_prod.audit_logs
CREATE TABLE IF NOT EXISTS ja4_processing.audit_logs
(
`timestamp` DateTime DEFAULT now(),
`user_name` LowCardinality(String) DEFAULT 'soc_user',

67
shared/clickhouse/deploy_schema.sh Executable file
View File

@ -0,0 +1,67 @@
#!/usr/bin/env bash
# =============================================================================
# deploy_schema.sh — Apply ClickHouse schema migrations with configurable
# database names via environment variables.
#
# Usage:
# CLICKHOUSE_DB_LOGS=my_logs CLICKHOUSE_DB_PROCESSING=my_proc ./deploy_schema.sh
#
# Env vars:
# CLICKHOUSE_DB_LOGS — logs database name (default: ja4_logs)
# CLICKHOUSE_DB_PROCESSING — processing database name (default: ja4_processing)
# CLICKHOUSE_HOST — ClickHouse host (default: localhost)
# CLICKHOUSE_PORT — ClickHouse native port (default: 9000)
# CLICKHOUSE_USER — ClickHouse user (default: default)
# CLICKHOUSE_PASSWORD — ClickHouse password (default: empty)
# =============================================================================
set -euo pipefail
CLICKHOUSE_DB_LOGS="${CLICKHOUSE_DB_LOGS:-ja4_logs}"
CLICKHOUSE_DB_PROCESSING="${CLICKHOUSE_DB_PROCESSING:-ja4_processing}"
CLICKHOUSE_HOST="${CLICKHOUSE_HOST:-localhost}"
CLICKHOUSE_PORT="${CLICKHOUSE_PORT:-9000}"
CLICKHOUSE_USER="${CLICKHOUSE_USER:-default}"
CLICKHOUSE_PASSWORD="${CLICKHOUSE_PASSWORD:-}"
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CH_ARGS=(
--host "$CLICKHOUSE_HOST"
--port "$CLICKHOUSE_PORT"
--user "$CLICKHOUSE_USER"
)
if [[ -n "$CLICKHOUSE_PASSWORD" ]]; then
CH_ARGS+=(--password "$CLICKHOUSE_PASSWORD")
fi
SQL_FILES=(
00_database.sql
01_raw_tables.sql
02_dictionaries.sql
03_anubis_tables.sql
04_mv_http_logs.sql
05_aggregation_tables.sql
06_ml_tables.sql
07_ai_features_view.sql
08_users.sql
09_audit_table.sql
)
for f in "${SQL_FILES[@]}"; do
filepath="${SCRIPT_DIR}/${f}"
if [[ ! -f "$filepath" ]]; then
echo "WARN: ${f} not found, skipping" >&2
continue
fi
echo ">>> Applying ${f} ..."
sed \
-e "s/ja4_logs/${CLICKHOUSE_DB_LOGS}/g" \
-e "s/ja4_processing/${CLICKHOUSE_DB_PROCESSING}/g" \
"$filepath" \
| clickhouse-client "${CH_ARGS[@]}" --multiquery
done
echo "=== Schema deployment complete ==="
echo " logs db: ${CLICKHOUSE_DB_LOGS}"
echo " processing db: ${CLICKHOUSE_DB_PROCESSING}"