code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(anubis): simplify to IP/CIDR + ASN only, remove UA and Country rules

Browse Source 
- Remove UA regex extraction (extract_ua_regex, _extract_ua_from_all/any)
- Remove Country rule collection from parse_bot_policies_inline
- Simplify fetch_rules.py: collect_all_rules returns (ip_rules, asn_rules)
- Remove insert_ua_rules and insert_country_rules functions
- reload_dicts now only reloads dict_anubis_ip + dict_anubis_asn
- Simplify CASE blocks in 04_mv_http_logs.sql, 07_ai_features_view.sql,
  view_ai_features_anubis.sql, mv_http_logs.sql: IP > ASN (was 5-level
  UA+IP > UA > IP > ASN > Country cascade)
- Remove dict_anubis_country + dict_anubis_ua from 03_anubis_tables.sql
  (UA table kept as stub for REGEXP_TREE catch-all compatibility)
- Remove anubis_country_rules table from schema
- Remove Anubis UA and Country tabs from dashboard reflists page
- Remove anubis_ua_rules/country_rules from API reflist queries
- deploy_schema.sql simplified from 339 to 122 lines
- 764 lines removed across 9 files

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-09 15:25:33 +02:00
parent 98abbc80c7
commit 8180f4af04
9 changed files with 136 additions and 762 deletions
Download Patch File Download Diff File Expand all files Collapse all files

86
shared/clickhouse/03_anubis_tables.sql
View File

@ -1,10 +1,14 @@
-- =============================================================================
-- 03_anubis_tables.sql — Anubis crawler rule tables and dictionaries
-- Items 1–8 from bot_detector/anubis/deploy_schema.sql
-- Only IP/CIDR and ASN rules are populated by fetch_rules.py.
-- UA and Country dictionaries are kept as stubs (required by MV references)
-- but are never populated with real data.
-- =============================================================================
-- -----------------------------------------------------------------------------
-- 1. TABLE SOURCE — User-Agent rules (for REGEXP_TREE dictionary)
-- 1. TABLE SOURCE — User-Agent rules (REGEXP_TREE stub)
-- REGEXP_TREE requires ≥1 rule; the catch-all is seeded at init time.
-- This table is NOT populated by fetch_rules.py.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_ua_rules
(
@ -20,6 +24,7 @@ ORDER BY id;
-- -----------------------------------------------------------------------------
-- 2. TABLE SOURCE — IP/CIDR rules (for IP_TRIE dictionary)
-- Populated by fetch_rules.py from Anubis GitHub data.
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_ip_rules
(
@ -35,30 +40,23 @@ ORDER BY prefix;
-- -----------------------------------------------------------------------------
-- 3. DICTIONARY — UA REGEXP_TREE
-- dictGet('ja4_processing.dict_anubis_ua', 'bot_name', header_user_agent)
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- 3. TABLE SOURCE — ASN rules (for Flat dictionary)
-- Populated by fetch_rules.py from botPolicies.yaml.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_ua;
CREATE DICTIONARY ja4_processing.dict_anubis_ua
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_asn_rules
(
regexp String,
bot_name String,
action String,
has_ip String,
rule_id String,
category String
asn UInt32,
bot_name LowCardinality(String),
action LowCardinality(String),
category LowCardinality(String)
)
PRIMARY KEY regexp
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'ja4_processing' TABLE 'anubis_ua_rules'))
LAYOUT(REGEXP_TREE)
LIFETIME(MIN 300 MAX 600);
ENGINE = ReplacingMergeTree()
ORDER BY asn;
-- -----------------------------------------------------------------------------
-- 4. DICTIONARY — IP IP_TRIE
-- 4. DICTIONARY — IP IP_TRIE (active)
-- dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', toIPv6(src_ip), '')
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_ip;
CREATE DICTIONARY ja4_processing.dict_anubis_ip
@ -77,37 +75,8 @@ LIFETIME(MIN 300 MAX 600);
-- -----------------------------------------------------------------------------
-- 5. TABLE SOURCE — ASN rules (for Flat dictionary)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_asn_rules
(
asn UInt32,
bot_name LowCardinality(String),
action LowCardinality(String),
category LowCardinality(String)
)
ENGINE = ReplacingMergeTree()
ORDER BY asn;
-- -----------------------------------------------------------------------------
-- 6. TABLE SOURCE — Country rules (for Flat dictionary)
-- -----------------------------------------------------------------------------
CREATE TABLE IF NOT EXISTS ja4_processing.anubis_country_rules
(
country_code LowCardinality(String),
bot_name LowCardinality(String),
action LowCardinality(String),
category LowCardinality(String)
)
ENGINE = ReplacingMergeTree()
ORDER BY country_code;
-- -----------------------------------------------------------------------------
-- 7. DICTIONARY — ASN Flat
-- 5. DICTIONARY — ASN Flat (active)
-- dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', src_asn, '')
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_asn;
CREATE DICTIONARY ja4_processing.dict_anubis_asn
@ -121,22 +90,3 @@ PRIMARY KEY asn
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'ja4_processing' TABLE 'anubis_asn_rules'))
LAYOUT(FLAT())
LIFETIME(MIN 300 MAX 600);
-- -----------------------------------------------------------------------------
-- 8. DICTIONARY — Country COMPLEX_KEY_HASHED
-- dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', src_country_code, '')
-- NOTE: Change 'CHANGE_ME' to the actual ClickHouse admin password before use.
-- -----------------------------------------------------------------------------
DROP DICTIONARY IF EXISTS ja4_processing.dict_anubis_country;
CREATE DICTIONARY ja4_processing.dict_anubis_country
(
country_code String,
bot_name String,
action String,
category String
)
PRIMARY KEY country_code
SOURCE(CLICKHOUSE(HOST 'localhost' PORT 9000 USER 'admin' PASSWORD 'CHANGE_ME' DB 'ja4_processing' TABLE 'anubis_country_rules'))
LAYOUT(COMPLEX_KEY_HASHED())
LIFETIME(MIN 300 MAX 600);

75
shared/clickhouse/04_mv_http_logs.sql
View File

@ -175,66 +175,23 @@ SELECT
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Mode'), '') AS header_sec_fetch_mode,
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Site'), '') AS header_sec_fetch_site,
-- Anubis enrichment: combined UA+IP priority logic
-- Priority: (1) UA+IP [same rule_id] > (2) UA only > (3) IP only > (4) ASN > (5) Country
CASE
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '')
ELSE ''
END AS anubis_bot_name,
-- Anubis enrichment: IP/CIDR > ASN (simplified — UA and Country rules removed)
COALESCE(
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, ''), ''),
''
) AS anubis_bot_name,
CASE
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', _ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', _ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'action', _ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'action', _asn, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'action', _cc, '')
ELSE ''
END AS anubis_bot_action,
COALESCE(
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_ip', 'action', _ip, ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_asn', 'action', _asn, ''), ''),
''
) AS anubis_bot_action,
CASE
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', _ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', _ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', _ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', _ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', _ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', _ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', _ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', _ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'category', _ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', _asn, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'category', _asn, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', _cc, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'category', _cc, '')
ELSE ''
END AS anubis_bot_category
COALESCE(
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_ip', 'category', _ip, ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_asn', 'category', _asn, ''), ''),
''
) AS anubis_bot_category
FROM ja4_logs.http_logs_raw;

74
shared/clickhouse/07_ai_features_view.sql
View File

@ -18,64 +18,22 @@ WITH base_data AS (
''
) AS bot_name,
dictGetOrDefault('ja4_processing.dict_browser_ja4', 'browser_family', tuple(a.ja4), '') AS browser_family,
-- Anubis: combined UA+IP priority logic > UA only > IP only > ASN > Country
CASE
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '')
ELSE ''
END AS anubis_bot_name,
CASE
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', a.first_ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'action', a.first_ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'action', a.src_ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'action', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'action', a.src_country_code, '')
ELSE ''
END AS anubis_bot_action,
CASE
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '1'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
AND toUInt64OrZero(dictGet('ja4_processing.dict_anubis_ua', 'rule_id', a.first_ua))
= dictGetOrDefault('ja4_processing.dict_anubis_ip', 'rule_id', a.src_ip, toUInt64(0))
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', a.first_ua)
WHEN dictGet('ja4_processing.dict_anubis_ua', 'has_ip', a.first_ua) = '0'
AND dictGet('ja4_processing.dict_anubis_ua', 'bot_name', a.first_ua) != ''
THEN dictGet('ja4_processing.dict_anubis_ua', 'category', a.first_ua)
WHEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'has_ua', a.src_ip, toUInt8(0)) = 0
AND dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_ip', 'category', a.src_ip, '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_asn', 'category', toUInt32(a.src_asn), '')
WHEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'bot_name', a.src_country_code, '') != ''
THEN dictGetOrDefault('ja4_processing.dict_anubis_country', 'category', a.src_country_code, '')
ELSE ''
END AS anubis_bot_category,
-- Anubis: IP/CIDR > ASN (simplified — UA and Country rules removed)
COALESCE(
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_ip', 'bot_name', a.src_ip, ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_asn', 'bot_name', toUInt32(a.src_asn), ''), ''),
''
) AS anubis_bot_name,
COALESCE(
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_ip', 'action', a.src_ip, ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_asn', 'action', toUInt32(a.src_asn), ''), ''),
''
) AS anubis_bot_action,
COALESCE(
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_ip', 'category', a.src_ip, ''), ''),
nullIf(dictGetOrDefault('ja4_processing.dict_anubis_asn', 'category', toUInt32(a.src_asn), ''), ''),
''
) AS anubis_bot_category,
a.hits AS hits,
sum(a.hits) OVER (PARTITION BY a.src_ip) AS total_ip_hits,
a.correlated AS correlated,