feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized
Services: - ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap) - logcorrelator: JA4 log correlation engine (Go, ClickHouse) - mod_reqin_log: Apache module (C, JSON request logging) - bot_detector: ML bot detection pipeline (Python) - dashboard: FastAPI/Streamlit analytics UI (Python) Shared libraries: - shared/go/ja4common: logger, config, shutdown, ipfilter (Go module) - shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package) - shared/clickhouse/: canonical SQL migrations (10 files) Build & packaging: - Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10) - go.work workspace linking sentinel, correlator, ja4common - Makefile with test-all, build-all, rpm-* targets Fixes applied: - go.work: 1.21 → 1.24.6 (required by sentinel) - correlator Dockerfiles: golang:1.21 → golang:1.24 - replace directives in go.mod for ja4common local path - pyproject.toml: setuptools.backends → setuptools.build_meta - Removed static libpcap linking (unavailable on Rocky 9) - Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32) - Rewrote corrupted test files (logger_test.go × 2) Test coverage: - correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%) - sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse) Documentation: - README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
21
services/correlator/sql/bots.sql
Normal file
21
services/correlator/sql/bots.sql
Normal file
@ -0,0 +1,21 @@
|
||||
DROP TABLE IF EXISTS mabase_prod.ref_bot_networks;
|
||||
|
||||
CREATE TABLE mabase_prod.ref_bot_networks (
|
||||
-- On utilise IPv6CIDR car il accepte aussi les IPv4 au format ::ffff:1.2.3.4/120
|
||||
network IPv6CIDR,
|
||||
bot_name LowCardinality(String),
|
||||
is_legitimate UInt8,
|
||||
last_update DateTime
|
||||
) ENGINE = ReplacingMergeTree(last_update)
|
||||
ORDER BY (network, bot_name);
|
||||
|
||||
|
||||
-- Création de la table lisant le fichier des IPs
|
||||
CREATE TABLE mabase_prod.bot_ip (
|
||||
ip String
|
||||
) ENGINE = File(CSV, 'bot_ip.csv');
|
||||
|
||||
-- Création de la table lisant le fichier des signatures JA4
|
||||
CREATE TABLE mabase_prod.bot_ja4 (
|
||||
ja4 String
|
||||
) ENGINE = File(CSV, 'bot_ja4.csv');
|
||||
234
services/correlator/sql/init.sql
Normal file
234
services/correlator/sql/init.sql
Normal file
@ -0,0 +1,234 @@
|
||||
-- =============================================================================
|
||||
-- logcorrelator - Initialisation ClickHouse
|
||||
-- =============================================================================
|
||||
-- Ce fichier crée la base de données, les tables, la vue matérialisée
|
||||
-- et les utilisateurs nécessaires au fonctionnement de logcorrelator.
|
||||
--
|
||||
-- Usage :
|
||||
-- clickhouse-client --multiquery < sql/init.sql
|
||||
-- =============================================================================
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Base de données
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE DATABASE IF NOT EXISTS mabase_prod;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Table brute : cible directe des inserts du service
|
||||
-- Le service n'insère que dans cette table (colonne raw_json).
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS mabase_prod.http_logs_raw
|
||||
(
|
||||
`raw_json` String CODEC(ZSTD(3)),
|
||||
`ingest_time` DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(ingest_time)
|
||||
ORDER BY ingest_time
|
||||
TTL ingest_time + INTERVAL 1 DAY
|
||||
SETTINGS
|
||||
index_granularity = 8192,
|
||||
ttl_only_drop_parts = 1;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Table parsée : alimentée automatiquement par la vue matérialisée
|
||||
-- -----------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE mabase_prod.http_logs
|
||||
(
|
||||
-- Temporel
|
||||
`time` DateTime,
|
||||
`log_date` Date DEFAULT toDate(time),
|
||||
|
||||
-- Réseau
|
||||
`src_ip` IPv4,
|
||||
`src_port` UInt16,
|
||||
`dst_ip` IPv4,
|
||||
`dst_port` UInt16,
|
||||
|
||||
-- Enrichissement IPLocate
|
||||
`src_asn` UInt32,
|
||||
`src_country_code` LowCardinality(String),
|
||||
`src_as_name` LowCardinality(String),
|
||||
`src_org` LowCardinality(String),
|
||||
`src_domain` LowCardinality(String),
|
||||
|
||||
-- HTTP
|
||||
`method` LowCardinality(String),
|
||||
`scheme` LowCardinality(String),
|
||||
`host` LowCardinality(String),
|
||||
`path` String CODEC(ZSTD(3)),
|
||||
`query` String CODEC(ZSTD(3)),
|
||||
`http_version` LowCardinality(String),
|
||||
|
||||
-- Corrélation
|
||||
`orphan_side` LowCardinality(String),
|
||||
`correlated` UInt8,
|
||||
`keepalives` UInt16,
|
||||
`a_timestamp` UInt64,
|
||||
`b_timestamp` UInt64,
|
||||
`conn_id` String CODEC(ZSTD(3)),
|
||||
|
||||
-- Métadonnées IP
|
||||
`ip_meta_df` UInt8,
|
||||
`ip_meta_id` UInt16,
|
||||
`ip_meta_total_length` UInt16,
|
||||
`ip_meta_ttl` UInt8,
|
||||
|
||||
-- Métadonnées TCP
|
||||
`tcp_meta_options` LowCardinality(String),
|
||||
`tcp_meta_window_size` UInt32,
|
||||
`tcp_meta_mss` UInt16,
|
||||
`tcp_meta_window_scale` UInt8,
|
||||
`syn_to_clienthello_ms` Int32,
|
||||
|
||||
-- TLS / fingerprint
|
||||
`tls_version` LowCardinality(String),
|
||||
`tls_sni` LowCardinality(String),
|
||||
`tls_alpn` LowCardinality(String),
|
||||
`ja3` String CODEC(ZSTD(3)),
|
||||
`ja3_hash` String CODEC(ZSTD(3)),
|
||||
`ja4` String CODEC(ZSTD(3)),
|
||||
|
||||
-- En-têtes HTTP
|
||||
`client_headers` String CODEC(ZSTD(3)),
|
||||
`header_user_agent` String CODEC(ZSTD(3)),
|
||||
`header_accept` String CODEC(ZSTD(3)),
|
||||
`header_accept_encoding` String CODEC(ZSTD(3)),
|
||||
`header_accept_language` String CODEC(ZSTD(3)),
|
||||
`header_content_type` String CODEC(ZSTD(3)),
|
||||
`header_x_request_id` String CODEC(ZSTD(3)),
|
||||
`header_x_trace_id` String CODEC(ZSTD(3)),
|
||||
`header_x_forwarded_for` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua_mobile` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua_platform` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_dest` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_mode` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_site` String CODEC(ZSTD(3))
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY log_date
|
||||
ORDER BY (time, src_ip, dst_ip, ja4)
|
||||
TTL log_date + INTERVAL 7 DAY
|
||||
SETTINGS
|
||||
index_granularity = 8192,
|
||||
ttl_only_drop_parts = 1;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vue matérialisée : parse le JSON de http_logs_raw vers http_logs
|
||||
-- -----------------------------------------------------------------------------
|
||||
DROP VIEW IF EXISTS mabase_prod.mv_http_logs;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS mabase_prod.mv_http_logs
|
||||
TO mabase_prod.http_logs
|
||||
AS
|
||||
SELECT
|
||||
parseDateTimeBestEffort(coalesce(JSONExtractString(raw_json, 'time'), '1970-01-01T00:00:00Z')) AS time,
|
||||
toDate(time) AS log_date,
|
||||
|
||||
toIPv4(coalesce(JSONExtractString(raw_json, 'src_ip'), '0.0.0.0')) AS src_ip,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'src_port'), 0)) AS src_port,
|
||||
toIPv4(coalesce(JSONExtractString(raw_json, 'dst_ip'), '0.0.0.0')) AS dst_ip,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'dst_port'), 0)) AS dst_port,
|
||||
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'asn',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
toUInt32(0)
|
||||
) AS src_asn,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'country_code',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_country_code,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'name',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_as_name,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'org',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_org,
|
||||
dictGetOrDefault(
|
||||
'mabase_prod.dict_iplocate_asn',
|
||||
'domain',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_domain,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'method'), '') AS method,
|
||||
coalesce(JSONExtractString(raw_json, 'scheme'), '') AS scheme,
|
||||
coalesce(JSONExtractString(raw_json, 'host'), '') AS host,
|
||||
coalesce(JSONExtractString(raw_json, 'path'), '') AS path,
|
||||
coalesce(JSONExtractString(raw_json, 'query'), '') AS query,
|
||||
coalesce(JSONExtractString(raw_json, 'http_version'), '') AS http_version,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'orphan_side'), '') AS orphan_side,
|
||||
toUInt8(coalesce(JSONExtractBool(raw_json, 'correlated'), 0)) AS correlated,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'keepalives'), 0)) AS keepalives,
|
||||
coalesce(JSONExtractUInt(raw_json, 'a_timestamp'), 0) AS a_timestamp,
|
||||
coalesce(JSONExtractUInt(raw_json, 'b_timestamp'), 0) AS b_timestamp,
|
||||
coalesce(JSONExtractString(raw_json, 'conn_id'), '') AS conn_id,
|
||||
|
||||
toUInt8(coalesce(JSONExtractBool(raw_json, 'ip_meta_df'), 0)) AS ip_meta_df,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'ip_meta_id'), 0)) AS ip_meta_id,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'ip_meta_total_length'), 0)) AS ip_meta_total_length,
|
||||
toUInt8(coalesce(JSONExtractUInt(raw_json, 'ip_meta_ttl'), 0)) AS ip_meta_ttl,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'tcp_meta_options'), '') AS tcp_meta_options,
|
||||
toUInt32(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_window_size'), 0)) AS tcp_meta_window_size,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_mss'), 0)) AS tcp_meta_mss,
|
||||
toUInt8(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_window_scale'), 0)) AS tcp_meta_window_scale,
|
||||
toInt32(coalesce(JSONExtractInt(raw_json, 'syn_to_clienthello_ms'), 0)) AS syn_to_clienthello_ms,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'tls_version'), '') AS tls_version,
|
||||
coalesce(JSONExtractString(raw_json, 'tls_sni'), '') AS tls_sni,
|
||||
coalesce(JSONExtractString(raw_json, 'tls_alpn'), '') AS tls_alpn,
|
||||
coalesce(JSONExtractString(raw_json, 'ja3'), '') AS ja3,
|
||||
coalesce(JSONExtractString(raw_json, 'ja3_hash'), '') AS ja3_hash,
|
||||
coalesce(JSONExtractString(raw_json, 'ja4'), '') AS ja4,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'client_headers'), '') AS client_headers,
|
||||
coalesce(JSONExtractString(raw_json, 'header_User-Agent'), '') AS header_user_agent,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Accept'), '') AS header_accept,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Accept-Encoding'), '') AS header_accept_encoding,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Accept-Language'), '') AS header_accept_language,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Content-Type'), '') AS header_content_type,
|
||||
coalesce(JSONExtractString(raw_json, 'header_X-Request-Id'), '') AS header_x_request_id,
|
||||
coalesce(JSONExtractString(raw_json, 'header_X-Trace-Id'), '') AS header_x_trace_id,
|
||||
coalesce(JSONExtractString(raw_json, 'header_X-Forwarded-For'), '') AS header_x_forwarded_for,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA'), '') AS header_sec_ch_ua,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA-Mobile'), '') AS header_sec_ch_ua_mobile,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA-Platform'), '') AS header_sec_ch_ua_platform,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Dest'), '') AS header_sec_fetch_dest,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Mode'), '') AS header_sec_fetch_mode,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Site'), '') AS header_sec_fetch_site
|
||||
|
||||
FROM mabase_prod.http_logs_raw;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Utilisateurs et permissions
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE USER IF NOT EXISTS data_writer IDENTIFIED WITH plaintext_password BY 'ChangeMe';
|
||||
CREATE USER IF NOT EXISTS analyst IDENTIFIED WITH plaintext_password BY 'ChangeMe';
|
||||
|
||||
-- data_writer : INSERT uniquement sur la table brute
|
||||
GRANT INSERT ON mabase_prod.http_logs_raw TO data_writer;
|
||||
GRANT SELECT ON mabase_prod.http_logs_raw TO data_writer;
|
||||
|
||||
-- analyst : lecture sur la table parsée
|
||||
GRANT SELECT ON mabase_prod.http_logs TO analyst;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vérifications post-installation
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- SELECT count(*), min(ingest_time), max(ingest_time) FROM mabase_prod.http_logs_raw;
|
||||
-- SELECT count(*), min(time), max(time) FROM mabase_prod.http_logs;
|
||||
-- SELECT time, src_ip, dst_ip, method, host, path, ja4 FROM mabase_prod.http_logs ORDER BY time DESC LIMIT 10;
|
||||
29
services/correlator/sql/tables.sql
Normal file
29
services/correlator/sql/tables.sql
Normal file
@ -0,0 +1,29 @@
|
||||
DROP DICTIONARY IF EXISTS mabase_prod.dict_iplocate_asn;
|
||||
|
||||
CREATE DICTIONARY IF NOT EXISTS mabase_prod.dict_iplocate_asn
|
||||
(
|
||||
network String,
|
||||
asn UInt32,
|
||||
country_code String,
|
||||
name String,
|
||||
org String,
|
||||
domain String
|
||||
)
|
||||
PRIMARY KEY network
|
||||
SOURCE(FILE(path '/var/lib/clickhouse/user_files/iplocate-ip-to-asn.csv' format 'CSVWithNames'))
|
||||
LAYOUT(IP_TRIE())
|
||||
LIFETIME(MIN 3600 MAX 7200);
|
||||
|
||||
|
||||
|
||||
-- Suppression si existe pour reconfiguration
|
||||
DROP TABLE IF EXISTS mabase_prod.ref_bot_networks;
|
||||
|
||||
-- Table optimisée pour le filtrage binaire de CIDR
|
||||
CREATE TABLE mabase_prod.ref_bot_networks (
|
||||
network IPv6CIDR, -- Gère nativement '1.2.3.0/24' et '2001:db8::/32'
|
||||
bot_name LowCardinality(String),
|
||||
is_legitimate UInt8, -- 1 = Whitelist, 0 = Blacklist
|
||||
last_update DateTime
|
||||
) ENGINE = ReplacingMergeTree(last_update)
|
||||
ORDER BY (network, bot_name)
|
||||
Reference in New Issue
Block a user