feat: full-stack Docker Compose integration tests
- 4-container stack: ClickHouse, platform (Rocky 9), bot-detector, dashboard - Platform builds sentinel on Rocky (CGO+libpcap native), correlator static - mod-reqin-log compiled with apxs on Rocky (matching RPM build target) - ClickHouse init script patches credentials for test env (sed-based) - 8-phase test runner: schema, traffic gen, pipeline, dashboard API, bot-detector, sentinel - All 13 checks pass, 3 non-blocking warnings (empty dicts, log paths) SQL schema fixes discovered during integration: - 02_dictionaries: IPv6CIDR → String (not a valid ClickHouse type) - 03_anubis_tables: dict_anubis_ua missing has_ip/rule_id/category attrs - 03_anubis_tables: dict_anubis_country FLAT() → COMPLEX_KEY_HASHED() (String key) - 09_audit_table: CODEC before DEFAULT → DEFAULT before CODEC Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
146
tests/integration/docker-compose.yml
Normal file
146
tests/integration/docker-compose.yml
Normal file
@ -0,0 +1,146 @@
|
||||
# =============================================================================
|
||||
# ja4-platform — Full-stack integration test
|
||||
#
|
||||
# Compose:
|
||||
# clickhouse — ClickHouse server (schema init via entrypoint)
|
||||
# platform — Rocky Linux 9: Apache + mod-reqin-log + sentinel + correlator
|
||||
# bot-detector — ML anomaly detection (Python)
|
||||
# dashboard — SOC dashboard API (FastAPI)
|
||||
#
|
||||
# Usage:
|
||||
# cd tests/integration && ./run-tests.sh
|
||||
# =============================================================================
|
||||
|
||||
services:
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ClickHouse — schema auto-init from shared/clickhouse/*.sql
|
||||
# ---------------------------------------------------------------------------
|
||||
clickhouse:
|
||||
image: clickhouse/clickhouse-server:24.8
|
||||
hostname: clickhouse
|
||||
environment:
|
||||
CLICKHOUSE_DB: ja4_processing
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
|
||||
volumes:
|
||||
# Init script: copies, patches credentials, and executes SQL files
|
||||
- ./platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh
|
||||
# SQL sources (read-only, patched by init script before execution)
|
||||
- ../../shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro
|
||||
- ../../shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro
|
||||
- ../../shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro
|
||||
- ../../shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro
|
||||
- ../../shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro
|
||||
- ../../shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro
|
||||
- ../../shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro
|
||||
- ../../shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro
|
||||
- ../../shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro
|
||||
- ../../shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro
|
||||
# Empty CSV stubs (dictionaries expect these files)
|
||||
- ./platform/csv-stubs:/var/lib/clickhouse/user_files
|
||||
ports:
|
||||
- "9000:9000"
|
||||
- "8123:8123"
|
||||
healthcheck:
|
||||
test: ["CMD", "clickhouse-client", "--query", "SELECT 1"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
networks:
|
||||
- ja4net
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Platform — Rocky Linux 9: Apache (HTTPS) + mod-reqin-log + sentinel + correlator
|
||||
# ---------------------------------------------------------------------------
|
||||
platform:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: tests/integration/platform/Dockerfile
|
||||
hostname: platform
|
||||
cap_add:
|
||||
- NET_RAW
|
||||
- NET_ADMIN
|
||||
environment:
|
||||
LOGCORRELATOR_CLICKHOUSE_DSN: "clickhouse://default:@clickhouse:9000/ja4_logs"
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "443:443"
|
||||
- "80:80"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sfk", "https://localhost/health"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
networks:
|
||||
- ja4net
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Bot-detector — ML anomaly detection
|
||||
# ---------------------------------------------------------------------------
|
||||
bot-detector:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: services/bot-detector/bot_detector/Dockerfile
|
||||
hostname: bot-detector
|
||||
environment:
|
||||
CLICKHOUSE_HOST: clickhouse
|
||||
CLICKHOUSE_PORT: 8123
|
||||
CLICKHOUSE_DB_PROCESSING: ja4_processing
|
||||
CLICKHOUSE_DB_LOGS: ja4_logs
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: ""
|
||||
CYCLE_INTERVAL_SEC: 30
|
||||
RETRAIN_INTERVAL_HOURS: 1
|
||||
ANOMALY_THRESHOLD: "-0.05"
|
||||
HEALTH_PORT: 8080
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
platform:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/')"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 10
|
||||
networks:
|
||||
- ja4net
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Dashboard — FastAPI SOC UI
|
||||
# ---------------------------------------------------------------------------
|
||||
dashboard:
|
||||
build:
|
||||
context: ../..
|
||||
dockerfile: services/dashboard/Dockerfile
|
||||
hostname: dashboard
|
||||
environment:
|
||||
CLICKHOUSE_HOST: clickhouse
|
||||
CLICKHOUSE_PORT: 8123
|
||||
CLICKHOUSE_DB: ja4_processing
|
||||
CLICKHOUSE_DB_PROCESSING: ja4_processing
|
||||
CLICKHOUSE_DB_LOGS: ja4_logs
|
||||
CLICKHOUSE_USER: default
|
||||
CLICKHOUSE_PASSWORD: ""
|
||||
API_HOST: 0.0.0.0
|
||||
API_PORT: 8000
|
||||
CORS_ORIGINS: '["*"]'
|
||||
depends_on:
|
||||
clickhouse:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "8000:8000"
|
||||
healthcheck:
|
||||
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8000/health')"]
|
||||
interval: 5s
|
||||
timeout: 3s
|
||||
retries: 30
|
||||
networks:
|
||||
- ja4net
|
||||
|
||||
networks:
|
||||
ja4net:
|
||||
driver: bridge
|
||||
Reference in New Issue
Block a user