9ea36ad22e
Schema cleanup:
- Remove anubis_ua_rules table stub from 03_anubis_tables.sql
- Remove anubis_ua_rules from bot-detector deploy_schema.sql
- Remove UA seed step from clickhouse-init.sh (no more REGEXP_TREE dependency)
- Drop dict_anubis_ua, dict_anubis_country, anubis_ua_rules, anubis_country_rules
New scripts:
- scripts/init-stack.sh: comprehensive ClickHouse init (13 SQL files + migrations
+ validation + cleanup of obsolete tables). Supports --reset, --import-prod.
- scripts/import-prod-data.sh: imports pre-exported prod data (Native format)
with dynamic date shift (max(time) → now). Supports --shift, --no-truncate.
- scripts/data/prod-export/: directory for cached Native format exports
Makefile targets: init-stack, import-prod-data, init-and-import
Tested: init-stack.sh passes all 13 SQL + 7 critical tables + 7 dicts
import-prod-data.sh: 3M rows in ~37s with auto date shift
Dashboard: 55 routes OK, bot-detector: 36/36 tests pass
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
156 lines
5.4 KiB
Makefile
156 lines
5.4 KiB
Makefile
# =============================================================================
|
||
# ja4-platform — Monorepo Makefile
|
||
# All targets use new service names:
|
||
# sentinel, correlator, bot-detector, dashboard, mod-reqin-log
|
||
# =============================================================================
|
||
|
||
.PHONY: build-all test-all rpm-all dist \
|
||
build-sentinel test-sentinel rpm-sentinel \
|
||
test-integration \
|
||
test-mod-reqin-log rpm-mod-reqin-log \
|
||
build-correlator test-correlator rpm-correlator \
|
||
build-bot-detector test-bot-detector \
|
||
build-dashboard test-dashboard \
|
||
test-ja4common-python \
|
||
reload-prod-logs init-stack import-prod-data init-and-import
|
||
|
||
# --- Root -------------------------------------------------------------------
|
||
|
||
build-all: build-sentinel build-correlator build-bot-detector build-dashboard
|
||
@echo "All services built."
|
||
|
||
test-all: test-sentinel test-correlator test-bot-detector test-dashboard test-ja4common-python
|
||
@echo "All tests completed."
|
||
|
||
rpm-all: rpm-sentinel rpm-correlator rpm-mod-reqin-log
|
||
@echo "All RPMs built."
|
||
|
||
dist: rpm-all
|
||
@echo "Distribution packages ready in services/*/dist/"
|
||
|
||
# --- sentinel (was ja4sentinel) ---------------------------------------------
|
||
|
||
build-sentinel:
|
||
docker build \
|
||
--build-arg VERSION=$$(git -C services/sentinel describe --tags --always 2>/dev/null || echo dev) \
|
||
--build-arg GIT_COMMIT=$$(git rev-parse --short HEAD 2>/dev/null || echo unknown) \
|
||
--build-arg BUILD_TIME=$$(date -u +%Y-%m-%dT%H:%M:%SZ) \
|
||
-f services/sentinel/Dockerfile \
|
||
-t ja4-platform/sentinel:latest \
|
||
.
|
||
|
||
test-sentinel:
|
||
# Tests run inside Docker — no native Go required on the host
|
||
docker build -f services/sentinel/Dockerfile.dev -t ja4-platform/sentinel-tests:latest .
|
||
docker run --rm --cap-add=NET_RAW --cap-add=NET_ADMIN ja4-platform/sentinel-tests:latest
|
||
|
||
rpm-sentinel:
|
||
# Méthode: Dockerfile.package → builder Go → rpm-builder (rpmbuild ×3) → output alpine
|
||
docker build \
|
||
-f services/sentinel/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/sentinel/dist \
|
||
--build-arg VERSION=$(shell git -C services/sentinel describe --tags --always 2>/dev/null || echo dev) \
|
||
.
|
||
@echo "📦 RPMs sentinel dans services/sentinel/dist/"
|
||
|
||
# --- mod-reqin-log (was mod_reqin_log) --------------------------------------
|
||
|
||
test-mod-reqin-log:
|
||
docker build -f services/mod-reqin-log/Dockerfile.tests -t ja4-platform/mod-reqin-log-tests:latest .
|
||
docker run --rm ja4-platform/mod-reqin-log-tests:latest
|
||
|
||
rpm-mod-reqin-log:
|
||
# Méthode: Dockerfile.package → builder C (×3 distros) → rpm-builder (rpmbuild ×3) → output alpine
|
||
docker build \
|
||
-f services/mod-reqin-log/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/mod-reqin-log/dist \
|
||
.
|
||
@echo "📦 RPMs mod-reqin-log dans services/mod-reqin-log/dist/"
|
||
|
||
# --- correlator (was logcorrelator) -----------------------------------------
|
||
|
||
build-correlator:
|
||
docker build \
|
||
-f services/correlator/Dockerfile \
|
||
-t ja4-platform/correlator:latest \
|
||
.
|
||
|
||
test-correlator:
|
||
# Tests run inside the Dockerfile builder stage (80% coverage gate enforced)
|
||
docker build --target builder -f services/correlator/Dockerfile -t ja4-platform/correlator-tests:latest .
|
||
|
||
rpm-correlator:
|
||
# Méthode: Dockerfile.package → builder Go → rpm-builder (rpmbuild ×3) → output alpine
|
||
docker build \
|
||
-f services/correlator/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/correlator/dist \
|
||
--build-arg VERSION=$(shell git -C services/correlator describe --tags --always 2>/dev/null || echo dev) \
|
||
.
|
||
@echo "📦 RPMs correlator dans services/correlator/dist/"
|
||
|
||
# --- bot-detector (was bot_detector) ----------------------------------------
|
||
|
||
build-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile \
|
||
-t ja4-platform/bot-detector:latest \
|
||
.
|
||
|
||
test-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile.tests \
|
||
-t ja4-platform/bot-detector-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/bot-detector-tests:latest
|
||
|
||
# --- dashboard --------------------------------------------------------------
|
||
|
||
build-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile \
|
||
-t ja4-platform/dashboard:latest \
|
||
.
|
||
|
||
test-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile.tests \
|
||
-t ja4-platform/dashboard-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/dashboard-tests:latest
|
||
|
||
# --- shared/python/ja4_common -----------------------------------------------
|
||
|
||
test-ja4common-python:
|
||
docker build \
|
||
-f shared/python/ja4_common/Dockerfile.tests \
|
||
-t ja4-platform/ja4common-python-tests:latest \
|
||
shared/python/ja4_common/
|
||
docker run --rm ja4-platform/ja4common-python-tests:latest
|
||
|
||
# --- integration (full-stack) -----------------------------------------------
|
||
|
||
test-integration:
|
||
cd tests/integration && ./run-tests.sh
|
||
|
||
test-integration-keep:
|
||
cd tests/integration && ./run-tests.sh --no-down
|
||
|
||
test-integration-down:
|
||
cd tests/integration && docker compose down -v --remove-orphans
|
||
|
||
# ── Dev data ─────────────────────────────────────────────────────────────────
|
||
reload-prod-logs:
|
||
./scripts/reload-prod-logs.sh
|
||
|
||
init-stack:
|
||
./scripts/init-stack.sh
|
||
|
||
import-prod-data:
|
||
./scripts/import-prod-data.sh
|
||
|
||
init-and-import:
|
||
./scripts/init-stack.sh --import-prod
|