d8ca804a55
Exports http_logs from prod ClickHouse via HTTP API, imports into dev with dynamic date shifting (max(time) → now() by default). Features: - Batch export in Native format (200K rows/batch, ~10s each) - Auto date shift: prod max(time) aligned to current time - --shift N: manual override (seconds) - --days N: filter to last N days only - --cron: silent mode for scheduled runs - Staging table approach: export → staging → INSERT SELECT with shift → cleanup Tested: 3,054,122 rows imported in ~3 minutes, dates 2026-04-03→2026-04-09. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
147 lines
5.2 KiB
Makefile
147 lines
5.2 KiB
Makefile
# =============================================================================
|
||
# ja4-platform — Monorepo Makefile
|
||
# All targets use new service names:
|
||
# sentinel, correlator, bot-detector, dashboard, mod-reqin-log
|
||
# =============================================================================
|
||
|
||
.PHONY: build-all test-all rpm-all dist \
|
||
build-sentinel test-sentinel rpm-sentinel \
|
||
test-integration \
|
||
test-mod-reqin-log rpm-mod-reqin-log \
|
||
build-correlator test-correlator rpm-correlator \
|
||
build-bot-detector test-bot-detector \
|
||
build-dashboard test-dashboard \
|
||
test-ja4common-python \
|
||
reload-prod-logs
|
||
|
||
# --- Root -------------------------------------------------------------------
|
||
|
||
build-all: build-sentinel build-correlator build-bot-detector build-dashboard
|
||
@echo "All services built."
|
||
|
||
test-all: test-sentinel test-correlator test-bot-detector test-dashboard test-ja4common-python
|
||
@echo "All tests completed."
|
||
|
||
rpm-all: rpm-sentinel rpm-correlator rpm-mod-reqin-log
|
||
@echo "All RPMs built."
|
||
|
||
dist: rpm-all
|
||
@echo "Distribution packages ready in services/*/dist/"
|
||
|
||
# --- sentinel (was ja4sentinel) ---------------------------------------------
|
||
|
||
build-sentinel:
|
||
docker build \
|
||
--build-arg VERSION=$$(git -C services/sentinel describe --tags --always 2>/dev/null || echo dev) \
|
||
--build-arg GIT_COMMIT=$$(git rev-parse --short HEAD 2>/dev/null || echo unknown) \
|
||
--build-arg BUILD_TIME=$$(date -u +%Y-%m-%dT%H:%M:%SZ) \
|
||
-f services/sentinel/Dockerfile \
|
||
-t ja4-platform/sentinel:latest \
|
||
.
|
||
|
||
test-sentinel:
|
||
# Tests run inside Docker — no native Go required on the host
|
||
docker build -f services/sentinel/Dockerfile.dev -t ja4-platform/sentinel-tests:latest .
|
||
docker run --rm --cap-add=NET_RAW --cap-add=NET_ADMIN ja4-platform/sentinel-tests:latest
|
||
|
||
rpm-sentinel:
|
||
# Méthode: Dockerfile.package → builder Go → rpm-builder (rpmbuild ×3) → output alpine
|
||
docker build \
|
||
-f services/sentinel/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/sentinel/dist \
|
||
--build-arg VERSION=$(shell git -C services/sentinel describe --tags --always 2>/dev/null || echo dev) \
|
||
.
|
||
@echo "📦 RPMs sentinel dans services/sentinel/dist/"
|
||
|
||
# --- mod-reqin-log (was mod_reqin_log) --------------------------------------
|
||
|
||
test-mod-reqin-log:
|
||
docker build -f services/mod-reqin-log/Dockerfile.tests -t ja4-platform/mod-reqin-log-tests:latest .
|
||
docker run --rm ja4-platform/mod-reqin-log-tests:latest
|
||
|
||
rpm-mod-reqin-log:
|
||
# Méthode: Dockerfile.package → builder C (×3 distros) → rpm-builder (rpmbuild ×3) → output alpine
|
||
docker build \
|
||
-f services/mod-reqin-log/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/mod-reqin-log/dist \
|
||
.
|
||
@echo "📦 RPMs mod-reqin-log dans services/mod-reqin-log/dist/"
|
||
|
||
# --- correlator (was logcorrelator) -----------------------------------------
|
||
|
||
build-correlator:
|
||
docker build \
|
||
-f services/correlator/Dockerfile \
|
||
-t ja4-platform/correlator:latest \
|
||
.
|
||
|
||
test-correlator:
|
||
# Tests run inside the Dockerfile builder stage (80% coverage gate enforced)
|
||
docker build --target builder -f services/correlator/Dockerfile -t ja4-platform/correlator-tests:latest .
|
||
|
||
rpm-correlator:
|
||
# Méthode: Dockerfile.package → builder Go → rpm-builder (rpmbuild ×3) → output alpine
|
||
docker build \
|
||
-f services/correlator/Dockerfile.package \
|
||
--target output \
|
||
--output type=local,dest=services/correlator/dist \
|
||
--build-arg VERSION=$(shell git -C services/correlator describe --tags --always 2>/dev/null || echo dev) \
|
||
.
|
||
@echo "📦 RPMs correlator dans services/correlator/dist/"
|
||
|
||
# --- bot-detector (was bot_detector) ----------------------------------------
|
||
|
||
build-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile \
|
||
-t ja4-platform/bot-detector:latest \
|
||
.
|
||
|
||
test-bot-detector:
|
||
docker build \
|
||
-f services/bot-detector/bot_detector/Dockerfile.tests \
|
||
-t ja4-platform/bot-detector-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/bot-detector-tests:latest
|
||
|
||
# --- dashboard --------------------------------------------------------------
|
||
|
||
build-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile \
|
||
-t ja4-platform/dashboard:latest \
|
||
.
|
||
|
||
test-dashboard:
|
||
docker build \
|
||
-f services/dashboard/Dockerfile.tests \
|
||
-t ja4-platform/dashboard-tests:latest \
|
||
.
|
||
docker run --rm ja4-platform/dashboard-tests:latest
|
||
|
||
# --- shared/python/ja4_common -----------------------------------------------
|
||
|
||
test-ja4common-python:
|
||
docker build \
|
||
-f shared/python/ja4_common/Dockerfile.tests \
|
||
-t ja4-platform/ja4common-python-tests:latest \
|
||
shared/python/ja4_common/
|
||
docker run --rm ja4-platform/ja4common-python-tests:latest
|
||
|
||
# --- integration (full-stack) -----------------------------------------------
|
||
|
||
test-integration:
|
||
cd tests/integration && ./run-tests.sh
|
||
|
||
test-integration-keep:
|
||
cd tests/integration && ./run-tests.sh --no-down
|
||
|
||
test-integration-down:
|
||
cd tests/integration && docker compose down -v --remove-orphans
|
||
|
||
# ── Dev data ─────────────────────────────────────────────────────────────────
|
||
reload-prod-logs:
|
||
./scripts/reload-prod-logs.sh
|