feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized
Services: - ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap) - logcorrelator: JA4 log correlation engine (Go, ClickHouse) - mod_reqin_log: Apache module (C, JSON request logging) - bot_detector: ML bot detection pipeline (Python) - dashboard: FastAPI/Streamlit analytics UI (Python) Shared libraries: - shared/go/ja4common: logger, config, shutdown, ipfilter (Go module) - shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package) - shared/clickhouse/: canonical SQL migrations (10 files) Build & packaging: - Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10) - go.work workspace linking sentinel, correlator, ja4common - Makefile with test-all, build-all, rpm-* targets Fixes applied: - go.work: 1.21 → 1.24.6 (required by sentinel) - correlator Dockerfiles: golang:1.21 → golang:1.24 - replace directives in go.mod for ja4common local path - pyproject.toml: setuptools.backends → setuptools.build_meta - Removed static libpcap linking (unavailable on Rocky 9) - Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32) - Rewrote corrupted test files (logger_test.go × 2) Test coverage: - correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%) - sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse) Documentation: - README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
148
services/correlator/Makefile
Normal file
148
services/correlator/Makefile
Normal file
@ -0,0 +1,148 @@
|
||||
.PHONY: build build-docker test test-docker lint clean help docker-build-dev docker-build-runtime package package-rpm
|
||||
|
||||
# Docker parameters
|
||||
DOCKER=docker
|
||||
# Use buildx for better cache management and parallel builds
|
||||
DOCKER_BUILD=$(DOCKER) build
|
||||
DOCKER_BUILDX=$(DOCKER) buildx
|
||||
DOCKER_RUN=$(DOCKER) run
|
||||
|
||||
# Image names
|
||||
DEV_IMAGE=logcorrelator-dev:latest
|
||||
RUNTIME_IMAGE=logcorrelator:latest
|
||||
PACKAGER_IMAGE=logcorrelator-packager:latest
|
||||
PACKAGER_IMAGE_EL8=logcorrelator-packager-el8:latest
|
||||
PACKAGER_IMAGE_EL9=logcorrelator-packager-el9:latest
|
||||
PACKAGER_IMAGE_EL10=logcorrelator-packager-el10:latest
|
||||
|
||||
# Binary name
|
||||
BINARY_NAME=logcorrelator
|
||||
DIST_DIR=dist
|
||||
|
||||
# Package version
|
||||
PKG_VERSION ?= 1.1.22
|
||||
|
||||
# Enable BuildKit for better performance
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
## build: Build the logcorrelator binary locally
|
||||
build:
|
||||
mkdir -p $(DIST_DIR)
|
||||
go build -ldflags="-w -s" -o $(DIST_DIR)/$(BINARY_NAME) ./cmd/$(BINARY_NAME)
|
||||
|
||||
## docker-build-dev: Build the development Docker image (with tests and coverage)
|
||||
docker-build-dev:
|
||||
$(DOCKER_BUILD) --target builder -t $(DEV_IMAGE) -f Dockerfile .
|
||||
|
||||
## docker-build-dev-no-test: Build the development Docker image WITHOUT tests (faster)
|
||||
docker-build-dev-no-test:
|
||||
$(DOCKER_BUILD) --target builder --no-cache --build-arg SKIP_TESTS=true -t $(DEV_IMAGE) -f Dockerfile .
|
||||
|
||||
## docker-build-runtime: Build the runtime Docker image (fast, no tests)
|
||||
docker-build-runtime:
|
||||
$(DOCKER_BUILD) --target runtime -t $(RUNTIME_IMAGE) -f Dockerfile .
|
||||
|
||||
## test: Run unit tests locally
|
||||
test:
|
||||
go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
## test-docker: Run unit tests inside Docker container
|
||||
test-docker: docker-build-dev
|
||||
@echo "Tests already run in builder stage"
|
||||
|
||||
## lint: Run linters
|
||||
lint:
|
||||
go vet ./...
|
||||
gofmt -l .
|
||||
|
||||
## fmt: Format all Go files
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
## package: Build RPM packages for all target distributions
|
||||
package: package-rpm
|
||||
|
||||
## package-rpm: Build RPM packages for Rocky Linux 8/9, AlmaLinux 10 (requires Docker)
|
||||
## Uses buildx for parallel builds (el8, el9, el10 built simultaneously)
|
||||
package-rpm:
|
||||
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
|
||||
@echo "Starting parallel RPM builds for el8, el9, el10..."
|
||||
# Build all three distributions in parallel using buildx
|
||||
$(DOCKER_BUILDX) build --target output -t $(PACKAGER_IMAGE) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package . \
|
||||
--load
|
||||
@echo "Extracting RPM packages from Docker image..."
|
||||
$(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm $(PACKAGER_IMAGE) sh -c \
|
||||
"cp -r /packages/rpm/el8 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el9 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el10 /output/rpm/"
|
||||
@echo "RPM packages created:"
|
||||
@echo " Enterprise Linux 8 (el8):"
|
||||
ls -la $(DIST_DIR)/rpm/el8/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " Enterprise Linux 9 (el9):"
|
||||
ls -la $(DIST_DIR)/rpm/el9/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " Enterprise Linux 10 (el10):"
|
||||
ls -la $(DIST_DIR)/rpm/el10/ 2>/dev/null || echo " (no packages)"
|
||||
|
||||
## package-rpm-sequential: Build RPM packages sequentially (fallback if parallel fails)
|
||||
package-rpm-sequential:
|
||||
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
|
||||
@echo "Building RPM for el8..."
|
||||
$(DOCKER_BUILD) --target rpm-el8-builder -t $(PACKAGER_IMAGE_EL8) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Building RPM for el9..."
|
||||
$(DOCKER_BUILD) --target rpm-el9-builder -t $(PACKAGER_IMAGE_EL9) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Building RPM for el10..."
|
||||
$(DOCKER_BUILD) --target rpm-el10-builder -t $(PACKAGER_IMAGE_EL10) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Extracting RPM packages..."
|
||||
$(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm \
|
||||
-v $(PACKAGER_IMAGE_EL8):/el8:ro \
|
||||
-v $(PACKAGER_IMAGE_EL9):/el9:ro \
|
||||
-v $(PACKAGER_IMAGE_EL10):/el10:ro \
|
||||
alpine:latest sh -c \
|
||||
"cp -r /el8/packages/rpm/el8 /output/rpm/ && \
|
||||
cp -r /el9/packages/rpm/el9 /output/rpm/ && \
|
||||
cp -r /el10/packages/rpm/el10 /output/rpm/"
|
||||
|
||||
## test-package-rpm: Test RPM package installation in Docker
|
||||
test-package-rpm: package-rpm
|
||||
./packaging/test/test-rpm.sh
|
||||
|
||||
## test-package: Test RPM package installation
|
||||
test-package: test-package-rpm
|
||||
|
||||
## ci: Full CI pipeline (tests, build, packages, package tests)
|
||||
ci: ci-test ci-build ci-package ci-package-test
|
||||
|
||||
## ci-test: Run all tests for CI
|
||||
ci-test: test lint
|
||||
|
||||
## ci-build: Build for CI (production binary)
|
||||
ci-build: build
|
||||
|
||||
## ci-package: Build all packages for CI
|
||||
ci-package: package
|
||||
|
||||
## ci-package-test: Test all packages for CI
|
||||
ci-package-test: test-package
|
||||
|
||||
## clean: Clean build artifacts and Docker images
|
||||
clean:
|
||||
rm -rf $(DIST_DIR)/
|
||||
rm -f coverage.out
|
||||
$(DOCKER) rmi $(DEV_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(RUNTIME_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(PACKAGER_IMAGE) 2>/dev/null || true
|
||||
|
||||
## help: Show this help message
|
||||
help:
|
||||
@echo "Usage: make [target]"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@sed -n 's/^##//p' $(MAKEFILE_LIST) | column -t -s ':' | sed -e 's/^/ /'
|
||||
Reference in New Issue
Block a user