feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized
Services: - ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap) - logcorrelator: JA4 log correlation engine (Go, ClickHouse) - mod_reqin_log: Apache module (C, JSON request logging) - bot_detector: ML bot detection pipeline (Python) - dashboard: FastAPI/Streamlit analytics UI (Python) Shared libraries: - shared/go/ja4common: logger, config, shutdown, ipfilter (Go module) - shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package) - shared/clickhouse/: canonical SQL migrations (10 files) Build & packaging: - Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10) - go.work workspace linking sentinel, correlator, ja4common - Makefile with test-all, build-all, rpm-* targets Fixes applied: - go.work: 1.21 → 1.24.6 (required by sentinel) - correlator Dockerfiles: golang:1.21 → golang:1.24 - replace directives in go.mod for ja4common local path - pyproject.toml: setuptools.backends → setuptools.build_meta - Removed static libpcap linking (unavailable on Rocky 9) - Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32) - Rewrote corrupted test files (logger_test.go × 2) Test coverage: - correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%) - sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse) Documentation: - README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
158
services/sentinel/Makefile
Normal file
158
services/sentinel/Makefile
Normal file
@ -0,0 +1,158 @@
|
||||
.PHONY: build build-docker test test-docker test-integration lint clean help docker-build-dev docker-build-runtime package package-rpm
|
||||
|
||||
# Docker parameters
|
||||
DOCKER=docker
|
||||
DOCKER_BUILD=$(DOCKER) build
|
||||
DOCKER_RUN=$(DOCKER) run
|
||||
DOCKER_COMPOSE=docker compose
|
||||
|
||||
# Image names
|
||||
DEV_IMAGE=ja4sentinel-dev:latest
|
||||
RUNTIME_IMAGE=ja4sentinel-runtime:latest
|
||||
TEST_SERVER_IMAGE=ja4sentinel-test-server:latest
|
||||
|
||||
# Binary name
|
||||
BINARY_NAME=ja4sentinel
|
||||
BINARY_PATH=./cmd/ja4sentinel
|
||||
DIST_DIR=dist
|
||||
BUILD_DIR=build
|
||||
|
||||
# RPM build directory
|
||||
RPM_DIR=$(DIST_DIR)/rpm
|
||||
|
||||
# Package version (extract default from spec file, can be overridden)
|
||||
PKG_VERSION ?= $(shell grep '^%define spec_version' packaging/rpm/ja4sentinel.spec | tail -1 | awk '{print $$3}')
|
||||
|
||||
# Build flags
|
||||
VERSION=$(PKG_VERSION)
|
||||
BUILD_TIME=$(shell date -u '+%Y-%m-%d_%H:%M:%S')
|
||||
GIT_COMMIT=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
LDFLAGS=-ldflags "-X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT)"
|
||||
|
||||
# Default target
|
||||
all: docker-build-dev test-docker
|
||||
|
||||
## build: Build the ja4sentinel binary locally
|
||||
build:
|
||||
mkdir -p $(DIST_DIR)
|
||||
go build -buildvcs=false $(LDFLAGS) -o $(DIST_DIR)/$(BINARY_NAME) $(BINARY_PATH)
|
||||
|
||||
## build-linux: Build for Linux (amd64)
|
||||
build-linux:
|
||||
mkdir -p $(DIST_DIR)
|
||||
GOOS=linux GOARCH=amd64 go build -buildvcs=false $(LDFLAGS) -o $(DIST_DIR)/$(BINARY_NAME)-linux-amd64 $(BINARY_PATH)
|
||||
|
||||
## docker-build-dev: Build the development Docker image
|
||||
docker-build-dev:
|
||||
$(DOCKER_BUILD) -t $(DEV_IMAGE) -f Dockerfile.dev .
|
||||
|
||||
## docker-build-runtime: Build the runtime Docker image (multi-stage build)
|
||||
docker-build-runtime:
|
||||
$(DOCKER_BUILD) -t $(RUNTIME_IMAGE) -f Dockerfile .
|
||||
|
||||
## docker-build-test-server: Build the test server image
|
||||
docker-build-test-server:
|
||||
$(DOCKER_BUILD) -t $(TEST_SERVER_IMAGE) -f Dockerfile.test-server .
|
||||
|
||||
## test: Run unit tests locally
|
||||
test:
|
||||
go test -v ./...
|
||||
|
||||
## test-docker: Run unit tests inside Docker container
|
||||
test-docker: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) go test -v ./...
|
||||
|
||||
## test-race: Run tests with race detector in Docker
|
||||
test-race: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) go test -race -v ./...
|
||||
|
||||
## test-coverage: Run tests with coverage report in Docker
|
||||
test-coverage: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) sh -c \
|
||||
"go test -v -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html"
|
||||
|
||||
## test-integration: Run integration tests in Docker
|
||||
test-integration: docker-build-dev docker-build-test-server
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml build --no-cache
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from ja4sentinel-test
|
||||
|
||||
## test-integration-clean: Run integration tests and clean up afterward
|
||||
test-integration-clean: docker-build-dev docker-build-test-server
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml build --no-cache
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from ja4sentinel-test
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml down -v
|
||||
|
||||
## lint: Run linters in Docker
|
||||
lint: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) sh -c \
|
||||
"go vet ./... && echo 'Running gofmt check...' && gofmt -l . | grep -v '^vendor/' | grep -v '^path/' || true"
|
||||
|
||||
## fmt: Format all Go files
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
## package: Build RPM packages for all target distributions
|
||||
package: package-rpm
|
||||
|
||||
## package-rpm: Build RPM packages for Rocky Linux 8/9/10, AlmaLinux (requires Docker)
|
||||
package-rpm:
|
||||
mkdir -p $(RPM_DIR)/el8 $(RPM_DIR)/el9 $(RPM_DIR)/el10
|
||||
@echo "Building RPM packages for Rocky Linux 8/9, AlmaLinux 10..."
|
||||
docker build --target output -t ja4sentinel-rpm-packager:latest \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Extracting RPM packages from Docker image..."
|
||||
@docker run --rm -v $(PWD)/$(RPM_DIR):/output/rpm ja4sentinel-rpm-packager:latest sh -c \
|
||||
'cp -r /packages/rpm/el8 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el9 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el10 /output/rpm/'
|
||||
@echo "RPM packages created:"
|
||||
@echo " Rocky Linux 8 (el8):"
|
||||
ls -la $(RPM_DIR)/el8/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " Rocky Linux 9 (el9):"
|
||||
ls -la $(RPM_DIR)/el9/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " AlmaLinux/Rocky 10 (el10):"
|
||||
ls -la $(RPM_DIR)/el10/ 2>/dev/null || echo " (no packages)"
|
||||
|
||||
## test-package-rpm: Test RPM package installation in Docker
|
||||
test-package-rpm: package-rpm
|
||||
./packaging/test/test-rpm.sh
|
||||
|
||||
## test-package: Test RPM package installation
|
||||
test-package: test-package-rpm
|
||||
|
||||
## ci: Full CI pipeline (tests, build, packages, package tests)
|
||||
ci: ci-test ci-build ci-package ci-package-test
|
||||
|
||||
## ci-test: Run all tests for CI
|
||||
ci-test: test lint
|
||||
|
||||
## ci-build: Build for CI (production binary)
|
||||
ci-build: build-linux
|
||||
|
||||
## ci-package: Build all packages for CI
|
||||
ci-package: package
|
||||
|
||||
## ci-package-test: Test all packages for CI
|
||||
ci-package-test: test-package
|
||||
|
||||
## clean: Clean build artifacts and Docker images
|
||||
clean:
|
||||
rm -rf $(DIST_DIR)/
|
||||
rm -rf $(BUILD_DIR)/
|
||||
rm -f coverage.out coverage.html
|
||||
$(DOCKER) rmi $(DEV_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(RUNTIME_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(TEST_SERVER_IMAGE) 2>/dev/null || true
|
||||
|
||||
## clean-all: Clean everything including containers and volumes
|
||||
clean-all: clean
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml down -v --remove-orphans
|
||||
|
||||
## help: Show this help message
|
||||
help:
|
||||
@echo "Usage: make [target]"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@sed -n 's/^##//p' $(MAKEFILE_LIST) | column -t -s ':' | sed -e 's/^/ /'
|
||||
Reference in New Issue
Block a user