chore: suppression des services obsolètes (sentinel, correlator, mod-reqin-log)
Remplacés par l'agent ja4ebpf (eBPF CO-RE). Nettoyage complet : Supprimé : - old/ (archive de l'ancienne architecture) - services/correlator/ (logcorrelator Go) - services/sentinel/ (capture pcap Go) - services/mod-reqin-log/ (module Apache C) - shared/go/ja4common/ (lib Go partagée — plus importée par ja4ebpf) - tests/integration/platform/ (test correlator+sentinel+httpd) - tests/integration/docker-compose.yml (compose ancienne archi) - tests/integration/run-tests.sh (runner correlator/sentinel) - tests/integration/verify_mvs.py (script orphelin) Nettoyé : - go.work : retire ./shared/go/ja4common - services/ja4ebpf/go.mod : retire replace ja4common (jamais importé) - services/ja4ebpf/Dockerfile* : retire les COPY ja4common inutiles - Makefile : retire test-ja4common-python, test-integration*, targets obsolètes - tests/integration/README.md : réécrit pour l'architecture ja4ebpf Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
29
Makefile
29
Makefile
@ -10,10 +10,8 @@ VERSION ?= $(shell git describe --tags --always 2>/dev/null || echo dev)
|
||||
build-ja4ebpf test-ja4ebpf rpm-ja4ebpf \
|
||||
build-bot-detector test-bot-detector \
|
||||
build-dashboard test-dashboard \
|
||||
test-ja4common-python \
|
||||
test-all-stacks test-nginx test-nginx-varnish test-hitch-varnish test-apache \
|
||||
test-matrix \
|
||||
test-integration test-integration-keep test-integration-down \
|
||||
reload-prod-logs init-stack import-prod-data init-and-import \
|
||||
purge-db
|
||||
|
||||
@ -35,7 +33,6 @@ help: ## Affiche cette aide
|
||||
@echo " make test-ja4ebpf Tests Go ja4ebpf"
|
||||
@echo " make test-bot-detector Tests Python bot-detector"
|
||||
@echo " make test-dashboard Tests Python dashboard"
|
||||
@echo " make test-ja4common-python Tests Python ja4_common"
|
||||
@echo ""
|
||||
@echo " Tests d'intégration (par stack)"
|
||||
@echo " make test-all-stacks Toutes les stacks sur Rocky Linux 9"
|
||||
@ -66,7 +63,7 @@ help: ## Affiche cette aide
|
||||
build-all: build-ja4ebpf build-bot-detector build-dashboard
|
||||
@echo "All services built."
|
||||
|
||||
test-all: test-ja4ebpf test-bot-detector test-dashboard test-ja4common-python
|
||||
test-all: test-ja4ebpf test-bot-detector test-dashboard
|
||||
@echo "All unit tests completed."
|
||||
|
||||
rpm-all: rpm-ja4ebpf
|
||||
@ -134,15 +131,6 @@ test-dashboard:
|
||||
.
|
||||
docker run --rm ja4-platform/dashboard-tests:latest
|
||||
|
||||
# ── shared/python/ja4_common ─────────────────────────────────────────────────
|
||||
|
||||
test-ja4common-python:
|
||||
docker build \
|
||||
-f shared/python/ja4_common/Dockerfile.tests \
|
||||
-t ja4-platform/ja4common-python-tests:latest \
|
||||
shared/python/ja4_common/
|
||||
docker run --rm ja4-platform/ja4common-python-tests:latest
|
||||
|
||||
# ── Tests d'intégration par stack ────────────────────────────────────────────
|
||||
|
||||
test-all-stacks: ## Toutes les stacks sur la distro par défaut (Rocky Linux 9)
|
||||
@ -167,21 +155,6 @@ test-matrix: ## Toutes stacks × el8 + el9 + el10
|
||||
$${MATRIX_STACKS:+--stacks=$${MATRIX_STACKS}} \
|
||||
$${MATRIX_DISTROS:+--distros=$${MATRIX_DISTROS}}
|
||||
|
||||
# ── Compat : anciens targets d'intégration ───────────────────────────────────
|
||||
|
||||
test-integration: ## Ancien target — alias vers test-all-stacks
|
||||
$(MAKE) test-all-stacks
|
||||
|
||||
test-integration-keep:
|
||||
cd tests/integration && bash run-all-stacks.sh --no-down
|
||||
|
||||
test-integration-down:
|
||||
cd tests/integration && \
|
||||
for stack in apache nginx nginx-varnish hitch-varnish; do \
|
||||
[ -f "$$stack/docker-compose.yml" ] && \
|
||||
docker compose -f "$$stack/docker-compose.yml" down -v --remove-orphans 2>/dev/null || true; \
|
||||
done
|
||||
|
||||
# ── Base de données ───────────────────────────────────────────────────────────
|
||||
|
||||
reload-prod-logs:
|
||||
|
||||
5
go.work
5
go.work
@ -1,6 +1,3 @@
|
||||
go 1.24.6
|
||||
|
||||
use (
|
||||
./shared/go/ja4common
|
||||
./services/ja4ebpf
|
||||
)
|
||||
use ./services/ja4ebpf
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
github.com/ClickHouse/clickhouse-go v1.5.4 h1:cKjXeYLNWVJIx2J1K6H2CqyRmfwVJVY1OV1coaaFcI0=
|
||||
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
|
||||
@ -1,19 +0,0 @@
|
||||
# Build outputs
|
||||
dist/
|
||||
|
||||
# Dependency directories
|
||||
vendor/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Aider cache
|
||||
.aider*
|
||||
@ -1,2 +0,0 @@
|
||||
# correlator configuration — DO NOT COMMIT real values
|
||||
LOGCORRELATOR_CLICKHOUSE_DSN=clickhouse://data_writer:ChangeMe@clickhouse:9000/ja4_logs
|
||||
73
old/services/correlator/.github/workflows/ci.yml
vendored
73
old/services/correlator/.github/workflows/ci.yml
vendored
@ -1,73 +0,0 @@
|
||||
name: Build and Test
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ master ]
|
||||
pull_request:
|
||||
branches: [ master ]
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Download dependencies
|
||||
run: go mod download
|
||||
|
||||
- name: Run tests with coverage
|
||||
run: |
|
||||
go test -race -coverprofile=coverage.txt -covermode=atomic ./...
|
||||
TOTAL=$(go tool cover -func=coverage.txt | grep total | awk '{gsub(/%/, "", $3); print $3}')
|
||||
echo "Coverage: ${TOTAL}%"
|
||||
if (( $(echo "$TOTAL < 80" | bc -l) )); then
|
||||
echo "Coverage ${TOTAL}% is below 80% threshold"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Upload coverage to Codecov
|
||||
uses: codecov/codecov-action@v3
|
||||
with:
|
||||
file: ./coverage.txt
|
||||
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Set up Go
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: '1.21'
|
||||
|
||||
- name: Build binary
|
||||
run: |
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-ldflags="-w -s" \
|
||||
-o logcorrelator \
|
||||
./cmd/logcorrelator
|
||||
|
||||
- name: Upload binary artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: logcorrelator-linux-amd64
|
||||
path: logcorrelator
|
||||
|
||||
docker:
|
||||
runs-on: ubuntu-latest
|
||||
needs: test
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Build Docker image
|
||||
run: docker build -t logcorrelator:latest .
|
||||
|
||||
- name: Run tests in Docker
|
||||
run: |
|
||||
docker run --rm logcorrelator:latest --help || true
|
||||
32
old/services/correlator/.gitignore
vendored
32
old/services/correlator/.gitignore
vendored
@ -1,32 +0,0 @@
|
||||
# Build directory
|
||||
/build/
|
||||
/dist/
|
||||
|
||||
# Binaries
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
/logcorrelator
|
||||
|
||||
# Test binary
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool
|
||||
*.out
|
||||
|
||||
# Dependency directories
|
||||
vendor/
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.aider*
|
||||
@ -1,43 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
FROM golang:1.24 AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends git bc && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY go.work go.work.sum* ./
|
||||
COPY shared/go/ja4common/ ./shared/go/ja4common/
|
||||
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
|
||||
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
|
||||
|
||||
WORKDIR /build/services/correlator
|
||||
RUN --mount=type=cache,target=/go/pkg/mod go mod download
|
||||
|
||||
COPY services/correlator/ /build/services/correlator/
|
||||
|
||||
ARG SKIP_TESTS=false
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
if [ "$SKIP_TESTS" = "false" ]; then \
|
||||
go test -race -coverprofile=coverage.txt -covermode=atomic ./... && \
|
||||
echo "=== Coverage Report ===" && \
|
||||
go tool cover -func=coverage.txt | grep total && \
|
||||
TOTAL=$(go tool cover -func=coverage.txt | grep total | awk '{gsub(/%/, "", $3); print $3}') && \
|
||||
echo "Total coverage: ${TOTAL}%" && \
|
||||
if (( $(echo "$TOTAL < 60" | bc -l) )); then \
|
||||
echo "ERROR: Coverage ${TOTAL}% is below 60% threshold"; \
|
||||
exit 1; \
|
||||
fi && \
|
||||
echo "Coverage check passed!"; \
|
||||
else \
|
||||
echo "Skipping tests (SKIP_TESTS=true)"; \
|
||||
fi
|
||||
|
||||
RUN --mount=type=cache,target=/go/pkg/mod \
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build \
|
||||
-ldflags="-w -s" -o /usr/bin/correlator ./cmd/logcorrelator
|
||||
|
||||
FROM scratch AS runtime
|
||||
COPY --from=builder /usr/bin/correlator /usr/bin/correlator
|
||||
COPY --from=builder /build/services/correlator/config.example.yml /etc/correlator/correlator.yml
|
||||
ENTRYPOINT ["/usr/bin/correlator"]
|
||||
CMD ["-config", "/etc/correlator/correlator.yml"]
|
||||
@ -1,108 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# correlator — Dockerfile de packaging RPM (Rocky Linux 8/9, AlmaLinux 10)
|
||||
# Build context: monorepo root (ja4-platform/)
|
||||
# Méthode: 1 builder Rocky → 1 rpm-builder (rpmbuild, 3 × dist) → 1 output alpine
|
||||
# =============================================================================
|
||||
|
||||
# =============================================================================
|
||||
# Stage 1: Builder — compilation du binaire Go sur Rocky Linux 9
|
||||
# CGO_ENABLED=0 → binaire statique, mais compilé sur la même distro cible
|
||||
# =============================================================================
|
||||
FROM rockylinux:9 AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN dnf install -y golang git && dnf clean all
|
||||
|
||||
# Copie du workspace Go et du module partagé en premier (meilleur cache)
|
||||
COPY go.work go.work.sum* ./
|
||||
COPY shared/go/ja4common/ ./shared/go/ja4common/
|
||||
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
|
||||
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
|
||||
|
||||
WORKDIR /build/services/correlator
|
||||
RUN go mod download
|
||||
|
||||
COPY services/correlator/ /build/services/correlator/
|
||||
|
||||
ARG VERSION=dev
|
||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
|
||||
go build -ldflags="-w -s -X main.Version=${VERSION}" \
|
||||
-o /tmp/correlator \
|
||||
./cmd/logcorrelator
|
||||
|
||||
# =============================================================================
|
||||
# Stage 2: rpm-builder — construction des RPMs avec rpmbuild
|
||||
# Un seul stage, trois appels rpmbuild successifs (el8, el9, el10).
|
||||
# Le spec lit les fichiers depuis %{_builddir} (répertoire BUILD de rpmbuild).
|
||||
# =============================================================================
|
||||
FROM rockylinux:9 AS rpm-builder
|
||||
|
||||
WORKDIR /package
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
RUN dnf install -y rpm-build rpmdevtools systemd-rpm-macros && dnf clean all
|
||||
|
||||
RUN mkdir -p /root/rpmbuild/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} && \
|
||||
mkdir -p /packages/rpm/{el8,el9,el10}
|
||||
|
||||
# Disposition des fichiers dans BUILD/ (attendue par le spec correlator)
|
||||
RUN mkdir -p /root/rpmbuild/BUILD/usr/bin \
|
||||
/root/rpmbuild/BUILD/etc/logcorrelator \
|
||||
/root/rpmbuild/BUILD/etc/systemd/system \
|
||||
/root/rpmbuild/BUILD/etc/logrotate.d
|
||||
|
||||
COPY --from=builder /tmp/correlator /root/rpmbuild/BUILD/usr/bin/logcorrelator
|
||||
COPY services/correlator/config.example.yml /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml
|
||||
COPY services/correlator/config.example.yml /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml.example
|
||||
COPY services/correlator/logcorrelator.service /root/rpmbuild/BUILD/etc/systemd/system/logcorrelator.service
|
||||
COPY services/correlator/packaging/rpm/logrotate /root/rpmbuild/BUILD/etc/logrotate.d/logcorrelator
|
||||
|
||||
RUN chmod 755 /root/rpmbuild/BUILD/usr/bin/logcorrelator && \
|
||||
chmod 640 /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml && \
|
||||
chmod 640 /root/rpmbuild/BUILD/etc/logcorrelator/logcorrelator.yml.example && \
|
||||
chmod 644 /root/rpmbuild/BUILD/etc/systemd/system/logcorrelator.service && \
|
||||
chmod 644 /root/rpmbuild/BUILD/etc/logrotate.d/logcorrelator
|
||||
|
||||
COPY services/correlator/packaging/rpm/logcorrelator.spec /root/rpmbuild/SPECS/logcorrelator.spec
|
||||
|
||||
# el8
|
||||
RUN rpmbuild --define "_topdir /root/rpmbuild" \
|
||||
--define "dist .el8" \
|
||||
--define "build_version ${VERSION}" \
|
||||
--target x86_64 \
|
||||
-bb /root/rpmbuild/SPECS/logcorrelator.spec && \
|
||||
cp /root/rpmbuild/RPMS/x86_64/*.el8.x86_64.rpm /packages/rpm/el8/
|
||||
|
||||
# el9
|
||||
RUN rpmbuild --define "_topdir /root/rpmbuild" \
|
||||
--define "dist .el9" \
|
||||
--define "build_version ${VERSION}" \
|
||||
--target x86_64 \
|
||||
-bb /root/rpmbuild/SPECS/logcorrelator.spec && \
|
||||
cp /root/rpmbuild/RPMS/x86_64/*.el9.x86_64.rpm /packages/rpm/el9/
|
||||
|
||||
# el10
|
||||
RUN rpmbuild --define "_topdir /root/rpmbuild" \
|
||||
--define "dist .el10" \
|
||||
--define "build_version ${VERSION}" \
|
||||
--target x86_64 \
|
||||
-bb /root/rpmbuild/SPECS/logcorrelator.spec && \
|
||||
cp /root/rpmbuild/RPMS/x86_64/*.el10.x86_64.rpm /packages/rpm/el10/
|
||||
|
||||
# =============================================================================
|
||||
# Stage 3: output — image finale contenant uniquement les RPMs
|
||||
# =============================================================================
|
||||
FROM alpine:latest AS output
|
||||
|
||||
WORKDIR /packages
|
||||
COPY --from=rpm-builder /packages/rpm/el8/*.rpm /packages/rpm/el8/
|
||||
COPY --from=rpm-builder /packages/rpm/el9/*.rpm /packages/rpm/el9/
|
||||
COPY --from=rpm-builder /packages/rpm/el10/*.rpm /packages/rpm/el10/
|
||||
|
||||
CMD ["sh", "-c", \
|
||||
"echo '=== RPM el8 ===' && ls -la /packages/rpm/el8/ && \
|
||||
echo '' && echo '=== RPM el9 ===' && ls -la /packages/rpm/el9/ && \
|
||||
echo '' && echo '=== RPM el10 ===' && ls -la /packages/rpm/el10/"]
|
||||
@ -1,148 +0,0 @@
|
||||
.PHONY: build build-docker test test-docker lint clean help docker-build-dev docker-build-runtime package package-rpm
|
||||
|
||||
# Docker parameters
|
||||
DOCKER=docker
|
||||
# Use buildx for better cache management and parallel builds
|
||||
DOCKER_BUILD=$(DOCKER) build
|
||||
DOCKER_BUILDX=$(DOCKER) buildx
|
||||
DOCKER_RUN=$(DOCKER) run
|
||||
|
||||
# Image names
|
||||
DEV_IMAGE=logcorrelator-dev:latest
|
||||
RUNTIME_IMAGE=logcorrelator:latest
|
||||
PACKAGER_IMAGE=logcorrelator-packager:latest
|
||||
PACKAGER_IMAGE_EL8=logcorrelator-packager-el8:latest
|
||||
PACKAGER_IMAGE_EL9=logcorrelator-packager-el9:latest
|
||||
PACKAGER_IMAGE_EL10=logcorrelator-packager-el10:latest
|
||||
|
||||
# Binary name
|
||||
BINARY_NAME=logcorrelator
|
||||
DIST_DIR=dist
|
||||
|
||||
# Package version
|
||||
PKG_VERSION ?= 1.1.22
|
||||
|
||||
# Enable BuildKit for better performance
|
||||
export DOCKER_BUILDKIT=1
|
||||
|
||||
## build: Build the logcorrelator binary locally
|
||||
build:
|
||||
mkdir -p $(DIST_DIR)
|
||||
go build -ldflags="-w -s" -o $(DIST_DIR)/$(BINARY_NAME) ./cmd/$(BINARY_NAME)
|
||||
|
||||
## docker-build-dev: Build the development Docker image (with tests and coverage)
|
||||
docker-build-dev:
|
||||
$(DOCKER_BUILD) --target builder -t $(DEV_IMAGE) -f Dockerfile .
|
||||
|
||||
## docker-build-dev-no-test: Build the development Docker image WITHOUT tests (faster)
|
||||
docker-build-dev-no-test:
|
||||
$(DOCKER_BUILD) --target builder --no-cache --build-arg SKIP_TESTS=true -t $(DEV_IMAGE) -f Dockerfile .
|
||||
|
||||
## docker-build-runtime: Build the runtime Docker image (fast, no tests)
|
||||
docker-build-runtime:
|
||||
$(DOCKER_BUILD) --target runtime -t $(RUNTIME_IMAGE) -f Dockerfile .
|
||||
|
||||
## test: Run unit tests locally
|
||||
test:
|
||||
go test -race -coverprofile=coverage.out ./...
|
||||
|
||||
## test-docker: Run unit tests inside Docker container
|
||||
test-docker: docker-build-dev
|
||||
@echo "Tests already run in builder stage"
|
||||
|
||||
## lint: Run linters
|
||||
lint:
|
||||
go vet ./...
|
||||
gofmt -l .
|
||||
|
||||
## fmt: Format all Go files
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
## package: Build RPM packages for all target distributions
|
||||
package: package-rpm
|
||||
|
||||
## package-rpm: Build RPM packages for Rocky Linux 8/9, AlmaLinux 10 (requires Docker)
|
||||
## Uses buildx for parallel builds (el8, el9, el10 built simultaneously)
|
||||
package-rpm:
|
||||
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
|
||||
@echo "Starting parallel RPM builds for el8, el9, el10..."
|
||||
# Build all three distributions in parallel using buildx
|
||||
$(DOCKER_BUILDX) build --target output -t $(PACKAGER_IMAGE) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package . \
|
||||
--load
|
||||
@echo "Extracting RPM packages from Docker image..."
|
||||
$(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm $(PACKAGER_IMAGE) sh -c \
|
||||
"cp -r /packages/rpm/el8 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el9 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el10 /output/rpm/"
|
||||
@echo "RPM packages created:"
|
||||
@echo " Enterprise Linux 8 (el8):"
|
||||
ls -la $(DIST_DIR)/rpm/el8/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " Enterprise Linux 9 (el9):"
|
||||
ls -la $(DIST_DIR)/rpm/el9/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " Enterprise Linux 10 (el10):"
|
||||
ls -la $(DIST_DIR)/rpm/el10/ 2>/dev/null || echo " (no packages)"
|
||||
|
||||
## package-rpm-sequential: Build RPM packages sequentially (fallback if parallel fails)
|
||||
package-rpm-sequential:
|
||||
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
|
||||
@echo "Building RPM for el8..."
|
||||
$(DOCKER_BUILD) --target rpm-el8-builder -t $(PACKAGER_IMAGE_EL8) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Building RPM for el9..."
|
||||
$(DOCKER_BUILD) --target rpm-el9-builder -t $(PACKAGER_IMAGE_EL9) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Building RPM for el10..."
|
||||
$(DOCKER_BUILD) --target rpm-el10-builder -t $(PACKAGER_IMAGE_EL10) \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Extracting RPM packages..."
|
||||
$(DOCKER_RUN) --rm -v $(PWD)/$(DIST_DIR)/rpm:/output/rpm \
|
||||
-v $(PACKAGER_IMAGE_EL8):/el8:ro \
|
||||
-v $(PACKAGER_IMAGE_EL9):/el9:ro \
|
||||
-v $(PACKAGER_IMAGE_EL10):/el10:ro \
|
||||
alpine:latest sh -c \
|
||||
"cp -r /el8/packages/rpm/el8 /output/rpm/ && \
|
||||
cp -r /el9/packages/rpm/el9 /output/rpm/ && \
|
||||
cp -r /el10/packages/rpm/el10 /output/rpm/"
|
||||
|
||||
## test-package-rpm: Test RPM package installation in Docker
|
||||
test-package-rpm: package-rpm
|
||||
./packaging/test/test-rpm.sh
|
||||
|
||||
## test-package: Test RPM package installation
|
||||
test-package: test-package-rpm
|
||||
|
||||
## ci: Full CI pipeline (tests, build, packages, package tests)
|
||||
ci: ci-test ci-build ci-package ci-package-test
|
||||
|
||||
## ci-test: Run all tests for CI
|
||||
ci-test: test lint
|
||||
|
||||
## ci-build: Build for CI (production binary)
|
||||
ci-build: build
|
||||
|
||||
## ci-package: Build all packages for CI
|
||||
ci-package: package
|
||||
|
||||
## ci-package-test: Test all packages for CI
|
||||
ci-package-test: test-package
|
||||
|
||||
## clean: Clean build artifacts and Docker images
|
||||
clean:
|
||||
rm -rf $(DIST_DIR)/
|
||||
rm -f coverage.out
|
||||
$(DOCKER) rmi $(DEV_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(RUNTIME_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(PACKAGER_IMAGE) 2>/dev/null || true
|
||||
|
||||
## help: Show this help message
|
||||
help:
|
||||
@echo "Usage: make [target]"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@sed -n 's/^##//p' $(MAKEFILE_LIST) | column -t -s ':' | sed -e 's/^/ /'
|
||||
@ -1,426 +0,0 @@
|
||||
# logcorrelator
|
||||
|
||||
Service de corrélation de logs HTTP et réseau écrit en Go.
|
||||
|
||||
## Description
|
||||
|
||||
**logcorrelator** reçoit deux flux de logs JSON via des sockets Unix datagrammes (SOCK_DGRAM) :
|
||||
- **Source A** : logs HTTP applicatifs (Apache, reverse proxy)
|
||||
- **Source B** : logs réseau (métadonnées IP/TCP, JA3/JA4, etc.)
|
||||
|
||||
Il corrèle les événements sur la base de `src_ip + src_port` dans une fenêtre temporelle configurable, et produit des logs corrélés vers :
|
||||
- Un fichier local (JSON lines)
|
||||
- ClickHouse (pour analyse et archivage)
|
||||
|
||||
Les logs opérationnels du service (démarrage, erreurs, métriques) sont écrits sur **stderr** et collectés par journald. Aucune donnée corrélée n'apparaît sur stdout.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────────┐ ┌──────────────────┐ ┌─────────────────┐
|
||||
│ Source A │────▶│ │────▶│ File Sink │
|
||||
│ HTTP/Apache │ │ Correlation │ │ (JSON lines) │
|
||||
│ (Unix DGRAM) │ │ Service │ └─────────────────┘
|
||||
└─────────────────┘ │ │
|
||||
│ - Buffers │ ┌─────────────────┐
|
||||
┌─────────────────┐ │ - Time Window │────▶│ ClickHouse │
|
||||
│ Source B │────▶│ - Orphan Policy │ │ Sink │
|
||||
│ Réseau/JA4 │ │ - Keep-Alive │ └─────────────────┘
|
||||
│ (Unix DGRAM) │ └──────────────────┘
|
||||
└─────────────────┘
|
||||
```
|
||||
|
||||
Architecture hexagonale : domaine pur (`internal/domain`), ports abstraits (`internal/ports`), adaptateurs (`internal/adapters`), orchestration (`internal/app`).
|
||||
|
||||
## Build (100% Docker)
|
||||
|
||||
Tout le build, les tests et le packaging RPM s'exécutent dans des conteneurs :
|
||||
|
||||
```bash
|
||||
# Build complet avec tests (builder stage)
|
||||
make docker-build-dev
|
||||
|
||||
# Packaging RPM (el8, el9, el10)
|
||||
make package-rpm
|
||||
|
||||
# Build rapide sans tests
|
||||
make docker-build-dev-no-test
|
||||
|
||||
# Tests en local (nécessite Go 1.21+)
|
||||
make test
|
||||
```
|
||||
|
||||
### Prérequis
|
||||
|
||||
- Docker 20.10+
|
||||
|
||||
## Installation
|
||||
|
||||
### Packages RPM
|
||||
|
||||
```bash
|
||||
# Générer les packages
|
||||
make package-rpm
|
||||
|
||||
# Installer (Rocky Linux / AlmaLinux)
|
||||
sudo dnf install -y dist/rpm/el8/logcorrelator-1.1.12-1.el8.x86_64.rpm
|
||||
sudo dnf install -y dist/rpm/el9/logcorrelator-1.1.12-1.el9.x86_64.rpm
|
||||
sudo dnf install -y dist/rpm/el10/logcorrelator-1.1.12-1.el10.x86_64.rpm
|
||||
|
||||
# Démarrer
|
||||
sudo systemctl enable --now logcorrelator
|
||||
sudo systemctl status logcorrelator
|
||||
```
|
||||
|
||||
### Build manuel
|
||||
|
||||
```bash
|
||||
# Binaire local (nécessite Go 1.21+)
|
||||
go build -o logcorrelator ./cmd/logcorrelator
|
||||
./logcorrelator -config config.example.yml
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Fichier YAML. Voir `config.example.yml` pour un exemple complet.
|
||||
|
||||
```yaml
|
||||
log:
|
||||
level: INFO # DEBUG, INFO, WARN, ERROR
|
||||
|
||||
inputs:
|
||||
unix_sockets:
|
||||
- name: http
|
||||
source_type: A # Source HTTP
|
||||
path: /var/run/logcorrelator/http.socket
|
||||
format: json
|
||||
socket_permissions: "0666"
|
||||
- name: network
|
||||
source_type: B # Source réseau
|
||||
path: /var/run/logcorrelator/network.socket
|
||||
format: json
|
||||
socket_permissions: "0666"
|
||||
|
||||
outputs:
|
||||
file:
|
||||
path: /var/log/logcorrelator/correlated.log
|
||||
clickhouse:
|
||||
enabled: false
|
||||
dsn: clickhouse://user:pass@localhost:9000/db
|
||||
table: http_logs_raw
|
||||
batch_size: 500
|
||||
flush_interval_ms: 200
|
||||
max_buffer_size: 5000
|
||||
drop_on_overflow: true
|
||||
timeout_ms: 1000
|
||||
stdout:
|
||||
enabled: false # no-op pour les données ; logs opérationnels toujours sur stderr
|
||||
|
||||
correlation:
|
||||
time_window:
|
||||
value: 10
|
||||
unit: s
|
||||
orphan_policy:
|
||||
apache_always_emit: true
|
||||
apache_emit_delay_ms: 500 # délai avant émission orphelin A (ms)
|
||||
network_emit: false
|
||||
matching:
|
||||
mode: one_to_many # Keep-Alive : un B peut corréler plusieurs A successifs
|
||||
buffers:
|
||||
max_http_items: 10000
|
||||
max_network_items: 20000
|
||||
ttl:
|
||||
network_ttl_s: 120 # TTL remis à zéro à chaque corrélation (Keep-Alive)
|
||||
# Exclure des IPs source (IPs uniques ou plages CIDR)
|
||||
exclude_source_ips:
|
||||
- 10.0.0.1
|
||||
- 172.16.0.0/12
|
||||
# Restreindre la corrélation à certains ports de destination (optionnel)
|
||||
# Si la liste est vide, tous les ports sont corrélés
|
||||
include_dest_ports:
|
||||
- 80
|
||||
- 443
|
||||
|
||||
metrics:
|
||||
enabled: false
|
||||
addr: ":8080"
|
||||
```
|
||||
|
||||
### Format du DSN ClickHouse
|
||||
|
||||
```
|
||||
clickhouse://username:password@host:port/database
|
||||
```
|
||||
|
||||
Ports : `9000` (natif, recommandé) ou `8123` (HTTP).
|
||||
|
||||
## Format des logs
|
||||
|
||||
### Source A (HTTP)
|
||||
|
||||
```json
|
||||
{
|
||||
"src_ip": "192.168.1.1", "src_port": 8080,
|
||||
"dst_ip": "10.0.0.1", "dst_port": 443,
|
||||
"timestamp": 1704110400000000000,
|
||||
"method": "GET", "path": "/api/test"
|
||||
}
|
||||
```
|
||||
|
||||
### Source B (Réseau)
|
||||
|
||||
```json
|
||||
{
|
||||
"src_ip": "192.168.1.1", "src_port": 8080,
|
||||
"dst_ip": "10.0.0.1", "dst_port": 443,
|
||||
"ja3": "abc123", "ja4": "xyz789"
|
||||
}
|
||||
```
|
||||
|
||||
### Log corrélé (sortie)
|
||||
|
||||
Structure JSON plate — tous les champs A et B sont fusionnés à la racine :
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": "2024-01-01T12:00:00Z",
|
||||
"src_ip": "192.168.1.1", "src_port": 8080,
|
||||
"dst_ip": "10.0.0.1", "dst_port": 443,
|
||||
"correlated": true,
|
||||
"method": "GET", "path": "/api/test",
|
||||
"ja3": "abc123", "ja4": "xyz789"
|
||||
}
|
||||
```
|
||||
|
||||
En cas de collision de champ entre A et B, les deux valeurs sont conservées avec préfixes `a_` et `b_`.
|
||||
|
||||
Les orphelins A (sans B correspondant) sont émis avec `"correlated": false, "orphan_side": "A"`.
|
||||
|
||||
## Schema ClickHouse
|
||||
|
||||
Le fichier `sql/init.sql` contient le schéma complet prêt à l'emploi.
|
||||
|
||||
```bash
|
||||
clickhouse-client --multiquery < sql/init.sql
|
||||
```
|
||||
|
||||
### Architecture des tables
|
||||
|
||||
```
|
||||
http_logs_raw ← inserts du service (raw_json String)
|
||||
│
|
||||
└─ mv_http_logs ← vue matérialisée (parse JSON → colonnes typées)
|
||||
│
|
||||
▼
|
||||
http_logs ← table requêtable par les analystes
|
||||
```
|
||||
|
||||
### Table `http_logs` — colonnes
|
||||
|
||||
| Groupe | Colonnes |
|
||||
|---|---|
|
||||
| Temporel | `time` DateTime, `log_date` Date |
|
||||
| Réseau | `src_ip` IPv4, `src_port` UInt16, `dst_ip` IPv4, `dst_port` UInt16 |
|
||||
| HTTP | `method`, `scheme`, `host`, `path`, `query`, `http_version` (LowCardinality) |
|
||||
| Corrélation | `orphan_side`, `correlated` UInt8, `keepalives` UInt16, `a_timestamp`/`b_timestamp` UInt64, `conn_id` |
|
||||
| IP meta | `ip_meta_df` UInt8, `ip_meta_id` UInt16, `ip_meta_total_length` UInt16, `ip_meta_ttl` UInt8 |
|
||||
| TCP meta | `tcp_meta_options`, `tcp_meta_window_size` UInt32, `tcp_meta_mss` UInt16, `tcp_meta_window_scale` UInt8, `syn_to_clienthello_ms` Int32 |
|
||||
| TLS / fingerprint | `tls_version`, `tls_sni`, `tls_alpn` (LowCardinality), `ja3`, `ja3_hash`, `ja4` |
|
||||
| En-têtes HTTP | `header_user_agent`, `header_accept`, `header_accept_encoding`, `header_accept_language`, `header_x_request_id`, `header_x_trace_id`, `header_x_forwarded_for`, `header_sec_ch_ua*`, `header_sec_fetch_*` |
|
||||
|
||||
### Utilisateurs et permissions
|
||||
|
||||
```sql
|
||||
-- data_writer : INSERT sur http_logs_raw uniquement (compte du service)
|
||||
GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;
|
||||
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
|
||||
|
||||
-- analyst : lecture sur la table parsée
|
||||
GRANT SELECT ON ja4_logs.http_logs TO analyst;
|
||||
```
|
||||
|
||||
### Vérification de l'ingestion
|
||||
|
||||
```sql
|
||||
-- Données brutes reçues
|
||||
SELECT count(*), min(ingest_time), max(ingest_time) FROM ja4_logs.http_logs_raw;
|
||||
|
||||
-- Données parsées par la vue matérialisée
|
||||
SELECT count(*), min(time), max(time) FROM ja4_logs.http_logs;
|
||||
|
||||
-- Derniers logs corrélés
|
||||
SELECT time, src_ip, dst_ip, method, host, path, ja4
|
||||
FROM ja4_logs.http_logs
|
||||
WHERE correlated = 1
|
||||
ORDER BY time DESC LIMIT 10;
|
||||
```
|
||||
|
||||
## Signaux
|
||||
|
||||
| Signal | Comportement |
|
||||
|--------|--------------|
|
||||
| `SIGINT` / `SIGTERM` | Arrêt gracieux (drain buffers, flush sinks) |
|
||||
| `SIGHUP` | Réouverture des fichiers de sortie (log rotation) |
|
||||
|
||||
## Logs internes
|
||||
|
||||
Les logs opérationnels vont sur **stderr** :
|
||||
|
||||
```bash
|
||||
# Systemd
|
||||
journalctl -u logcorrelator -f
|
||||
|
||||
# Docker
|
||||
docker logs -f logcorrelator
|
||||
```
|
||||
|
||||
## Structure du projet
|
||||
|
||||
```
|
||||
cmd/logcorrelator/ # Point d'entrée
|
||||
internal/
|
||||
adapters/
|
||||
inbound/unixsocket/ # Lecture SOCK_DGRAM → NormalizedEvent
|
||||
outbound/
|
||||
clickhouse/ # Sink ClickHouse (batch, retry, logging complet)
|
||||
file/ # Sink fichier (JSON lines, SIGHUP reopen)
|
||||
multi/ # Fan-out vers plusieurs sinks
|
||||
stdout/ # No-op pour les données (logs opérationnels sur stderr)
|
||||
app/ # Orchestrator (sources → corrélation → sinks)
|
||||
config/ # Chargement/validation YAML
|
||||
domain/ # CorrelationService, NormalizedEvent, CorrelatedLog
|
||||
observability/ # Logger, métriques, serveur HTTP /metrics /health
|
||||
ports/ # Interfaces EventSource, CorrelatedLogSink, CorrelationProcessor
|
||||
config.example.yml # Exemple de configuration
|
||||
Dockerfile # Build multi-stage (builder, runtime, dev)
|
||||
Dockerfile.package # Packaging RPM multi-distros (el8, el9, el10)
|
||||
Makefile # Cibles de build
|
||||
architecture.yml # Spécification architecture
|
||||
logcorrelator.service # Unité systemd
|
||||
```
|
||||
|
||||
## Débogage
|
||||
|
||||
### Logs DEBUG
|
||||
|
||||
```yaml
|
||||
log:
|
||||
level: DEBUG
|
||||
```
|
||||
|
||||
Exemples de logs produits :
|
||||
```
|
||||
[unixsocket:http] DEBUG event received: source=A src_ip=192.168.1.1 src_port=8080
|
||||
[correlation] DEBUG processing A event: key=192.168.1.1:8080
|
||||
[correlation] DEBUG correlation found: A(src_ip=... src_port=... ts=...) + B(...)
|
||||
[correlation] DEBUG A event has no matching B key in buffer: key=...
|
||||
[correlation] DEBUG event excluded by IP filter: source=A src_ip=10.0.0.1 src_port=8080
|
||||
[correlation] DEBUG event excluded by dest port filter: source=A dst_port=22
|
||||
[correlation] DEBUG TTL reset for B event (Keep-Alive): key=... new_ttl=120s
|
||||
[clickhouse] DEBUG batch sent: rows=42 table=http_logs_raw
|
||||
```
|
||||
|
||||
### Serveur de métriques
|
||||
|
||||
```yaml
|
||||
metrics:
|
||||
enabled: true
|
||||
addr: ":8080"
|
||||
```
|
||||
|
||||
`GET /health` → `{"status":"healthy"}`
|
||||
|
||||
`GET /metrics` :
|
||||
|
||||
```json
|
||||
{
|
||||
"events_received_a": 1542, "events_received_b": 1498,
|
||||
"correlations_success": 1450, "correlations_failed": 92,
|
||||
"failed_no_match_key": 45, "failed_time_window": 23,
|
||||
"failed_buffer_eviction": 5, "failed_ttl_expired": 12,
|
||||
"failed_ip_excluded": 7, "failed_dest_port_filtered": 3,
|
||||
"buffer_a_size": 23, "buffer_b_size": 18,
|
||||
"orphans_emitted_a": 92, "orphans_pending_a": 4,
|
||||
"keepalive_resets": 892
|
||||
}
|
||||
```
|
||||
|
||||
### Diagnostic par métriques
|
||||
|
||||
| Métrique élevée | Cause | Solution |
|
||||
|---|---|---|
|
||||
| `failed_no_match_key` | A et B n'ont pas le même `src_ip:src_port` | Vérifier les deux sources |
|
||||
| `failed_time_window` | Timestamps trop éloignés | Augmenter `time_window.value` ou vérifier NTP |
|
||||
| `failed_ttl_expired` | B expire avant corrélation | Augmenter `ttl.network_ttl_s` |
|
||||
| `failed_buffer_eviction` | Buffers trop petits | Augmenter `buffers.max_http_items` / `max_network_items` |
|
||||
| `failed_ip_excluded` | Traffic depuis IPs exclues | Normal si attendu |
|
||||
| `failed_dest_port_filtered` | Traffic sur ports non listés | Vérifier `include_dest_ports` |
|
||||
| `orphans_emitted_a` élevé | Beaucoup de A sans B | Vérifier que la source B envoie des événements |
|
||||
|
||||
### Filtrage par IP source
|
||||
|
||||
```yaml
|
||||
correlation:
|
||||
exclude_source_ips:
|
||||
- 10.0.0.1 # IP unique (health checks)
|
||||
- 172.16.0.0/12 # Plage CIDR
|
||||
```
|
||||
|
||||
Les événements depuis ces IPs sont silencieusement ignorés (non corrélés, non émis en orphelin). La métrique `failed_ip_excluded` comptabilise les exclusions.
|
||||
|
||||
### Filtrage par port de destination
|
||||
|
||||
```yaml
|
||||
correlation:
|
||||
include_dest_ports:
|
||||
- 80 # HTTP
|
||||
- 443 # HTTPS
|
||||
- 8080
|
||||
- 8443
|
||||
```
|
||||
|
||||
Si la liste est non vide, seuls les événements dont le `dst_port` est dans la liste participent à la corrélation. Les autres sont silencieusement ignorés. Liste vide = tous les ports corrélés (comportement par défaut). La métrique `failed_dest_port_filtered` comptabilise les exclusions.
|
||||
|
||||
### Scripts de test
|
||||
|
||||
```bash
|
||||
# Script Bash (simple)
|
||||
./scripts/test-correlation.sh -c 10 -v
|
||||
|
||||
# Script Python (scénarios complets : basic, time window, keepalive, différentes IPs)
|
||||
pip install requests
|
||||
python3 scripts/test-correlation-advanced.py --all
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### ClickHouse : erreurs d'insertion
|
||||
|
||||
- **`No such column`** : vérifier que la table `http_logs_raw` utilise la colonne unique `raw_json` (pas de colonnes séparées)
|
||||
- **`ACCESS_DENIED`** : `GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;`
|
||||
- Les erreurs de flush sont loggées en ERROR dans les logs du service
|
||||
|
||||
### Vue matérialisée vide
|
||||
|
||||
Si `http_logs_raw` a des données mais `http_logs` est vide :
|
||||
```sql
|
||||
-- Vérifier la vue
|
||||
SHOW CREATE TABLE ja4_logs.mv_http_logs;
|
||||
-- Vérifier les permissions (la MV s'exécute sous le compte du service)
|
||||
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
|
||||
```
|
||||
|
||||
### Sockets Unix : permission denied
|
||||
|
||||
Vérifier que `socket_permissions: "0666"` est configuré et que le répertoire `/var/run/logcorrelator` appartient à l'utilisateur `logcorrelator`.
|
||||
|
||||
### Service systemd ne démarre pas
|
||||
|
||||
```bash
|
||||
journalctl -u logcorrelator -n 50 --no-pager
|
||||
/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@ -1,974 +0,0 @@
|
||||
service:
|
||||
name: logcorrelator
|
||||
context: http-network-correlation
|
||||
language: go
|
||||
pattern: hexagonal
|
||||
description: >
|
||||
logcorrelator est un service système (lancé par systemd) écrit en Go, chargé
|
||||
de recevoir deux flux de logs JSON via des sockets Unix, de corréler les
|
||||
événements HTTP applicatifs (source A, typiquement Apache ou reverse proxy)
|
||||
avec des événements réseau (source B, métadonnées IP/TCP, JA3/JA4, etc.)
|
||||
sur la base de la combinaison strictement définie src_ip + src_port, avec
|
||||
une fenêtre temporelle configurable. Le service supporte les connexions
|
||||
HTTP Keep-Alive : un log réseau peut être corrélé à plusieurs logs HTTP
|
||||
successifs (stratégie 1‑à‑N). La rétention en mémoire est bornée par des
|
||||
tailles de caches configurables et un TTL dynamique pour la source B. Le
|
||||
service émet toujours les événements A même lorsqu'aucun événement B n'est
|
||||
disponible, n'émet jamais de logs B seuls, et pousse les résultats vers
|
||||
ClickHouse et/ou un fichier local.
|
||||
|
||||
Fonctionnalités de débogage incluses :
|
||||
- Serveur de métriques HTTP (/metrics, /health)
|
||||
- Logs DEBUG détaillés avec raisons des échecs de corrélation
|
||||
- Filtrage des IPs source (exclude_source_ips)
|
||||
- Scripts de test (Bash et Python)
|
||||
- Métriques : événements reçus, corrélations, échecs par raison, buffers, orphelins
|
||||
|
||||
runtime:
|
||||
deployment:
|
||||
unit_type: systemd
|
||||
description: >
|
||||
logcorrelator est livré sous forme de binaire autonome, exécuté comme un
|
||||
service systemd. L'unité systemd assure le démarrage automatique au boot,
|
||||
le redémarrage en cas de crash, et une intégration standard dans l'écosystème
|
||||
Linux.
|
||||
binary_path: /usr/bin/logcorrelator
|
||||
config_path: /etc/logcorrelator/logcorrelator.yml
|
||||
user: logcorrelator
|
||||
group: logcorrelator
|
||||
restart: on-failure
|
||||
systemd_unit:
|
||||
path: /etc/systemd/system/logcorrelator.service
|
||||
content_example: |
|
||||
[Unit]
|
||||
Description=logcorrelator service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=logcorrelator
|
||||
Group=logcorrelator
|
||||
ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/var/log/logcorrelator /var/run/logcorrelator /etc/logcorrelator
|
||||
|
||||
# Resource limits
|
||||
LimitNOFILE=65536
|
||||
|
||||
# Systemd timeouts
|
||||
TimeoutStartSec=10
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
os:
|
||||
supported:
|
||||
- rocky-linux-8
|
||||
- rocky-linux-9
|
||||
- almalinux-10
|
||||
- autres-linux-recentes
|
||||
logs:
|
||||
stdout_stderr: journald
|
||||
structured: true
|
||||
description: >
|
||||
Les logs internes du service (erreurs, messages d'information) sont envoyés
|
||||
vers stdout/stderr et collectés par journald. Ils sont structurés et ne
|
||||
contiennent pas de données personnelles.
|
||||
signals:
|
||||
graceful_shutdown:
|
||||
- SIGINT
|
||||
- SIGTERM
|
||||
reload:
|
||||
- SIGHUP
|
||||
description: >
|
||||
SIGINT/SIGTERM : arrêt propre (arrêt des sockets, vidage des buffers, fermeture
|
||||
des sinks). SIGHUP : réouverture des fichiers de sortie (utile pour la
|
||||
rotation des logs via logrotate) sans arrêter le service.
|
||||
filesystem:
|
||||
description: >
|
||||
Permissions et propriété des fichiers et répertoires utilisés par logcorrelator.
|
||||
directories:
|
||||
- path: /var/run/logcorrelator
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0755"
|
||||
purpose: >
|
||||
Contient les sockets Unix (http.socket, network.socket).
|
||||
Les sockets sont créés avec des permissions 0666 (world read/write).
|
||||
- path: /var/log/logcorrelator
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0750"
|
||||
purpose: >
|
||||
Contient les logs corrélés (correlated.log).
|
||||
- path: /var/lib/logcorrelator
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0750"
|
||||
purpose: >
|
||||
Répertoire home du service (données internes).
|
||||
- path: /etc/logcorrelator
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0750"
|
||||
purpose: >
|
||||
Contient la configuration (logcorrelator.yml, logcorrelator.yml.example).
|
||||
files:
|
||||
- path: /etc/logcorrelator/logcorrelator.yml
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0640"
|
||||
rpm_directive: "%config(noreplace)"
|
||||
- path: /etc/logcorrelator/logcorrelator.yml.example
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0640"
|
||||
- path: /etc/systemd/system/logcorrelator.service
|
||||
owner: root:root
|
||||
permissions: "0644"
|
||||
- path: /etc/logrotate.d/logcorrelator
|
||||
owner: root:root
|
||||
permissions: "0644"
|
||||
rpm_directive: "%config(noreplace)"
|
||||
sockets:
|
||||
- path: /var/run/logcorrelator/http.socket
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0666"
|
||||
type: unix_datagram
|
||||
purpose: "Source A - logs HTTP applicatifs"
|
||||
- path: /var/run/logcorrelator/network.socket
|
||||
owner: logcorrelator:logcorrelator
|
||||
permissions: "0666"
|
||||
type: unix_datagram
|
||||
purpose: "Source B - logs réseau"
|
||||
|
||||
packaging:
|
||||
description: >
|
||||
logcorrelator est distribué sous forme de packages .rpm (Rocky Linux, AlmaLinux,
|
||||
RHEL), construits intégralement dans des conteneurs. Le changelog RPM est mis
|
||||
à jour à chaque changement de version.
|
||||
Tous les numéros de version doivent être cohérents entre le spec RPM, le Makefile
|
||||
(PKG_VERSION), le CHANGELOG.md et les tags git.
|
||||
|
||||
Politique de mise à jour de la configuration :
|
||||
- Le fichier logcorrelator.yml est marqué %config(noreplace) : il n'est JAMAIS
|
||||
écrasé lors d'une mise à jour. La configuration existante est préservée.
|
||||
- Le fichier logcorrelator.yml.example est TOUJOURS mis à jour pour refléter
|
||||
les nouvelles options de configuration disponibles.
|
||||
- Lors de la première installation, si logcorrelator.yml n'existe pas, il est
|
||||
créé à partir de logcorrelator.yml.example.
|
||||
formats:
|
||||
- rpm
|
||||
target_distros:
|
||||
- rocky-linux-8
|
||||
- rocky-linux-9
|
||||
- almalinux-10
|
||||
- rhel-8
|
||||
- rhel-9
|
||||
- rhel-10
|
||||
rpm:
|
||||
tool: fpm
|
||||
changelog:
|
||||
source: git # ou CHANGELOG.md
|
||||
description: >
|
||||
À chaque build, un script génère un fichier de changelog RPM à partir de
|
||||
l'historique (tags/commits) et le passe à fpm (option --rpm-changelog).
|
||||
contents:
|
||||
- path: /usr/bin/logcorrelator
|
||||
type: binary
|
||||
- path: /etc/logcorrelator/logcorrelator.yml
|
||||
type: config
|
||||
directives: "%config(noreplace)"
|
||||
behavior: >
|
||||
Jamais écrasé lors des mises à jour. Préservé automatiquement par RPM.
|
||||
Créé uniquement lors de la première installation s'il n'existe pas.
|
||||
- path: /etc/logcorrelator/logcorrelator.yml.example
|
||||
type: doc
|
||||
behavior: >
|
||||
TOUJOURS mis à jour lors des mises à jour. Sert de référence pour les
|
||||
nouvelles options de configuration disponibles.
|
||||
- path: /etc/systemd/system/logcorrelator.service
|
||||
type: systemd_unit
|
||||
- path: /etc/logrotate.d/logcorrelator
|
||||
type: logrotate_script
|
||||
directives: "%config(noreplace)"
|
||||
logrotate_example: |
|
||||
/var/log/logcorrelator/correlated.log {
|
||||
daily
|
||||
rotate 7
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 logcorrelator logcorrelator
|
||||
sharedscripts
|
||||
postrotate
|
||||
/bin/systemctl reload logcorrelator > /dev/null 2>&1 || true
|
||||
endscript
|
||||
}
|
||||
|
||||
config:
|
||||
format: yaml
|
||||
location: /etc/logcorrelator/logcorrelator.yml
|
||||
reload_strategy: signal_sighup_for_files
|
||||
description: >
|
||||
Toute la configuration est centralisée dans un fichier YAML lisible. Le RPM
|
||||
fournit aussi un fichier d'exemple mis à jour à chaque version.
|
||||
example: |
|
||||
# /etc/logcorrelator/logcorrelator.yml
|
||||
|
||||
log:
|
||||
level: INFO # DEBUG, INFO, WARN, ERROR
|
||||
|
||||
inputs:
|
||||
unix_sockets:
|
||||
# Source HTTP (A) : logs applicatifs en JSON, 1 datagramme = 1 log.
|
||||
- name: http
|
||||
source_type: A
|
||||
path: /var/run/logcorrelator/http.socket
|
||||
format: json
|
||||
socket_permissions: "0666"
|
||||
|
||||
# Source réseau (B) : logs IP/TCP/JA3... en JSON, 1 datagramme = 1 log.
|
||||
- name: network
|
||||
source_type: B
|
||||
path: /var/run/logcorrelator/network.socket
|
||||
format: json
|
||||
socket_permissions: "0666"
|
||||
|
||||
outputs:
|
||||
file:
|
||||
enabled: true
|
||||
path: /var/log/logcorrelator/correlated.log
|
||||
|
||||
clickhouse:
|
||||
enabled: false
|
||||
dsn: clickhouse://user:pass@localhost:9000/db
|
||||
table: correlated_logs_http_network
|
||||
batch_size: 500
|
||||
flush_interval_ms: 200
|
||||
max_buffer_size: 5000
|
||||
drop_on_overflow: true
|
||||
async_insert: true
|
||||
timeout_ms: 1000
|
||||
|
||||
stdout:
|
||||
enabled: false
|
||||
level: INFO # DEBUG: tous les logs (y compris orphelins), INFO: seulement corrélés, WARN: corrélés seulement, ERROR: aucun
|
||||
|
||||
correlation:
|
||||
# Fenêtre de corrélation : si le log HTTP arrive avant le réseau, il attend
|
||||
# au plus cette durée (sauf éviction du cache HTTP).
|
||||
# Augmentée à 10s pour supporter le Keep-Alive HTTP.
|
||||
time_window:
|
||||
value: 10
|
||||
unit: s
|
||||
|
||||
orphan_policy:
|
||||
apache_always_emit: true # Toujours émettre les événements A, même sans correspondance B
|
||||
network_emit: false # Ne jamais émettre les événements B seuls
|
||||
|
||||
matching:
|
||||
mode: one_to_many # Keep‑Alive : un B peut corréler plusieurs A.
|
||||
|
||||
buffers:
|
||||
# Tailles max des caches en mémoire (en nombre de logs).
|
||||
max_http_items: 10000
|
||||
max_network_items: 20000
|
||||
|
||||
ttl:
|
||||
# Durée de vie standard d'un log réseau (B) en mémoire. Chaque corrélation
|
||||
# réussie avec un A réinitialise ce TTL.
|
||||
# Augmenté à 120s pour supporter les sessions HTTP Keep-Alive longues.
|
||||
network_ttl_s: 120
|
||||
|
||||
# Filtrage des IPs source à exclure (optionnel)
|
||||
exclude_source_ips:
|
||||
- 10.0.0.1 # IP unique
|
||||
- 172.16.0.0/12 # Plage CIDR
|
||||
# Les événements depuis ces IPs sont silencieusement ignorés
|
||||
|
||||
# Serveur de métriques HTTP (optionnel, pour débogage et monitoring)
|
||||
metrics:
|
||||
enabled: false
|
||||
addr: ":8080" # Adresse d'écoute du serveur HTTP
|
||||
# Endpoints:
|
||||
# GET /metrics - Retourne les métriques de corrélation en JSON
|
||||
# GET /health - Health check
|
||||
|
||||
inputs:
|
||||
description: >
|
||||
Deux flux de logs JSON via sockets Unix datagram (SOCK_DGRAM). Chaque datagramme
|
||||
contient un JSON complet. Le champ source_type ("A" ou "B") doit être spécifié
|
||||
pour chaque socket. À défaut, la source est déduite automatiquement (présence de
|
||||
headers = source A, sinon source B).
|
||||
unix_sockets:
|
||||
- name: http
|
||||
id: A
|
||||
description: >
|
||||
Source A, logs HTTP applicatifs (Apache, reverse proxy, etc.). Schéma JSON
|
||||
variable, champ timestamp (int64, nanosecondes) obligatoire, headers dynamiques (header_*).
|
||||
path: /var/run/logcorrelator/http.socket
|
||||
source_type: A
|
||||
permissions: "0666"
|
||||
protocol: unix
|
||||
socket_type: dgram
|
||||
mode: datagram
|
||||
format: json
|
||||
framing: message
|
||||
max_datagram_bytes: 65535
|
||||
retry_on_error: true
|
||||
|
||||
- name: network
|
||||
id: B
|
||||
description: >
|
||||
Source B, logs réseau (métadonnées IP/TCP, JA3/JA4, etc.). Seuls src_ip
|
||||
et src_port sont requis pour la corrélation. Le champ timestamp est optionnel ;
|
||||
s'il est absent, l'heure de réception est utilisée.
|
||||
path: /var/run/logcorrelator/network.socket
|
||||
source_type: B
|
||||
permissions: "0666"
|
||||
protocol: unix
|
||||
socket_type: dgram
|
||||
mode: datagram
|
||||
format: json
|
||||
framing: message
|
||||
max_datagram_bytes: 65535
|
||||
retry_on_error: true
|
||||
|
||||
outputs:
|
||||
description: >
|
||||
Les logs corrélés sont envoyés vers un ou plusieurs sinks (MultiSink).
|
||||
sinks:
|
||||
file:
|
||||
enabled: true
|
||||
description: >
|
||||
Sink fichier local. Un JSON par ligne. Rotation gérée par logrotate,
|
||||
réouverture du fichier sur SIGHUP. Le champ `enabled: false` coupe
|
||||
completement l'ecriture du fichier (le sink n'est pas cree).
|
||||
path: /var/log/logcorrelator/correlated.log
|
||||
format: json_lines
|
||||
rotate_managed_by: external_logrotate
|
||||
clickhouse:
|
||||
enabled: false
|
||||
description: >
|
||||
Sink principal pour l'archivage et l'analyse quasi temps réel. Inserts
|
||||
batch asynchrones, drop en cas de saturation. Le service insère uniquement
|
||||
dans une table RAW (raw_json String, ingest_time DateTime DEFAULT now()).
|
||||
La table parsée et la vue matérialisée sont gérées en externe (DDL séparés).
|
||||
Toutes les erreurs de connexion, de flush et de retry sont loggées :
|
||||
INFO à la connexion, ERROR sur échec de flush, WARN sur drop/retry, DEBUG sur envoi réussi.
|
||||
dsn: clickhouse://user:pass@host:9000/db
|
||||
table: correlated_logs_http_network
|
||||
batch_size: 500
|
||||
flush_interval_ms: 200
|
||||
max_buffer_size: 5000
|
||||
drop_on_overflow: true
|
||||
async_insert: true
|
||||
timeout_ms: 1000
|
||||
stdout:
|
||||
enabled: false
|
||||
description: >
|
||||
Sink no-op pour les données. Aucune donnée corrélée ou orpheline n'est
|
||||
jamais écrite sur stdout. Ce sink existe uniquement pour satisfaire
|
||||
l'interface CorrelatedLogSink. Les logs opérationnels du service
|
||||
(démarrage, erreurs, métriques de débogage) sont toujours sur stderr
|
||||
via observability.Logger, indépendamment de ce sink.
|
||||
|
||||
correlation:
|
||||
description: >
|
||||
Corrélation stricte basée sur src_ip + src_port et une fenêtre temporelle
|
||||
configurable. Aucun autre champ n'est utilisé pour la décision de corrélation.
|
||||
key:
|
||||
- src_ip
|
||||
- src_port
|
||||
time_window:
|
||||
value: 10
|
||||
unit: s
|
||||
description: >
|
||||
Fenêtre de temps appliquée aux timestamps de A et B. Si B n'arrive pas dans
|
||||
ce délai, A est émis comme orphelin. Augmentée à 10s pour le Keep-Alive.
|
||||
retention_limits:
|
||||
max_http_items: 10000
|
||||
max_network_items: 20000
|
||||
description: >
|
||||
Limites des caches. Si max_http_items est atteint, le plus ancien A est
|
||||
évincé et émis orphelin. Si max_network_items est atteint, le plus ancien B
|
||||
est supprimé silencieusement.
|
||||
ttl_management:
|
||||
network_ttl_s: 120
|
||||
description: >
|
||||
TTL des logs réseau. Chaque fois qu'un B est corrélé à un A (Keep-Alive),
|
||||
son TTL est remis à cette valeur. Augmenté à 120s pour les sessions longues.
|
||||
timestamp_source:
|
||||
apache: timestamp (champ int64, nanosecondes)
|
||||
network: timestamp (champ int64, nanosecondes) si présent, sinon time (RFC3339),
|
||||
sinon reception_time (time.Now())
|
||||
orphan_policy:
|
||||
apache_always_emit: true
|
||||
network_emit: false
|
||||
matching:
|
||||
mode: one_to_many
|
||||
description: >
|
||||
Stratégie 1‑à‑N : un log réseau peut être utilisé pour plusieurs logs HTTP
|
||||
successifs tant qu'il n'a pas expiré ni été évincé.
|
||||
ip_filtering:
|
||||
directive: exclude_source_ips
|
||||
description: >
|
||||
Liste d'IPs source (exactes ou plages CIDR) à ignorer silencieusement.
|
||||
Événements non corrélés, non émis en orphelin. Métrique : failed_ip_excluded.
|
||||
dest_port_filtering:
|
||||
directive: include_dest_ports
|
||||
description: >
|
||||
Liste blanche de ports de destination. Si non vide, seuls les événements
|
||||
dont le dst_port est dans la liste participent à la corrélation. Les autres
|
||||
sont silencieusement ignorés (non corrélés, non émis en orphelin).
|
||||
Liste vide = tous les ports autorisés (comportement par défaut).
|
||||
Métrique : failed_dest_port_filtered.
|
||||
example:
|
||||
include_dest_ports: [80, 443, 8080, 8443]
|
||||
|
||||
schema:
|
||||
description: >
|
||||
Schémas variables pour A et B. Quelques champs seulement sont obligatoires
|
||||
pour la corrélation, les autres sont acceptés sans modification de code.
|
||||
source_A:
|
||||
description: >
|
||||
Logs HTTP applicatifs au format JSON.
|
||||
required_fields:
|
||||
- name: src_ip
|
||||
type: string
|
||||
- name: src_port
|
||||
type: int
|
||||
- name: timestamp
|
||||
type: int64
|
||||
unit: ns
|
||||
optional_fields:
|
||||
- name: dst_ip
|
||||
type: string
|
||||
- name: dst_port
|
||||
type: int
|
||||
- name: method
|
||||
type: string
|
||||
- name: path
|
||||
type: string
|
||||
- name: host
|
||||
type: string
|
||||
- name: http_version
|
||||
type: string
|
||||
dynamic_fields:
|
||||
- pattern: header_*
|
||||
target_map: headers
|
||||
- pattern: "*"
|
||||
target_map: extra
|
||||
source_B:
|
||||
description: Logs réseau JSON (IP/TCP, JA3/JA4...).
|
||||
required_fields:
|
||||
- name: src_ip
|
||||
type: string
|
||||
- name: src_port
|
||||
type: int
|
||||
optional_fields:
|
||||
- name: dst_ip
|
||||
type: string
|
||||
- name: dst_port
|
||||
type: int
|
||||
- name: timestamp
|
||||
type: int64
|
||||
unit: ns
|
||||
- name: time
|
||||
type: string
|
||||
format: RFC3339 ou RFC3339Nano
|
||||
dynamic_fields:
|
||||
- pattern: "*"
|
||||
target_map: extra
|
||||
|
||||
normalized_event:
|
||||
description: >
|
||||
Représentation interne unifiée des événements A/B.
|
||||
fields:
|
||||
- name: source
|
||||
type: enum("A","B")
|
||||
- name: timestamp
|
||||
type: time.Time
|
||||
- name: src_ip
|
||||
type: string
|
||||
- name: src_port
|
||||
type: int
|
||||
- name: dst_ip
|
||||
type: string
|
||||
optional: true
|
||||
- name: dst_port
|
||||
type: int
|
||||
optional: true
|
||||
- name: headers
|
||||
type: map[string]string
|
||||
optional: true
|
||||
- name: extra
|
||||
type: map[string]any
|
||||
|
||||
correlated_log:
|
||||
description: >
|
||||
Structure du log corrélé émis vers les sinks.
|
||||
fields:
|
||||
- name: timestamp
|
||||
type: time.Time
|
||||
- name: src_ip
|
||||
type: string
|
||||
- name: src_port
|
||||
type: int
|
||||
- name: dst_ip
|
||||
type: string
|
||||
optional: true
|
||||
- name: dst_port
|
||||
type: int
|
||||
optional: true
|
||||
- name: correlated
|
||||
type: bool
|
||||
- name: orphan_side
|
||||
type: string
|
||||
- name: "*"
|
||||
type: map[string]any
|
||||
|
||||
clickhouse_schema:
|
||||
strategy: external_ddls
|
||||
database: ja4_processing
|
||||
description: >
|
||||
La table ClickHouse est gérée en dehors du service. Le service insère dans une
|
||||
table RAW avec une seule colonne raw_json contenant le log corrélé complet
|
||||
sérialisé en JSON. La colonne ingest_time utilise DEFAULT now().
|
||||
Toute extraction de champs (table parsée, vue matérialisée) est gérée en externe
|
||||
via des DDL séparés, non implémentés dans le service.
|
||||
tables:
|
||||
- name: http_logs_raw
|
||||
description: >
|
||||
Table d'ingestion brute. Une seule colonne raw_json contient le log corrélé
|
||||
complet sérialisé en JSON. La colonne ingest_time est auto-générée avec
|
||||
DEFAULT now(). Partitionnée par jour pour optimiser le TTL.
|
||||
engine: MergeTree
|
||||
partition_by: toDate(ingest_time)
|
||||
order_by: ingest_time
|
||||
columns:
|
||||
- name: raw_json
|
||||
type: String
|
||||
- name: ingest_time
|
||||
type: DateTime
|
||||
default: now()
|
||||
insert_format: |
|
||||
INSERT INTO ja4_processing.http_logs_raw (raw_json) VALUES
|
||||
('{...log corrélé sérialisé en JSON...}')
|
||||
notes: >
|
||||
Le service utilise l'API native clickhouse-go/v2 (PrepareBatch + Append + Send).
|
||||
La colonne ingest_time n'est PAS explicitement insérée (DEFAULT now() est utilisé).
|
||||
|
||||
- name: http_logs
|
||||
description: >
|
||||
Table parsée (optionnelle, gérée en externe). Le service n'implémente PAS
|
||||
l'extraction des champs suivants. Si cette table est utilisée, elle doit être
|
||||
alimentée par une vue matérialisée ou un traitement ETL externe.
|
||||
engine: MergeTree
|
||||
partition_by: log_date
|
||||
order_by: (time, src_ip, dst_ip, ja4)
|
||||
columns:
|
||||
- name: time
|
||||
type: DateTime
|
||||
- name: log_date
|
||||
type: Date
|
||||
default: toDate(time)
|
||||
- name: src_ip
|
||||
type: IPv4
|
||||
- name: src_port
|
||||
type: UInt16
|
||||
- name: dst_ip
|
||||
type: IPv4
|
||||
- name: dst_port
|
||||
type: UInt16
|
||||
- name: method
|
||||
type: LowCardinality(String)
|
||||
- name: scheme
|
||||
type: LowCardinality(String)
|
||||
- name: host
|
||||
type: LowCardinality(String)
|
||||
- name: path
|
||||
type: String
|
||||
- name: query
|
||||
type: String
|
||||
- name: http_version
|
||||
type: LowCardinality(String)
|
||||
- name: orphan_side
|
||||
type: LowCardinality(String)
|
||||
- name: correlated
|
||||
type: UInt8
|
||||
- name: keepalives
|
||||
type: UInt16
|
||||
status: non_implémenté
|
||||
- name: a_timestamp
|
||||
type: UInt64
|
||||
status: non_implémenté
|
||||
- name: b_timestamp
|
||||
type: UInt64
|
||||
status: non_implémenté
|
||||
- name: conn_id
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: ip_meta_df
|
||||
type: UInt8
|
||||
status: non_implémenté
|
||||
- name: ip_meta_id
|
||||
type: UInt32
|
||||
status: non_implémenté
|
||||
- name: ip_meta_total_length
|
||||
type: UInt32
|
||||
status: non_implémenté
|
||||
- name: ip_meta_ttl
|
||||
type: UInt8
|
||||
status: non_implémenté
|
||||
- name: tcp_meta_options
|
||||
type: LowCardinality(String)
|
||||
status: non_implémenté
|
||||
- name: tcp_meta_window_size
|
||||
type: UInt32
|
||||
status: non_implémenté
|
||||
- name: syn_to_clienthello_ms
|
||||
type: Int32
|
||||
status: non_implémenté
|
||||
- name: tls_version
|
||||
type: LowCardinality(String)
|
||||
status: non_implémenté
|
||||
- name: tls_sni
|
||||
type: LowCardinality(String)
|
||||
status: non_implémenté
|
||||
- name: ja3
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: ja3_hash
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: ja4
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_user_agent
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_accept
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_accept_encoding
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_accept_language
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_x_request_id
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_x_trace_id
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_x_forwarded_for
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_sec_ch_ua
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_sec_ch_ua_mobile
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_sec_ch_ua_platform
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_sec_fetch_dest
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_sec_fetch_mode
|
||||
type: String
|
||||
status: non_implémenté
|
||||
- name: header_sec_fetch_site
|
||||
type: String
|
||||
status: non_implémenté
|
||||
notes: >
|
||||
Cette table et la vue matérialisée associée sont gérées en externe (DDL séparés).
|
||||
Le service se contente d'insérer le JSON brut dans http_logs_raw.
|
||||
Les champs marqués "non_implémenté" ne sont PAS extraits par le service.
|
||||
|
||||
users:
|
||||
description: >
|
||||
La gestion des utilisateurs ClickHouse est externe au service. Le DSN est
|
||||
configuré dans le fichier de configuration YAML.
|
||||
notes: >
|
||||
Cette section est fournie à titre indicatif pour l'administration ClickHouse.
|
||||
|
||||
migration:
|
||||
description: >
|
||||
Aucune migration n'est implémentée dans le service. La gestion des schémas
|
||||
(tables, vues matérialisées) est entièrement externe (DDL séparés).
|
||||
|
||||
architecture:
|
||||
description: >
|
||||
Architecture hexagonale : domaine de corrélation indépendant, ports abstraits
|
||||
pour les sources/sinks, adaptateurs pour sockets Unix, fichier, ClickHouse et
|
||||
stdout, couche application d'orchestration, et modules infra (config, observabilité).
|
||||
modules:
|
||||
- name: cmd/logcorrelator
|
||||
type: entrypoint
|
||||
responsibilities:
|
||||
- Chargement de la configuration YAML.
|
||||
- Initialisation des adaptateurs d'entrée/sortie.
|
||||
- Création du CorrelationService.
|
||||
- Démarrage de l'orchestrateur.
|
||||
- Gestion des signaux (SIGINT, SIGTERM, SIGHUP).
|
||||
- Versioning via -ldflags (main.Version).
|
||||
- name: internal/domain
|
||||
type: domain
|
||||
responsibilities:
|
||||
- Modèles NormalizedEvent et CorrelatedLog.
|
||||
- CorrelationService (fenêtre, TTL, buffers bornés, one-to-many/Keep-Alive, orphelins).
|
||||
- Filtrage par IP source (exclude_source_ips, CIDR).
|
||||
- Filtrage par port destination (include_dest_ports, liste blanche).
|
||||
- Custom JSON marshaling pour CorrelatedLog (structure plate).
|
||||
- name: internal/ports
|
||||
type: ports
|
||||
responsibilities:
|
||||
- Interfaces EventSource, CorrelatedLogSink, CorrelationProcessor.
|
||||
- name: internal/app
|
||||
type: application
|
||||
responsibilities:
|
||||
- Orchestrator : EventSource → CorrelationService → MultiSink.
|
||||
- Gestion du contexte de shutdown et drain des événements.
|
||||
- name: internal/adapters/inbound/unixsocket
|
||||
type: adapter_inbound
|
||||
responsibilities:
|
||||
- Lecture Unix datagram (SOCK_DGRAM) et parsing JSON → NormalizedEvent.
|
||||
- Détection automatique de la source (A/B) via source_type ou headers.
|
||||
- Gestion des permissions de socket (défaut 0666).
|
||||
- Cleanup du fichier socket à l'arrêt.
|
||||
- name: internal/adapters/outbound/file
|
||||
type: adapter_outbound
|
||||
responsibilities:
|
||||
- Écriture JSON lines.
|
||||
- Réouverture du fichier sur SIGHUP (log rotation).
|
||||
- Validation des chemins (répertoire autorisé).
|
||||
- name: internal/adapters/outbound/clickhouse
|
||||
type: adapter_outbound
|
||||
responsibilities:
|
||||
- Bufferisation + inserts batch asynchrones.
|
||||
- Gestion du drop_on_overflow.
|
||||
- Retry avec backoff exponentiel (MaxRetries=3).
|
||||
- API native clickhouse-go/v2 (PrepareBatch + Append + Send).
|
||||
- Logging complet via observability.Logger (SetLogger) : INFO à la connexion,
|
||||
DEBUG sur envoi réussi (rows/table), WARN sur drop buffer et retries,
|
||||
ERROR sur échec de flush (périodique, batch, fermeture).
|
||||
- name: internal/adapters/outbound/stdout
|
||||
type: adapter_outbound
|
||||
responsibilities:
|
||||
- Sink no-op pour les données corrélées.
|
||||
- Write/Flush/Close ne font rien : les données ne passent jamais par stdout.
|
||||
- Les logs opérationnels sont sur stderr via observability.Logger (indépendant de ce sink).
|
||||
- name: internal/adapters/outbound/multi
|
||||
type: adapter_outbound
|
||||
responsibilities:
|
||||
- Fan-out vers plusieurs sinks.
|
||||
- Implémentation de Reopen() pour la rotation des logs.
|
||||
- name: internal/config
|
||||
type: infrastructure
|
||||
responsibilities:
|
||||
- Chargement/validation de la configuration YAML.
|
||||
- Valeurs par défaut et fallback pour champs dépréciés.
|
||||
- name: internal/observability
|
||||
type: infrastructure
|
||||
responsibilities:
|
||||
- Logger structuré avec niveaux (DEBUG, INFO, WARN, ERROR).
|
||||
- CorrelationMetrics : suivi des statistiques de corrélation.
|
||||
- MetricsServer : serveur HTTP pour exposition des métriques (/metrics, /health).
|
||||
- Traçage des événements exclus (exclude_source_ips).
|
||||
- Logs pour : événements reçus, corrélations, orphelins, buffer plein.
|
||||
|
||||
testing:
|
||||
unit:
|
||||
description: >
|
||||
Tests unitaires table‑driven, couverture cible ≥ 80 %. La couverture actuelle
|
||||
est d'environ 74-80% selon les versions. Les tests se concentrent sur la logique
|
||||
de corrélation, les caches, les sinks et le parsing des datagrammes.
|
||||
coverage_minimum: 0.8
|
||||
coverage_actual: ~0.74-0.80
|
||||
focus:
|
||||
- CorrelationService (fenêtre, TTL, évictions, one-to-many/Keep-Alive)
|
||||
- Parsing A/B → NormalizedEvent (datagrammes JSON)
|
||||
- ClickHouseSink (batching, retry, overflow, logging erreurs/succès)
|
||||
- FileSink (réouverture sur SIGHUP)
|
||||
- MultiSink (fan-out)
|
||||
- StdoutSink (no-op data, test stdout reste vide)
|
||||
- Config (validation, valeurs par défaut, exclude_source_ips)
|
||||
- UnixSocketSource (lecture, permissions, cleanup)
|
||||
- CorrelationMetrics (suivi des statistiques)
|
||||
- MetricsServer (endpoints /metrics et /health)
|
||||
integration:
|
||||
description: >
|
||||
Tests d'intégration limités. Le flux complet A+B → corrélation → sinks est
|
||||
testé via des tests unitaires avec mocks. ClickHouse est mocké (pas de tests
|
||||
avec vrai ClickHouse). Scénarios Keep-Alive testés dans correlation_service_test.go.
|
||||
Scripts de test fournis : scripts/test-correlation.sh et scripts/test-correlation-advanced.py.
|
||||
|
||||
docker:
|
||||
description: >
|
||||
Build, tests et packaging RPM sont exécutés intégralement dans des conteneurs
|
||||
via un multi‑stage build. Deux Dockerfiles : Dockerfile (build + runtime + dev)
|
||||
et Dockerfile.package (RPM multi-distros : el8, el9, el10).
|
||||
build_pipeline:
|
||||
multi_stage: true
|
||||
stages:
|
||||
- name: builder
|
||||
base: golang:1.21
|
||||
description: >
|
||||
go test -race -coverprofile=coverage.txt ./... avec vérification de couverture
|
||||
(échec si < 80 %). Compilation d'un binaire statique (CGO_ENABLED=0,
|
||||
GOOS=linux, GOARCH=amd64).
|
||||
- name: runtime
|
||||
base: scratch
|
||||
description: >
|
||||
Image minimale contenant uniquement le binaire et la config exemple.
|
||||
- name: rpm_builder_el8
|
||||
base: rockylinux:8
|
||||
description: >
|
||||
Installation de fpm (via Ruby), construction RPM pour Enterprise Linux 8.
|
||||
- name: rpm_builder_el9
|
||||
base: rockylinux:9
|
||||
description: >
|
||||
Installation de fpm (via Ruby), construction RPM pour Enterprise Linux 9.
|
||||
- name: rpm_builder_el10
|
||||
base: almalinux:10
|
||||
description: >
|
||||
Installation de fpm (via Ruby), construction RPM pour Enterprise Linux 10.
|
||||
- name: output_export
|
||||
base: alpine:latest
|
||||
description: >
|
||||
Export des paquets RPM produits pour les 3 distributions (el8, el9, el10).
|
||||
files:
|
||||
- path: Dockerfile
|
||||
description: Build principal (builder, runtime, dev) et packaging RPM mono-distro.
|
||||
- path: Dockerfile.package
|
||||
description: Packaging RPM multi-distros (el8, el9, el10) avec scripts post/preun/postun.
|
||||
|
||||
observability:
|
||||
description: >
|
||||
Le service inclut des fonctionnalités complètes de débogage et de monitoring
|
||||
pour diagnostiquer les problèmes de corrélation et surveiller les performances.
|
||||
logging:
|
||||
levels:
|
||||
- DEBUG: Tous les événements reçus, tentatives de corrélation, raisons des échecs
|
||||
- INFO: Événements corrélés, démarrage/arrêt du service
|
||||
- WARN: Orphelins émis, buffer plein, TTL expiré
|
||||
- ERROR: Erreurs de parsing, échecs de sink, erreurs critiques
|
||||
debug_logs:
|
||||
- "event received: source=A src_ip=192.168.1.1 src_port=8080 timestamp=..."
|
||||
- "processing A event: key=192.168.1.1:8080 timestamp=..."
|
||||
- "correlation found: A(src_ip=... src_port=... ts=...) + B(src_ip=... src_port=... ts=...)"
|
||||
- "A event has no matching B key in buffer: key=..."
|
||||
- "A event has same key as B but outside time window: key=... time_diff=5s window=10s"
|
||||
- "event excluded by IP filter: source=A src_ip=10.0.0.1 src_port=8080"
|
||||
- "event excluded by dest port filter: source=A dst_port=22"
|
||||
- "TTL reset for B event (Keep-Alive): key=... new_ttl=120s"
|
||||
- "[clickhouse] DEBUG batch sent: rows=42 table=correlated_logs_http_network"
|
||||
info_logs:
|
||||
- "[clickhouse] INFO connected to ClickHouse: table=... batch_size=500 flush_interval_ms=200"
|
||||
warn_logs:
|
||||
- "[clickhouse] WARN buffer full, dropping log: table=... buffer_size=5000"
|
||||
- "[clickhouse] WARN retrying batch insert: attempt=2/3 delay=100ms rows=42 err=connection refused"
|
||||
error_logs:
|
||||
- "[clickhouse] ERROR periodic flush failed: ..."
|
||||
- "[clickhouse] ERROR batch flush failed: ..."
|
||||
- "[clickhouse] ERROR final flush on close failed: ..."
|
||||
metrics_server:
|
||||
enabled: true
|
||||
endpoints:
|
||||
- path: /metrics
|
||||
method: GET
|
||||
description: Retourne les métriques de corrélation au format JSON
|
||||
response_example: |
|
||||
{
|
||||
"events_received_a": 1542,
|
||||
"events_received_b": 1498,
|
||||
"correlations_success": 1450,
|
||||
"correlations_failed": 92,
|
||||
"failed_no_match_key": 45,
|
||||
"failed_time_window": 23,
|
||||
"failed_buffer_eviction": 5,
|
||||
"failed_ttl_expired": 12,
|
||||
"failed_ip_excluded": 7,
|
||||
"failed_dest_port_filtered": 3,
|
||||
"buffer_a_size": 23,
|
||||
"buffer_b_size": 18,
|
||||
"orphans_emitted_a": 92,
|
||||
"keepalive_resets": 892
|
||||
}
|
||||
- path: /health
|
||||
method: GET
|
||||
description: Health check
|
||||
response_example: |
|
||||
{"status":"healthy"}
|
||||
metrics_tracked:
|
||||
events_received:
|
||||
- events_received_a: Nombre d'événements HTTP (source A) reçus
|
||||
- events_received_b: Nombre d'événements réseau (source B) reçus
|
||||
correlations:
|
||||
- correlations_success: Corrélations réussies
|
||||
- correlations_failed: Échecs de corrélation
|
||||
failure_reasons:
|
||||
- failed_no_match_key: Clé src_ip:src_port non trouvée dans le buffer
|
||||
- failed_time_window: Événements hors fenêtre temporelle
|
||||
- failed_buffer_eviction: Buffer plein, événement évincé
|
||||
- failed_ttl_expired: TTL du événement B expiré
|
||||
- failed_ip_excluded: Événement exclu par filtre IP (exclude_source_ips)
|
||||
- failed_dest_port_filtered: Événement exclu par filtre port destination (include_dest_ports)
|
||||
buffers:
|
||||
- buffer_a_size: Taille actuelle du buffer HTTP
|
||||
- buffer_b_size: Taille actuelle du buffer réseau
|
||||
orphans:
|
||||
- orphans_emitted_a: Orphelins A émis (sans correspondance B)
|
||||
- orphans_emitted_b: Orphelins B émis (toujours 0, policy: network_emit=false)
|
||||
- orphans_pending_a: Orphelins A en attente (délai avant émission)
|
||||
- pending_orphan_match: B a corrélé avec un orphelin A en attente
|
||||
keepalive:
|
||||
- keepalive_resets: Resets TTL pour mode Keep-Alive (one-to-many)
|
||||
troubleshooting:
|
||||
description: >
|
||||
Guide de diagnostic basé sur les métriques et logs
|
||||
common_issues:
|
||||
- symptom: failed_no_match_key élevé
|
||||
cause: Les logs A et B n'ont pas le même src_ip + src_port
|
||||
solution: Vérifier que les deux sources utilisent la même combinaison IP/port
|
||||
- symptom: failed_time_window élevé
|
||||
cause: Timestamps trop éloignés (> time_window.value)
|
||||
solution: Augmenter correlation.time_window.value ou synchroniser les horloges (NTP)
|
||||
- symptom: failed_ttl_expired élevé
|
||||
cause: Les événements B expirent avant corrélation
|
||||
solution: Augmenter correlation.ttl.network_ttl_s
|
||||
- symptom: failed_buffer_eviction élevé
|
||||
cause: Buffers trop petits pour le volume de logs
|
||||
solution: Augmenter correlation.buffers.max_http_items et max_network_items
|
||||
- symptom: failed_ip_excluded élevé
|
||||
cause: Traffic depuis des IPs configurées dans exclude_source_ips
|
||||
solution: Vérifier la configuration, c'est normal si attendu
|
||||
- symptom: failed_dest_port_filtered élevé
|
||||
cause: Traffic sur des ports non listés dans include_dest_ports
|
||||
solution: Vérifier la configuration include_dest_ports, ou vider la liste pour tout accepter
|
||||
- symptom: orphans_emitted_a élevé
|
||||
cause: Beaucoup de logs A sans correspondance B
|
||||
solution: Vérifier que la source B envoie bien les événements attendus
|
||||
test_scripts:
|
||||
- name: scripts/test-correlation.sh
|
||||
description: Script Bash pour tester la corrélation avec des événements synthétiques
|
||||
features:
|
||||
- Envoi de paires A+B avec mêmes src_ip:src_port
|
||||
- Vérification des métriques avant/après
|
||||
- Options: -c (count), -d (delay), -v (verbose), -m (metrics-url)
|
||||
- name: scripts/test-correlation-advanced.py
|
||||
description: Script Python avancé avec multiples scénarios de test
|
||||
features:
|
||||
- Basic test: corrélations simples
|
||||
- Time window test: vérifie l'expiration de la fenêtre temporelle
|
||||
- Different IP test: vérifie non-corrélation avec IPs différentes
|
||||
- Keep-Alive test: vérifie le mode one-to-many
|
||||
- Métriques en temps réel
|
||||
|
||||
@ -1,204 +0,0 @@
|
||||
// Package main initialise et démarre le service logcorrelator.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/adapters/inbound/unixsocket"
|
||||
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/clickhouse"
|
||||
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/file"
|
||||
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/multi"
|
||||
"github.com/antitbone/ja4/correlator/internal/adapters/outbound/stdout"
|
||||
"github.com/antitbone/ja4/correlator/internal/app"
|
||||
"github.com/antitbone/ja4/correlator/internal/config"
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
"github.com/antitbone/ja4/correlator/internal/observability"
|
||||
"github.com/antitbone/ja4/correlator/internal/ports"
|
||||
)
|
||||
|
||||
var Version = "dev"
|
||||
|
||||
// main configure les sources, les puits et le service de corrélation, puis démarre l'orchestrateur.
|
||||
func main() {
|
||||
configPath := flag.String("config", "config.yml", "path to configuration file")
|
||||
version := flag.Bool("version", false, "print version and exit")
|
||||
flag.Parse()
|
||||
|
||||
if *version {
|
||||
fmt.Println(Version)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Load configuration
|
||||
cfg, err := config.Load(*configPath)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "Error loading configuration: %v\n", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Initialize logger with configured level
|
||||
logger := observability.NewLoggerWithLevel("logcorrelator", cfg.Log.GetLevel())
|
||||
|
||||
logger.Info(fmt.Sprintf("Starting logcorrelator version %s (log_level=%s)", Version, cfg.Log.GetLevel()))
|
||||
|
||||
// Create sources
|
||||
sources := make([]ports.EventSource, 0, len(cfg.Inputs.UnixSockets))
|
||||
for _, inputCfg := range cfg.Inputs.UnixSockets {
|
||||
source := unixsocket.NewUnixSocketSource(unixsocket.Config{
|
||||
Name: inputCfg.Name,
|
||||
Path: inputCfg.Path,
|
||||
SourceType: inputCfg.SourceType,
|
||||
SocketPermissions: inputCfg.GetSocketPermissions(),
|
||||
})
|
||||
// Set logger for debug logging
|
||||
source.SetLogger(logger)
|
||||
sources = append(sources, source)
|
||||
logger.Info(fmt.Sprintf("Configured input source: name=%s, path=%s, permissions=%o", inputCfg.Name, inputCfg.Path, inputCfg.GetSocketPermissions()))
|
||||
}
|
||||
|
||||
// Create sinks
|
||||
sinks := make([]ports.CorrelatedLogSink, 0)
|
||||
|
||||
if cfg.Outputs.File.Enabled && cfg.Outputs.File.Path != "" {
|
||||
fileSink, err := file.NewFileSink(file.Config{
|
||||
Path: cfg.Outputs.File.Path,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("Failed to create file sink", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
sinks = append(sinks, fileSink)
|
||||
logger.Info(fmt.Sprintf("Configured file sink: path=%s", cfg.Outputs.File.Path))
|
||||
}
|
||||
|
||||
if cfg.Outputs.ClickHouse.Enabled {
|
||||
clickHouseSink, err := clickhouse.NewClickHouseSink(clickhouse.Config{
|
||||
DSN: cfg.Outputs.ClickHouse.DSN,
|
||||
Table: cfg.Outputs.ClickHouse.Table,
|
||||
BatchSize: cfg.Outputs.ClickHouse.BatchSize,
|
||||
FlushIntervalMs: cfg.Outputs.ClickHouse.FlushIntervalMs,
|
||||
MaxBufferSize: cfg.Outputs.ClickHouse.MaxBufferSize,
|
||||
DropOnOverflow: cfg.Outputs.ClickHouse.DropOnOverflow,
|
||||
AsyncInsert: cfg.Outputs.ClickHouse.AsyncInsert,
|
||||
TimeoutMs: cfg.Outputs.ClickHouse.TimeoutMs,
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error("Failed to create ClickHouse sink", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
clickHouseSink.SetLogger(logger)
|
||||
sinks = append(sinks, clickHouseSink)
|
||||
logger.Info(fmt.Sprintf("Configured ClickHouse sink: table=%s", cfg.Outputs.ClickHouse.Table))
|
||||
}
|
||||
|
||||
if cfg.Outputs.Stdout.Enabled {
|
||||
stdoutSink := stdout.NewStdoutSink(stdout.Config{Enabled: true})
|
||||
sinks = append(sinks, stdoutSink)
|
||||
logger.Info("Configured stdout sink (operational logs on stderr)")
|
||||
}
|
||||
|
||||
// Create multi-sink wrapper
|
||||
multiSink := multi.NewMultiSink(sinks...)
|
||||
|
||||
// Create correlation service
|
||||
correlationSvc := domain.NewCorrelationService(domain.CorrelationConfig{
|
||||
TimeWindow: cfg.Correlation.GetTimeWindow(),
|
||||
ApacheAlwaysEmit: cfg.Correlation.GetApacheAlwaysEmit(),
|
||||
ApacheEmitDelayMs: cfg.Correlation.GetApacheEmitDelayMs(),
|
||||
NetworkEmit: false,
|
||||
MaxHTTPBufferSize: cfg.Correlation.GetMaxHTTPBufferSize(),
|
||||
MaxNetworkBufferSize: cfg.Correlation.GetMaxNetworkBufferSize(),
|
||||
NetworkTTLS: cfg.Correlation.GetNetworkTTLS(),
|
||||
MatchingMode: cfg.Correlation.GetMatchingMode(),
|
||||
ExcludeSourceIPs: cfg.Correlation.GetExcludeSourceIPs(),
|
||||
IncludeDestPorts: cfg.Correlation.GetIncludeDestPorts(),
|
||||
}, &domain.RealTimeProvider{})
|
||||
|
||||
// Set logger for correlation service
|
||||
correlationSvc.SetLogger(logger.WithFields(map[string]any{"component": "correlation"}))
|
||||
|
||||
logger.Info(fmt.Sprintf("Correlation service initialized: time_window=%s, emit_orphans=%v, emit_delay_ms=%d",
|
||||
cfg.Correlation.GetTimeWindow().String(),
|
||||
cfg.Correlation.GetApacheAlwaysEmit(),
|
||||
cfg.Correlation.GetApacheEmitDelayMs()))
|
||||
|
||||
// Start metrics server if enabled
|
||||
var metricsServer *observability.MetricsServer
|
||||
if cfg.Metrics.Enabled {
|
||||
addr := cfg.Metrics.Addr
|
||||
if addr == "" {
|
||||
addr = ":8080" // Default address
|
||||
}
|
||||
var err error
|
||||
metricsServer, err = observability.NewMetricsServer(addr, correlationSvc.GetMetricsSnapshot)
|
||||
if err != nil {
|
||||
logger.Error("Failed to create metrics server", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
if err := metricsServer.Start(); err != nil {
|
||||
logger.Error("Failed to start metrics server", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
logger.Info(fmt.Sprintf("Metrics server started: addr=%s", metricsServer.Addr()))
|
||||
logger.Info("Metrics endpoints: /metrics (JSON), /health")
|
||||
}
|
||||
|
||||
// Create orchestrator
|
||||
orchestrator := app.NewOrchestrator(app.OrchestratorConfig{
|
||||
Sources: sources,
|
||||
Sink: multiSink,
|
||||
}, correlationSvc)
|
||||
|
||||
// Start the application
|
||||
if err := orchestrator.Start(); err != nil {
|
||||
logger.Error("Failed to start orchestrator", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
logger.Info("logcorrelator started successfully")
|
||||
|
||||
// Wait for shutdown signal
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
|
||||
for {
|
||||
sig := <-sigChan
|
||||
|
||||
if sig == syscall.SIGHUP {
|
||||
// Reopen file sinks for log rotation
|
||||
logger.Info("SIGHUP received, reopening file sinks...")
|
||||
if err := multiSink.Reopen(); err != nil {
|
||||
logger.Error("Error reopening file sinks", err)
|
||||
} else {
|
||||
logger.Info("File sinks reopened successfully")
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Shutdown signal received
|
||||
logger.Info(fmt.Sprintf("Shutdown signal received: %v", sig))
|
||||
break
|
||||
}
|
||||
|
||||
// Graceful shutdown
|
||||
if err := orchestrator.Stop(); err != nil {
|
||||
logger.Error("Error during shutdown", err)
|
||||
}
|
||||
|
||||
// Stop metrics server
|
||||
if metricsServer != nil {
|
||||
shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
if err := metricsServer.Stop(shutdownCtx); err != nil {
|
||||
logger.Error("Error stopping metrics server", err)
|
||||
}
|
||||
}
|
||||
|
||||
logger.Info("logcorrelator stopped")
|
||||
}
|
||||
@ -1,92 +0,0 @@
|
||||
# logcorrelator configuration file
|
||||
# Format: YAML
|
||||
|
||||
# Logging configuration
|
||||
log:
|
||||
level: INFO # DEBUG, INFO, WARN, ERROR
|
||||
|
||||
inputs:
|
||||
unix_sockets:
|
||||
- name: http
|
||||
source_type: A
|
||||
path: /var/run/logcorrelator/http.socket
|
||||
format: json
|
||||
socket_permissions: "0666" # world read/write
|
||||
- name: network
|
||||
source_type: B
|
||||
path: /var/run/logcorrelator/network.socket
|
||||
format: json
|
||||
socket_permissions: "0666"
|
||||
|
||||
outputs:
|
||||
file:
|
||||
enabled: true
|
||||
path: /var/log/logcorrelator/correlated.log
|
||||
|
||||
clickhouse:
|
||||
enabled: false
|
||||
dsn: clickhouse://user:pass@localhost:9000/ja4_logs
|
||||
table: http_logs_raw
|
||||
batch_size: 500
|
||||
flush_interval_ms: 200
|
||||
max_buffer_size: 5000
|
||||
drop_on_overflow: true
|
||||
async_insert: true
|
||||
timeout_ms: 1000
|
||||
|
||||
stdout:
|
||||
enabled: false
|
||||
|
||||
correlation:
|
||||
# Time window for correlation (A and B must be within this window)
|
||||
# Increased to 10s to support HTTP Keep-Alive scenarios
|
||||
time_window:
|
||||
value: 10
|
||||
unit: s
|
||||
|
||||
# Orphan policy: what to do when no match is found
|
||||
orphan_policy:
|
||||
apache_always_emit: true # Always emit A events, even without B match
|
||||
apache_emit_delay_ms: 500 # Wait 500ms before emitting as orphan (allows B to arrive)
|
||||
network_emit: false # Never emit B events alone
|
||||
|
||||
# Matching mode: one_to_one or one_to_many (Keep-Alive)
|
||||
matching:
|
||||
mode: one_to_many
|
||||
|
||||
# Buffer limits (max events in memory)
|
||||
buffers:
|
||||
max_http_items: 10000
|
||||
max_network_items: 20000
|
||||
|
||||
# TTL for network events (source B)
|
||||
# Increased to 120s to support long-lived HTTP Keep-Alive sessions
|
||||
ttl:
|
||||
network_ttl_s: 120
|
||||
|
||||
# Exclude specific source IPs or CIDR ranges from correlation
|
||||
# Events from these IPs will be silently dropped (not correlated, not emitted)
|
||||
# Useful for excluding health checks, internal traffic, or known bad actors
|
||||
exclude_source_ips:
|
||||
- 10.0.0.1 # Single IP
|
||||
- 192.168.1.100 # Another single IP
|
||||
- 172.16.0.0/12 # CIDR range (private network)
|
||||
- 10.10.10.0/24 # Another CIDR range
|
||||
|
||||
# Restrict correlation to specific destination ports (optional)
|
||||
# If non-empty, only events whose dst_port matches one of these values will be correlated
|
||||
# Events on other ports are silently ignored (not correlated, not emitted as orphans)
|
||||
# Useful to focus on HTTP/HTTPS traffic only and ignore unrelated connections
|
||||
# include_dest_ports:
|
||||
# - 80 # HTTP
|
||||
# - 443 # HTTPS
|
||||
# - 8080 # HTTP alt
|
||||
# - 8443 # HTTPS alt
|
||||
|
||||
# Metrics server configuration (optional, for debugging/monitoring)
|
||||
metrics:
|
||||
enabled: false
|
||||
addr: ":8080" # Address to listen on (e.g., ":8080", "localhost:8080")
|
||||
# Endpoints:
|
||||
# GET /metrics - Returns correlation metrics as JSON
|
||||
# GET /health - Health check endpoint
|
||||
@ -1,224 +0,0 @@
|
||||
# Architecture de détection — logcorrelator
|
||||
|
||||
## Vue d'ensemble
|
||||
|
||||
Le système de détection est composé de **trois couches** qui s'enchaînent en pipeline :
|
||||
|
||||
```
|
||||
Trafic HTTP/TLS capturé
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ ClickHouse │ Stockage, agrégation, vues heuristiques
|
||||
│ (SQL pipeline) │
|
||||
└────────┬──────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ bot_detector.py │ Modèle IA (Isolation Forest, cycle 5 min)
|
||||
│ (Python / ML) │
|
||||
└────────┬──────────┘
|
||||
│
|
||||
▼
|
||||
┌───────────────────┐
|
||||
│ ml_detected_ │ Table de résultats (ReplacingMergeTree)
|
||||
│ anomalies │
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 1. Ingestion des logs (`http_logs_raw` → `http_logs`)
|
||||
|
||||
Les logs bruts arrivent en JSON dans la table `http_logs_raw`. Une **vue matérialisée** (`mv_http_logs`) les parse en temps réel et alimente la table `http_logs`, qui contient les champs structurés suivants :
|
||||
|
||||
| Catégorie | Champs clés |
|
||||
|---|---|
|
||||
| Réseau | `src_ip`, `src_port`, `dst_ip`, `dst_port` |
|
||||
| Enrichissement | `src_asn`, `src_country_code`, `src_as_name` (via dictionnaire IPLocate) |
|
||||
| HTTP | `method`, `host`, `path`, `query`, `http_version` |
|
||||
| Corrélation | `correlated`, `orphan_side`, `conn_id`, `keepalives` |
|
||||
| Métadonnées IP | `ip_meta_ttl`, `ip_meta_id`, `ip_meta_df`, `ip_meta_total_length` |
|
||||
| Métadonnées TCP | `tcp_meta_window_size`, `tcp_meta_mss`, `tcp_meta_window_scale`, `tcp_meta_options` |
|
||||
| TLS / Fingerprint | `tls_version`, `tls_sni`, `tls_alpn`, `ja3`, `ja3_hash`, `ja4` |
|
||||
| En-têtes HTTP | `header_user_agent`, `header_sec_ch_ua*`, `header_sec_fetch_*`, … |
|
||||
|
||||
L'enrichissement IP est réalisé via le dictionnaire `dict_iplocate_asn` (fichier CSV chargé en mémoire, rechargé toutes les 1-2 heures).
|
||||
|
||||
---
|
||||
|
||||
## 2. Agrégation comportementale (fenêtre horaire)
|
||||
|
||||
Deux tables d'agrégation `AggregatingMergeTree` sont alimentées en continu par des vues matérialisées.
|
||||
|
||||
### 2.1 `agg_host_ip_ja4_1h` — Comportement réseau & applicatif
|
||||
|
||||
Agrège par triplet **(window_start, src_ip, ja4, host)** toutes les heures :
|
||||
|
||||
| Métrique agrégée | Signification |
|
||||
|---|---|
|
||||
| `hits` | Nombre total de requêtes |
|
||||
| `count_post` | Requêtes POST |
|
||||
| `uniq_paths` | Chemins distincts visités |
|
||||
| `uniq_query_params` | Paramètres de query distincts |
|
||||
| `unique_src_ports` | Ports sources distincts |
|
||||
| `unique_conn_id` | Connexions TCP distinctes |
|
||||
| `max_keepalives` | Réutilisation maximale d'une connexion |
|
||||
| `orphan_count` | Requêtes sans corrélation TCP complète |
|
||||
| `ip_id_zero_count` | Paquets avec IP ID = 0 (spoofing potentiel) |
|
||||
| `tcp_fp_raw` | Hash de l'empreinte TCP (window, MSS, scale, options) |
|
||||
| `tcp_jitter_variance` | Variance du délai SYN→ClientHello (jitter TLS) |
|
||||
| `total_ip_length_var` | Variance de la taille des paquets IP |
|
||||
| `mss_1460_count` | Requêtes avec MSS = 1460 (signature Ethernet/desktop) |
|
||||
|
||||
### 2.2 `agg_header_fingerprint_1h` — Empreinte des en-têtes HTTP
|
||||
|
||||
Agrège par **(window_start, src_ip)** :
|
||||
|
||||
| Métrique | Signification |
|
||||
|---|---|
|
||||
| `header_order_hash` | Hash de l'ordre des en-têtes (fingerprint JA4H) |
|
||||
| `header_count` | Nombre d'en-têtes distincts |
|
||||
| `has_accept_language` | Présence de `Accept-Language` |
|
||||
| `has_cookie` | Présence de `Cookie` |
|
||||
| `has_referer` | Présence de `Referer` |
|
||||
| `modern_browser_score` | Score 0/50/100 selon présence UA et `Sec-CH-UA` |
|
||||
| `ua_ch_mismatch` | Incohérence entre `User-Agent` et `Sec-CH-UA-Platform` |
|
||||
| `sec_fetch_mode/dest` | Contexte de navigation déclaré |
|
||||
|
||||
---
|
||||
|
||||
## 3. Exclusions (listes blanches)
|
||||
|
||||
Avant toute analyse, deux tables permettent d'**exclure les robots légitimes** connus :
|
||||
|
||||
- `bot_ip` (fichier `bot_ip.csv`) — IPs à ignorer (crawlers, monitoring…)
|
||||
- `bot_ja4` (fichier `bot_ja4.csv`) — Fingerprints JA4 à ignorer
|
||||
- `ref_bot_networks` — Réseaux CIDR IPv4/IPv6 catégorisés (légitimes ou malveillants)
|
||||
|
||||
Ces exclusions sont appliquées dans la vue `view_ai_features_1h`.
|
||||
|
||||
---
|
||||
|
||||
## 4. Vue IA : `view_ai_features_1h`
|
||||
|
||||
Cette vue consolidée **sur 24 heures glissantes** calcule les **28 features** passées au modèle ML. Elle joint les deux tables d'agrégation et dérive les métriques suivantes :
|
||||
|
||||
| Feature | Calcul | Signal détecté |
|
||||
|---|---|---|
|
||||
| `hit_velocity` | `hits / durée_en_secondes` | Volume de requêtes anormalement élevé |
|
||||
| `fuzzing_index` | `uniq_query_params / uniq_paths` | Exploration paramétrique (fuzzing) |
|
||||
| `post_ratio` | `count_post / hits` | Soumission de formulaires en masse |
|
||||
| `port_exhaustion_ratio` | `unique_src_ports / hits` | Rotation de ports (scan) |
|
||||
| `orphan_ratio` | `orphan_count / hits` | Requêtes sans handshake complet |
|
||||
| `ip_id_zero_ratio` | `ip_id_zero_count / hits` | Spoofing d'adresse IP |
|
||||
| `multiplexing_efficiency` | `hits / unique_conn_id` | Réutilisation des connexions (H2/H3) |
|
||||
| `true_window_size` | `tcp_win * 2^tcp_scale` | Taille réelle de la fenêtre TCP |
|
||||
| `window_mss_ratio` | `tcp_win / tcp_mss` | Cohérence TCP stack |
|
||||
| `tcp_jitter_variance` | Variance SYN→ClientHello | Irrégularité du timing TLS |
|
||||
| `alpn_http_mismatch` | ALPN=h2 mais HTTP/1.1 | Négociation TLS mensongère |
|
||||
| `is_alpn_missing` | ALPN absent ou `00` | Client non-standard |
|
||||
| `sni_host_mismatch` | SNI ≠ Host header | Proxy transparent / bot |
|
||||
| `mss_mobile_mismatch` | MSS=1460 + score navigateur élevé | Client mobile simulé depuis desktop |
|
||||
| `is_fake_navigation` | `sec_fetch_mode=navigate` mais `sec_fetch_dest≠document` | Navigation simulée |
|
||||
| `tcp_shared_count` | Nb d'IPs partageant la même empreinte TCP | Infrastructure partagée / botnet |
|
||||
| `header_order_shared_count` | Nb d'IPs partageant le même ordre d'en-têtes | Outil automatisé commun |
|
||||
|
||||
---
|
||||
|
||||
## 5. Modèle IA : Isolation Forest (`bot_detector.py`)
|
||||
|
||||
### Cycle d'exécution
|
||||
|
||||
Le service tourne en boucle avec un **cycle de 5 minutes** :
|
||||
|
||||
```
|
||||
fetch_and_analyze()
|
||||
│
|
||||
├─ Requête SELECT * FROM view_ai_features_1h
|
||||
│
|
||||
├─ Nettoyage des données (fillna)
|
||||
│
|
||||
├─ Dual-Model routing :
|
||||
│ ├─ [Complet] correlated=1 → 23 features (réseau + TLS + headers)
|
||||
│ └─ [Applicatif] correlated=0 → 19 features (headers + comportement)
|
||||
│
|
||||
└─ INSERT INTO ml_detected_anomalies
|
||||
```
|
||||
|
||||
### Paramétrage du modèle
|
||||
|
||||
| Paramètre | Valeur | Signification |
|
||||
|---|---|---|
|
||||
| `n_estimators` | 200 | Nombre d'arbres d'isolation |
|
||||
| `contamination` | 0.2% | Proportion de bots attendue dans le trafic |
|
||||
| `seuil de score` | < -0.05 | Score en dessous duquel une session est marquée anomalie |
|
||||
| `volume minimum` | 500 sessions | En dessous, le modèle est ignoré (trop peu de données) |
|
||||
|
||||
### Dual-Model routing
|
||||
|
||||
Le trafic est **séparé en deux populations** selon le champ `correlated` :
|
||||
|
||||
- **Modèle Complet** (`correlated=1`) : la corrélation TCP↔HTTP est disponible → les features réseau (TTL, jitter TLS, ALPN, SNI) sont fiables et ajoutées à l'analyse.
|
||||
- **Modèle Applicatif** (`correlated=0`) : seule la couche HTTP est disponible → l'analyse se concentre sur le comportement applicatif (headers, paths, POST ratio…).
|
||||
|
||||
---
|
||||
|
||||
## 6. Vues heuristiques statiques
|
||||
|
||||
En parallèle du modèle IA, cinq vues SQL fournissent des **détections déterministes** sans ML, sur fenêtre 24h :
|
||||
|
||||
| Vue | Règle de détection |
|
||||
|---|---|
|
||||
| `view_host_ip_ja4_rotation` | IP avec ≥ 5 fingerprints JA4 distincts et > 100 requêtes → rotation d'identité |
|
||||
| `view_host_ja4_anomalies` | Fingerprint JA4 vu depuis ≥ 20 IPs sur ≥ 3 hôtes → outil de scan distribué |
|
||||
| `view_form_bruteforce_detected` | ≥ 10 query params distincts et ≥ 20 hits → brute-force de formulaire |
|
||||
| `view_alpn_mismatch_detected` | HTTP/1.1 avec ALPN h2 ou h3 et ≥ 10 hits → négociation TLS frauduleuse |
|
||||
| `view_tcp_spoofing_detected` | TTL ≤ 64 avec User-Agent Windows ou iPhone → empreinte OS incohérente |
|
||||
|
||||
---
|
||||
|
||||
## 7. Résultats : `ml_detected_anomalies`
|
||||
|
||||
Les anomalies détectées sont stockées dans une table `ReplacingMergeTree(detected_at)` avec **TTL 30 jours**. La clé d'ordre `(src_ip, ja4, host)` garantit que chaque triplet ne conserve que la **détection la plus récente** (dédoublonnage automatique).
|
||||
|
||||
Chaque enregistrement contient :
|
||||
- Les scores et features ayant conduit à la détection
|
||||
- Le champ `reason` : texte lisible avec score, vélocité, et indice de fuzzing
|
||||
- Le champ `is_headless` : déduit de l'incohérence `sec_fetch_mode`
|
||||
|
||||
---
|
||||
|
||||
## 8. Schéma de flux complet
|
||||
|
||||
```
|
||||
┌─────────────────────────────────────┐
|
||||
│ http_logs_raw (JSON) │
|
||||
└──────────────┬──────────────────────┘
|
||||
│ mv_http_logs (MV)
|
||||
▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ http_logs (parsée) │
|
||||
└────────┬──────────────┬─────────────┘
|
||||
│ │
|
||||
mv_agg_host_ip_ja4 │ │ mv_agg_header_fingerprint
|
||||
▼ ▼
|
||||
┌──────────────────┐ ┌──────────────────────────┐
|
||||
│ agg_host_ip_ja4 │ │ agg_header_fingerprint │
|
||||
│ _1h │ │ _1h │
|
||||
└────────┬─────────┘ └──────────┬──────────────┘
|
||||
│ │
|
||||
└──────────┬─────────────┘
|
||||
│ view_ai_features_1h (JOIN + calculs)
|
||||
▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ bot_detector.py (Isolation Forest) │
|
||||
│ Cycle : 5 min | Fenêtre : 24h │
|
||||
└──────────────┬──────────────────────┘
|
||||
│
|
||||
▼
|
||||
┌─────────────────────────────────────┐
|
||||
│ ml_detected_anomalies │
|
||||
│ (ReplacingMergeTree, TTL 30j) │
|
||||
└─────────────────────────────────────┘
|
||||
```
|
||||
@ -1,29 +0,0 @@
|
||||
module github.com/antitbone/ja4/correlator
|
||||
|
||||
go 1.21
|
||||
|
||||
require (
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.23.0
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/ClickHouse/ch-go v0.61.5 // indirect
|
||||
github.com/andybalholm/brotli v1.1.0 // indirect
|
||||
github.com/go-faster/city v1.0.1 // indirect
|
||||
github.com/go-faster/errors v0.7.1 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/klauspost/compress v1.17.7 // indirect
|
||||
github.com/paulmach/orb v0.11.1 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.21 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/segmentio/asm v1.2.0 // indirect
|
||||
github.com/shopspring/decimal v1.3.1 // indirect
|
||||
go.opentelemetry.io/otel v1.24.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.24.0 // indirect
|
||||
golang.org/x/sys v0.18.0 // indirect
|
||||
)
|
||||
|
||||
require github.com/antitbone/ja4/ja4common v0.1.0
|
||||
|
||||
replace github.com/antitbone/ja4/ja4common => ../../shared/go/ja4common
|
||||
@ -1,110 +0,0 @@
|
||||
github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4=
|
||||
github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.23.0 h1:srmRrkS0BR8gEut87u8jpcZ7geOob6nGj9ifrb+aKmg=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.23.0/go.mod h1:tBhdF3f3RdP7sS59+oBAtTyhWpy0024ZxDMhgxra0QE=
|
||||
github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
|
||||
github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
|
||||
github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
|
||||
github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg=
|
||||
github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
|
||||
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg=
|
||||
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
|
||||
github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU=
|
||||
github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU=
|
||||
github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
|
||||
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
|
||||
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
|
||||
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
|
||||
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
|
||||
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
|
||||
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
|
||||
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
|
||||
go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
|
||||
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
|
||||
go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
|
||||
go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
|
||||
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@ -1,111 +0,0 @@
|
||||
time
|
||||
log_date
|
||||
|
||||
src_ip
|
||||
- ip source de la connexion
|
||||
src_port
|
||||
- port source de la connexion
|
||||
dst_ip
|
||||
- ip de destination de la connexion
|
||||
dst_port
|
||||
- port de destination de la connexion
|
||||
|
||||
src_asn
|
||||
- Numero d'AS de l'ip source
|
||||
src_country_code
|
||||
- Code Pays de l'ip source
|
||||
src_as_name
|
||||
- Nom de l'AS de l ip source
|
||||
src_org
|
||||
- Organisation de l AS source
|
||||
src_domain
|
||||
- domaine de l'AS de l ip source
|
||||
|
||||
method
|
||||
- Methode HTTP [GET, POST, ... ]
|
||||
scheme
|
||||
- Type de connexion http [http, https]
|
||||
host
|
||||
- Hostname demandé dans l'url
|
||||
path
|
||||
- Path demandé dans l'url
|
||||
query
|
||||
- Query demandé dans l'url
|
||||
http_version
|
||||
- Version du protocol http utilisé
|
||||
|
||||
orphan_side
|
||||
- Indique si le log HTTP a pu etre enrichi avec les informations ip_, tcp, ja3_ et ja4_
|
||||
- "A" indique que seul le log HTTP est present, sans enrichissement
|
||||
correlated
|
||||
- l'algorithm de correlation log http + parametres tcp a il réussi (tcp + ja4/3)
|
||||
keepalives
|
||||
- Numero de desquance dans une connexion http avec keepalive.
|
||||
a_timestamp
|
||||
b_timestamp
|
||||
conn_id
|
||||
|
||||
ip_meta_df
|
||||
- Flag dont fragement
|
||||
ip_meta_id
|
||||
- id du packet ip
|
||||
ip_meta_total_length
|
||||
- Taille des metadata dans pe packet ip
|
||||
ip_meta_ttl
|
||||
- TTL du packet ip vu par le serveur destinataire du packet
|
||||
|
||||
tcp_meta_options
|
||||
- options du packet TCP vu par le serveur destinataire du packet
|
||||
tcp_meta_window_size
|
||||
- TCP window size vu par le serveur destinataire du packet
|
||||
tcp_meta_mss
|
||||
- TCP mss vu par le serveur destinataire du packet
|
||||
tcp_meta_window_scale
|
||||
- TCP windows scale vu par le serveur destinataire du packet
|
||||
syn_to_clienthello_ms
|
||||
- durée en ms entre le 1er packet SYN et le ClienHello du TLS
|
||||
|
||||
tls_version
|
||||
- Version de TLS negocié avec le serveur destinataire du packet
|
||||
tls_sni
|
||||
- SNI, nom de domaine demandé pour le cerificat TLS
|
||||
tls_alpn
|
||||
- ALPN annoncé lors du TLS
|
||||
ja3
|
||||
- liste des agos utiliés pour la signature ja3
|
||||
ja3_hash
|
||||
- hash ja3
|
||||
ja4
|
||||
- hash ja4
|
||||
|
||||
client_headers
|
||||
- liste des headers envoyés par le client http sous forme de liste Header,Header2,Header3,...
|
||||
|
||||
header_user_agent
|
||||
- Header HTTP User-Agent
|
||||
header_accept
|
||||
- Header HTTP Accept
|
||||
header_accept_encoding
|
||||
- Header HTTP Accept-Encoding
|
||||
header_accept_language
|
||||
- Header HTTP Accept-Language
|
||||
header_content_type
|
||||
- Header Content-Type
|
||||
header_x_request_id
|
||||
- Header X-Request-ID
|
||||
header_x_trace_id
|
||||
- Header X-Trace-ID
|
||||
header_x_forwarded_for
|
||||
- Header X-Forwarded-For
|
||||
header_sec_ch_ua
|
||||
- Header Sec-Ch-UA
|
||||
header_sec_ch_ua_mobile
|
||||
- Header -Sec-Ch-UA-Mobile
|
||||
header_sec_ch_ua_platform
|
||||
- Header Sec-Ch-UA-Plateform
|
||||
header_sec_fetch_dest
|
||||
- Header -Sec-Fetch-Dest
|
||||
header_sec_fetch_mode
|
||||
- Header Sec-Fetch-Mode
|
||||
header_sec_fetch_site
|
||||
- Header Sec-Fetch-Site
|
||||
@ -1,30 +0,0 @@
|
||||
1. Incohérences de Signatures (Spoofing)
|
||||
|
||||
User-Agent vs TLS : Le header_user_agent prétend être un navigateur (Chrome/Safari) mais le ja3/ja4 correspond à un outil de script.
|
||||
User-Agent vs Headers modernes : Le header_user_agent indique un navigateur récent, mais les headers header_sec_ch_ua_* sont vides ou absents de client_headers.
|
||||
User-Agent vs ALPN : Le navigateur déclaré ne correspond pas au protocole négocié dans tls_alpn (ex: Chrome sans h2).
|
||||
OS vs TTL TCP : L'OS déclaré dans le header_user_agent (ex: Windows) contredit la valeur de ip_meta_ttl (ex: 64, typique de Linux).
|
||||
Host vs SNI : Le nom de domaine dans le header host ne correspond pas au tls_sni demandé lors du handshake TLS.
|
||||
|
||||
2. Anomalies de Headers (HTTP Fingerprinting)
|
||||
|
||||
Empreinte d'ordre (Fingerprint) : Apparition soudaine d'une disposition de client_headers (ordre exact) très rare, générant beaucoup de trafic.
|
||||
Pauvreté des headers : Le nombre total de headers dans client_headers est anormalement bas (ex: < 5), typique des scripts basiques.
|
||||
Absence de headers vitaux : Le trafic prétend être humain mais n'envoie pas header_accept_language ou header_accept_encoding.
|
||||
Combinaison fatale : Le croisement d'un ja4 spécifique avec un ordre de client_headers inédit (détection de bots modifiant leur TLS mais trahis par l'applicatif).
|
||||
|
||||
3. Anomalies Réseau et TCP (Couche 3 & 4)
|
||||
|
||||
Mécanique TCP de masse : Une même combinaison (tcp_meta_window_size, tcp_meta_window_scale, tcp_meta_mss) vue sur des milliers d'IP différentes.
|
||||
Handshake robotique : Un délai syn_to_clienthello_ms anormalement constant (variance quasi nulle) sur un grand nombre de connexions, typique d'un bot en datacenter.
|
||||
Options TCP atypiques : Des paramètres tcp_meta_options inhabituels pour le trafic web classique de tes vrais utilisateurs.
|
||||
|
||||
4. Anomalies Comportementales et Volumétriques (Côté Requête)
|
||||
|
||||
Rafale de requêtes (Spike) : Volume d'appels (count) par src_ip ou par ja4 dépassant drastiquement le 99ème percentile historique sur 5 minutes.
|
||||
Scraping furtif distribué : Un même ja4 (non standard) utilisé par des centaines de src_ip différentes, chacune faisant très peu de requêtes.
|
||||
Balayage aveugle (Scanner) : Un volume anormal de path uniques (ou path + query) visités par une même IP ou un même ja4 en quelques minutes (remplace la détection des erreurs 404).
|
||||
Acharnement sur cible (Brute force aveugle) : Une concentration extrême de requêtes ciblant uniquement les path sensibles (login, API, password-reset) sans navigation normale sur le reste du site (remplace la détection des 401/403).
|
||||
Méthodes suspectes : Utilisation massive ou inhabituelle de method non standards (PUT, DELETE, OPTIONS, TRACE) par rapport à la baseline.
|
||||
Payloads suspects : Présence de patterns d'injection ou de caractères très inhabituels dans query ou path (longueur extrême, encodages multiples).
|
||||
Bot "Low and Slow" : IP ou ja4 qui passe sous les radars sur 5 minutes, mais dont le volume cumulé sur 24h ou 7 jours est mathématiquement improbable pour un humain.
|
||||
@ -1,521 +0,0 @@
|
||||
# 🛡️ Manuel de Référence Technique : Moteur de Détection Antispam & Bot
|
||||
|
||||
Ce document détaille les algorithmes de détection implémentés dans les vues ClickHouse pour la plateforme.
|
||||
|
||||
---
|
||||
|
||||
## 1. Analyse de la Couche Transport (L4) : La "Trace Physique"
|
||||
Avant même d'analyser l'URL, le moteur inspecte la manière dont la connexion a été établie. C'est la couche la plus difficile à falsifier pour un attaquant.
|
||||
|
||||
### A. Fingerprint de la Pile TCP (`tcp_fingerprint`)
|
||||
* **Fonctionnement :** Nous utilisons `cityHash64` pour créer un identifiant unique basé sur trois paramètres immuables du handshake : le **MSS** (Maximum Segment Size), la **Window Size** et le **Window Scale**.
|
||||
* **Ce que ça détecte :** L'unicité logicielle. Un bot tournant sur une image Alpine Linux aura une signature TCP différente d'un utilisateur sur iOS 17 ou Windows 11.
|
||||
* **Détection de botnet :** Si 500 IPs différentes partagent exactement le même `tcp_fingerprint` ET le même `ja4`, il y a une probabilité de 99% qu'il s'agisse d'un cluster de bots clonés.
|
||||
|
||||
|
||||
|
||||
### B. Analyse de la gigue (Jitter) et Handshake
|
||||
* **Fonctionnement :** On calcule la variance (`varPop`) du délai entre le `SYN` et le `ClientHello` TLS.
|
||||
* **Ce que ça détecte :** La stabilité robotique.
|
||||
* **Humain :** Latence variable (4G, Wi-Fi, mouvements). La variance est élevée.
|
||||
* **Bot Datacenter :** Latence ultra-stable (fibre optique dédiée). Une variance proche de 0 indique une connexion automatisée depuis une infrastructure cloud.
|
||||
|
||||
---
|
||||
|
||||
## 2. Analyse de la Session (L5) : Le "Passeport TLS"
|
||||
Le handshake TLS est une mine d'or pour identifier la bibliothèque logicielle (OpenSSL, Go-TLS, etc.).
|
||||
|
||||
### A. Incohérence UA vs JA4
|
||||
* **Fonctionnement :** Le moteur croise le `header_user_agent` (déclaratif) avec le `ja4` (structurel).
|
||||
* **Ce que ça détecte :** Le **Spoofing de Browser**. Un script Python peut facilement écrire `User-Agent: Mozilla/5.0...Chrome/120`, mais il ne peut pas simuler l'ordre exact des extensions TLS et des algorithmes de chiffrement d'un vrai Chrome sans une ingénierie complexe (comme `utls`).
|
||||
* **Logique de score :** Si UA = Chrome mais JA4 != Signature_Chrome -> **+50 points de risque**.
|
||||
|
||||
### B. Discordance Host vs SNI
|
||||
* **Fonctionnement :** Comparaison entre le champ `tls_sni` (négocié en clair lors du handshake) et le header `Host` (envoyé plus tard dans la requête chiffrée).
|
||||
* **Ce que ça détecte :** Le **Domain Fronting** ou les attaques par tunnel. Un bot peut demander un certificat pour `domaine-innocent.com` (SNI) mais tenter d'attaquer `api-critique.com` (Host).
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 3. Analyse Applicative (L7) : Le "Comportement HTTP"
|
||||
Une fois le tunnel établi, on analyse la structure de la requête HTTP.
|
||||
|
||||
### A. Empreinte d'ordre des Headers (`http_fp`)
|
||||
* **Fonctionnement :** Nous hashons la liste ordonnée des clés de headers (`Accept`, `User-Agent`, `Connection`, etc.).
|
||||
* **Ce que ça détecte :** La signature du moteur de rendu. Chaque navigateur (Firefox, Safari, Chromium) a un ordre immuable pour envoyer ses headers.
|
||||
* **Détection :** Si un client envoie les headers dans un ordre inhabituel ou minimaliste (pauvreté des headers < 6), il est marqué comme suspect.
|
||||
|
||||
### B. Analyse des Payloads et Entropie
|
||||
* **Fonctionnement :** Recherche de patterns via regex dans `query` et `path` (détection SQLi, XSS, Path Traversal).
|
||||
* **Complexité :** Nous détectons les encodages multiples (ex: `%2520`) qui tentent de tromper les pare-feux simples.
|
||||
|
||||
---
|
||||
|
||||
## 4. Corrélation Temporelle & Baseline : Le "Voisinage Statistique"
|
||||
Le score final dépend du passé de la signature TLS.
|
||||
|
||||
### A. Le Malus de Nouveauté (`agg_novelty`)
|
||||
* **Logique :** Une signature (JA4 + FP) vue pour la première fois aujourd'hui est "froide".
|
||||
* **Traitement :** On applique un malus si `first_seen` date de moins de 2 heures. Un botnet qui vient de lancer une campagne de rotation de signatures sera immédiatement pénalisé par son manque d'historique.
|
||||
|
||||
### B. Le Dépassement de Baseline (`tbl_baseline_ja4_7d`)
|
||||
* **Fonctionnement :** On compare les `hits` actuels au 99ème percentile (`p99`) historique de cette signature précise.
|
||||
* **Exemple :** Si le JA4 de "Chrome 122" fait habituellement 10 requêtes/min/IP sur votre site, et qu'une IP en fait soudainement 300, le score explose même si la requête est techniquement parfaite.
|
||||
|
||||
---
|
||||
|
||||
## 5. Synthèse du Scoring (Le Verdict)
|
||||
|
||||
| Algorithme | Signal | Impact Score |
|
||||
| :--- | :--- | :--- |
|
||||
| **Fingerprint Mismatch** | UA vs TLS (Spoofing) | **Haut (50)** |
|
||||
| **L4 Anomaly** | Variance latence < 0.5ms | **Moyen (30)** |
|
||||
| **Path Sensitivity** | Hit sur `/admin` ou `/config` | **Haut (40)** |
|
||||
| **Payload Security** | Caractères d'injection (SQL/XSS) | **Critique (60)** |
|
||||
| **Mass Distribution** | 1 JA4 sur > 50 IPs différentes | **Moyen (30)** |
|
||||
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 6. Identification des Hosts par IP et JA4 (sql/hosts.sql)
|
||||
|
||||
Cette section détaille les vues d'agrégation et de détection pour identifier quels hosts sont associés à quelles signatures (IP + JA4).
|
||||
|
||||
### A. Agrégats de Base
|
||||
|
||||
| Table | Granularité | Description |
|
||||
|-------|-------------|-------------|
|
||||
| `agg_host_ip_ja4_1h` | heure | Hits, paths uniques, query params, méthodes par (IP, JA4, host) |
|
||||
| `agg_host_ip_ja4_24h` | jour | Rollup quotidien pour historique long terme |
|
||||
|
||||
### B. Vues d'Identification
|
||||
|
||||
**`view_host_identification`** - Top hosts par signature
|
||||
```sql
|
||||
-- Quel host est associé à cette IP/JA4 ?
|
||||
SELECT src_ip, ja4, host, total_hits, unique_paths, user_agent
|
||||
FROM ja4_processing.view_host_identification
|
||||
WHERE src_ip = '1.2.3.4'
|
||||
ORDER BY total_hits DESC;
|
||||
```
|
||||
|
||||
**`view_host_ja4_anomalies`** - JA4 partagé par plusieurs hosts (botnet)
|
||||
```sql
|
||||
-- Ce JA4 est-il utilisé par plusieurs hosts différents ?
|
||||
SELECT ja4, hosts, unique_hosts, unique_ips
|
||||
FROM ja4_processing.view_host_ja4_anomalies
|
||||
HAVING unique_hosts >= 3;
|
||||
-- Interprétation : 1 JA4 sur 3+ hosts = botnet cloné probable
|
||||
```
|
||||
|
||||
**`view_host_ip_ja4_rotation`** - IP avec rotation de fingerprints
|
||||
```sql
|
||||
-- Cette IP change-t-elle de JA4 fréquemment ?
|
||||
SELECT src_ip, ja4s, unique_ja4s
|
||||
FROM ja4_processing.view_host_ip_ja4_rotation
|
||||
HAVING unique_ja4s >= 5;
|
||||
-- Interprétation : 1 IP avec 5+ JA4 différents = fingerprint spoofing
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7. Détection de Brute Force (sql/hosts.sql)
|
||||
|
||||
### A. Brute Force sur POST (endpoints sensibles)
|
||||
|
||||
**Table :** `agg_bruteforce_post_5m` - Fenêtres de 5 minutes
|
||||
|
||||
**Vue :** `view_bruteforce_post_detected`
|
||||
```sql
|
||||
-- Détecter les tentatives de brute force sur les login
|
||||
SELECT window, src_ip, ja4, host, path, attempts, attempts_per_minute
|
||||
FROM ja4_processing.view_bruteforce_post_detected
|
||||
WHERE host = 'api.example.com'
|
||||
ORDER BY attempts DESC;
|
||||
|
||||
-- Threshold : ≥10 POST en 5 minutes sur endpoints sensibles
|
||||
-- Endpoints ciblés : login, auth, signin, password, admin, wp-login, etc.
|
||||
```
|
||||
|
||||
### B. Brute Force sur Formulaire (Query params variables)
|
||||
|
||||
**Table :** `agg_form_bruteforce_5m`
|
||||
|
||||
**Vue :** `view_form_bruteforce_detected`
|
||||
```sql
|
||||
-- Détecter les requêtes avec query params hautement variables
|
||||
SELECT window, src_ip, ja4, host, path, requests, unique_query_patterns
|
||||
FROM ja4_processing.view_form_bruteforce_detected
|
||||
HAVING requests >= 20 AND unique_query_patterns >= 10;
|
||||
|
||||
-- Interprétation : 20+ requêtes avec 10+ patterns query différents
|
||||
-- = tentative de fuzzing ou brute force sur paramètres
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Header Fingerprinting (sql/hosts.sql)
|
||||
|
||||
Le champ `client_headers` contient la liste comma-separated des headers présents.
|
||||
Exemple : `"Accept,Accept-Encoding,Sec-CH-UA,Sec-Fetch-Dest,User-Agent"`
|
||||
|
||||
### A. Signature par Ordre de Headers
|
||||
|
||||
**Table :** `agg_header_fingerprint_1h`
|
||||
|
||||
| Champ | Description |
|
||||
|-------|-------------|
|
||||
| `header_count` | Nombre total de headers (virgules + 1) |
|
||||
| `has_*` | Flags pour chaque header moderne (Sec-CH-UA, Sec-Fetch-*, etc.) |
|
||||
| `header_order_hash` | MD5(client_headers) = signature unique de l'ordre |
|
||||
| `modern_browser_score` | Score 0-100 basé sur les headers modernes présents |
|
||||
|
||||
### B. Vues de Détection
|
||||
|
||||
**`view_header_missing_modern_headers`** - Headers modernes manquants
|
||||
```sql
|
||||
-- Navigateurs "modernes" avec headers manquants
|
||||
SELECT src_ip, ja4, header_user_agent, modern_browser_score, header_count
|
||||
FROM ja4_processing.view_header_missing_modern_headers
|
||||
WHERE header_user_agent ILIKE '%Chrome%';
|
||||
|
||||
-- Threshold : score < 70 pour Chrome/Firefox = suspect
|
||||
-- Un vrai Chrome envoie automatiquement Sec-CH-UA, Sec-Fetch-*, etc.
|
||||
```
|
||||
|
||||
**`view_header_ua_order_mismatch`** - Spoofing détecté
|
||||
```sql
|
||||
-- Même User-Agent avec ordre de headers différent
|
||||
SELECT header_user_agent, ja4, unique_hashes, unique_ips
|
||||
FROM ja4_processing.view_header_ua_order_mismatch
|
||||
HAVING unique_hashes > 1;
|
||||
|
||||
-- Interprétation : 1 UA avec 2+ ordres de headers = spoofing ou outil custom
|
||||
```
|
||||
|
||||
**`view_header_minimalist_count`** - Bot minimaliste
|
||||
```sql
|
||||
-- Clients avec trop peu de headers
|
||||
SELECT src_ip, ja4, header_count, header_user_agent
|
||||
FROM ja4_processing.view_header_minimalist_count
|
||||
WHERE header_count < 6;
|
||||
|
||||
-- Threshold : < 6 headers = bot scripté (curl, Python requests, etc.)
|
||||
```
|
||||
|
||||
**`view_header_sec_ch_missing`** - Incohérence Chrome
|
||||
```sql
|
||||
-- Chrome sans Sec-CH-UA (impossible pour un vrai Chrome)
|
||||
SELECT src_ip, ja4, header_user_agent
|
||||
FROM ja4_processing.view_header_sec_ch_missing
|
||||
WHERE header_user_agent ILIKE '%Chrome/%';
|
||||
```
|
||||
|
||||
**`view_header_known_bot_signature`** - Signature botnet
|
||||
```sql
|
||||
-- Même ordre de headers sur 10+ IPs différentes
|
||||
SELECT header_order_hash, header_user_agent, unique_ips, total_hits
|
||||
FROM ja4_processing.view_header_known_bot_signature
|
||||
HAVING unique_ips >= 10;
|
||||
|
||||
-- Interprétation : 1 signature sur 10+ IPs = cluster de bots clonés
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9. ALPN Mismatch Detection (sql/hosts.sql)
|
||||
|
||||
### Principe
|
||||
|
||||
ALPN (Application-Layer Protocol Negotiation) est une extension TLS qui négocie le protocole HTTP **avant** la requête.
|
||||
|
||||
| ALPN déclaré | HTTP réel | Interprétation |
|
||||
|--------------|-----------|----------------|
|
||||
| `h2` | `HTTP/2` | ✅ Normal |
|
||||
| `h2` | `HTTP/1.1` | ❌ Bot mal configuré |
|
||||
| `http/1.1` | `HTTP/1.1` | ✅ Normal |
|
||||
|
||||
### Vue de Détection
|
||||
|
||||
**`view_alpn_mismatch_detected`**
|
||||
```sql
|
||||
-- Clients déclarant h2 mais parlant HTTP/1.1
|
||||
SELECT src_ip, ja4, declared_alpn, actual_http_version, mismatches, mismatch_pct
|
||||
FROM ja4_processing.view_alpn_mismatch_detected
|
||||
HAVING mismatch_pct >= 80;
|
||||
|
||||
-- Threshold : ≥5 requêtes avec ≥80% d'incohérence
|
||||
-- Cause : curl mal configuré, Python requests, bots spoofant ALPN
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. Rate Limiting & Burst Detection (sql/hosts.sql)
|
||||
|
||||
### A. Rate Limiting (1 minute)
|
||||
|
||||
**Table :** `agg_rate_limit_1m`
|
||||
|
||||
**Vue :** `view_rate_limit_exceeded`
|
||||
```sql
|
||||
-- IPs dépassant 50 requêtes/minute
|
||||
SELECT minute, src_ip, ja4, requests_per_min, unique_paths
|
||||
FROM ja4_processing.view_rate_limit_exceeded
|
||||
ORDER BY requests_per_min DESC;
|
||||
|
||||
-- Threshold : > 50 req/min = trafic automatisé
|
||||
-- Un humain ne peut pas soutenir 50+ req/min de manière cohérente
|
||||
```
|
||||
|
||||
### B. Burst Detection (10 secondes)
|
||||
|
||||
**Table :** `agg_burst_10s`
|
||||
|
||||
**Vue :** `view_burst_detected`
|
||||
```sql
|
||||
-- Pics soudains de trafic
|
||||
SELECT window, src_ip, ja4, burst_count
|
||||
FROM ja4_processing.view_burst_detected
|
||||
HAVING burst_count > 20;
|
||||
|
||||
-- Threshold : > 20 requêtes en 10 secondes = burst suspect
|
||||
-- Utile pour détecter les attaques par vagues
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 11. Path Enumeration / Scanning (sql/hosts.sql)
|
||||
|
||||
### Vue de Détection
|
||||
|
||||
**`view_path_scan_detected`**
|
||||
```sql
|
||||
-- Détection de scanning de paths sensibles
|
||||
SELECT window, src_ip, ja4, host, sensitive_hits, sensitive_ratio
|
||||
FROM ja4_processing.view_path_scan_detected
|
||||
HAVING sensitive_hits >= 5;
|
||||
|
||||
-- Paths surveillés : admin, backup, config, .env, .git, wp-admin,
|
||||
-- phpinfo, test, debug, log, sql, dump, passwd, shadow, htaccess, etc.
|
||||
|
||||
-- Threshold : ≥5 paths sensibles en 5 minutes = scanning
|
||||
```
|
||||
|
||||
### Exemple de Résultat
|
||||
|
||||
| src_ip | ja4 | host | sensitive_hits | sensitive_ratio |
|
||||
|--------|-----|------|----------------|-----------------|
|
||||
| 1.2.3.4 | t13d... | api.example.com | 47 | 94.00 |
|
||||
| 5.6.7.8 | t13d... | www.example.com | 12 | 80.00 |
|
||||
|
||||
**Interprétation :** Ces IPs testent systématiquement les paths sensibles = outils comme Nikto, Dirb, Gobuster.
|
||||
|
||||
---
|
||||
|
||||
## 12. Payload Attack Detection (sql/hosts.sql)
|
||||
|
||||
### A. Types d'Attaques Détectées
|
||||
|
||||
| Type | Patterns Détectés |
|
||||
|------|-------------------|
|
||||
| **SQL Injection** | `UNION SELECT`, `OR 1=1`, `DROP TABLE`, `; --`, `/* */`, `WAITFOR DELAY`, `SLEEP()` |
|
||||
| **XSS** | `<script>`, `javascript:`, `onerror=`, `onload=`, `<img src=data:`, `<svg onload>` |
|
||||
| **Path Traversal** | `../`, `..\\`, `%2e%2e%2f`, `%252e%252e`, `%%32%65%%32%65` |
|
||||
|
||||
### Vue de Détection
|
||||
|
||||
**`view_payload_attacks_detected`**
|
||||
```sql
|
||||
-- Toutes les tentatives d'injection
|
||||
SELECT window, src_ip, ja4, host, path,
|
||||
sqli_attempts, xss_attempts, traversal_attempts
|
||||
FROM ja4_processing.view_payload_attacks_detected
|
||||
ORDER BY sqli_attempts DESC, xss_attempts DESC, traversal_attempts DESC;
|
||||
|
||||
-- Threshold : ≥1 tentative = alerte (zero tolerance)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 13. JA4 Botnet Detection (sql/hosts.sql)
|
||||
|
||||
### Principe
|
||||
|
||||
Un vrai navigateur a un fingerprint TLS unique. Un bot déployé sur 100 machines aura le **même JA4**.
|
||||
|
||||
### Vue de Détection
|
||||
|
||||
**`view_ja4_botnet_suspected`**
|
||||
```sql
|
||||
-- JA4 partagé par 20+ IPs différentes
|
||||
SELECT ja4, ja3_hash, unique_ips, unique_asns, unique_countries, total_hits
|
||||
FROM ja4_processing.view_ja4_botnet_suspected
|
||||
HAVING unique_ips >= 20;
|
||||
|
||||
-- Threshold : ≥20 IPs avec le même JA4 = botnet cloné
|
||||
```
|
||||
|
||||
### Exemple de Résultat
|
||||
|
||||
| ja4 | ja3_hash | unique_ips | unique_asns | unique_countries |
|
||||
|-----|----------|------------|-------------|------------------|
|
||||
| t13d1512... | a3b5c7... | 147 | 12 | 8 |
|
||||
| t13d0918... | f1e2d3... | 52 | 3 | 2 |
|
||||
|
||||
**Interprétation :** 147 IPs différentes avec le même fingerprint = cluster de bots clonés.
|
||||
|
||||
---
|
||||
|
||||
## 14. Correlation Quality (sql/hosts.sql)
|
||||
|
||||
### Principe
|
||||
|
||||
Mesure le ratio d'événements non-corrélés (orphelins). Un trafic légitime a une bonne corrélation HTTP/TCP.
|
||||
|
||||
### Vue de Détection
|
||||
|
||||
**`view_high_orphan_ratio`**
|
||||
```sql
|
||||
-- Trafic avec >80% d'événements non-corrélés
|
||||
SELECT hour, src_ip, ja4, host, correlated, orphans, orphan_pct
|
||||
FROM ja4_processing.view_high_orphan_ratio
|
||||
ORDER BY orphan_pct DESC;
|
||||
|
||||
-- Threshold : orphan_pct > 80% = trafic suspect
|
||||
-- Peut indiquer du trafic généré artificiellement
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 15. Maintenance et Faux Positifs
|
||||
|
||||
### Exceptions Connues
|
||||
|
||||
| Source | Faux Positif | Solution |
|
||||
|--------|--------------|----------|
|
||||
| **Googlebot/Bingbot** | Scan agressif mais légitime | Filtrer par ASN + Reverse DNS |
|
||||
| **Monitoring interne** | Rate limit élevé | Whitelist par IP/ASN |
|
||||
| **CDN/Proxy** | JA4 partagé (clients derrière proxy) | Vérifier ASN (Cloudflare, Akamai) |
|
||||
| **Navigateurs anciens** | Headers modernes manquants | Vérifier UA version |
|
||||
|
||||
### Reset des Scores
|
||||
|
||||
Les agrégats sont automatiquement purgés par TTL :
|
||||
- `agg_*_1h` : TTL 7 jours
|
||||
- `agg_*_5m` : TTL 1 jour
|
||||
- `agg_*_1m` : TTL 1 jour
|
||||
|
||||
Un IP bloquée par erreur retrouvera un score normal après expiration du TTL.
|
||||
|
||||
---
|
||||
|
||||
## 16. Synthèse des Vues de Détection
|
||||
|
||||
| Vue | Détection | Threshold | Impact |
|
||||
|-----|-----------|-----------|--------|
|
||||
| `view_bruteforce_post_detected` | POST endpoints sensibles | ≥10 en 5min | 🔴 Haut |
|
||||
| `view_form_bruteforce_detected` | Query params variables | ≥20 req, ≥10 patterns | 🔴 Haut |
|
||||
| `view_header_missing_modern_headers` | Headers modernes manquants | score < 70 | 🔴 Haut |
|
||||
| `view_header_ua_order_mismatch` | UA spoofing (ordre) | >1 hash | 🔴 Haut |
|
||||
| `view_header_minimalist_count` | Bot minimaliste | < 6 headers | 🔴 Haut |
|
||||
| `view_header_sec_ch_missing` | Chrome sans Sec-CH | absent | 🟡 Moyen |
|
||||
| `view_header_known_bot_signature` | Signature connue (botnet) | 10+ IPs | 🔴 Haut |
|
||||
| `view_alpn_mismatch_detected` | h2 déclaré, HTTP/1.1 parlé | ≥80% mismatch | 🔴 Haut |
|
||||
| `view_rate_limit_exceeded` | Rate limit dépassé | >50 req/min | 🔴 Haut |
|
||||
| `view_burst_detected` | Burst soudain | >20 req/10s | 🟡 Moyen |
|
||||
| `view_path_scan_detected` | Scanning de paths | ≥5 sensibles | 🔴 Haut |
|
||||
| `view_payload_attacks_detected` | Injections SQLi/XSS | ≥1 tentative | 🔴 Critique |
|
||||
| `view_ja4_botnet_suspected` | JA4 partagé (botnet) | ≥20 IPs | 🔴 Haut |
|
||||
| `view_high_orphan_ratio` | Trafic non-corrélé | >80% orphans | 🟡 Moyen |
|
||||
| `view_host_ja4_anomalies` | JA4 sur plusieurs hosts | ≥3 hosts | 🟡 Moyen |
|
||||
| `view_host_ip_ja4_rotation` | IP rotate JA4 | ≥5 JA4 | 🟡 Moyen |
|
||||
|
||||
---
|
||||
|
||||
## 17. Exemples de Requêtes d'Investigation
|
||||
|
||||
### Top 10 des IPs les plus suspectes (score cumulé)
|
||||
```sql
|
||||
WITH threats AS (
|
||||
SELECT src_ip, ja4, 'bruteforce' AS type, sum(attempts) AS score
|
||||
FROM ja4_processing.view_bruteforce_post_detected GROUP BY src_ip, ja4
|
||||
UNION ALL
|
||||
SELECT src_ip, ja4, 'path_scan', sum(sensitive_hits)
|
||||
FROM ja4_processing.view_path_scan_detected GROUP BY src_ip, ja4
|
||||
UNION ALL
|
||||
SELECT src_ip, ja4, 'payload', sum(sqli_attempts + xss_attempts)
|
||||
FROM ja4_processing.view_payload_attacks_detected GROUP BY src_ip, ja4
|
||||
)
|
||||
SELECT src_ip, ja4, sum(score) AS total_score, groupArray(type) AS threat_types
|
||||
FROM threats
|
||||
GROUP BY src_ip, ja4
|
||||
ORDER BY total_score DESC
|
||||
LIMIT 10;
|
||||
```
|
||||
|
||||
### Historique d'une IP suspecte
|
||||
```sql
|
||||
SELECT
|
||||
hour,
|
||||
host,
|
||||
countMerge(hits) AS requests,
|
||||
uniqMerge(uniq_paths) AS unique_paths
|
||||
FROM ja4_processing.agg_host_ip_ja4_1h
|
||||
WHERE src_ip = '1.2.3.4'
|
||||
AND hour >= now() - INTERVAL 24 HOUR
|
||||
GROUP BY hour, host
|
||||
ORDER BY hour DESC;
|
||||
```
|
||||
|
||||
### Corrélation JA4 → User-Agent → Hosts
|
||||
```sql
|
||||
SELECT
|
||||
ja4,
|
||||
any(first_ua) AS user_agent,
|
||||
groupArray(DISTINCT host) AS hosts,
|
||||
sum(countMerge(hits)) AS total_requests
|
||||
FROM ja4_processing.agg_host_ip_ja4_1h
|
||||
WHERE hour >= now() - INTERVAL 1 HOUR
|
||||
GROUP BY ja4
|
||||
ORDER BY total_requests DESC
|
||||
LIMIT 20;
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 18. Installation et Maintenance
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
# Exécuter après init.sql
|
||||
clickhouse-client --multiquery < sql/hosts.sql
|
||||
```
|
||||
|
||||
### Vérification
|
||||
```sql
|
||||
-- Compter les enregistrements
|
||||
SELECT count(*) FROM ja4_processing.agg_host_ip_ja4_1h;
|
||||
SELECT count(*) FROM ja4_processing.agg_header_fingerprint_1h;
|
||||
|
||||
-- Tester les vues
|
||||
SELECT * FROM ja4_processing.view_host_identification LIMIT 10;
|
||||
SELECT * FROM ja4_processing.view_bruteforce_post_detected LIMIT 10;
|
||||
SELECT * FROM ja4_processing.view_payload_attacks_detected LIMIT 10;
|
||||
```
|
||||
|
||||
### Monitoring
|
||||
```sql
|
||||
-- Vues les plus actives (dernière heure)
|
||||
SELECT
|
||||
'bruteforce_post' AS view_name, count() AS alerts
|
||||
FROM ja4_processing.view_bruteforce_post_detected
|
||||
UNION ALL
|
||||
SELECT 'path_scan', count() FROM ja4_processing.view_path_scan_detected
|
||||
UNION ALL
|
||||
SELECT 'payload_attacks', count() FROM ja4_processing.view_payload_attacks_detected
|
||||
UNION ALL
|
||||
SELECT 'ja4_botnet', count() FROM ja4_processing.view_ja4_botnet_suspected
|
||||
ORDER BY alerts DESC;
|
||||
```
|
||||
@ -1,386 +0,0 @@
|
||||
package unixsocket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
"github.com/antitbone/ja4/correlator/internal/observability"
|
||||
)
|
||||
|
||||
const (
|
||||
// Maximum datagram size for JSON logs (64KB - Unix datagram limit)
|
||||
MaxDatagramSize = 65535
|
||||
// Rate limit: max events per second
|
||||
MaxEventsPerSecond = 10000
|
||||
)
|
||||
|
||||
// Config holds the Unix socket source configuration.
|
||||
type Config struct {
|
||||
Name string
|
||||
Path string
|
||||
SourceType string // "A" for Apache/HTTP, "B" for Network, "" for auto-detect
|
||||
SocketPermissions os.FileMode
|
||||
}
|
||||
|
||||
// UnixSocketSource reads JSON events from a Unix datagram socket.
|
||||
type UnixSocketSource struct {
|
||||
config Config
|
||||
mu sync.Mutex
|
||||
conn *net.UnixConn
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
stopOnce sync.Once
|
||||
logger *observability.Logger
|
||||
}
|
||||
|
||||
// NewUnixSocketSource creates a new Unix socket source.
|
||||
func NewUnixSocketSource(config Config) *UnixSocketSource {
|
||||
return &UnixSocketSource{
|
||||
config: config,
|
||||
done: make(chan struct{}),
|
||||
logger: observability.NewLogger("unixsocket:" + config.Name),
|
||||
}
|
||||
}
|
||||
|
||||
// SetLogger sets the logger for the source (for debug mode).
|
||||
func (s *UnixSocketSource) SetLogger(logger *observability.Logger) {
|
||||
s.logger = logger.WithFields(map[string]any{"source": s.config.Name})
|
||||
}
|
||||
|
||||
// Name returns the source name.
|
||||
func (s *UnixSocketSource) Name() string {
|
||||
return s.config.Name
|
||||
}
|
||||
|
||||
// Start begins listening on the Unix datagram socket.
|
||||
func (s *UnixSocketSource) Start(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) error {
|
||||
if strings.TrimSpace(s.config.Path) == "" {
|
||||
return fmt.Errorf("socket path cannot be empty")
|
||||
}
|
||||
|
||||
// Create parent directory if it doesn't exist
|
||||
socketDir := filepath.Dir(s.config.Path)
|
||||
if err := os.MkdirAll(socketDir, 0755); err != nil {
|
||||
return fmt.Errorf("failed to create socket directory %s: %w", socketDir, err)
|
||||
}
|
||||
|
||||
// Remove existing socket file if present
|
||||
if info, err := os.Stat(s.config.Path); err == nil {
|
||||
if info.Mode()&os.ModeSocket != 0 {
|
||||
if err := os.Remove(s.config.Path); err != nil {
|
||||
return fmt.Errorf("failed to remove existing socket: %w", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("path exists but is not a socket: %s", s.config.Path)
|
||||
}
|
||||
}
|
||||
|
||||
// Create Unix datagram socket
|
||||
addr, err := net.ResolveUnixAddr("unixgram", s.config.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve unix socket address: %w", err)
|
||||
}
|
||||
|
||||
conn, err := net.ListenUnixgram("unixgram", addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create unix datagram socket: %w", err)
|
||||
}
|
||||
s.conn = conn
|
||||
|
||||
// Set permissions - fail if we can't
|
||||
permissions := s.config.SocketPermissions
|
||||
if permissions == 0 {
|
||||
permissions = 0666 // default
|
||||
}
|
||||
if err := os.Chmod(s.config.Path, permissions); err != nil {
|
||||
_ = conn.Close()
|
||||
_ = os.Remove(s.config.Path)
|
||||
return fmt.Errorf("failed to set socket permissions: %w", err)
|
||||
}
|
||||
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
s.readDatagrams(ctx, eventChan)
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// readDatagrams lit en continu les datagrammes sur la socket Unix et envoie les événements normalisés sur le canal.
|
||||
func (s *UnixSocketSource) readDatagrams(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) {
|
||||
buf := make([]byte, MaxDatagramSize)
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.done:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
// Set read deadline to allow periodic context checks
|
||||
_ = s.conn.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
|
||||
|
||||
n, _, err := s.conn.ReadFromUnix(buf)
|
||||
if err != nil {
|
||||
if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
|
||||
// Read timeout, continue to check context
|
||||
continue
|
||||
}
|
||||
// Other errors (e.g., closed socket)
|
||||
select {
|
||||
case <-s.done:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
s.logger.Warnf("read error: %v", err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
data := make([]byte, n)
|
||||
copy(data, buf[:n])
|
||||
|
||||
event, err := parseJSONEvent(data, s.config.SourceType)
|
||||
if err != nil {
|
||||
// Log parse errors with the raw data for debugging
|
||||
s.logger.Warnf("parse error: %v | raw: %s", err, string(data))
|
||||
continue
|
||||
}
|
||||
|
||||
// Debug: log raw events with all key details
|
||||
s.logger.Debugf("event received: source=%s src_ip=%s src_port=%d timestamp=%v raw_timestamp=%v",
|
||||
event.Source, event.SrcIP, event.SrcPort, event.Timestamp, event.Raw["timestamp"])
|
||||
|
||||
select {
|
||||
case eventChan <- event:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// resolveSource détermine la source d'un événement à partir du type déclaré ou de la présence d'en-têtes HTTP.
|
||||
func resolveSource(sourceType string, headers map[string]string) domain.EventSource {
|
||||
switch strings.ToLower(strings.TrimSpace(sourceType)) {
|
||||
case "a", "apache", "http":
|
||||
return domain.SourceA
|
||||
case "b", "network", "net":
|
||||
return domain.SourceB
|
||||
default:
|
||||
// fallback compat
|
||||
if len(headers) > 0 {
|
||||
return domain.SourceA
|
||||
}
|
||||
return domain.SourceB
|
||||
}
|
||||
}
|
||||
|
||||
// parseJSONEvent désérialise un datagramme JSON et construit un NormalizedEvent validé avec ses champs obligatoires.
|
||||
func parseJSONEvent(data []byte, sourceType string) (*domain.NormalizedEvent, error) {
|
||||
var raw map[string]any
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON: %w", err)
|
||||
}
|
||||
|
||||
event := &domain.NormalizedEvent{
|
||||
Raw: raw,
|
||||
Extra: make(map[string]any),
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// Extract headers (header_* fields) first
|
||||
for k, v := range raw {
|
||||
if strings.HasPrefix(k, "header_") {
|
||||
if sv, ok := v.(string); ok {
|
||||
event.Headers[k[7:]] = sv
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve source first (strict timestamp logic depends on source)
|
||||
event.Source = resolveSource(sourceType, event.Headers)
|
||||
|
||||
// Extract and validate src_ip
|
||||
if v, ok := getString(raw, "src_ip"); ok {
|
||||
v = strings.TrimSpace(v)
|
||||
if v == "" {
|
||||
return nil, fmt.Errorf("src_ip cannot be empty")
|
||||
}
|
||||
event.SrcIP = v
|
||||
} else {
|
||||
return nil, fmt.Errorf("missing required field: src_ip")
|
||||
}
|
||||
|
||||
// Extract and validate src_port
|
||||
if v, ok := getInt(raw, "src_port"); ok {
|
||||
if v < 1 || v > 65535 {
|
||||
return nil, fmt.Errorf("src_port must be between 1 and 65535, got %d", v)
|
||||
}
|
||||
event.SrcPort = v
|
||||
} else {
|
||||
return nil, fmt.Errorf("missing required field: src_port")
|
||||
}
|
||||
|
||||
// Extract dst_ip (optional)
|
||||
if v, ok := getString(raw, "dst_ip"); ok {
|
||||
event.DstIP = strings.TrimSpace(v)
|
||||
}
|
||||
|
||||
// Extract dst_port (optional)
|
||||
if v, ok := getInt(raw, "dst_port"); ok {
|
||||
if v < 0 || v > 65535 {
|
||||
return nil, fmt.Errorf("dst_port must be between 0 and 65535, got %d", v)
|
||||
}
|
||||
event.DstPort = v
|
||||
}
|
||||
|
||||
// Extract timestamp based on source contract
|
||||
switch event.Source {
|
||||
case domain.SourceA:
|
||||
ts, ok := getInt64(raw, "timestamp_ns")
|
||||
if !ok {
|
||||
// Fallback to legacy "timestamp" field name
|
||||
ts, ok = getInt64(raw, "timestamp")
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("missing required numeric field: timestamp/timestamp_ns for source A")
|
||||
}
|
||||
// Assume nanoseconds
|
||||
event.Timestamp = time.Unix(0, ts)
|
||||
case domain.SourceB:
|
||||
// For network source, try to use event timestamp if available,
|
||||
// fallback to reception time. This improves correlation accuracy
|
||||
// when network logs include their own timestamp (e.g., from packet capture).
|
||||
if ts, ok := getInt64(raw, "timestamp"); ok {
|
||||
event.Timestamp = time.Unix(0, ts)
|
||||
} else if timeStr, ok := getString(raw, "time"); ok {
|
||||
// Try RFC3339 format
|
||||
if t, err := time.Parse(time.RFC3339, timeStr); err == nil {
|
||||
event.Timestamp = t
|
||||
} else if t, err := time.Parse(time.RFC3339Nano, timeStr); err == nil {
|
||||
event.Timestamp = t
|
||||
} else {
|
||||
event.Timestamp = time.Now()
|
||||
}
|
||||
} else {
|
||||
event.Timestamp = time.Now()
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported source type: %s", event.Source)
|
||||
}
|
||||
|
||||
// Extra fields
|
||||
knownFields := map[string]bool{
|
||||
"src_ip": true, "src_port": true, "dst_ip": true, "dst_port": true,
|
||||
"timestamp": true, "time": true,
|
||||
}
|
||||
for k, v := range raw {
|
||||
if knownFields[k] {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, "header_") {
|
||||
continue
|
||||
}
|
||||
event.Extra[k] = v
|
||||
}
|
||||
|
||||
return event, nil
|
||||
}
|
||||
|
||||
// getString extrait la valeur d'une clé sous forme de chaîne depuis une map JSON désérialisée.
|
||||
func getString(m map[string]any, key string) (string, bool) {
|
||||
if v, ok := m[key]; ok {
|
||||
if s, ok := v.(string); ok {
|
||||
return s, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// getInt extrait la valeur d'une clé sous forme d'entier depuis une map JSON en gérant les conversions de types courants.
|
||||
func getInt(m map[string]any, key string) (int, bool) {
|
||||
if v, ok := m[key]; ok {
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
if math.Trunc(val) != val {
|
||||
return 0, false
|
||||
}
|
||||
return int(val), true
|
||||
case int:
|
||||
return val, true
|
||||
case int64:
|
||||
return int(val), true
|
||||
case string:
|
||||
if i, err := strconv.Atoi(val); err == nil {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// getInt64 extrait la valeur d'une clé sous forme d'entier 64 bits depuis une map JSON en gérant les conversions de types courants.
|
||||
func getInt64(m map[string]any, key string) (int64, bool) {
|
||||
if v, ok := m[key]; ok {
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
if math.Trunc(val) != val {
|
||||
return 0, false
|
||||
}
|
||||
return int64(val), true
|
||||
case int:
|
||||
return int64(val), true
|
||||
case int64:
|
||||
return val, true
|
||||
case string:
|
||||
if i, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Stop gracefully stops the source.
|
||||
func (s *UnixSocketSource) Stop() error {
|
||||
var stopErr error
|
||||
|
||||
s.stopOnce.Do(func() {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
close(s.done)
|
||||
|
||||
if s.conn != nil {
|
||||
_ = s.conn.Close()
|
||||
}
|
||||
|
||||
s.wg.Wait()
|
||||
|
||||
// Clean up socket file
|
||||
if err := os.Remove(s.config.Path); err != nil && !os.IsNotExist(err) {
|
||||
stopErr = fmt.Errorf("failed to remove socket file: %w", err)
|
||||
return
|
||||
}
|
||||
})
|
||||
|
||||
return stopErr
|
||||
}
|
||||
@ -1,596 +0,0 @@
|
||||
package unixsocket
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
)
|
||||
|
||||
func TestParseJSONEvent_Apache(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"dst_ip": "10.0.0.1",
|
||||
"dst_port": 80,
|
||||
"timestamp": 1704110400000000000,
|
||||
"method": "GET",
|
||||
"path": "/api/test",
|
||||
"header_host": "example.com",
|
||||
"header_user_agent": "Mozilla/5.0"
|
||||
}`)
|
||||
|
||||
event, err := parseJSONEvent(data, "A")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if event.SrcIP != "192.168.1.1" {
|
||||
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
|
||||
}
|
||||
if event.SrcPort != 8080 {
|
||||
t.Errorf("expected src_port 8080, got %d", event.SrcPort)
|
||||
}
|
||||
if event.Headers["host"] != "example.com" {
|
||||
t.Errorf("expected header host example.com, got %s", event.Headers["host"])
|
||||
}
|
||||
if event.Headers["user_agent"] != "Mozilla/5.0" {
|
||||
t.Errorf("expected header_user_agent Mozilla/5.0, got %s", event.Headers["user_agent"])
|
||||
}
|
||||
if event.Source != domain.SourceA {
|
||||
t.Errorf("expected source A, got %s", event.Source)
|
||||
}
|
||||
expectedTs := time.Unix(0, 1704110400000000000)
|
||||
if !event.Timestamp.Equal(expectedTs) {
|
||||
t.Errorf("expected timestamp %v, got %v", expectedTs, event.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_Network(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"dst_ip": "10.0.0.1",
|
||||
"dst_port": 443,
|
||||
"timestamp": 1704110400000000000,
|
||||
"ja3": "abc123def456",
|
||||
"ja4": "xyz789",
|
||||
"tcp_meta_flags": "SYN"
|
||||
}`)
|
||||
|
||||
event, err := parseJSONEvent(data, "B")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if event.SrcIP != "192.168.1.1" {
|
||||
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
|
||||
}
|
||||
if event.Extra["ja3"] != "abc123def456" {
|
||||
t.Errorf("expected ja3 abc123def456, got %v", event.Extra["ja3"])
|
||||
}
|
||||
if event.Source != domain.SourceB {
|
||||
t.Errorf("expected source B, got %s", event.Source)
|
||||
}
|
||||
// Network source now uses payload timestamp if available
|
||||
expectedTs := time.Unix(0, 1704110400000000000)
|
||||
if !event.Timestamp.Equal(expectedTs) {
|
||||
t.Errorf("expected network timestamp %v, got %v", expectedTs, event.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_InvalidJSON(t *testing.T) {
|
||||
data := []byte(`{invalid json}`)
|
||||
|
||||
_, err := parseJSONEvent(data, "")
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid JSON")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_MissingFields(t *testing.T) {
|
||||
data := []byte(`{"other_field": "value"}`)
|
||||
|
||||
_, err := parseJSONEvent(data, "")
|
||||
if err == nil {
|
||||
t.Error("expected error for missing src_ip/src_port")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_SourceARequiresNumericTimestamp(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"time": "2024-01-01T12:00:00Z"
|
||||
}`)
|
||||
|
||||
_, err := parseJSONEvent(data, "A")
|
||||
if err == nil {
|
||||
t.Fatal("expected error for source A without numeric timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_SourceBUsesPayloadTimestamp(t *testing.T) {
|
||||
expectedTs := int64(1704110400000000000)
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"timestamp": 1704110400000000000
|
||||
}`)
|
||||
|
||||
event, err := parseJSONEvent(data, "B")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
expectedTime := time.Unix(0, expectedTs)
|
||||
if !event.Timestamp.Equal(expectedTime) {
|
||||
t.Errorf("expected source B to use payload timestamp %v, got %v", expectedTime, event.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_SourceBUsesTimeField(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"time": "2024-01-01T12:00:00Z"
|
||||
}`)
|
||||
|
||||
event, err := parseJSONEvent(data, "B")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
expectedTime := time.Unix(0, 1704110400000000000)
|
||||
if !event.Timestamp.Equal(expectedTime) {
|
||||
t.Errorf("expected source B to use time field %v, got %v", expectedTime, event.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_SourceBFallbackToNow(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080
|
||||
}`)
|
||||
|
||||
before := time.Now()
|
||||
event, err := parseJSONEvent(data, "B")
|
||||
after := time.Now()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if event.Timestamp.Before(before.Add(-2*time.Second)) || event.Timestamp.After(after.Add(2*time.Second)) {
|
||||
t.Errorf("expected source B timestamp near now, got %v", event.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_ExplicitSourceType(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
sourceType string
|
||||
expected domain.EventSource
|
||||
}{
|
||||
{
|
||||
name: "explicit A",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000}`,
|
||||
sourceType: "A",
|
||||
expected: domain.SourceA,
|
||||
},
|
||||
{
|
||||
name: "explicit B",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
|
||||
sourceType: "B",
|
||||
expected: domain.SourceB,
|
||||
},
|
||||
{
|
||||
name: "explicit apache",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000}`,
|
||||
sourceType: "apache",
|
||||
expected: domain.SourceA,
|
||||
},
|
||||
{
|
||||
name: "explicit network",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
|
||||
sourceType: "network",
|
||||
expected: domain.SourceB,
|
||||
},
|
||||
{
|
||||
name: "auto-detect A with headers",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000, "header_host": "example.com"}`,
|
||||
sourceType: "",
|
||||
expected: domain.SourceA,
|
||||
},
|
||||
{
|
||||
name: "auto-detect B without headers",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "ja3": "abc"}`,
|
||||
sourceType: "",
|
||||
expected: domain.SourceB,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
event, err := parseJSONEvent([]byte(tt.data), tt.sourceType)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if event.Source != tt.expected {
|
||||
t.Errorf("expected source %s, got %s", tt.expected, event.Source)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixSocketSource_Name(t *testing.T) {
|
||||
source := NewUnixSocketSource(Config{
|
||||
Name: "test_source",
|
||||
Path: "/tmp/test.sock",
|
||||
})
|
||||
|
||||
if source.Name() != "test_source" {
|
||||
t.Errorf("expected name 'test_source', got %s", source.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixSocketSource_StopWithoutStart(t *testing.T) {
|
||||
source := NewUnixSocketSource(Config{
|
||||
Name: "test_source",
|
||||
Path: "/tmp/test.sock",
|
||||
})
|
||||
|
||||
// Should not panic
|
||||
err := source.Stop()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error on stop without start, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixSocketSource_EmptyPath(t *testing.T) {
|
||||
source := NewUnixSocketSource(Config{
|
||||
Name: "test_source",
|
||||
Path: "",
|
||||
})
|
||||
|
||||
ctx := context.Background()
|
||||
eventChan := make(chan *domain.NormalizedEvent, 10)
|
||||
|
||||
err := source.Start(ctx, eventChan)
|
||||
if err == nil {
|
||||
t.Error("expected error for empty path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetString(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"string": "hello",
|
||||
"int": 42,
|
||||
"nil": nil,
|
||||
}
|
||||
|
||||
v, ok := getString(m, "string")
|
||||
if !ok || v != "hello" {
|
||||
t.Errorf("expected 'hello', got %v, %v", v, ok)
|
||||
}
|
||||
|
||||
_, ok = getString(m, "int")
|
||||
if ok {
|
||||
t.Error("expected false for int")
|
||||
}
|
||||
|
||||
_, ok = getString(m, "missing")
|
||||
if ok {
|
||||
t.Error("expected false for missing key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetInt(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"float": 42.5,
|
||||
"int": 42,
|
||||
"int64": int64(42),
|
||||
"string": "42",
|
||||
"bad": "not a number",
|
||||
"nil": nil,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
key string
|
||||
expected int
|
||||
ok bool
|
||||
}{
|
||||
{"float", 0, false},
|
||||
{"int", 42, true},
|
||||
{"int64", 42, true},
|
||||
{"string", 42, true},
|
||||
{"bad", 0, false},
|
||||
{"nil", 0, false},
|
||||
{"missing", 0, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.key, func(t *testing.T) {
|
||||
v, ok := getInt(m, tt.key)
|
||||
if ok != tt.ok {
|
||||
t.Errorf("getInt(%q) ok = %v, want %v", tt.key, ok, tt.ok)
|
||||
}
|
||||
if v != tt.expected {
|
||||
t.Errorf("getInt(%q) = %v, want %v", tt.key, v, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetInt64(t *testing.T) {
|
||||
m := map[string]any{
|
||||
"float": 42.5,
|
||||
"int": 42,
|
||||
"int64": int64(42),
|
||||
"string": "42",
|
||||
"bad": "not a number",
|
||||
"nil": nil,
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
key string
|
||||
expected int64
|
||||
ok bool
|
||||
}{
|
||||
{"float", 0, false},
|
||||
{"int", 42, true},
|
||||
{"int64", 42, true},
|
||||
{"string", 42, true},
|
||||
{"bad", 0, false},
|
||||
{"nil", 0, false},
|
||||
{"missing", 0, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.key, func(t *testing.T) {
|
||||
v, ok := getInt64(m, tt.key)
|
||||
if ok != tt.ok {
|
||||
t.Errorf("getInt64(%q) ok = %v, want %v", tt.key, ok, tt.ok)
|
||||
}
|
||||
if v != tt.expected {
|
||||
t.Errorf("getInt64(%q) = %v, want %v", tt.key, v, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_PortValidation(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
data string
|
||||
sourceType string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "valid src_port",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
|
||||
sourceType: "B",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "src_port zero",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 0}`,
|
||||
sourceType: "B",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "src_port negative",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": -1}`,
|
||||
sourceType: "B",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "src_port too high",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 70000}`,
|
||||
sourceType: "B",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "valid dst_port zero",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 0}`,
|
||||
sourceType: "B",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "dst_port too high",
|
||||
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 70000}`,
|
||||
sourceType: "B",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
_, err := parseJSONEvent([]byte(tt.data), tt.sourceType)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("parseJSONEvent() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_TimestampFallback(t *testing.T) {
|
||||
data := []byte(`{"src_ip": "192.168.1.1", "src_port": 8080}`)
|
||||
event, err := parseJSONEvent(data, "B")
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
// For source B, timestamp is reception time
|
||||
if event.Timestamp.IsZero() {
|
||||
t.Error("expected non-zero timestamp")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixSocketSource_StartStopDatagram(t *testing.T) {
|
||||
tmpPath := "/tmp/test_logcorrelator_datagram.sock"
|
||||
// Clean up any existing socket
|
||||
os.Remove(tmpPath)
|
||||
|
||||
source := NewUnixSocketSource(Config{
|
||||
Name: "test_datagram",
|
||||
Path: tmpPath,
|
||||
SourceType: "B",
|
||||
SocketPermissions: 0666,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||
defer cancel()
|
||||
|
||||
eventChan := make(chan *domain.NormalizedEvent, 10)
|
||||
|
||||
err := source.Start(ctx, eventChan)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start source: %v", err)
|
||||
}
|
||||
|
||||
// Give socket time to start
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Verify socket file exists
|
||||
if _, err := os.Stat(tmpPath); os.IsNotExist(err) {
|
||||
t.Error("socket file should exist")
|
||||
}
|
||||
|
||||
// Stop the source
|
||||
err = source.Stop()
|
||||
if err != nil {
|
||||
t.Errorf("failed to stop source: %v", err)
|
||||
}
|
||||
|
||||
// Socket file should be cleaned up
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
if _, err := os.Stat(tmpPath); !os.IsNotExist(err) {
|
||||
t.Error("socket file should be removed after stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixSocketSource_SendDatagram(t *testing.T) {
|
||||
tmpPath := "/tmp/test_logcorrelator_send.sock"
|
||||
os.Remove(tmpPath)
|
||||
|
||||
source := NewUnixSocketSource(Config{
|
||||
Name: "test_send",
|
||||
Path: tmpPath,
|
||||
SourceType: "B",
|
||||
SocketPermissions: 0666,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
eventChan := make(chan *domain.NormalizedEvent, 10)
|
||||
|
||||
err := source.Start(ctx, eventChan)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start source: %v", err)
|
||||
}
|
||||
|
||||
// Give socket time to start
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Connect and send a datagram
|
||||
conn, err := net.Dial("unixgram", tmpPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to dial socket: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
data := []byte(`{"src_ip": "192.168.1.1", "src_port": 8080, "ja3": "test"}`)
|
||||
_, err = conn.Write(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write: %v", err)
|
||||
}
|
||||
|
||||
// Wait for event
|
||||
select {
|
||||
case event := <-eventChan:
|
||||
if event.SrcIP != "192.168.1.1" {
|
||||
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
|
||||
}
|
||||
if event.SrcPort != 8080 {
|
||||
t.Errorf("expected src_port 8080, got %d", event.SrcPort)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Error("timeout waiting for event")
|
||||
case <-ctx.Done():
|
||||
t.Error("context cancelled")
|
||||
}
|
||||
|
||||
err = source.Stop()
|
||||
if err != nil {
|
||||
t.Errorf("failed to stop source: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixSocketSource_MultipleDatagrams(t *testing.T) {
|
||||
tmpPath := "/tmp/test_logcorrelator_multi.sock"
|
||||
os.Remove(tmpPath)
|
||||
|
||||
source := NewUnixSocketSource(Config{
|
||||
Name: "test_multi",
|
||||
Path: tmpPath,
|
||||
SourceType: "B",
|
||||
SocketPermissions: 0666,
|
||||
})
|
||||
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
defer cancel()
|
||||
|
||||
eventChan := make(chan *domain.NormalizedEvent, 100)
|
||||
|
||||
err := source.Start(ctx, eventChan)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to start source: %v", err)
|
||||
}
|
||||
|
||||
// Give socket time to start
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
// Connect and send multiple datagrams
|
||||
conn, err := net.Dial("unixgram", tmpPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to dial socket: %v", err)
|
||||
}
|
||||
defer conn.Close()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
data := []byte(fmt.Sprintf(`{"src_ip": "192.168.1.%d", "src_port": %d, "ja3": "test%d"}`, i+1, 8080+i, i))
|
||||
_, err = conn.Write(data)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write datagram %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for all events
|
||||
received := 0
|
||||
timeout := time.After(3 * time.Second)
|
||||
for received < 5 {
|
||||
select {
|
||||
case event := <-eventChan:
|
||||
received++
|
||||
t.Logf("received event %d: src_ip=%s", received, event.SrcIP)
|
||||
case <-timeout:
|
||||
t.Errorf("timeout waiting for events, received %d/5", received)
|
||||
goto done
|
||||
case <-ctx.Done():
|
||||
t.Error("context cancelled")
|
||||
goto done
|
||||
}
|
||||
}
|
||||
|
||||
done:
|
||||
err = source.Stop()
|
||||
if err != nil {
|
||||
t.Errorf("failed to stop source: %v", err)
|
||||
}
|
||||
}
|
||||
@ -1,391 +0,0 @@
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
"github.com/antitbone/ja4/correlator/internal/observability"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultBatchSize is the default number of records per batch
|
||||
DefaultBatchSize = 500
|
||||
// DefaultFlushIntervalMs is the default flush interval in milliseconds
|
||||
DefaultFlushIntervalMs = 200
|
||||
// DefaultMaxBufferSize is the default maximum buffer size
|
||||
DefaultMaxBufferSize = 5000
|
||||
// DefaultTimeoutMs is the default timeout for operations in milliseconds
|
||||
DefaultTimeoutMs = 1000
|
||||
// DefaultPingTimeoutMs is the timeout for initial connection ping
|
||||
DefaultPingTimeoutMs = 5000
|
||||
// MaxRetries is the maximum number of retry attempts for failed inserts
|
||||
MaxRetries = 3
|
||||
// RetryBaseDelay is the base delay between retries
|
||||
RetryBaseDelay = 100 * time.Millisecond
|
||||
)
|
||||
|
||||
// Config holds the ClickHouse sink configuration.
|
||||
type Config struct {
|
||||
DSN string
|
||||
Table string
|
||||
BatchSize int
|
||||
FlushIntervalMs int
|
||||
MaxBufferSize int
|
||||
DropOnOverflow bool
|
||||
AsyncInsert bool
|
||||
TimeoutMs int
|
||||
}
|
||||
|
||||
// ClickHouseSink writes correlated logs to ClickHouse.
|
||||
type ClickHouseSink struct {
|
||||
config Config
|
||||
conn clickhouse.Conn
|
||||
mu sync.Mutex
|
||||
buffer []domain.CorrelatedLog
|
||||
flushChan chan struct{}
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
closeOnce sync.Once
|
||||
logger *observability.Logger
|
||||
}
|
||||
|
||||
// SetLogger sets the logger used by the sink.
|
||||
func (s *ClickHouseSink) SetLogger(logger *observability.Logger) {
|
||||
s.logger = logger.WithFields(map[string]any{"sink": "clickhouse"})
|
||||
}
|
||||
|
||||
// NewClickHouseSink creates a new ClickHouse sink.
|
||||
func NewClickHouseSink(config Config) (*ClickHouseSink, error) {
|
||||
if strings.TrimSpace(config.DSN) == "" {
|
||||
return nil, fmt.Errorf("clickhouse DSN is required")
|
||||
}
|
||||
if strings.TrimSpace(config.Table) == "" {
|
||||
return nil, fmt.Errorf("clickhouse table is required")
|
||||
}
|
||||
|
||||
// Apply defaults
|
||||
if config.BatchSize <= 0 {
|
||||
config.BatchSize = DefaultBatchSize
|
||||
}
|
||||
if config.FlushIntervalMs <= 0 {
|
||||
config.FlushIntervalMs = DefaultFlushIntervalMs
|
||||
}
|
||||
if config.MaxBufferSize <= 0 {
|
||||
config.MaxBufferSize = DefaultMaxBufferSize
|
||||
}
|
||||
if config.TimeoutMs <= 0 {
|
||||
config.TimeoutMs = DefaultTimeoutMs
|
||||
}
|
||||
|
||||
s := &ClickHouseSink{
|
||||
config: config,
|
||||
buffer: make([]domain.CorrelatedLog, 0, config.BatchSize),
|
||||
flushChan: make(chan struct{}, 1),
|
||||
done: make(chan struct{}),
|
||||
logger: observability.NewLogger("clickhouse"),
|
||||
}
|
||||
|
||||
// Parse DSN and create options
|
||||
options, err := clickhouse.ParseDSN(config.DSN)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to parse ClickHouse DSN: %w", err)
|
||||
}
|
||||
|
||||
// Connect to ClickHouse using native API
|
||||
conn, err := clickhouse.Open(options)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to connect to ClickHouse: %w", err)
|
||||
}
|
||||
|
||||
// Ping with timeout to verify connection
|
||||
pingCtx, pingCancel := context.WithTimeout(context.Background(), time.Duration(DefaultPingTimeoutMs)*time.Millisecond)
|
||||
defer pingCancel()
|
||||
|
||||
if err := conn.Ping(pingCtx); err != nil {
|
||||
_ = conn.Close()
|
||||
return nil, fmt.Errorf("failed to ping ClickHouse: %w", err)
|
||||
}
|
||||
|
||||
s.conn = conn
|
||||
s.log().Infof("connected to ClickHouse: table=%s batch_size=%d flush_interval_ms=%d",
|
||||
config.Table, config.BatchSize, config.FlushIntervalMs)
|
||||
|
||||
// Start flush goroutine
|
||||
s.wg.Add(1)
|
||||
go s.flushLoop()
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Name returns the sink name.
|
||||
func (s *ClickHouseSink) Name() string {
|
||||
return "clickhouse"
|
||||
}
|
||||
|
||||
// log returns the logger, initializing a default one if not set (e.g. in tests).
|
||||
func (s *ClickHouseSink) log() *observability.Logger {
|
||||
if s.logger == nil {
|
||||
s.logger = observability.NewLogger("clickhouse")
|
||||
}
|
||||
return s.logger
|
||||
}
|
||||
|
||||
// Reopen is a no-op for ClickHouse (connection is managed internally).
|
||||
func (s *ClickHouseSink) Reopen() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write adds a log to the buffer.
|
||||
func (s *ClickHouseSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
|
||||
deadline := time.Now().Add(time.Duration(s.config.TimeoutMs) * time.Millisecond)
|
||||
|
||||
for {
|
||||
s.mu.Lock()
|
||||
if len(s.buffer) < s.config.MaxBufferSize {
|
||||
s.buffer = append(s.buffer, log)
|
||||
if len(s.buffer) >= s.config.BatchSize {
|
||||
select {
|
||||
case s.flushChan <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
drop := s.config.DropOnOverflow
|
||||
s.mu.Unlock()
|
||||
|
||||
if drop {
|
||||
s.log().Warnf("buffer full, dropping log: table=%s buffer_size=%d", s.config.Table, s.config.MaxBufferSize)
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("buffer full, timeout exceeded")
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(10 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Flush flushes the buffer to ClickHouse.
|
||||
func (s *ClickHouseSink) Flush(ctx context.Context) error {
|
||||
return s.doFlush(ctx)
|
||||
}
|
||||
|
||||
// Close closes the sink.
|
||||
func (s *ClickHouseSink) Close() error {
|
||||
var closeErr error
|
||||
|
||||
s.closeOnce.Do(func() {
|
||||
if s.done != nil {
|
||||
close(s.done)
|
||||
}
|
||||
s.wg.Wait()
|
||||
|
||||
flushCtx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
|
||||
defer cancel()
|
||||
if err := s.doFlush(flushCtx); err != nil {
|
||||
closeErr = err
|
||||
}
|
||||
|
||||
if s.conn != nil {
|
||||
if err := s.conn.Close(); err != nil && closeErr == nil {
|
||||
closeErr = err
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
return closeErr
|
||||
}
|
||||
|
||||
func (s *ClickHouseSink) flushLoop() {
|
||||
defer s.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(time.Duration(s.config.FlushIntervalMs) * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-s.done:
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
|
||||
if err := s.doFlush(ctx); err != nil {
|
||||
s.log().Error("final flush on close failed", err)
|
||||
}
|
||||
cancel()
|
||||
return
|
||||
|
||||
case <-ticker.C:
|
||||
s.mu.Lock()
|
||||
needsFlush := len(s.buffer) > 0
|
||||
s.mu.Unlock()
|
||||
|
||||
if needsFlush {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
|
||||
if err := s.doFlush(ctx); err != nil {
|
||||
s.log().Error("periodic flush failed", err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
|
||||
case <-s.flushChan:
|
||||
s.mu.Lock()
|
||||
needsFlush := len(s.buffer) >= s.config.BatchSize
|
||||
s.mu.Unlock()
|
||||
|
||||
if needsFlush {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Duration(s.config.TimeoutMs)*time.Millisecond)
|
||||
if err := s.doFlush(ctx); err != nil {
|
||||
s.log().Error("batch flush failed", err)
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *ClickHouseSink) doFlush(ctx context.Context) error {
|
||||
s.mu.Lock()
|
||||
if len(s.buffer) == 0 {
|
||||
s.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// Copy buffer to flush
|
||||
buffer := make([]domain.CorrelatedLog, len(s.buffer))
|
||||
copy(buffer, s.buffer)
|
||||
s.buffer = make([]domain.CorrelatedLog, 0, s.config.BatchSize)
|
||||
s.mu.Unlock()
|
||||
|
||||
if s.conn == nil {
|
||||
return fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
batchSize := len(buffer)
|
||||
|
||||
// Retry logic with exponential backoff
|
||||
var lastErr error
|
||||
for attempt := 0; attempt < MaxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
delay := RetryBaseDelay * time.Duration(1<<uint(attempt-1))
|
||||
s.log().Warnf("retrying batch insert: attempt=%d/%d delay=%s rows=%d err=%v",
|
||||
attempt+1, MaxRetries, delay, batchSize, lastErr)
|
||||
select {
|
||||
case <-time.After(delay):
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
}
|
||||
|
||||
lastErr = s.executeBatch(ctx, buffer)
|
||||
if lastErr == nil {
|
||||
s.log().Debugf("batch sent: rows=%d table=%s", batchSize, s.config.Table)
|
||||
return nil
|
||||
}
|
||||
|
||||
if !isRetryableError(lastErr) {
|
||||
return fmt.Errorf("non-retryable error: %w", lastErr)
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("failed after %d retries (batch size: %d): %w", MaxRetries, batchSize, lastErr)
|
||||
}
|
||||
|
||||
func (s *ClickHouseSink) executeBatch(ctx context.Context, buffer []domain.CorrelatedLog) error {
|
||||
if s.conn == nil {
|
||||
return fmt.Errorf("clickhouse connection is not initialized")
|
||||
}
|
||||
|
||||
// Table schema: http_logs_raw (raw_json String)
|
||||
// Single column insert - the entire log is serialized as JSON string
|
||||
query := fmt.Sprintf(`INSERT INTO %s (raw_json)`, s.config.Table)
|
||||
|
||||
// Prepare batch using native clickhouse-go/v2 API
|
||||
batch, err := s.conn.PrepareBatch(ctx, query)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to prepare batch: %w", err)
|
||||
}
|
||||
|
||||
for i, log := range buffer {
|
||||
// Marshal the entire CorrelatedLog to JSON
|
||||
logJSON, marshalErr := json.Marshal(log)
|
||||
if marshalErr != nil {
|
||||
return fmt.Errorf("failed to marshal log %d to JSON: %w", i, marshalErr)
|
||||
}
|
||||
|
||||
// Append the JSON string as the raw_json column value
|
||||
appendErr := batch.Append(string(logJSON))
|
||||
if appendErr != nil {
|
||||
return fmt.Errorf("failed to append log %d to batch: %w", i, appendErr)
|
||||
}
|
||||
}
|
||||
|
||||
// Send the batch - DO NOT FORGET this step
|
||||
sendErr := batch.Send()
|
||||
if sendErr != nil {
|
||||
return fmt.Errorf("failed to send batch (%d rows): %w", len(buffer), sendErr)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// isRetryableError checks if an error is retryable.
|
||||
func isRetryableError(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if errors.Is(err, context.DeadlineExceeded) {
|
||||
return true
|
||||
}
|
||||
|
||||
if errors.Is(err, context.Canceled) {
|
||||
return false
|
||||
}
|
||||
|
||||
var netErr net.Error
|
||||
if errors.As(err, &netErr) {
|
||||
if netErr.Timeout() {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
errStr := strings.ToLower(err.Error())
|
||||
|
||||
// Explicit non-retryable SQL/schema errors
|
||||
if strings.Contains(errStr, "syntax error") ||
|
||||
strings.Contains(errStr, "unknown table") ||
|
||||
strings.Contains(errStr, "unknown column") ||
|
||||
(strings.Contains(errStr, "table") && strings.Contains(errStr, "not found")) {
|
||||
return false
|
||||
}
|
||||
|
||||
// Fallback network/transient errors
|
||||
retryableErrors := []string{
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"timeout",
|
||||
"temporary failure",
|
||||
"network is unreachable",
|
||||
"broken pipe",
|
||||
"no route to host",
|
||||
}
|
||||
for _, re := range retryableErrors {
|
||||
if strings.Contains(errStr, re) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
@ -1,538 +0,0 @@
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
"github.com/antitbone/ja4/correlator/internal/observability"
|
||||
)
|
||||
|
||||
func TestClickHouseSink_Name(t *testing.T) {
|
||||
sink := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
},
|
||||
}
|
||||
|
||||
if sink.Name() != "clickhouse" {
|
||||
t.Errorf("expected name 'clickhouse', got %s", sink.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseSink_ConfigDefaults(t *testing.T) {
|
||||
// Test that defaults are applied correctly
|
||||
config := Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
// Other fields are zero, should get defaults
|
||||
}
|
||||
|
||||
// Verify defaults would be applied (we can't actually connect in tests)
|
||||
if config.BatchSize <= 0 {
|
||||
config.BatchSize = DefaultBatchSize
|
||||
}
|
||||
if config.FlushIntervalMs <= 0 {
|
||||
config.FlushIntervalMs = DefaultFlushIntervalMs
|
||||
}
|
||||
if config.MaxBufferSize <= 0 {
|
||||
config.MaxBufferSize = DefaultMaxBufferSize
|
||||
}
|
||||
if config.TimeoutMs <= 0 {
|
||||
config.TimeoutMs = DefaultTimeoutMs
|
||||
}
|
||||
|
||||
if config.BatchSize != DefaultBatchSize {
|
||||
t.Errorf("expected BatchSize %d, got %d", DefaultBatchSize, config.BatchSize)
|
||||
}
|
||||
if config.FlushIntervalMs != DefaultFlushIntervalMs {
|
||||
t.Errorf("expected FlushIntervalMs %d, got %d", DefaultFlushIntervalMs, config.FlushIntervalMs)
|
||||
}
|
||||
if config.MaxBufferSize != DefaultMaxBufferSize {
|
||||
t.Errorf("expected MaxBufferSize %d, got %d", DefaultMaxBufferSize, config.MaxBufferSize)
|
||||
}
|
||||
if config.TimeoutMs != DefaultTimeoutMs {
|
||||
t.Errorf("expected TimeoutMs %d, got %d", DefaultTimeoutMs, config.TimeoutMs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseSink_Write_BufferOverflow(t *testing.T) {
|
||||
// This test verifies the buffer overflow logic without actually connecting
|
||||
config := Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
BatchSize: 10,
|
||||
MaxBufferSize: 10,
|
||||
DropOnOverflow: true,
|
||||
TimeoutMs: 100,
|
||||
FlushIntervalMs: 1000,
|
||||
}
|
||||
|
||||
// We can't test actual writes without a ClickHouse instance,
|
||||
// but we can verify the config is valid
|
||||
if config.BatchSize > config.MaxBufferSize {
|
||||
t.Error("BatchSize should not exceed MaxBufferSize")
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseSink_IsRetryableError(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
err error
|
||||
expected bool
|
||||
}{
|
||||
{"nil error", nil, false},
|
||||
{"connection refused", &mockError{"connection refused"}, true},
|
||||
{"connection reset", &mockError{"connection reset by peer"}, true},
|
||||
{"timeout", &mockError{"timeout waiting for response"}, true},
|
||||
{"network unreachable", &mockError{"network is unreachable"}, true},
|
||||
{"broken pipe", &mockError{"broken pipe"}, true},
|
||||
{"syntax error", &mockError{"syntax error in SQL"}, false},
|
||||
{"table not found", &mockError{"table test not found"}, false},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := isRetryableError(tt.err)
|
||||
if result != tt.expected {
|
||||
t.Errorf("expected %v, got %v", tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseSink_FlushEmpty(t *testing.T) {
|
||||
// Test that flushing an empty buffer doesn't cause issues
|
||||
// (We can't test actual ClickHouse operations without a real instance)
|
||||
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0),
|
||||
}
|
||||
|
||||
// Should not panic or error on empty flush
|
||||
ctx := context.Background()
|
||||
err := s.Flush(ctx)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error on empty flush, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseSink_CloseWithoutConnect(t *testing.T) {
|
||||
// Test that closing without connecting doesn't panic
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
err := s.Close()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error on close without connect, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClickHouseSink_Constants(t *testing.T) {
|
||||
// Verify constants have reasonable values
|
||||
if DefaultBatchSize <= 0 {
|
||||
t.Error("DefaultBatchSize should be positive")
|
||||
}
|
||||
if DefaultFlushIntervalMs <= 0 {
|
||||
t.Error("DefaultFlushIntervalMs should be positive")
|
||||
}
|
||||
if DefaultMaxBufferSize <= 0 {
|
||||
t.Error("DefaultMaxBufferSize should be positive")
|
||||
}
|
||||
if DefaultTimeoutMs <= 0 {
|
||||
t.Error("DefaultTimeoutMs should be positive")
|
||||
}
|
||||
if DefaultPingTimeoutMs <= 0 {
|
||||
t.Error("DefaultPingTimeoutMs should be positive")
|
||||
}
|
||||
if MaxRetries <= 0 {
|
||||
t.Error("MaxRetries should be positive")
|
||||
}
|
||||
if RetryBaseDelay <= 0 {
|
||||
t.Error("RetryBaseDelay should be positive")
|
||||
}
|
||||
}
|
||||
|
||||
// mockError implements error for testing
|
||||
type mockError struct {
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *mockError) Error() string {
|
||||
return e.msg
|
||||
}
|
||||
|
||||
// Test the doFlush function with empty buffer (no actual DB connection)
|
||||
func TestClickHouseSink_DoFlushEmpty(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0),
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
err := s.doFlush(ctx)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error when flushing empty buffer, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that buffer is properly managed (without actual DB operations)
|
||||
func TestClickHouseSink_BufferManagement(t *testing.T) {
|
||||
log := domain.CorrelatedLog{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
Correlated: 1,
|
||||
}
|
||||
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
MaxBufferSize: 100, // Allow more than 1 element
|
||||
DropOnOverflow: false,
|
||||
TimeoutMs: 1000,
|
||||
},
|
||||
buffer: []domain.CorrelatedLog{log},
|
||||
}
|
||||
|
||||
// Verify buffer has data
|
||||
if len(s.buffer) != 1 {
|
||||
t.Fatalf("expected buffer length 1, got %d", len(s.buffer))
|
||||
}
|
||||
|
||||
// Test that Write properly adds to buffer
|
||||
ctx := context.Background()
|
||||
err := s.Write(ctx, log)
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error on Write: %v", err)
|
||||
}
|
||||
|
||||
if len(s.buffer) != 2 {
|
||||
t.Errorf("expected buffer length 2 after Write, got %d", len(s.buffer))
|
||||
}
|
||||
}
|
||||
|
||||
// Test Write with context cancellation
|
||||
func TestClickHouseSink_Write_ContextCancel(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
MaxBufferSize: 1,
|
||||
DropOnOverflow: false,
|
||||
TimeoutMs: 10,
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0, 1),
|
||||
}
|
||||
|
||||
// Fill the buffer
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
|
||||
s.buffer = append(s.buffer, log)
|
||||
|
||||
// Try to write with cancelled context
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // Cancel immediately
|
||||
|
||||
err := s.Write(ctx, log)
|
||||
if err == nil {
|
||||
t.Error("expected error when writing with cancelled context")
|
||||
}
|
||||
}
|
||||
|
||||
// Test DropOnOverflow behavior
|
||||
func TestClickHouseSink_Write_DropOnOverflow(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
MaxBufferSize: 1,
|
||||
DropOnOverflow: true,
|
||||
TimeoutMs: 10,
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0, 1),
|
||||
}
|
||||
|
||||
// Fill the buffer
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
|
||||
s.buffer = append(s.buffer, log)
|
||||
|
||||
// Try to write when buffer is full - should drop silently
|
||||
ctx := context.Background()
|
||||
err := s.Write(ctx, log)
|
||||
if err != nil {
|
||||
t.Errorf("expected no error when DropOnOverflow is true, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_ContextDeadlineExceeded tests context.DeadlineExceeded is retryable.
|
||||
func TestIsRetryableError_ContextDeadlineExceeded(t *testing.T) {
|
||||
if !isRetryableError(context.DeadlineExceeded) {
|
||||
t.Error("context.DeadlineExceeded should be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_ContextCanceled tests context.Canceled is NOT retryable.
|
||||
func TestIsRetryableError_ContextCanceled(t *testing.T) {
|
||||
if isRetryableError(context.Canceled) {
|
||||
t.Error("context.Canceled should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_NetTimeout tests net.Error with Timeout() = true is retryable.
|
||||
func TestIsRetryableError_NetTimeout(t *testing.T) {
|
||||
err := &mockNetError{timeout: true, temporary: false}
|
||||
if !isRetryableError(err) {
|
||||
t.Error("net.Error with Timeout()=true should be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_NetNoTimeout tests net.Error with Timeout() = false is NOT retryable.
|
||||
func TestIsRetryableError_NetNoTimeout(t *testing.T) {
|
||||
err := &mockNetError{timeout: false, temporary: false}
|
||||
if isRetryableError(err) {
|
||||
t.Error("net.Error with Timeout()=false should not be retryable (unless msg matches)")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_UnknownTable tests "unknown table" is NOT retryable.
|
||||
func TestIsRetryableError_UnknownTable(t *testing.T) {
|
||||
if isRetryableError(&mockError{"unknown table users"}) {
|
||||
t.Error("unknown table error should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_UnknownColumn tests "unknown column" is NOT retryable.
|
||||
func TestIsRetryableError_UnknownColumn(t *testing.T) {
|
||||
if isRetryableError(&mockError{"unknown column foo"}) {
|
||||
t.Error("unknown column error should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_RandomError tests a random error is NOT retryable.
|
||||
func TestIsRetryableError_RandomError(t *testing.T) {
|
||||
if isRetryableError(&mockError{"some random unrecognized error"}) {
|
||||
t.Error("random error should not be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_NoRouteToHost tests "no route to host" is retryable.
|
||||
func TestIsRetryableError_NoRouteToHost(t *testing.T) {
|
||||
if !isRetryableError(&mockError{"no route to host"}) {
|
||||
t.Error("'no route to host' should be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// TestIsRetryableError_TemporaryFailure tests "temporary failure" is retryable.
|
||||
func TestIsRetryableError_TemporaryFailure(t *testing.T) {
|
||||
if !isRetryableError(&mockError{"temporary failure in name resolution"}) {
|
||||
t.Error("'temporary failure' should be retryable")
|
||||
}
|
||||
}
|
||||
|
||||
// mockNetError implements net.Error for testing.
|
||||
type mockNetError struct {
|
||||
timeout bool
|
||||
temporary bool
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *mockNetError) Error() string { return e.msg }
|
||||
func (e *mockNetError) Timeout() bool { return e.timeout }
|
||||
func (e *mockNetError) Temporary() bool { return e.temporary }
|
||||
|
||||
// TestNewClickHouseSink_EmptyDSN tests that empty DSN returns error.
|
||||
func TestNewClickHouseSink_EmptyDSN(t *testing.T) {
|
||||
_, err := NewClickHouseSink(Config{
|
||||
DSN: "",
|
||||
Table: "test_table",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("expected error for empty DSN")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClickHouseSink_WhitespaceDSN tests that whitespace DSN returns error.
|
||||
func TestNewClickHouseSink_WhitespaceDSN(t *testing.T) {
|
||||
_, err := NewClickHouseSink(Config{
|
||||
DSN: " ",
|
||||
Table: "test_table",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("expected error for whitespace-only DSN")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClickHouseSink_EmptyTable tests that empty Table returns error.
|
||||
func TestNewClickHouseSink_EmptyTable(t *testing.T) {
|
||||
_, err := NewClickHouseSink(Config{
|
||||
DSN: "clickhouse://localhost:9000/test",
|
||||
Table: "",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("expected error for empty Table")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClickHouseSink_WhitespaceTable tests that whitespace Table returns error.
|
||||
func TestNewClickHouseSink_WhitespaceTable(t *testing.T) {
|
||||
_, err := NewClickHouseSink(Config{
|
||||
DSN: "clickhouse://localhost:9000/test",
|
||||
Table: " ",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("expected error for whitespace-only Table")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewClickHouseSink_InvalidDSN tests that an invalid DSN (no real connection) returns error.
|
||||
func TestNewClickHouseSink_InvalidDSN(t *testing.T) {
|
||||
_, err := NewClickHouseSink(Config{
|
||||
DSN: "not-a-valid-dsn",
|
||||
Table: "test_table",
|
||||
})
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid DSN")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseSink_SetLogger tests that SetLogger sets a logger.
|
||||
func TestClickHouseSink_SetLogger(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{Table: "test_table"},
|
||||
buffer: make([]domain.CorrelatedLog, 0),
|
||||
}
|
||||
|
||||
testLogger := observability.NewLogger("test")
|
||||
s.SetLogger(testLogger)
|
||||
|
||||
if s.logger == nil {
|
||||
t.Error("expected logger to be set")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseSink_LogNilLogger tests that log() returns a logger even when s.logger is nil.
|
||||
func TestClickHouseSink_LogNilLogger(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{Table: "test_table"},
|
||||
buffer: make([]domain.CorrelatedLog, 0),
|
||||
}
|
||||
s.logger = nil
|
||||
|
||||
// log() should auto-initialize
|
||||
logger := s.log()
|
||||
if logger == nil {
|
||||
t.Error("expected non-nil logger from log()")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseSink_Reopen tests that Reopen is a no-op and returns nil.
|
||||
func TestClickHouseSink_Reopen(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{Table: "test_table"},
|
||||
buffer: make([]domain.CorrelatedLog, 0),
|
||||
}
|
||||
if err := s.Reopen(); err != nil {
|
||||
t.Errorf("Reopen() should return nil, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseSink_DoFlushNilConn tests doFlush returns error when conn is nil and buffer non-empty.
|
||||
func TestClickHouseSink_DoFlushNilConn(t *testing.T) {
|
||||
log := domain.CorrelatedLog{SrcIP: "1.2.3.4", SrcPort: 1234}
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
Table: "test_table",
|
||||
BatchSize: DefaultBatchSize,
|
||||
},
|
||||
buffer: []domain.CorrelatedLog{log},
|
||||
conn: nil,
|
||||
}
|
||||
|
||||
err := s.doFlush(context.Background())
|
||||
if err == nil {
|
||||
t.Error("expected error from doFlush when conn is nil")
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseSink_CloseTwice tests that calling Close() twice does not panic or error.
|
||||
func TestClickHouseSink_CloseTwice(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
Table: "test_table",
|
||||
TimeoutMs: DefaultTimeoutMs,
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0),
|
||||
done: make(chan struct{}),
|
||||
}
|
||||
|
||||
if err := s.Close(); err != nil {
|
||||
t.Errorf("first Close() should not error, got: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Errorf("second Close() should not error (closeOnce), got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestClickHouseSink_WriteTimeout tests that Write returns error when buffer is full and timeout exceeded.
|
||||
func TestClickHouseSink_Write_Timeout(t *testing.T) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
Table: "test_table",
|
||||
MaxBufferSize: 1,
|
||||
DropOnOverflow: false,
|
||||
TimeoutMs: 1, // 1ms timeout
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0, 1),
|
||||
}
|
||||
|
||||
log := domain.CorrelatedLog{SrcIP: "1.2.3.4", SrcPort: 1234}
|
||||
// Fill the buffer
|
||||
s.buffer = append(s.buffer, log)
|
||||
|
||||
ctx := context.Background()
|
||||
err := s.Write(ctx, log)
|
||||
if err == nil {
|
||||
t.Error("expected error when buffer full and timeout exceeded")
|
||||
}
|
||||
}
|
||||
|
||||
// Benchmark Write operation (without actual DB)
|
||||
func BenchmarkClickHouseSink_Write(b *testing.B) {
|
||||
s := &ClickHouseSink{
|
||||
config: Config{
|
||||
DSN: "clickhouse://test:test@localhost:9000/test",
|
||||
Table: "test_table",
|
||||
MaxBufferSize: 10000,
|
||||
DropOnOverflow: true,
|
||||
},
|
||||
buffer: make([]domain.CorrelatedLog, 0, 10000),
|
||||
}
|
||||
|
||||
log := domain.CorrelatedLog{
|
||||
Timestamp: time.Now(),
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
Correlated: 1,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
s.Write(ctx, log)
|
||||
}
|
||||
}
|
||||
@ -1,191 +0,0 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultFilePermissions for output files
|
||||
DefaultFilePermissions os.FileMode = 0644
|
||||
// DefaultDirPermissions for output directories
|
||||
DefaultDirPermissions os.FileMode = 0750
|
||||
)
|
||||
|
||||
// Config holds the file sink configuration.
|
||||
type Config struct {
|
||||
Path string
|
||||
}
|
||||
|
||||
// FileSink writes correlated logs to a file as JSON lines.
|
||||
type FileSink struct {
|
||||
config Config
|
||||
mu sync.Mutex
|
||||
file *os.File
|
||||
}
|
||||
|
||||
// NewFileSink creates a new file sink.
|
||||
func NewFileSink(config Config) (*FileSink, error) {
|
||||
// Validate path
|
||||
if err := validateFilePath(config.Path); err != nil {
|
||||
return nil, fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
s := &FileSink{
|
||||
config: config,
|
||||
}
|
||||
|
||||
// Open file on creation
|
||||
if err := s.openFile(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
// Name returns the sink name.
|
||||
func (s *FileSink) Name() string {
|
||||
return "file"
|
||||
}
|
||||
|
||||
// Reopen closes and reopens the file (for log rotation on SIGHUP).
|
||||
func (s *FileSink) Reopen() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.file != nil {
|
||||
if err := s.file.Close(); err != nil {
|
||||
return fmt.Errorf("failed to close file: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return s.openFile()
|
||||
}
|
||||
|
||||
// Write writes a correlated log to the file.
|
||||
func (s *FileSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.file == nil {
|
||||
if err := s.openFile(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
data, err := json.Marshal(log)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal log: %w", err)
|
||||
}
|
||||
|
||||
line := append(data, '\n')
|
||||
if _, err := s.file.Write(line); err != nil {
|
||||
return fmt.Errorf("failed to write log line: %w", err)
|
||||
}
|
||||
if err := s.file.Sync(); err != nil {
|
||||
return fmt.Errorf("failed to sync log line: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush flushes any buffered data.
|
||||
func (s *FileSink) Flush(ctx context.Context) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.file != nil {
|
||||
return s.file.Sync()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the sink.
|
||||
func (s *FileSink) Close() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
if s.file != nil {
|
||||
err := s.file.Close()
|
||||
s.file = nil
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *FileSink) openFile() error {
|
||||
// Validate path again before opening
|
||||
if err := validateFilePath(s.config.Path); err != nil {
|
||||
return fmt.Errorf("invalid file path: %w", err)
|
||||
}
|
||||
|
||||
// Ensure directory exists
|
||||
dir := filepath.Dir(s.config.Path)
|
||||
if err := os.MkdirAll(dir, DefaultDirPermissions); err != nil {
|
||||
return fmt.Errorf("failed to create directory: %w", err)
|
||||
}
|
||||
|
||||
file, err := os.OpenFile(s.config.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, DefaultFilePermissions)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open file: %w", err)
|
||||
}
|
||||
|
||||
s.file = file
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateFilePath validates that the file path is safe and allowed.
|
||||
func validateFilePath(path string) error {
|
||||
if strings.TrimSpace(path) == "" {
|
||||
return fmt.Errorf("path cannot be empty")
|
||||
}
|
||||
|
||||
cleanPath := filepath.Clean(path)
|
||||
|
||||
// Allow relative paths for testing/dev
|
||||
if !filepath.IsAbs(cleanPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
absPath, err := filepath.Abs(cleanPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to resolve absolute path: %w", err)
|
||||
}
|
||||
|
||||
allowedRoots := []string{
|
||||
"/var/log/logcorrelator",
|
||||
"/var/log",
|
||||
"/tmp",
|
||||
}
|
||||
|
||||
for _, root := range allowedRoots {
|
||||
absRoot, err := filepath.Abs(filepath.Clean(root))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
rel, err := filepath.Rel(absRoot, absPath)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if rel == "." {
|
||||
return nil
|
||||
}
|
||||
if rel == ".." {
|
||||
continue
|
||||
}
|
||||
if !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
return fmt.Errorf("path must be under allowed directories: %v", allowedRoots)
|
||||
}
|
||||
@ -1,524 +0,0 @@
|
||||
package file
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
)
|
||||
|
||||
func TestFileSink_Write(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
log := domain.CorrelatedLog{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
Correlated: 1,
|
||||
}
|
||||
|
||||
if err := sink.Write(context.Background(), log); err != nil {
|
||||
t.Fatalf("failed to write: %v", err)
|
||||
}
|
||||
|
||||
if err := sink.Flush(context.Background()); err != nil {
|
||||
t.Fatalf("failed to flush: %v", err)
|
||||
}
|
||||
|
||||
// Verify file exists and contains data
|
||||
data, err := os.ReadFile(testPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file: %v", err)
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
t.Error("expected non-empty file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_WriteImmediatePersist_NoFlushNeeded(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
log := domain.CorrelatedLog{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
Correlated: 1,
|
||||
}
|
||||
|
||||
if err := sink.Write(context.Background(), log); err != nil {
|
||||
t.Fatalf("failed to write: %v", err)
|
||||
}
|
||||
|
||||
// Must be visible immediately without Flush()
|
||||
data, err := os.ReadFile(testPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file: %v", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
t.Error("expected data to be present immediately after Write without Flush")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_MultipleWrites(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
for i := 0; i < 5; i++ {
|
||||
log := domain.CorrelatedLog{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080 + i,
|
||||
}
|
||||
if err := sink.Write(context.Background(), log); err != nil {
|
||||
t.Fatalf("failed to write: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
sink.Close()
|
||||
|
||||
// Verify file has 5 lines
|
||||
data, err := os.ReadFile(testPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file: %v", err)
|
||||
}
|
||||
|
||||
lines := 0
|
||||
for _, b := range data {
|
||||
if b == '\n' {
|
||||
lines++
|
||||
}
|
||||
}
|
||||
|
||||
if lines != 5 {
|
||||
t.Errorf("expected 5 lines, got %d", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_Name(t *testing.T) {
|
||||
sink, err := NewFileSink(Config{Path: "/tmp/test.log"})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
|
||||
if sink.Name() != "file" {
|
||||
t.Errorf("expected name 'file', got %s", sink.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_ValidateFilePath(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
path string
|
||||
wantErr bool
|
||||
}{
|
||||
{"empty path", "", true},
|
||||
{"valid /var/log/logcorrelator", "/var/log/logcorrelator/test.log", false},
|
||||
{"valid /var/log", "/var/log/test.log", false},
|
||||
{"valid /tmp", "/tmp/test.log", false},
|
||||
{"reject lookalike /var/logevil", "/var/logevil/test.log", true},
|
||||
{"invalid directory", "/etc/logcorrelator/test.log", true},
|
||||
{"relative path", "test.log", false}, // Allowed for testing
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateFilePath(tt.path)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("validateFilePath(%q) error = %v, wantErr %v", tt.path, err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_OpenFile(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "subdir", "test.log")
|
||||
|
||||
sink := &FileSink{
|
||||
config: Config{Path: testPath},
|
||||
}
|
||||
|
||||
err := sink.openFile()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
if sink.file == nil {
|
||||
t.Error("expected file to be opened")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_WriteBeforeOpen(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
// Write should open file automatically
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
|
||||
err = sink.Write(context.Background(), log)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to write: %v", err)
|
||||
}
|
||||
|
||||
// Verify file was created
|
||||
if _, err := os.Stat(testPath); os.IsNotExist(err) {
|
||||
t.Error("expected file to be created")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_FlushBeforeOpen(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
// Flush before any write should not error
|
||||
err = sink.Flush(context.Background())
|
||||
if err != nil {
|
||||
t.Errorf("expected no error on flush before open, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_InvalidPath(t *testing.T) {
|
||||
// Test with invalid path (outside allowed directories)
|
||||
_, err := NewFileSink(Config{Path: "/etc/../passwd"})
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_Reopen(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
|
||||
// Write initial data
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
|
||||
if err := sink.Write(context.Background(), log); err != nil {
|
||||
t.Fatalf("failed to write: %v", err)
|
||||
}
|
||||
|
||||
// Reopen should close and reopen the file
|
||||
err = sink.Reopen()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error on Reopen, got %v", err)
|
||||
}
|
||||
|
||||
// Write after reopen
|
||||
log2 := domain.CorrelatedLog{SrcIP: "192.168.1.2", SrcPort: 8081}
|
||||
if err := sink.Write(context.Background(), log2); err != nil {
|
||||
t.Fatalf("failed to write after reopen: %v", err)
|
||||
}
|
||||
|
||||
sink.Close()
|
||||
|
||||
// Verify both writes are present
|
||||
data, err := os.ReadFile(testPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file: %v", err)
|
||||
}
|
||||
|
||||
lines := 0
|
||||
for _, b := range data {
|
||||
if b == '\n' {
|
||||
lines++
|
||||
}
|
||||
}
|
||||
|
||||
if lines != 2 {
|
||||
t.Errorf("expected 2 lines after reopen, got %d", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_Close(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
|
||||
// Close should succeed
|
||||
err = sink.Close()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error on Close, got %v", err)
|
||||
}
|
||||
|
||||
// Write after close should fail or reopen
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
|
||||
err = sink.Write(context.Background(), log)
|
||||
if err != nil {
|
||||
// Expected - file was closed
|
||||
t.Logf("write after close returned error (expected): %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_EmptyPath(t *testing.T) {
|
||||
_, err := NewFileSink(Config{Path: ""})
|
||||
if err == nil {
|
||||
t.Error("expected error for empty path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_WhitespacePath(t *testing.T) {
|
||||
_, err := NewFileSink(Config{Path: " "})
|
||||
if err == nil {
|
||||
t.Error("expected error for whitespace-only path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_ValidateFilePath_AllowedRoots(t *testing.T) {
|
||||
// Test paths under allowed roots
|
||||
allowedPaths := []string{
|
||||
"/var/log/logcorrelator/correlated.log",
|
||||
"/var/log/test.log",
|
||||
"/tmp/test.log",
|
||||
"/tmp/subdir/test.log",
|
||||
"relative/path/test.log",
|
||||
"./test.log",
|
||||
}
|
||||
|
||||
for _, path := range allowedPaths {
|
||||
err := validateFilePath(path)
|
||||
if err != nil {
|
||||
t.Errorf("validateFilePath(%q) unexpected error: %v", path, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_ValidateFilePath_RejectedPaths(t *testing.T) {
|
||||
// Test paths that should be rejected
|
||||
rejectedPaths := []string{
|
||||
"",
|
||||
" ",
|
||||
"/etc/passwd",
|
||||
"/etc/logcorrelator/test.log",
|
||||
"/root/test.log",
|
||||
"/home/user/test.log",
|
||||
"/var/logevil/test.log",
|
||||
}
|
||||
|
||||
for _, path := range rejectedPaths {
|
||||
err := validateFilePath(path)
|
||||
if err == nil {
|
||||
t.Errorf("validateFilePath(%q) should have been rejected", path)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_ConcurrentWrites(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
done := make(chan bool)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func(n int) {
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080 + n}
|
||||
sink.Write(context.Background(), log)
|
||||
done <- true
|
||||
}(i)
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
|
||||
// Verify all writes completed
|
||||
data, err := os.ReadFile(testPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file: %v", err)
|
||||
}
|
||||
|
||||
lines := 0
|
||||
for _, b := range data {
|
||||
if b == '\n' {
|
||||
lines++
|
||||
}
|
||||
}
|
||||
|
||||
if lines != 10 {
|
||||
t.Errorf("expected 10 lines from concurrent writes, got %d", lines)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_Flush(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1", SrcPort: 8080}
|
||||
if err := sink.Write(context.Background(), log); err != nil {
|
||||
t.Fatalf("failed to write: %v", err)
|
||||
}
|
||||
|
||||
// Flush should succeed
|
||||
err = sink.Flush(context.Background())
|
||||
if err != nil {
|
||||
t.Errorf("expected no error on Flush, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileSink_MarshalError(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
// Create a log with unmarshalable data (channel)
|
||||
log := domain.CorrelatedLog{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
Fields: map[string]any{"chan": make(chan int)},
|
||||
}
|
||||
|
||||
err = sink.Write(context.Background(), log)
|
||||
if err == nil {
|
||||
t.Error("expected error when marshaling unmarshalable data")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileSink_CloseTwice tests that closing an already-closed sink does not error.
|
||||
func TestFileSink_CloseTwice(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
|
||||
if err := sink.Close(); err != nil {
|
||||
t.Errorf("first Close() should not error, got: %v", err)
|
||||
}
|
||||
|
||||
// After close, file is nil, so second close should return nil
|
||||
if err := sink.Close(); err != nil {
|
||||
t.Errorf("second Close() on already-closed sink should not error, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileSink_WriteAfterClose tests that Write after Close re-opens the file.
|
||||
func TestFileSink_WriteAfterCloseReopens(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
|
||||
if err := sink.Close(); err != nil {
|
||||
t.Fatalf("Close() failed: %v", err)
|
||||
}
|
||||
|
||||
// Write after close: FileSink.Write reopens the file when file == nil
|
||||
log := domain.CorrelatedLog{SrcIP: "1.2.3.4", SrcPort: 80}
|
||||
if err := sink.Write(context.Background(), log); err != nil {
|
||||
t.Errorf("Write after close should succeed (auto-reopen), got: %v", err)
|
||||
}
|
||||
|
||||
// Verify data was written
|
||||
data, err := os.ReadFile(testPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file: %v", err)
|
||||
}
|
||||
if len(data) == 0 {
|
||||
t.Error("expected data to be present after write on re-opened file")
|
||||
}
|
||||
}
|
||||
|
||||
// TestFileSink_ReopenAfterWrite tests Reopen then write produces correct output.
|
||||
func TestFileSink_ReopenThenWrite(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
testPath := filepath.Join(tmpDir, "test.log")
|
||||
|
||||
sink, err := NewFileSink(Config{Path: testPath})
|
||||
if err != nil {
|
||||
t.Fatalf("failed to create sink: %v", err)
|
||||
}
|
||||
defer sink.Close()
|
||||
|
||||
// Write before reopen
|
||||
log1 := domain.CorrelatedLog{SrcIP: "1.1.1.1", SrcPort: 80}
|
||||
if err := sink.Write(context.Background(), log1); err != nil {
|
||||
t.Fatalf("first Write failed: %v", err)
|
||||
}
|
||||
|
||||
// Simulate log rotation
|
||||
if err := sink.Reopen(); err != nil {
|
||||
t.Fatalf("Reopen failed: %v", err)
|
||||
}
|
||||
|
||||
// Write after reopen
|
||||
log2 := domain.CorrelatedLog{SrcIP: "2.2.2.2", SrcPort: 443}
|
||||
if err := sink.Write(context.Background(), log2); err != nil {
|
||||
t.Fatalf("second Write failed: %v", err)
|
||||
}
|
||||
|
||||
sink.Close()
|
||||
|
||||
data, err := os.ReadFile(testPath)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read file: %v", err)
|
||||
}
|
||||
|
||||
lines := 0
|
||||
for _, b := range data {
|
||||
if b == '\n' {
|
||||
lines++
|
||||
}
|
||||
}
|
||||
if lines != 2 {
|
||||
t.Errorf("expected 2 lines after reopen+write, got %d", lines)
|
||||
}
|
||||
}
|
||||
@ -1,137 +0,0 @@
|
||||
package multi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
"github.com/antitbone/ja4/correlator/internal/ports"
|
||||
)
|
||||
|
||||
// MultiSink fans out correlated logs to multiple sinks.
|
||||
type MultiSink struct {
|
||||
mu sync.RWMutex
|
||||
sinks []ports.CorrelatedLogSink
|
||||
}
|
||||
|
||||
// NewMultiSink creates a new multi-sink.
|
||||
func NewMultiSink(sinks ...ports.CorrelatedLogSink) *MultiSink {
|
||||
return &MultiSink{
|
||||
sinks: sinks,
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the sink name.
|
||||
func (s *MultiSink) Name() string {
|
||||
return "multi"
|
||||
}
|
||||
|
||||
// AddSink adds a sink to the fan-out.
|
||||
func (s *MultiSink) AddSink(sink ports.CorrelatedLogSink) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.sinks = append(s.sinks, sink)
|
||||
}
|
||||
|
||||
// Write writes a correlated log to all sinks concurrently.
|
||||
// Returns the first error encountered (but all sinks are attempted).
|
||||
func (s *MultiSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
|
||||
s.mu.RLock()
|
||||
sinks := make([]ports.CorrelatedLogSink, len(s.sinks))
|
||||
copy(sinks, s.sinks)
|
||||
s.mu.RUnlock()
|
||||
|
||||
if len(sinks) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var firstErr error
|
||||
var firstErrMu sync.Mutex
|
||||
errChan := make(chan error, len(sinks))
|
||||
|
||||
for _, sink := range sinks {
|
||||
wg.Add(1)
|
||||
go func(sk ports.CorrelatedLogSink) {
|
||||
defer wg.Done()
|
||||
if err := sk.Write(ctx, log); err != nil {
|
||||
// Non-blocking send to errChan
|
||||
select {
|
||||
case errChan <- err:
|
||||
default:
|
||||
// Channel full, error will be handled via firstErr
|
||||
}
|
||||
}
|
||||
}(sink)
|
||||
}
|
||||
|
||||
// Wait for all writes to complete in a separate goroutine
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
|
||||
// Collect errors with timeout
|
||||
select {
|
||||
case <-done:
|
||||
close(errChan)
|
||||
// Collect first error
|
||||
for err := range errChan {
|
||||
if err != nil {
|
||||
firstErrMu.Lock()
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
firstErrMu.Unlock()
|
||||
}
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
}
|
||||
|
||||
firstErrMu.Lock()
|
||||
defer firstErrMu.Unlock()
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// Flush flushes all sinks.
|
||||
func (s *MultiSink) Flush(ctx context.Context) error {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
for _, sink := range s.sinks {
|
||||
if err := sink.Flush(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes all sinks.
|
||||
func (s *MultiSink) Close() error {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
var firstErr error
|
||||
for _, sink := range s.sinks {
|
||||
if err := sink.Close(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
|
||||
// Reopen reopens all sinks (for log rotation on SIGHUP).
|
||||
func (s *MultiSink) Reopen() error {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
var firstErr error
|
||||
for _, sink := range s.sinks {
|
||||
if err := sink.Reopen(); err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
}
|
||||
return firstErr
|
||||
}
|
||||
@ -1,233 +0,0 @@
|
||||
package multi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
)
|
||||
|
||||
type mockSink struct {
|
||||
name string
|
||||
mu sync.Mutex
|
||||
writeFunc func(domain.CorrelatedLog) error
|
||||
flushFunc func() error
|
||||
closeFunc func() error
|
||||
reopenFunc func() error
|
||||
}
|
||||
|
||||
func (m *mockSink) Name() string { return m.name }
|
||||
func (m *mockSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
return m.writeFunc(log)
|
||||
}
|
||||
func (m *mockSink) Flush(ctx context.Context) error { return m.flushFunc() }
|
||||
func (m *mockSink) Close() error { return m.closeFunc() }
|
||||
func (m *mockSink) Reopen() error {
|
||||
if m.reopenFunc != nil {
|
||||
return m.reopenFunc()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestMultiSink_Write(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
writeCount := 0
|
||||
|
||||
sink1 := &mockSink{
|
||||
name: "sink1",
|
||||
writeFunc: func(log domain.CorrelatedLog) error {
|
||||
mu.Lock()
|
||||
writeCount++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
},
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
sink2 := &mockSink{
|
||||
name: "sink2",
|
||||
writeFunc: func(log domain.CorrelatedLog) error {
|
||||
mu.Lock()
|
||||
writeCount++
|
||||
mu.Unlock()
|
||||
return nil
|
||||
},
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
ms := NewMultiSink(sink1, sink2)
|
||||
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
|
||||
err := ms.Write(context.Background(), log)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if writeCount != 2 {
|
||||
t.Errorf("expected 2 writes, got %d", writeCount)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Write_OneFails(t *testing.T) {
|
||||
sink1 := &mockSink{
|
||||
name: "sink1",
|
||||
writeFunc: func(log domain.CorrelatedLog) error {
|
||||
return nil
|
||||
},
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
sink2 := &mockSink{
|
||||
name: "sink2",
|
||||
writeFunc: func(log domain.CorrelatedLog) error {
|
||||
return context.Canceled
|
||||
},
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
ms := NewMultiSink(sink1, sink2)
|
||||
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
|
||||
err := ms.Write(context.Background(), log)
|
||||
if err == nil {
|
||||
t.Error("expected error when one sink fails")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_AddSink(t *testing.T) {
|
||||
ms := NewMultiSink()
|
||||
|
||||
sink := &mockSink{
|
||||
name: "dynamic",
|
||||
writeFunc: func(log domain.CorrelatedLog) error { return nil },
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
ms.AddSink(sink)
|
||||
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
|
||||
err := ms.Write(context.Background(), log)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Name(t *testing.T) {
|
||||
ms := NewMultiSink()
|
||||
if ms.Name() != "multi" {
|
||||
t.Errorf("expected name 'multi', got %s", ms.Name())
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Flush(t *testing.T) {
|
||||
flushed := false
|
||||
sink := &mockSink{
|
||||
name: "test",
|
||||
writeFunc: func(log domain.CorrelatedLog) error { return nil },
|
||||
flushFunc: func() error {
|
||||
flushed = true
|
||||
return nil
|
||||
},
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
ms := NewMultiSink(sink)
|
||||
err := ms.Flush(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !flushed {
|
||||
t.Error("expected sink to be flushed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Flush_Error(t *testing.T) {
|
||||
sink := &mockSink{
|
||||
name: "test",
|
||||
writeFunc: func(log domain.CorrelatedLog) error { return nil },
|
||||
flushFunc: func() error { return context.Canceled },
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
ms := NewMultiSink(sink)
|
||||
err := ms.Flush(context.Background())
|
||||
if err != context.Canceled {
|
||||
t.Errorf("expected context.Canceled error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Close(t *testing.T) {
|
||||
closed := false
|
||||
sink := &mockSink{
|
||||
name: "test",
|
||||
writeFunc: func(log domain.CorrelatedLog) error { return nil },
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error {
|
||||
closed = true
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
ms := NewMultiSink(sink)
|
||||
err := ms.Close()
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
if !closed {
|
||||
t.Error("expected sink to be closed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Close_Error(t *testing.T) {
|
||||
sink := &mockSink{
|
||||
name: "test",
|
||||
writeFunc: func(log domain.CorrelatedLog) error { return nil },
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error { return context.Canceled },
|
||||
}
|
||||
|
||||
ms := NewMultiSink(sink)
|
||||
err := ms.Close()
|
||||
if err != context.Canceled {
|
||||
t.Errorf("expected context.Canceled error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Write_EmptySinks(t *testing.T) {
|
||||
ms := NewMultiSink()
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
|
||||
err := ms.Write(context.Background(), log)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error with empty sinks: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiSink_Write_ContextCancelled(t *testing.T) {
|
||||
sink := &mockSink{
|
||||
name: "test",
|
||||
writeFunc: func(log domain.CorrelatedLog) error {
|
||||
<-context.Background().Done()
|
||||
return nil
|
||||
},
|
||||
flushFunc: func() error { return nil },
|
||||
closeFunc: func() error { return nil },
|
||||
}
|
||||
|
||||
ms := NewMultiSink(sink)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
log := domain.CorrelatedLog{SrcIP: "192.168.1.1"}
|
||||
err := ms.Write(ctx, log)
|
||||
if err != context.Canceled {
|
||||
t.Errorf("expected context.Canceled error, got %v", err)
|
||||
}
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
package stdout
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
)
|
||||
|
||||
// Config holds the stdout sink configuration.
|
||||
type Config struct {
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
// StdoutSink is a no-op data sink. Operational logs are written to stderr
|
||||
// by the observability.Logger; correlated data must never appear on stdout.
|
||||
type StdoutSink struct{}
|
||||
|
||||
// NewStdoutSink creates a new stdout sink.
|
||||
func NewStdoutSink(config Config) *StdoutSink {
|
||||
return &StdoutSink{}
|
||||
}
|
||||
|
||||
// Name returns the sink name.
|
||||
func (s *StdoutSink) Name() string {
|
||||
return "stdout"
|
||||
}
|
||||
|
||||
// Reopen is a no-op for stdout.
|
||||
func (s *StdoutSink) Reopen() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Write is a no-op: correlated data must never be written to stdout.
|
||||
func (s *StdoutSink) Write(_ context.Context, _ domain.CorrelatedLog) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Flush is a no-op for stdout.
|
||||
func (s *StdoutSink) Flush(_ context.Context) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close is a no-op for stdout.
|
||||
func (s *StdoutSink) Close() error {
|
||||
return nil
|
||||
}
|
||||
@ -1,81 +0,0 @@
|
||||
package stdout
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
)
|
||||
|
||||
func makeLog(correlated int) domain.CorrelatedLog {
|
||||
return domain.CorrelatedLog{
|
||||
Timestamp: time.Unix(1700000000, 0),
|
||||
SrcIP: "1.2.3.4",
|
||||
SrcPort: 12345,
|
||||
Correlated: correlated,
|
||||
}
|
||||
}
|
||||
|
||||
// captureStdout replaces os.Stdout temporarily and returns what was written.
|
||||
func captureStdout(t *testing.T, fn func()) string {
|
||||
t.Helper()
|
||||
r, w, err := os.Pipe()
|
||||
if err != nil {
|
||||
t.Fatalf("os.Pipe: %v", err)
|
||||
}
|
||||
old := os.Stdout
|
||||
os.Stdout = w
|
||||
|
||||
fn()
|
||||
|
||||
w.Close()
|
||||
os.Stdout = old
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.ReadFrom(r)
|
||||
r.Close()
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func TestStdoutSink_Name(t *testing.T) {
|
||||
s := NewStdoutSink(Config{Enabled: true})
|
||||
if s.Name() != "stdout" {
|
||||
t.Errorf("expected name 'stdout', got %q", s.Name())
|
||||
}
|
||||
}
|
||||
|
||||
// TestStdoutSink_WriteDoesNotProduceOutput verifies that no JSON data
|
||||
// (correlated or not) is ever written to stdout.
|
||||
func TestStdoutSink_WriteDoesNotProduceOutput(t *testing.T) {
|
||||
s := NewStdoutSink(Config{Enabled: true})
|
||||
|
||||
got := captureStdout(t, func() {
|
||||
if err := s.Write(context.Background(), makeLog(1)); err != nil {
|
||||
t.Fatalf("Write(correlated) returned error: %v", err)
|
||||
}
|
||||
if err := s.Write(context.Background(), makeLog(0)); err != nil {
|
||||
t.Fatalf("Write(orphan) returned error: %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
if got != "" {
|
||||
t.Errorf("stdout must be empty but got: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStdoutSink_NoopMethods(t *testing.T) {
|
||||
s := NewStdoutSink(Config{Enabled: true})
|
||||
|
||||
if err := s.Flush(context.Background()); err != nil {
|
||||
t.Errorf("Flush returned error: %v", err)
|
||||
}
|
||||
if err := s.Close(); err != nil {
|
||||
t.Errorf("Close returned error: %v", err)
|
||||
}
|
||||
if err := s.Reopen(); err != nil {
|
||||
t.Errorf("Reopen returned error: %v", err)
|
||||
}
|
||||
}
|
||||
@ -1,161 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
"github.com/antitbone/ja4/correlator/internal/ports"
|
||||
)
|
||||
|
||||
const (
|
||||
// DefaultEventChannelBufferSize is the default size for event channels
|
||||
DefaultEventChannelBufferSize = 1000
|
||||
// OrphanTickInterval is how often the orchestrator drains pending orphans.
|
||||
// Set to half the default emit delay (500ms/2) so orphans are emitted promptly
|
||||
// even when no new events arrive.
|
||||
OrphanTickInterval = 250 * time.Millisecond
|
||||
)
|
||||
|
||||
// OrchestratorConfig holds the orchestrator configuration.
|
||||
type OrchestratorConfig struct {
|
||||
Sources []ports.EventSource
|
||||
Sink ports.CorrelatedLogSink
|
||||
}
|
||||
|
||||
// Orchestrator connects sources to the correlation service and sinks.
|
||||
type Orchestrator struct {
|
||||
config OrchestratorConfig
|
||||
correlationSvc ports.CorrelationProcessor
|
||||
ctx context.Context
|
||||
cancel context.CancelFunc
|
||||
wg sync.WaitGroup
|
||||
running atomic.Bool
|
||||
}
|
||||
|
||||
// NewOrchestrator creates a new orchestrator.
|
||||
func NewOrchestrator(config OrchestratorConfig, correlationSvc ports.CorrelationProcessor) *Orchestrator {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
return &Orchestrator{
|
||||
config: config,
|
||||
correlationSvc: correlationSvc,
|
||||
ctx: ctx,
|
||||
cancel: cancel,
|
||||
}
|
||||
}
|
||||
|
||||
// Start begins the orchestration.
|
||||
func (o *Orchestrator) Start() error {
|
||||
if !o.running.CompareAndSwap(false, true) {
|
||||
return nil // Already running
|
||||
}
|
||||
|
||||
// Start each source
|
||||
for _, source := range o.config.Sources {
|
||||
eventChan := make(chan *domain.NormalizedEvent, DefaultEventChannelBufferSize)
|
||||
|
||||
o.wg.Add(1)
|
||||
go func(src ports.EventSource, evChan chan *domain.NormalizedEvent) {
|
||||
defer o.wg.Done()
|
||||
|
||||
// Start the source in a separate goroutine
|
||||
sourceErr := make(chan error, 1)
|
||||
go func() {
|
||||
if err := src.Start(o.ctx, evChan); err != nil {
|
||||
sourceErr <- err
|
||||
}
|
||||
}()
|
||||
|
||||
// Process events in the current goroutine
|
||||
o.processEvents(evChan)
|
||||
|
||||
// Check for source start errors
|
||||
if err := <-sourceErr; err != nil {
|
||||
// Source failed to start, log error and exit
|
||||
return
|
||||
}
|
||||
}(source, eventChan)
|
||||
}
|
||||
|
||||
// Start a periodic ticker to drain pending orphan A events independently of the
|
||||
// event flow. Without this, orphans are only emitted when a new event arrives,
|
||||
// causing them to accumulate silently when the source goes quiet.
|
||||
o.wg.Add(1)
|
||||
go func() {
|
||||
defer o.wg.Done()
|
||||
ticker := time.NewTicker(OrphanTickInterval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-o.ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
logs := o.correlationSvc.EmitPendingOrphans()
|
||||
for _, log := range logs {
|
||||
o.config.Sink.Write(o.ctx, log) //nolint:errcheck
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// processEvents lit les événements du canal, les soumet au service de corrélation et écrit les résultats dans le puits.
|
||||
func (o *Orchestrator) processEvents(eventChan <-chan *domain.NormalizedEvent) {
|
||||
for {
|
||||
select {
|
||||
case <-o.ctx.Done():
|
||||
// Drain remaining events before exiting
|
||||
for {
|
||||
select {
|
||||
case event, ok := <-eventChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
logs := o.correlationSvc.ProcessEvent(event)
|
||||
for _, log := range logs {
|
||||
o.config.Sink.Write(o.ctx, log)
|
||||
}
|
||||
default:
|
||||
return
|
||||
}
|
||||
}
|
||||
case event, ok := <-eventChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Process through correlation service
|
||||
logs := o.correlationSvc.ProcessEvent(event)
|
||||
|
||||
// Write correlated logs to sink
|
||||
for _, log := range logs {
|
||||
if err := o.config.Sink.Write(o.ctx, log); err != nil {
|
||||
// Log error but continue processing
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Stop gracefully stops the orchestrator.
|
||||
// It stops all sources and closes sinks immediately without waiting for queue drainage.
|
||||
// systemd TimeoutStopSec handles forced termination if needed.
|
||||
func (o *Orchestrator) Stop() error {
|
||||
if !o.running.CompareAndSwap(true, false) {
|
||||
return nil // Not running
|
||||
}
|
||||
|
||||
// Cancel context to stop accepting new events immediately
|
||||
o.cancel()
|
||||
|
||||
// Close sink (flush skipped - in-flight events are dropped)
|
||||
if err := o.config.Sink.Close(); err != nil {
|
||||
// Log error
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@ -1,300 +0,0 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
"github.com/antitbone/ja4/correlator/internal/ports"
|
||||
)
|
||||
|
||||
type mockEventSource struct {
|
||||
name string
|
||||
mu sync.RWMutex
|
||||
eventChan chan<- *domain.NormalizedEvent
|
||||
started bool
|
||||
stopped bool
|
||||
}
|
||||
|
||||
func (m *mockEventSource) Name() string { return m.name }
|
||||
func (m *mockEventSource) Start(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) error {
|
||||
m.mu.Lock()
|
||||
m.started = true
|
||||
m.eventChan = eventChan
|
||||
m.mu.Unlock()
|
||||
<-ctx.Done()
|
||||
m.mu.Lock()
|
||||
m.stopped = true
|
||||
m.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
func (m *mockEventSource) Stop() error { return nil }
|
||||
|
||||
func (m *mockEventSource) getEventChan() chan<- *domain.NormalizedEvent {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.eventChan
|
||||
}
|
||||
|
||||
func (m *mockEventSource) isStarted() bool {
|
||||
m.mu.RLock()
|
||||
defer m.mu.RUnlock()
|
||||
return m.started
|
||||
}
|
||||
|
||||
type mockSink struct {
|
||||
mu sync.Mutex
|
||||
written []domain.CorrelatedLog
|
||||
}
|
||||
|
||||
func (m *mockSink) Name() string { return "mock" }
|
||||
func (m *mockSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.written = append(m.written, log)
|
||||
return nil
|
||||
}
|
||||
func (m *mockSink) Flush(ctx context.Context) error { return nil }
|
||||
func (m *mockSink) Close() error { return nil }
|
||||
func (m *mockSink) Reopen() error { return nil }
|
||||
|
||||
func (m *mockSink) getWritten() []domain.CorrelatedLog {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
result := make([]domain.CorrelatedLog, len(m.written))
|
||||
copy(result, m.written)
|
||||
return result
|
||||
}
|
||||
|
||||
func TestOrchestrator_StartStop(t *testing.T) {
|
||||
source := &mockEventSource{name: "test"}
|
||||
sink := &mockSink{}
|
||||
|
||||
corrConfig := domain.CorrelationConfig{
|
||||
TimeWindow: time.Second,
|
||||
ApacheAlwaysEmit: true,
|
||||
NetworkEmit: false,
|
||||
}
|
||||
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
|
||||
|
||||
orchestrator := NewOrchestrator(OrchestratorConfig{
|
||||
Sources: []ports.EventSource{source},
|
||||
Sink: sink,
|
||||
}, correlationSvc)
|
||||
|
||||
if err := orchestrator.Start(); err != nil {
|
||||
t.Fatalf("failed to start: %v", err)
|
||||
}
|
||||
|
||||
// Let it run briefly
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if err := orchestrator.Stop(); err != nil {
|
||||
t.Fatalf("failed to stop: %v", err)
|
||||
}
|
||||
|
||||
if !source.isStarted() {
|
||||
t.Error("expected source to be started")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrchestrator_ProcessEvent(t *testing.T) {
|
||||
source := &mockEventSource{name: "test"}
|
||||
sink := &mockSink{}
|
||||
|
||||
corrConfig := domain.CorrelationConfig{
|
||||
TimeWindow: time.Second,
|
||||
ApacheAlwaysEmit: true,
|
||||
NetworkEmit: false,
|
||||
}
|
||||
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
|
||||
|
||||
orchestrator := NewOrchestrator(OrchestratorConfig{
|
||||
Sources: []ports.EventSource{source},
|
||||
Sink: sink,
|
||||
}, correlationSvc)
|
||||
|
||||
if err := orchestrator.Start(); err != nil {
|
||||
t.Fatalf("failed to start: %v", err)
|
||||
}
|
||||
|
||||
// Wait for source to start and get the channel
|
||||
var eventChan chan<- *domain.NormalizedEvent
|
||||
for i := 0; i < 50; i++ {
|
||||
eventChan = source.getEventChan()
|
||||
if eventChan != nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
|
||||
if eventChan == nil {
|
||||
t.Fatal("source did not start properly")
|
||||
}
|
||||
|
||||
// Send an event through the source
|
||||
event := &domain.NormalizedEvent{
|
||||
Source: domain.SourceA,
|
||||
Timestamp: time.Now(),
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
Raw: map[string]any{"method": "GET"},
|
||||
}
|
||||
|
||||
// Send event
|
||||
eventChan <- event
|
||||
|
||||
// Give it time to process
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
|
||||
if err := orchestrator.Stop(); err != nil {
|
||||
t.Fatalf("failed to stop: %v", err)
|
||||
}
|
||||
|
||||
// Should have written at least one log (the orphan A)
|
||||
written := sink.getWritten()
|
||||
if len(written) == 0 {
|
||||
t.Error("expected at least one log to be written")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOrchestrator_StartTwice tests that calling Start() twice is a no-op (already running).
|
||||
func TestOrchestrator_StartTwice(t *testing.T) {
|
||||
source := &mockEventSource{name: "test"}
|
||||
sink := &mockSink{}
|
||||
|
||||
corrConfig := domain.CorrelationConfig{
|
||||
TimeWindow: time.Second,
|
||||
ApacheAlwaysEmit: true,
|
||||
}
|
||||
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
|
||||
|
||||
o := NewOrchestrator(OrchestratorConfig{
|
||||
Sources: []ports.EventSource{source},
|
||||
Sink: sink,
|
||||
}, correlationSvc)
|
||||
|
||||
if err := o.Start(); err != nil {
|
||||
t.Fatalf("first Start() failed: %v", err)
|
||||
}
|
||||
if err := o.Start(); err != nil {
|
||||
t.Errorf("second Start() should be no-op, got: %v", err)
|
||||
}
|
||||
|
||||
o.Stop()
|
||||
}
|
||||
|
||||
// TestOrchestrator_StopTwice tests that calling Stop() twice is a no-op.
|
||||
func TestOrchestrator_StopTwice(t *testing.T) {
|
||||
source := &mockEventSource{name: "test"}
|
||||
sink := &mockSink{}
|
||||
|
||||
corrConfig := domain.CorrelationConfig{
|
||||
TimeWindow: time.Second,
|
||||
ApacheAlwaysEmit: true,
|
||||
}
|
||||
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
|
||||
|
||||
o := NewOrchestrator(OrchestratorConfig{
|
||||
Sources: []ports.EventSource{source},
|
||||
Sink: sink,
|
||||
}, correlationSvc)
|
||||
|
||||
o.Start()
|
||||
|
||||
if err := o.Stop(); err != nil {
|
||||
t.Errorf("first Stop() failed: %v", err)
|
||||
}
|
||||
if err := o.Stop(); err != nil {
|
||||
t.Errorf("second Stop() should be no-op, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOrchestrator_NoSources tests that Orchestrator works with no sources.
|
||||
func TestOrchestrator_NoSources(t *testing.T) {
|
||||
sink := &mockSink{}
|
||||
|
||||
corrConfig := domain.CorrelationConfig{TimeWindow: time.Second}
|
||||
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
|
||||
|
||||
o := NewOrchestrator(OrchestratorConfig{
|
||||
Sources: []ports.EventSource{},
|
||||
Sink: sink,
|
||||
}, correlationSvc)
|
||||
|
||||
if err := o.Start(); err != nil {
|
||||
t.Fatalf("Start() with no sources failed: %v", err)
|
||||
}
|
||||
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
|
||||
if err := o.Stop(); err != nil {
|
||||
t.Errorf("Stop() failed: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestOrchestrator_OrphanEmission tests that orphan A events are emitted via tick.
|
||||
func TestOrchestrator_OrphanEmission(t *testing.T) {
|
||||
source := &mockEventSource{name: "test"}
|
||||
sink := &mockSink{}
|
||||
|
||||
corrConfig := domain.CorrelationConfig{
|
||||
TimeWindow: 50 * time.Millisecond,
|
||||
ApacheAlwaysEmit: true,
|
||||
ApacheEmitDelayMs: 10, // Very short delay so orphans emit quickly
|
||||
}
|
||||
correlationSvc := domain.NewCorrelationService(corrConfig, &domain.RealTimeProvider{})
|
||||
|
||||
o := NewOrchestrator(OrchestratorConfig{
|
||||
Sources: []ports.EventSource{source},
|
||||
Sink: sink,
|
||||
}, correlationSvc)
|
||||
|
||||
if err := o.Start(); err != nil {
|
||||
t.Fatalf("Start() failed: %v", err)
|
||||
}
|
||||
|
||||
// Wait for source to be ready
|
||||
var eventChan chan<- *domain.NormalizedEvent
|
||||
for i := 0; i < 50; i++ {
|
||||
eventChan = source.getEventChan()
|
||||
if eventChan != nil {
|
||||
break
|
||||
}
|
||||
time.Sleep(5 * time.Millisecond)
|
||||
}
|
||||
if eventChan == nil {
|
||||
t.Fatal("source did not start")
|
||||
}
|
||||
|
||||
// Send a source A event (Apache/HTTP)
|
||||
eventChan <- &domain.NormalizedEvent{
|
||||
Source: domain.SourceA,
|
||||
Timestamp: time.Now(),
|
||||
SrcIP: "10.0.0.1",
|
||||
SrcPort: 12345,
|
||||
Raw: map[string]any{"method": "GET"},
|
||||
}
|
||||
|
||||
// Allow time for orphan ticker to fire (OrphanTickInterval = 250ms, but emit delay is 10ms)
|
||||
time.Sleep(600 * time.Millisecond)
|
||||
|
||||
o.Stop()
|
||||
|
||||
written := sink.getWritten()
|
||||
if len(written) == 0 {
|
||||
t.Error("expected at least one orphan log to be emitted")
|
||||
}
|
||||
}
|
||||
|
||||
// TestOrchestrator_Constants tests that constants have reasonable values.
|
||||
func TestOrchestrator_Constants(t *testing.T) {
|
||||
if DefaultEventChannelBufferSize <= 0 {
|
||||
t.Error("DefaultEventChannelBufferSize should be positive")
|
||||
}
|
||||
if OrphanTickInterval <= 0 {
|
||||
t.Error("OrphanTickInterval should be positive")
|
||||
}
|
||||
}
|
||||
@ -1,412 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
ja4config "github.com/antitbone/ja4/ja4common/config"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// Config holds the complete application configuration.
|
||||
type Config struct {
|
||||
Log LogConfig `yaml:"log"`
|
||||
Inputs InputsConfig `yaml:"inputs"`
|
||||
Outputs OutputsConfig `yaml:"outputs"`
|
||||
Correlation CorrelationConfig `yaml:"correlation"`
|
||||
Metrics MetricsConfig `yaml:"metrics"`
|
||||
}
|
||||
|
||||
// MetricsConfig holds metrics server configuration.
|
||||
type MetricsConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Addr string `yaml:"addr"` // e.g., ":8080", "localhost:8080"
|
||||
}
|
||||
|
||||
// LogConfig holds logging configuration.
|
||||
type LogConfig struct {
|
||||
Level string `yaml:"level" env:"LOG_LEVEL"` // DEBUG, INFO, WARN, ERROR
|
||||
}
|
||||
|
||||
// GetLogLevel returns the log level, defaulting to INFO if not set.
|
||||
func (c *LogConfig) GetLevel() string {
|
||||
if c.Level == "" {
|
||||
return "INFO"
|
||||
}
|
||||
return strings.ToUpper(c.Level)
|
||||
}
|
||||
|
||||
// ServiceConfig holds service-level configuration.
|
||||
type ServiceConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Language string `yaml:"language"`
|
||||
}
|
||||
|
||||
// InputsConfig holds input sources configuration.
|
||||
type InputsConfig struct {
|
||||
UnixSockets []UnixSocketConfig `yaml:"unix_sockets"`
|
||||
}
|
||||
|
||||
// UnixSocketConfig holds a Unix socket source configuration.
|
||||
type UnixSocketConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Path string `yaml:"path"`
|
||||
Format string `yaml:"format"`
|
||||
SourceType string `yaml:"source_type"` // "A" for Apache/HTTP, "B" for Network
|
||||
SocketPermissions string `yaml:"socket_permissions"` // octal string, e.g., "0660", "0666"
|
||||
}
|
||||
|
||||
// OutputsConfig holds output sinks configuration.
|
||||
type OutputsConfig struct {
|
||||
File FileOutputConfig `yaml:"file"`
|
||||
ClickHouse ClickHouseOutputConfig `yaml:"clickhouse"`
|
||||
Stdout StdoutOutputConfig `yaml:"stdout"`
|
||||
}
|
||||
|
||||
// FileOutputConfig holds file sink configuration.
|
||||
type FileOutputConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Path string `yaml:"path"`
|
||||
}
|
||||
|
||||
// ClickHouseOutputConfig holds ClickHouse sink configuration.
|
||||
type ClickHouseOutputConfig struct {
|
||||
Enabled bool `yaml:"enabled" env:"CLICKHOUSE_ENABLED"`
|
||||
DSN string `yaml:"dsn" env:"CLICKHOUSE_DSN"`
|
||||
Table string `yaml:"table" env:"CLICKHOUSE_TABLE"`
|
||||
BatchSize int `yaml:"batch_size" env:"CLICKHOUSE_BATCH_SIZE"`
|
||||
FlushIntervalMs int `yaml:"flush_interval_ms" env:"CLICKHOUSE_FLUSH_INTERVAL_MS"`
|
||||
MaxBufferSize int `yaml:"max_buffer_size" env:"CLICKHOUSE_MAX_BUFFER_SIZE"`
|
||||
DropOnOverflow bool `yaml:"drop_on_overflow"`
|
||||
AsyncInsert bool `yaml:"async_insert"`
|
||||
TimeoutMs int `yaml:"timeout_ms" env:"CLICKHOUSE_TIMEOUT_MS"`
|
||||
}
|
||||
|
||||
// StdoutOutputConfig holds stdout sink configuration.
|
||||
type StdoutOutputConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Level string `yaml:"level"` // DEBUG, INFO, WARN, ERROR - filters output verbosity
|
||||
}
|
||||
|
||||
// CorrelationConfig holds correlation configuration.
|
||||
type CorrelationConfig struct {
|
||||
TimeWindow TimeWindowConfig `yaml:"time_window"`
|
||||
OrphanPolicy OrphanPolicyConfig `yaml:"orphan_policy"`
|
||||
Matching MatchingConfig `yaml:"matching"`
|
||||
Buffers BuffersConfig `yaml:"buffers"`
|
||||
TTL TTLConfig `yaml:"ttl"`
|
||||
ExcludeSourceIPs []string `yaml:"exclude_source_ips"` // List of source IPs or CIDR ranges to exclude
|
||||
IncludeDestPorts []int `yaml:"include_dest_ports"` // If non-empty, only correlate events matching these destination ports
|
||||
// Deprecated: Use TimeWindow.Value instead
|
||||
TimeWindowS int `yaml:"time_window_s"`
|
||||
// Deprecated: Use OrphanPolicy.ApacheAlwaysEmit instead
|
||||
EmitOrphans bool `yaml:"emit_orphans"`
|
||||
}
|
||||
|
||||
// TimeWindowConfig holds time window configuration.
|
||||
type TimeWindowConfig struct {
|
||||
Value int `yaml:"value"`
|
||||
Unit string `yaml:"unit"` // s, ms, etc.
|
||||
}
|
||||
|
||||
// GetDuration returns the time window as a duration.
|
||||
func (c *TimeWindowConfig) GetDuration() time.Duration {
|
||||
value := c.Value
|
||||
if value <= 0 {
|
||||
value = 1
|
||||
}
|
||||
switch c.Unit {
|
||||
case "ms", "millisecond", "milliseconds":
|
||||
return time.Duration(value) * time.Millisecond
|
||||
case "s", "sec", "second", "seconds":
|
||||
fallthrough
|
||||
default:
|
||||
return time.Duration(value) * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
// OrphanPolicyConfig holds orphan event policy configuration.
|
||||
type OrphanPolicyConfig struct {
|
||||
ApacheAlwaysEmit bool `yaml:"apache_always_emit"`
|
||||
ApacheEmitDelayMs int `yaml:"apache_emit_delay_ms"` // Delay in ms before emitting orphan A
|
||||
NetworkEmit bool `yaml:"network_emit"`
|
||||
}
|
||||
|
||||
// MatchingConfig holds matching mode configuration.
|
||||
type MatchingConfig struct {
|
||||
Mode string `yaml:"mode"` // one_to_one or one_to_many
|
||||
}
|
||||
|
||||
// BuffersConfig holds buffer size configuration.
|
||||
type BuffersConfig struct {
|
||||
MaxHTTPItems int `yaml:"max_http_items"`
|
||||
MaxNetworkItems int `yaml:"max_network_items"`
|
||||
}
|
||||
|
||||
// TTLConfig holds TTL configuration.
|
||||
type TTLConfig struct {
|
||||
NetworkTTLS int `yaml:"network_ttl_s"`
|
||||
}
|
||||
|
||||
// Load loads configuration from a YAML file.
|
||||
func Load(path string) (*Config, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read config file: %w", err)
|
||||
}
|
||||
|
||||
cfg := defaultConfig()
|
||||
|
||||
if err := yaml.Unmarshal(data, cfg); err != nil {
|
||||
return nil, fmt.Errorf("failed to parse config file: %w", err)
|
||||
}
|
||||
|
||||
// Surcharge par variables d'environnement (préfixe LOGCORRELATOR_)
|
||||
if err := ja4config.OverrideFromEnv(cfg, "LOGCORRELATOR"); err != nil {
|
||||
return nil, fmt.Errorf("failed to apply env overrides: %w", err)
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid config: %w", err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// defaultConfig returns a Config with default values.
|
||||
func defaultConfig() *Config {
|
||||
return &Config{
|
||||
Log: LogConfig{
|
||||
Level: "INFO",
|
||||
},
|
||||
Inputs: InputsConfig{
|
||||
UnixSockets: make([]UnixSocketConfig, 0),
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{
|
||||
Enabled: true,
|
||||
Path: "/var/log/logcorrelator/correlated.log",
|
||||
},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: false,
|
||||
BatchSize: 500,
|
||||
FlushIntervalMs: 200,
|
||||
MaxBufferSize: 5000,
|
||||
DropOnOverflow: true,
|
||||
AsyncInsert: true,
|
||||
TimeoutMs: 1000,
|
||||
},
|
||||
Stdout: StdoutOutputConfig{Enabled: false},
|
||||
},
|
||||
Correlation: CorrelationConfig{
|
||||
TimeWindowS: 1,
|
||||
EmitOrphans: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Validate validates the configuration.
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.Inputs.UnixSockets) < 2 {
|
||||
return fmt.Errorf("at least two unix socket inputs are required")
|
||||
}
|
||||
|
||||
seenNames := make(map[string]struct{}, len(c.Inputs.UnixSockets))
|
||||
seenPaths := make(map[string]struct{}, len(c.Inputs.UnixSockets))
|
||||
|
||||
for i, input := range c.Inputs.UnixSockets {
|
||||
if strings.TrimSpace(input.Name) == "" {
|
||||
return fmt.Errorf("inputs.unix_sockets[%d].name is required", i)
|
||||
}
|
||||
if strings.TrimSpace(input.Path) == "" {
|
||||
return fmt.Errorf("inputs.unix_sockets[%d].path is required", i)
|
||||
}
|
||||
|
||||
if _, exists := seenNames[input.Name]; exists {
|
||||
return fmt.Errorf("duplicate unix socket input name: %s", input.Name)
|
||||
}
|
||||
seenNames[input.Name] = struct{}{}
|
||||
|
||||
if _, exists := seenPaths[input.Path]; exists {
|
||||
return fmt.Errorf("duplicate unix socket input path: %s", input.Path)
|
||||
}
|
||||
seenPaths[input.Path] = struct{}{}
|
||||
}
|
||||
|
||||
// At least one output must be enabled
|
||||
hasOutput := false
|
||||
if c.Outputs.File.Enabled && c.Outputs.File.Path != "" {
|
||||
hasOutput = true
|
||||
}
|
||||
if c.Outputs.ClickHouse.Enabled {
|
||||
hasOutput = true
|
||||
}
|
||||
if c.Outputs.Stdout.Enabled {
|
||||
hasOutput = true
|
||||
}
|
||||
|
||||
if !hasOutput {
|
||||
return fmt.Errorf("at least one output must be enabled (file, clickhouse, or stdout)")
|
||||
}
|
||||
|
||||
if c.Outputs.ClickHouse.Enabled {
|
||||
if strings.TrimSpace(c.Outputs.ClickHouse.DSN) == "" {
|
||||
return fmt.Errorf("clickhouse DSN is required when enabled")
|
||||
}
|
||||
if strings.TrimSpace(c.Outputs.ClickHouse.Table) == "" {
|
||||
return fmt.Errorf("clickhouse table is required when enabled")
|
||||
}
|
||||
if c.Outputs.ClickHouse.BatchSize <= 0 {
|
||||
return fmt.Errorf("clickhouse batch_size must be > 0")
|
||||
}
|
||||
if c.Outputs.ClickHouse.MaxBufferSize <= 0 {
|
||||
return fmt.Errorf("clickhouse max_buffer_size must be > 0")
|
||||
}
|
||||
if c.Outputs.ClickHouse.TimeoutMs <= 0 {
|
||||
return fmt.Errorf("clickhouse timeout_ms must be > 0")
|
||||
}
|
||||
}
|
||||
|
||||
if c.Correlation.TimeWindowS <= 0 {
|
||||
return fmt.Errorf("correlation.time_window_s must be > 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTimeWindow returns the time window as a duration.
|
||||
// Deprecated: Use TimeWindow.GetDuration() instead.
|
||||
func (c *CorrelationConfig) GetTimeWindow() time.Duration {
|
||||
// New config takes precedence
|
||||
if c.TimeWindow.Value > 0 {
|
||||
return c.TimeWindow.GetDuration()
|
||||
}
|
||||
// Fallback to deprecated field
|
||||
value := c.TimeWindowS
|
||||
if value <= 0 {
|
||||
value = 1
|
||||
}
|
||||
return time.Duration(value) * time.Second
|
||||
}
|
||||
|
||||
// GetApacheAlwaysEmit returns whether to always emit Apache events.
|
||||
func (c *CorrelationConfig) GetApacheAlwaysEmit() bool {
|
||||
if c.OrphanPolicy.ApacheAlwaysEmit {
|
||||
return true
|
||||
}
|
||||
// Fallback to deprecated field
|
||||
return c.EmitOrphans
|
||||
}
|
||||
|
||||
// GetApacheEmitDelayMs returns the delay in milliseconds before emitting orphan A events.
|
||||
func (c *CorrelationConfig) GetApacheEmitDelayMs() int {
|
||||
if c.OrphanPolicy.ApacheEmitDelayMs > 0 {
|
||||
return c.OrphanPolicy.ApacheEmitDelayMs
|
||||
}
|
||||
return domain.DefaultApacheEmitDelayMs // Default: 500ms
|
||||
}
|
||||
|
||||
// GetMatchingMode returns the matching mode.
|
||||
func (c *CorrelationConfig) GetMatchingMode() string {
|
||||
if c.Matching.Mode != "" {
|
||||
return c.Matching.Mode
|
||||
}
|
||||
return "one_to_many" // Default to Keep-Alive
|
||||
}
|
||||
|
||||
// GetMaxHTTPBufferSize returns the max HTTP buffer size.
|
||||
func (c *CorrelationConfig) GetMaxHTTPBufferSize() int {
|
||||
if c.Buffers.MaxHTTPItems > 0 {
|
||||
return c.Buffers.MaxHTTPItems
|
||||
}
|
||||
return domain.DefaultMaxHTTPBufferSize
|
||||
}
|
||||
|
||||
// GetMaxNetworkBufferSize returns the max network buffer size.
|
||||
func (c *CorrelationConfig) GetMaxNetworkBufferSize() int {
|
||||
if c.Buffers.MaxNetworkItems > 0 {
|
||||
return c.Buffers.MaxNetworkItems
|
||||
}
|
||||
return domain.DefaultMaxNetworkBufferSize
|
||||
}
|
||||
|
||||
// GetNetworkTTLS returns the network TTL in seconds.
|
||||
func (c *CorrelationConfig) GetNetworkTTLS() int {
|
||||
if c.TTL.NetworkTTLS > 0 {
|
||||
return c.TTL.NetworkTTLS
|
||||
}
|
||||
return domain.DefaultNetworkTTLS
|
||||
}
|
||||
|
||||
// GetSocketPermissions returns the socket permissions as os.FileMode.
|
||||
// Default is 0666 (world read/write).
|
||||
func (c *UnixSocketConfig) GetSocketPermissions() os.FileMode {
|
||||
trimmed := strings.TrimSpace(c.SocketPermissions)
|
||||
if trimmed == "" {
|
||||
return 0666
|
||||
}
|
||||
|
||||
// Parse octal string (e.g., "0660", "660", "0666")
|
||||
perms, err := strconv.ParseUint(trimmed, 8, 32)
|
||||
if err != nil {
|
||||
return 0666
|
||||
}
|
||||
|
||||
return os.FileMode(perms)
|
||||
}
|
||||
|
||||
// GetIncludeDestPorts returns the list of destination ports allowed for correlation.
|
||||
// An empty list means all ports are allowed.
|
||||
func (c *CorrelationConfig) GetIncludeDestPorts() []int {
|
||||
return c.IncludeDestPorts
|
||||
}
|
||||
|
||||
// GetExcludeSourceIPs returns the list of excluded source IPs or CIDR ranges.
|
||||
func (c *CorrelationConfig) GetExcludeSourceIPs() []string {
|
||||
return c.ExcludeSourceIPs
|
||||
}
|
||||
|
||||
// IsSourceIPExcluded checks if a source IP should be excluded.
|
||||
// Supports both exact IP matches and CIDR ranges.
|
||||
func (c *CorrelationConfig) IsSourceIPExcluded(ip string) bool {
|
||||
if len(c.ExcludeSourceIPs) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
// Parse the IP once
|
||||
parsedIP := net.ParseIP(ip)
|
||||
if parsedIP == nil {
|
||||
return false // Invalid IP
|
||||
}
|
||||
|
||||
for _, exclude := range c.ExcludeSourceIPs {
|
||||
// Try CIDR first
|
||||
if strings.Contains(exclude, "/") {
|
||||
_, cidr, err := net.ParseCIDR(exclude)
|
||||
if err != nil {
|
||||
continue // Invalid CIDR, skip
|
||||
}
|
||||
if cidr.Contains(parsedIP) {
|
||||
return true
|
||||
}
|
||||
} else {
|
||||
// Exact IP match
|
||||
if exclude == ip {
|
||||
return true
|
||||
}
|
||||
// Also try parsing as IP (handles different formats like 192.168.1.1 vs 192.168.001.001)
|
||||
if excludeIP := net.ParseIP(exclude); excludeIP != nil {
|
||||
if excludeIP.Equal(parsedIP) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return false
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,155 +0,0 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"reflect"
|
||||
"time"
|
||||
)
|
||||
|
||||
// CorrelatedLog represents the output correlated log entry.
|
||||
// All fields are flattened into a single-level structure.
|
||||
type CorrelatedLog struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
SrcIP string `json:"src_ip"`
|
||||
SrcPort int `json:"src_port"`
|
||||
DstIP string `json:"dst_ip,omitempty"`
|
||||
DstPort int `json:"dst_port,omitempty"`
|
||||
Correlated int `json:"correlated"` // 0 = orphelin, 1 = corrélé
|
||||
OrphanSide string `json:"orphan_side,omitempty"`
|
||||
Fields map[string]any `json:"-"` // Additional fields, merged at marshal time
|
||||
}
|
||||
|
||||
// MarshalJSON implements custom JSON marshaling to flatten the structure.
|
||||
func (c CorrelatedLog) MarshalJSON() ([]byte, error) {
|
||||
// Create a flat map with all fields
|
||||
flat := make(map[string]any)
|
||||
|
||||
// Add core fields
|
||||
flat["timestamp"] = c.Timestamp
|
||||
flat["src_ip"] = c.SrcIP
|
||||
flat["src_port"] = c.SrcPort
|
||||
if c.DstIP != "" {
|
||||
flat["dst_ip"] = c.DstIP
|
||||
}
|
||||
if c.DstPort != 0 {
|
||||
flat["dst_port"] = c.DstPort
|
||||
}
|
||||
flat["correlated"] = c.Correlated
|
||||
if c.OrphanSide != "" {
|
||||
flat["orphan_side"] = c.OrphanSide
|
||||
}
|
||||
|
||||
// Merge additional fields while preserving reserved keys
|
||||
reservedKeys := map[string]struct{}{
|
||||
"timestamp": {},
|
||||
"src_ip": {},
|
||||
"src_port": {},
|
||||
"dst_ip": {},
|
||||
"dst_port": {},
|
||||
"correlated": {},
|
||||
"orphan_side": {},
|
||||
}
|
||||
for k, v := range c.Fields {
|
||||
if _, reserved := reservedKeys[k]; reserved {
|
||||
continue
|
||||
}
|
||||
flat[k] = v
|
||||
}
|
||||
|
||||
return json.Marshal(flat)
|
||||
}
|
||||
|
||||
// NewCorrelatedLogFromEvent creates a correlated log from a single event (orphan).
|
||||
func NewCorrelatedLogFromEvent(event *NormalizedEvent, orphanSide string) CorrelatedLog {
|
||||
fields := extractFields(event)
|
||||
if event.KeepAliveSeq > 0 {
|
||||
fields["keepalives"] = event.KeepAliveSeq
|
||||
}
|
||||
return CorrelatedLog{
|
||||
Timestamp: event.Timestamp,
|
||||
SrcIP: event.SrcIP,
|
||||
SrcPort: event.SrcPort,
|
||||
DstIP: event.DstIP,
|
||||
DstPort: event.DstPort,
|
||||
Correlated: 0,
|
||||
OrphanSide: orphanSide,
|
||||
Fields: fields,
|
||||
}
|
||||
}
|
||||
|
||||
// NewCorrelatedLog creates a correlated log from two matched events.
|
||||
func NewCorrelatedLog(apacheEvent, networkEvent *NormalizedEvent) CorrelatedLog {
|
||||
ts := apacheEvent.Timestamp
|
||||
if networkEvent.Timestamp.After(ts) {
|
||||
ts = networkEvent.Timestamp
|
||||
}
|
||||
|
||||
fields := mergeFields(apacheEvent, networkEvent)
|
||||
if apacheEvent.KeepAliveSeq > 0 {
|
||||
fields["keepalives"] = apacheEvent.KeepAliveSeq
|
||||
}
|
||||
|
||||
return CorrelatedLog{
|
||||
Timestamp: ts,
|
||||
SrcIP: apacheEvent.SrcIP,
|
||||
SrcPort: apacheEvent.SrcPort,
|
||||
DstIP: coalesceString(apacheEvent.DstIP, networkEvent.DstIP),
|
||||
DstPort: coalesceInt(apacheEvent.DstPort, networkEvent.DstPort),
|
||||
Correlated: 1,
|
||||
OrphanSide: "",
|
||||
Fields: fields,
|
||||
}
|
||||
}
|
||||
|
||||
// extractFields copie l'ensemble des champs bruts d'un événement dans une nouvelle map.
|
||||
func extractFields(e *NormalizedEvent) map[string]any {
|
||||
result := make(map[string]any)
|
||||
for k, v := range e.Raw {
|
||||
result[k] = v
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// mergeFields fusionne les champs bruts de deux événements en préfixant les clés en collision par "a_" et "b_".
|
||||
func mergeFields(a, b *NormalizedEvent) map[string]any {
|
||||
result := make(map[string]any)
|
||||
|
||||
// Start with A fields
|
||||
for k, v := range a.Raw {
|
||||
result[k] = v
|
||||
}
|
||||
|
||||
// Merge B fields with collision handling
|
||||
for k, v := range b.Raw {
|
||||
if existing, exists := result[k]; exists {
|
||||
if reflect.DeepEqual(existing, v) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Collision with different values: keep both with prefixes
|
||||
delete(result, k)
|
||||
result["a_"+k] = existing
|
||||
result["b_"+k] = v
|
||||
continue
|
||||
}
|
||||
result[k] = v
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// coalesceString retourne la première chaîne non vide parmi les deux arguments.
|
||||
func coalesceString(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
// coalesceInt retourne le premier entier non nul parmi les deux arguments.
|
||||
func coalesceInt(a, b int) int {
|
||||
if a != 0 {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@ -1,365 +0,0 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestNormalizedEvent_CorrelationKey(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
event *NormalizedEvent
|
||||
expected string
|
||||
}{
|
||||
{
|
||||
name: "basic key",
|
||||
event: &NormalizedEvent{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
},
|
||||
expected: "192.168.1.1:8080",
|
||||
},
|
||||
{
|
||||
name: "different port",
|
||||
event: &NormalizedEvent{
|
||||
SrcIP: "10.0.0.1",
|
||||
SrcPort: 443,
|
||||
},
|
||||
expected: "10.0.0.1:443",
|
||||
},
|
||||
{
|
||||
name: "port zero",
|
||||
event: &NormalizedEvent{
|
||||
SrcIP: "127.0.0.1",
|
||||
SrcPort: 0,
|
||||
},
|
||||
expected: "127.0.0.1:0",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
key := tt.event.CorrelationKey()
|
||||
if key != tt.expected {
|
||||
t.Errorf("expected %s, got %s", tt.expected, key)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCorrelatedLogFromEvent(t *testing.T) {
|
||||
event := &NormalizedEvent{
|
||||
Source: SourceA,
|
||||
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 80,
|
||||
Raw: map[string]any{
|
||||
"method": "GET",
|
||||
"path": "/api/test",
|
||||
},
|
||||
}
|
||||
|
||||
log := NewCorrelatedLogFromEvent(event, "A")
|
||||
|
||||
if log.Correlated != 0 {
|
||||
t.Error("expected correlated to be 0")
|
||||
}
|
||||
if log.OrphanSide != "A" {
|
||||
t.Errorf("expected orphan_side A, got %s", log.OrphanSide)
|
||||
}
|
||||
if log.SrcIP != "192.168.1.1" {
|
||||
t.Errorf("expected src_ip 192.168.1.1, got %s", log.SrcIP)
|
||||
}
|
||||
if log.Fields == nil {
|
||||
t.Error("expected fields to be non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCorrelatedLog(t *testing.T) {
|
||||
apacheEvent := &NormalizedEvent{
|
||||
Source: SourceA,
|
||||
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 80,
|
||||
Raw: map[string]any{"method": "GET"},
|
||||
}
|
||||
|
||||
networkEvent := &NormalizedEvent{
|
||||
Source: SourceB,
|
||||
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 500000000, time.UTC),
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 8080,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 80,
|
||||
Raw: map[string]any{"ja3": "abc123"},
|
||||
}
|
||||
|
||||
log := NewCorrelatedLog(apacheEvent, networkEvent)
|
||||
|
||||
if log.Correlated != 1 {
|
||||
t.Error("expected correlated to be 1")
|
||||
}
|
||||
if log.OrphanSide != "" {
|
||||
t.Errorf("expected orphan_side to be empty, got %s", log.OrphanSide)
|
||||
}
|
||||
if log.Fields == nil {
|
||||
t.Error("expected fields to be non-nil")
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewCorrelatedLog_TimestampSelectionAEarlier verifies that when A is earlier the later (B) timestamp is used.
|
||||
func TestNewCorrelatedLog_TimestampSelectionAEarlier(t *testing.T) {
|
||||
tsA := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
tsB := time.Date(2024, 1, 1, 12, 0, 1, 0, time.UTC) // B is later
|
||||
|
||||
a := &NormalizedEvent{Source: SourceA, Timestamp: tsA, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
|
||||
b := &NormalizedEvent{Source: SourceB, Timestamp: tsB, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
|
||||
|
||||
log := NewCorrelatedLog(a, b)
|
||||
if !log.Timestamp.Equal(tsB) {
|
||||
t.Errorf("expected timestamp to be B's (later), got %v", log.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewCorrelatedLog_TimestampSelectionBEarlier verifies that when B is earlier, A's timestamp is used.
|
||||
func TestNewCorrelatedLog_TimestampSelectionBEarlier(t *testing.T) {
|
||||
tsA := time.Date(2024, 1, 1, 12, 0, 1, 0, time.UTC) // A is later
|
||||
tsB := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
|
||||
a := &NormalizedEvent{Source: SourceA, Timestamp: tsA, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
|
||||
b := &NormalizedEvent{Source: SourceB, Timestamp: tsB, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
|
||||
|
||||
log := NewCorrelatedLog(a, b)
|
||||
// The later timestamp wins. Since B is not After A, ts stays as A's timestamp.
|
||||
if !log.Timestamp.Equal(tsA) {
|
||||
t.Errorf("expected timestamp to be A's (later), got %v", log.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewCorrelatedLog_TimestampEqual verifies equal timestamps yield A's timestamp.
|
||||
func TestNewCorrelatedLog_TimestampEqual(t *testing.T) {
|
||||
ts := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
a := &NormalizedEvent{Source: SourceA, Timestamp: ts, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
|
||||
b := &NormalizedEvent{Source: SourceB, Timestamp: ts, SrcIP: "1.1.1.1", SrcPort: 100, Raw: map[string]any{}}
|
||||
|
||||
log := NewCorrelatedLog(a, b)
|
||||
if !log.Timestamp.Equal(ts) {
|
||||
t.Errorf("expected timestamp to be equal to both events' timestamp, got %v", log.Timestamp)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewCorrelatedLogFromEvent_WithKeepAlive verifies keepalives field is added when KeepAliveSeq > 0.
|
||||
func TestNewCorrelatedLogFromEvent_WithKeepAlive(t *testing.T) {
|
||||
event := &NormalizedEvent{
|
||||
Source: SourceA,
|
||||
Timestamp: time.Now(),
|
||||
SrcIP: "1.1.1.1",
|
||||
SrcPort: 9999,
|
||||
KeepAliveSeq: 3,
|
||||
Raw: map[string]any{"method": "GET"},
|
||||
}
|
||||
|
||||
log := NewCorrelatedLogFromEvent(event, "A")
|
||||
if log.Fields["keepalives"] != 3 {
|
||||
t.Errorf("expected keepalives=3, got %v", log.Fields["keepalives"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewCorrelatedLogFromEvent_NoKeepAlive verifies keepalives field is absent when KeepAliveSeq == 0.
|
||||
func TestNewCorrelatedLogFromEvent_NoKeepAlive(t *testing.T) {
|
||||
event := &NormalizedEvent{
|
||||
Source: SourceA,
|
||||
Timestamp: time.Now(),
|
||||
SrcIP: "1.1.1.1",
|
||||
SrcPort: 9999,
|
||||
KeepAliveSeq: 0,
|
||||
Raw: map[string]any{"method": "GET"},
|
||||
}
|
||||
|
||||
log := NewCorrelatedLogFromEvent(event, "A")
|
||||
if _, ok := log.Fields["keepalives"]; ok {
|
||||
t.Error("keepalives field should not be present when KeepAliveSeq == 0")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMergeFields_NoCollision verifies fields from A and B are merged without conflict.
|
||||
func TestMergeFields_NoCollision(t *testing.T) {
|
||||
a := &NormalizedEvent{Raw: map[string]any{"method": "GET", "path": "/foo"}}
|
||||
b := &NormalizedEvent{Raw: map[string]any{"ja4": "abc123", "proto": "TLS"}}
|
||||
|
||||
fields := mergeFields(a, b)
|
||||
if fields["method"] != "GET" {
|
||||
t.Errorf("expected method=GET, got %v", fields["method"])
|
||||
}
|
||||
if fields["ja4"] != "abc123" {
|
||||
t.Errorf("expected ja4=abc123, got %v", fields["ja4"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestMergeFields_SameValueNoPrefix verifies same-value fields are not prefixed.
|
||||
func TestMergeFields_SameValueNoPrefix(t *testing.T) {
|
||||
a := &NormalizedEvent{Raw: map[string]any{"proto": "TCP"}}
|
||||
b := &NormalizedEvent{Raw: map[string]any{"proto": "TCP"}}
|
||||
|
||||
fields := mergeFields(a, b)
|
||||
if fields["proto"] != "TCP" {
|
||||
t.Errorf("expected proto=TCP (no prefix), got %v", fields["proto"])
|
||||
}
|
||||
if _, ok := fields["a_proto"]; ok {
|
||||
t.Error("a_proto should not exist for same-value collision")
|
||||
}
|
||||
if _, ok := fields["b_proto"]; ok {
|
||||
t.Error("b_proto should not exist for same-value collision")
|
||||
}
|
||||
}
|
||||
|
||||
// TestMergeFields_DifferentValuePrefix verifies different-value fields get a_/b_ prefix.
|
||||
func TestMergeFields_DifferentValuePrefix(t *testing.T) {
|
||||
a := &NormalizedEvent{Raw: map[string]any{"port": 80}}
|
||||
b := &NormalizedEvent{Raw: map[string]any{"port": 443}}
|
||||
|
||||
fields := mergeFields(a, b)
|
||||
if fields["a_port"] != 80 {
|
||||
t.Errorf("expected a_port=80, got %v", fields["a_port"])
|
||||
}
|
||||
if fields["b_port"] != 443 {
|
||||
t.Errorf("expected b_port=443, got %v", fields["b_port"])
|
||||
}
|
||||
if _, ok := fields["port"]; ok {
|
||||
t.Error("original 'port' key should be removed on collision")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoalesceString_EmptyA tests that when a is empty, b is returned.
|
||||
func TestCoalesceString_EmptyA(t *testing.T) {
|
||||
result := coalesceString("", "fallback")
|
||||
if result != "fallback" {
|
||||
t.Errorf("expected 'fallback', got %q", result)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoalesceString_NonEmptyA tests that when a is non-empty, a is returned.
|
||||
func TestCoalesceString_NonEmptyA(t *testing.T) {
|
||||
result := coalesceString("primary", "fallback")
|
||||
if result != "primary" {
|
||||
t.Errorf("expected 'primary', got %q", result)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoalesceInt_ZeroA tests that when a is zero, b is returned.
|
||||
func TestCoalesceInt_ZeroA(t *testing.T) {
|
||||
result := coalesceInt(0, 443)
|
||||
if result != 443 {
|
||||
t.Errorf("expected 443, got %d", result)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCoalesceInt_NonZeroA tests that when a is non-zero, a is returned.
|
||||
func TestCoalesceInt_NonZeroA(t *testing.T) {
|
||||
result := coalesceInt(80, 443)
|
||||
if result != 80 {
|
||||
t.Errorf("expected 80, got %d", result)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarshalJSON_ReservedKeyProtection verifies reserved keys in Fields are not overwritten.
|
||||
func TestMarshalJSON_ReservedKeyProtection(t *testing.T) {
|
||||
log := CorrelatedLog{
|
||||
Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
|
||||
SrcIP: "1.2.3.4",
|
||||
SrcPort: 1234,
|
||||
Correlated: 1,
|
||||
Fields: map[string]any{
|
||||
"src_ip": "EVIL_OVERRIDE", // should be ignored
|
||||
"correlated": false, // should be ignored
|
||||
"extra": "value",
|
||||
},
|
||||
}
|
||||
|
||||
data, err := json.Marshal(log)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalJSON failed: %v", err)
|
||||
}
|
||||
|
||||
var flat map[string]any
|
||||
if err := json.Unmarshal(data, &flat); err != nil {
|
||||
t.Fatalf("Unmarshal failed: %v", err)
|
||||
}
|
||||
|
||||
if flat["src_ip"] != "1.2.3.4" {
|
||||
t.Errorf("reserved key src_ip should not be overwritten, got %v", flat["src_ip"])
|
||||
}
|
||||
if flat["correlated"] != float64(1) {
|
||||
t.Errorf("reserved key correlated should not be overwritten, got %v", flat["correlated"])
|
||||
}
|
||||
if flat["extra"] != "value" {
|
||||
t.Errorf("non-reserved key extra should be present, got %v", flat["extra"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestMarshalJSON_OptionalFieldsOmittedWhenZero verifies DstIP/DstPort are omitted when zero.
|
||||
func TestMarshalJSON_OptionalFieldsOmittedWhenZero(t *testing.T) {
|
||||
log := CorrelatedLog{
|
||||
Timestamp: time.Now(),
|
||||
SrcIP: "1.2.3.4",
|
||||
SrcPort: 1234,
|
||||
Correlated: 0,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(log)
|
||||
if err != nil {
|
||||
t.Fatalf("MarshalJSON failed: %v", err)
|
||||
}
|
||||
|
||||
var flat map[string]any
|
||||
if err := json.Unmarshal(data, &flat); err != nil {
|
||||
t.Fatalf("Unmarshal failed: %v", err)
|
||||
}
|
||||
|
||||
if _, ok := flat["dst_ip"]; ok {
|
||||
t.Error("dst_ip should be omitted when empty")
|
||||
}
|
||||
if _, ok := flat["dst_port"]; ok {
|
||||
t.Error("dst_port should be omitted when zero")
|
||||
}
|
||||
if _, ok := flat["orphan_side"]; ok {
|
||||
t.Error("orphan_side should be omitted when empty")
|
||||
}
|
||||
}
|
||||
|
||||
// TestExtractFields_Basic verifies extractFields copies Raw fields.
|
||||
func TestExtractFields_Basic(t *testing.T) {
|
||||
e := &NormalizedEvent{
|
||||
Raw: map[string]any{"key1": "val1", "key2": 42},
|
||||
}
|
||||
fields := extractFields(e)
|
||||
if fields["key1"] != "val1" {
|
||||
t.Errorf("expected key1=val1, got %v", fields["key1"])
|
||||
}
|
||||
if fields["key2"] != 42 {
|
||||
t.Errorf("expected key2=42, got %v", fields["key2"])
|
||||
}
|
||||
}
|
||||
|
||||
// TestNewCorrelatedLog_KeepAliveSeq verifies keepalives is set from apache event.
|
||||
func TestNewCorrelatedLog_KeepAliveSeq(t *testing.T) {
|
||||
a := &NormalizedEvent{
|
||||
Source: SourceA, Timestamp: time.Now(), SrcIP: "1.1.1.1", SrcPort: 100,
|
||||
KeepAliveSeq: 5,
|
||||
Raw: map[string]any{},
|
||||
}
|
||||
b := &NormalizedEvent{
|
||||
Source: SourceB, Timestamp: time.Now(), SrcIP: "1.1.1.1", SrcPort: 100,
|
||||
Raw: map[string]any{},
|
||||
}
|
||||
|
||||
log := NewCorrelatedLog(a, b)
|
||||
if log.Fields["keepalives"] != 5 {
|
||||
t.Errorf("expected keepalives=5, got %v", log.Fields["keepalives"])
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@ -1,33 +0,0 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// EventSource identifies the source of an event.
|
||||
type EventSource string
|
||||
|
||||
const (
|
||||
SourceA EventSource = "A" // Apache/HTTP source
|
||||
SourceB EventSource = "B" // Network source
|
||||
)
|
||||
|
||||
// NormalizedEvent represents a unified internal event from either source.
|
||||
type NormalizedEvent struct {
|
||||
Source EventSource
|
||||
Timestamp time.Time
|
||||
SrcIP string
|
||||
SrcPort int
|
||||
DstIP string
|
||||
DstPort int
|
||||
Headers map[string]string
|
||||
Extra map[string]any
|
||||
Raw map[string]any // Original raw data
|
||||
KeepAliveSeq int // Request sequence number within the Keep-Alive connection (1-based)
|
||||
}
|
||||
|
||||
// CorrelationKey returns the key used for correlation (src_ip + src_port).
|
||||
func (e *NormalizedEvent) CorrelationKey() string {
|
||||
return e.SrcIP + ":" + strconv.Itoa(e.SrcPort)
|
||||
}
|
||||
@ -1,27 +0,0 @@
|
||||
// Package observability provides structured logging for the correlator service.
|
||||
// Implementation is delegated to shared/go/ja4common/logger to avoid duplication.
|
||||
package observability
|
||||
|
||||
import jalogger "github.com/antitbone/ja4/ja4common/logger"
|
||||
|
||||
// Logger est un alias du type Logger de ja4common pour la journalisation structurée.
|
||||
type Logger = jalogger.Logger
|
||||
|
||||
// LogLevel est un alias du type LogLevel de ja4common pour le niveau de journalisation.
|
||||
type LogLevel = jalogger.LogLevel
|
||||
|
||||
const (
|
||||
DEBUG LogLevel = jalogger.DEBUG
|
||||
INFO LogLevel = jalogger.INFO
|
||||
WARN LogLevel = jalogger.WARN
|
||||
ERROR LogLevel = jalogger.ERROR
|
||||
)
|
||||
|
||||
// NewLogger creates a new Logger with INFO level.
|
||||
func NewLogger(prefix string) *Logger { return jalogger.New(prefix) }
|
||||
|
||||
// NewLoggerWithLevel creates a new Logger with the specified minimum level.
|
||||
func NewLoggerWithLevel(prefix, level string) *Logger { return jalogger.NewWithLevel(prefix, level) }
|
||||
|
||||
// ParseLogLevel converts a string to LogLevel.
|
||||
func ParseLogLevel(level string) LogLevel { return jalogger.ParseLogLevel(level) }
|
||||
@ -1,296 +0,0 @@
|
||||
// Package observability tests — behavioral tests for the Logger type alias.
|
||||
// Since Logger = jalogger.Logger, we test the observable API only.
|
||||
package observability_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/observability"
|
||||
)
|
||||
|
||||
func TestNewLogger_NonNil(t *testing.T) {
|
||||
logger := observability.NewLogger("test")
|
||||
if logger == nil {
|
||||
t.Fatal("expected non-nil logger")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_DefaultLevel_IsInfo(t *testing.T) {
|
||||
logger := observability.NewLogger("test")
|
||||
if !logger.ShouldLog(observability.INFO) {
|
||||
t.Error("INFO should be enabled by default")
|
||||
}
|
||||
if logger.ShouldLog(observability.DEBUG) {
|
||||
t.Error("DEBUG should be disabled by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_Info_NoPanic(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "INFO")
|
||||
if !logger.ShouldLog(observability.INFO) {
|
||||
t.Error("INFO should be enabled")
|
||||
}
|
||||
logger.Info("test message")
|
||||
}
|
||||
|
||||
func TestLogger_Error_NoPanic(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "ERROR")
|
||||
if !logger.ShouldLog(observability.ERROR) {
|
||||
t.Error("ERROR should be enabled")
|
||||
}
|
||||
logger.Error("error message", nil)
|
||||
}
|
||||
|
||||
func TestLogger_Debug_NoPanic(t *testing.T) {
|
||||
logger := observability.NewLogger("test")
|
||||
logger.SetLevel("DEBUG")
|
||||
if !logger.ShouldLog(observability.DEBUG) {
|
||||
t.Error("DEBUG should be enabled after SetLevel(DEBUG)")
|
||||
}
|
||||
logger.Debug("test message")
|
||||
}
|
||||
|
||||
func TestLogger_SetLevel(t *testing.T) {
|
||||
logger := observability.NewLogger("test")
|
||||
|
||||
logger.SetLevel("DEBUG")
|
||||
if !logger.ShouldLog(observability.DEBUG) {
|
||||
t.Error("DEBUG should be enabled after SetLevel(DEBUG)")
|
||||
}
|
||||
|
||||
logger.SetLevel("INFO")
|
||||
if logger.ShouldLog(observability.DEBUG) {
|
||||
t.Error("DEBUG should be disabled after SetLevel(INFO)")
|
||||
}
|
||||
|
||||
logger.SetLevel("WARN")
|
||||
if logger.ShouldLog(observability.INFO) {
|
||||
t.Error("INFO should be disabled after SetLevel(WARN)")
|
||||
}
|
||||
if !logger.ShouldLog(observability.WARN) {
|
||||
t.Error("WARN should be enabled after SetLevel(WARN)")
|
||||
}
|
||||
|
||||
logger.SetLevel("ERROR")
|
||||
if logger.ShouldLog(observability.WARN) {
|
||||
t.Error("WARN should be disabled after SetLevel(ERROR)")
|
||||
}
|
||||
if !logger.ShouldLog(observability.ERROR) {
|
||||
t.Error("ERROR should be enabled after SetLevel(ERROR)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLogLevel(t *testing.T) {
|
||||
cases := []struct {
|
||||
input string
|
||||
expected observability.LogLevel
|
||||
}{
|
||||
{"DEBUG", observability.DEBUG},
|
||||
{"debug", observability.DEBUG},
|
||||
{"INFO", observability.INFO},
|
||||
{"info", observability.INFO},
|
||||
{"WARN", observability.WARN},
|
||||
{"warn", observability.WARN},
|
||||
{"WARNING", observability.WARN},
|
||||
{"ERROR", observability.ERROR},
|
||||
{"error", observability.ERROR},
|
||||
{"", observability.INFO},
|
||||
{"invalid", observability.INFO},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
result := observability.ParseLogLevel(tt.input)
|
||||
if result != tt.expected {
|
||||
t.Errorf("ParseLogLevel(%q) = %v, want %v", tt.input, result, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_WithFields_NoPanic(t *testing.T) {
|
||||
logger := observability.NewLogger("test")
|
||||
child := logger.WithFields(map[string]any{"key1": "value1", "key2": 42})
|
||||
if child == logger {
|
||||
t.Error("expected different logger instance")
|
||||
}
|
||||
child.Info("message with fields")
|
||||
}
|
||||
|
||||
func TestLogLevel_String(t *testing.T) {
|
||||
cases := []struct {
|
||||
level observability.LogLevel
|
||||
expected string
|
||||
}{
|
||||
{observability.DEBUG, "DEBUG"},
|
||||
{observability.INFO, "INFO"},
|
||||
{observability.WARN, "WARN"},
|
||||
{observability.ERROR, "ERROR"},
|
||||
}
|
||||
for _, tt := range cases {
|
||||
t.Run(tt.expected, func(t *testing.T) {
|
||||
if got := tt.level.String(); got != tt.expected {
|
||||
t.Errorf("LogLevel(%d).String() = %q, want %q", tt.level, got, tt.expected)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_Warn_NoPanic(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "WARN")
|
||||
if !logger.ShouldLog(observability.WARN) {
|
||||
t.Error("WARN should be enabled")
|
||||
}
|
||||
logger.Warn("warning message")
|
||||
}
|
||||
|
||||
func TestLogger_Formatted_NoPanic(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "DEBUG")
|
||||
logger.Warnf("formatted %s %d", "message", 42)
|
||||
logger.Infof("formatted %s %d", "message", 42)
|
||||
logger.Debugf("formatted %s %d", "message", 42)
|
||||
}
|
||||
|
||||
func TestLogger_Error_WithError(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "ERROR")
|
||||
logger.Error("error occurred", &testErr{"test error"})
|
||||
}
|
||||
|
||||
func TestLogger_ShouldLog_Concurrent(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "DEBUG")
|
||||
done := make(chan bool)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func() {
|
||||
_ = logger.ShouldLog(observability.DEBUG)
|
||||
done <- true
|
||||
}()
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_Log_Concurrent(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "DEBUG")
|
||||
done := make(chan bool)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func(n int) {
|
||||
logger.Debugf("message %d", n)
|
||||
done <- true
|
||||
}(i)
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_WithFields_Concurrent(t *testing.T) {
|
||||
logger := observability.NewLogger("test")
|
||||
done := make(chan bool)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func(n int) {
|
||||
_ = logger.WithFields(map[string]any{"key": n})
|
||||
done <- true
|
||||
}(i)
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_SetLevel_Concurrent(t *testing.T) {
|
||||
logger := observability.NewLogger("test")
|
||||
done := make(chan bool)
|
||||
for i := 0; i < 10; i++ {
|
||||
go func() {
|
||||
logger.SetLevel("DEBUG")
|
||||
logger.SetLevel("INFO")
|
||||
done <- true
|
||||
}()
|
||||
}
|
||||
for i := 0; i < 10; i++ {
|
||||
<-done
|
||||
}
|
||||
}
|
||||
|
||||
type testErr struct{ msg string }
|
||||
|
||||
func (e *testErr) Error() string { return e.msg }
|
||||
|
||||
func TestNewLoggerWithLevel_AllLevels(t *testing.T) {
|
||||
levels := []string{"DEBUG", "INFO", "WARN", "WARNING", "ERROR", "invalid", ""}
|
||||
for _, level := range levels {
|
||||
t.Run(level, func(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", level)
|
||||
if logger == nil {
|
||||
t.Errorf("NewLoggerWithLevel(%q) returned nil", level)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogLevel_Constants(t *testing.T) {
|
||||
if observability.DEBUG >= observability.INFO {
|
||||
t.Error("DEBUG should be less than INFO")
|
||||
}
|
||||
if observability.INFO >= observability.WARN {
|
||||
t.Error("INFO should be less than WARN")
|
||||
}
|
||||
if observability.WARN >= observability.ERROR {
|
||||
t.Error("WARN should be less than ERROR")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_ShouldLog_AllLevels(t *testing.T) {
|
||||
cases := []struct {
|
||||
minLevel string
|
||||
level observability.LogLevel
|
||||
want bool
|
||||
}{
|
||||
{"DEBUG", observability.DEBUG, true},
|
||||
{"DEBUG", observability.INFO, true},
|
||||
{"DEBUG", observability.WARN, true},
|
||||
{"DEBUG", observability.ERROR, true},
|
||||
{"INFO", observability.DEBUG, false},
|
||||
{"INFO", observability.INFO, true},
|
||||
{"INFO", observability.WARN, true},
|
||||
{"WARN", observability.INFO, false},
|
||||
{"WARN", observability.WARN, true},
|
||||
{"WARN", observability.ERROR, true},
|
||||
{"ERROR", observability.WARN, false},
|
||||
{"ERROR", observability.ERROR, true},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.minLevel+"_"+tc.level.String(), func(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", tc.minLevel)
|
||||
got := logger.ShouldLog(tc.level)
|
||||
if got != tc.want {
|
||||
t.Errorf("ShouldLog(%v) with min=%s: expected %v, got %v",
|
||||
tc.level, tc.minLevel, tc.want, got)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLogLevel_WarningAlias(t *testing.T) {
|
||||
got := observability.ParseLogLevel("WARNING")
|
||||
if got != observability.WARN {
|
||||
t.Errorf("ParseLogLevel(WARNING) = %v, want WARN", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogger_Errorf_NoPanic(t *testing.T) {
|
||||
logger := observability.NewLoggerWithLevel("test", "DEBUG")
|
||||
// Errorf is not defined in the interface, but Warnf/Infof/Debugf are tested
|
||||
// Just ensure Error with a formatted message doesn't panic
|
||||
logger.Error("formatted error", &testErr{"err detail"})
|
||||
}
|
||||
|
||||
func TestNewLogger_PrefixIsUsed(t *testing.T) {
|
||||
logger := observability.NewLogger("my-prefix")
|
||||
if logger == nil {
|
||||
t.Fatal("expected non-nil logger")
|
||||
}
|
||||
// The logger should be usable
|
||||
logger.Infof("hello from %s", "my-prefix")
|
||||
}
|
||||
@ -1,176 +0,0 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
|
||||
// CorrelationMetrics tracks correlation statistics for debugging and monitoring.
|
||||
type CorrelationMetrics struct {
|
||||
mu sync.RWMutex
|
||||
|
||||
// Events received
|
||||
eventsReceivedA atomic.Int64
|
||||
eventsReceivedB atomic.Int64
|
||||
|
||||
// Correlation results
|
||||
correlationsSuccess atomic.Int64
|
||||
correlationsFailed atomic.Int64
|
||||
|
||||
// Failure reasons
|
||||
failedNoMatchKey atomic.Int64 // No event with same key in buffer
|
||||
failedTimeWindow atomic.Int64 // Key found but outside time window
|
||||
failedBufferEviction atomic.Int64 // Event evicted due to buffer full
|
||||
failedTTLExpired atomic.Int64 // B event TTL expired before match
|
||||
failedIPExcluded atomic.Int64 // Event excluded by IP filter
|
||||
|
||||
// Buffer stats
|
||||
bufferASize atomic.Int64
|
||||
bufferBSize atomic.Int64
|
||||
|
||||
// Orphan stats
|
||||
orphansEmittedA atomic.Int64
|
||||
orphansEmittedB atomic.Int64
|
||||
orphansPendingA atomic.Int64
|
||||
pendingOrphanMatch atomic.Int64 // B matched with pending orphan A
|
||||
|
||||
// Keep-Alive stats
|
||||
keepAliveResets atomic.Int64 // Number of TTL resets (one-to-many mode)
|
||||
}
|
||||
|
||||
// NewCorrelationMetrics creates a new metrics tracker.
|
||||
func NewCorrelationMetrics() *CorrelationMetrics {
|
||||
return &CorrelationMetrics{}
|
||||
}
|
||||
|
||||
// RecordEventReceived records an event received from a source.
|
||||
func (m *CorrelationMetrics) RecordEventReceived(source string) {
|
||||
if source == "A" {
|
||||
m.eventsReceivedA.Add(1)
|
||||
} else if source == "B" {
|
||||
m.eventsReceivedB.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
// RecordCorrelationSuccess records a successful correlation.
|
||||
func (m *CorrelationMetrics) RecordCorrelationSuccess() {
|
||||
m.correlationsSuccess.Add(1)
|
||||
}
|
||||
|
||||
// RecordCorrelationFailed records a failed correlation attempt with the reason.
|
||||
func (m *CorrelationMetrics) RecordCorrelationFailed(reason string) {
|
||||
m.correlationsFailed.Add(1)
|
||||
switch reason {
|
||||
case "no_match_key":
|
||||
m.failedNoMatchKey.Add(1)
|
||||
case "time_window":
|
||||
m.failedTimeWindow.Add(1)
|
||||
case "buffer_eviction":
|
||||
m.failedBufferEviction.Add(1)
|
||||
case "ttl_expired":
|
||||
m.failedTTLExpired.Add(1)
|
||||
case "ip_excluded":
|
||||
m.failedIPExcluded.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
// RecordBufferEviction records an event evicted from buffer.
|
||||
func (m *CorrelationMetrics) RecordBufferEviction(source string) {
|
||||
// Can be used for additional tracking if needed
|
||||
}
|
||||
|
||||
// RecordOrphanEmitted records an orphan event emitted.
|
||||
func (m *CorrelationMetrics) RecordOrphanEmitted(source string) {
|
||||
if source == "A" {
|
||||
m.orphansEmittedA.Add(1)
|
||||
} else if source == "B" {
|
||||
m.orphansEmittedB.Add(1)
|
||||
}
|
||||
}
|
||||
|
||||
// RecordPendingOrphan records an A event added to pending orphans.
|
||||
func (m *CorrelationMetrics) RecordPendingOrphan() {
|
||||
m.orphansPendingA.Add(1)
|
||||
}
|
||||
|
||||
// RecordPendingOrphanMatch records a B event matching a pending orphan A.
|
||||
func (m *CorrelationMetrics) RecordPendingOrphanMatch() {
|
||||
m.pendingOrphanMatch.Add(1)
|
||||
}
|
||||
|
||||
// RecordKeepAliveReset records a TTL reset for Keep-Alive.
|
||||
func (m *CorrelationMetrics) RecordKeepAliveReset() {
|
||||
m.keepAliveResets.Add(1)
|
||||
}
|
||||
|
||||
// UpdateBufferSizes updates the current buffer sizes.
|
||||
func (m *CorrelationMetrics) UpdateBufferSizes(sizeA, sizeB int64) {
|
||||
m.bufferASize.Store(sizeA)
|
||||
m.bufferBSize.Store(sizeB)
|
||||
}
|
||||
|
||||
// Snapshot returns a point-in-time snapshot of all metrics.
|
||||
func (m *CorrelationMetrics) Snapshot() MetricsSnapshot {
|
||||
return MetricsSnapshot{
|
||||
EventsReceivedA: m.eventsReceivedA.Load(),
|
||||
EventsReceivedB: m.eventsReceivedB.Load(),
|
||||
CorrelationsSuccess: m.correlationsSuccess.Load(),
|
||||
CorrelationsFailed: m.correlationsFailed.Load(),
|
||||
FailedNoMatchKey: m.failedNoMatchKey.Load(),
|
||||
FailedTimeWindow: m.failedTimeWindow.Load(),
|
||||
FailedBufferEviction: m.failedBufferEviction.Load(),
|
||||
FailedTTLExpired: m.failedTTLExpired.Load(),
|
||||
FailedIPExcluded: m.failedIPExcluded.Load(),
|
||||
BufferASize: m.bufferASize.Load(),
|
||||
BufferBSize: m.bufferBSize.Load(),
|
||||
OrphansEmittedA: m.orphansEmittedA.Load(),
|
||||
OrphansEmittedB: m.orphansEmittedB.Load(),
|
||||
OrphansPendingA: m.orphansPendingA.Load(),
|
||||
PendingOrphanMatch: m.pendingOrphanMatch.Load(),
|
||||
KeepAliveResets: m.keepAliveResets.Load(),
|
||||
}
|
||||
}
|
||||
|
||||
// MetricsSnapshot is a point-in-time snapshot of metrics.
|
||||
type MetricsSnapshot struct {
|
||||
EventsReceivedA int64 `json:"events_received_a"`
|
||||
EventsReceivedB int64 `json:"events_received_b"`
|
||||
CorrelationsSuccess int64 `json:"correlations_success"`
|
||||
CorrelationsFailed int64 `json:"correlations_failed"`
|
||||
FailedNoMatchKey int64 `json:"failed_no_match_key"`
|
||||
FailedTimeWindow int64 `json:"failed_time_window"`
|
||||
FailedBufferEviction int64 `json:"failed_buffer_eviction"`
|
||||
FailedTTLExpired int64 `json:"failed_ttl_expired"`
|
||||
FailedIPExcluded int64 `json:"failed_ip_excluded"`
|
||||
BufferASize int64 `json:"buffer_a_size"`
|
||||
BufferBSize int64 `json:"buffer_b_size"`
|
||||
OrphansEmittedA int64 `json:"orphans_emitted_a"`
|
||||
OrphansEmittedB int64 `json:"orphans_emitted_b"`
|
||||
OrphansPendingA int64 `json:"orphans_pending_a"`
|
||||
PendingOrphanMatch int64 `json:"pending_orphan_match"`
|
||||
KeepAliveResets int64 `json:"keepalive_resets"`
|
||||
}
|
||||
|
||||
// MarshalJSON implements json.Marshaler.
|
||||
func (m *CorrelationMetrics) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(m.Snapshot())
|
||||
}
|
||||
|
||||
// String returns a human-readable string of metrics.
|
||||
func (m *CorrelationMetrics) String() string {
|
||||
s := m.Snapshot()
|
||||
var b strings.Builder
|
||||
b.WriteString("Correlation Metrics:\n")
|
||||
fmt.Fprintf(&b, " Events Received: A=%d B=%d Total=%d\n", s.EventsReceivedA, s.EventsReceivedB, s.EventsReceivedA+s.EventsReceivedB)
|
||||
fmt.Fprintf(&b, " Correlations: Success=%d Failed=%d\n", s.CorrelationsSuccess, s.CorrelationsFailed)
|
||||
fmt.Fprintf(&b, " Failure Reasons: no_match_key=%d time_window=%d buffer_eviction=%d ttl_expired=%d ip_excluded=%d\n",
|
||||
s.FailedNoMatchKey, s.FailedTimeWindow, s.FailedBufferEviction, s.FailedTTLExpired, s.FailedIPExcluded)
|
||||
fmt.Fprintf(&b, " Buffer Sizes: A=%d B=%d\n", s.BufferASize, s.BufferBSize)
|
||||
fmt.Fprintf(&b, " Orphans: Emitted A=%d B=%d Pending A=%d\n", s.OrphansEmittedA, s.OrphansEmittedB, s.OrphansPendingA)
|
||||
fmt.Fprintf(&b, " Pending Orphan Match: %d\n", s.PendingOrphanMatch)
|
||||
fmt.Fprintf(&b, " Keep-Alive Resets: %d\n", s.KeepAliveResets)
|
||||
return b.String()
|
||||
}
|
||||
@ -1,128 +0,0 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MetricsServer exposes correlation metrics via HTTP.
|
||||
type MetricsServer struct {
|
||||
mu sync.Mutex
|
||||
server *http.Server
|
||||
listener net.Listener
|
||||
metricsFunc func() MetricsSnapshot
|
||||
running bool
|
||||
}
|
||||
|
||||
// NewMetricsServer creates a new metrics HTTP server.
|
||||
func NewMetricsServer(addr string, metricsFunc func() MetricsSnapshot) (*MetricsServer, error) {
|
||||
if metricsFunc == nil {
|
||||
return nil, fmt.Errorf("metricsFunc cannot be nil")
|
||||
}
|
||||
|
||||
ms := &MetricsServer{
|
||||
metricsFunc: metricsFunc,
|
||||
}
|
||||
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("/metrics", ms.handleMetrics)
|
||||
mux.HandleFunc("/health", ms.handleHealth)
|
||||
|
||||
ms.server = &http.Server{
|
||||
Addr: addr,
|
||||
Handler: mux,
|
||||
ReadTimeout: 5 * time.Second,
|
||||
WriteTimeout: 10 * time.Second,
|
||||
}
|
||||
|
||||
return ms, nil
|
||||
}
|
||||
|
||||
// Start begins listening on the configured address.
|
||||
func (ms *MetricsServer) Start() error {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
|
||||
if ms.running {
|
||||
return nil
|
||||
}
|
||||
|
||||
listener, err := net.Listen("tcp", ms.server.Addr)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to start metrics server: %w", err)
|
||||
}
|
||||
|
||||
ms.listener = listener
|
||||
ms.running = true
|
||||
|
||||
go func() {
|
||||
if err := ms.server.Serve(listener); err != nil && err != http.ErrServerClosed {
|
||||
// Server error or closed
|
||||
}
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Stop gracefully stops the metrics server.
|
||||
func (ms *MetricsServer) Stop(ctx context.Context) error {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
|
||||
if !ms.running {
|
||||
return nil
|
||||
}
|
||||
|
||||
ms.running = false
|
||||
return ms.server.Shutdown(ctx)
|
||||
}
|
||||
|
||||
// handleMetrics returns the correlation metrics as JSON.
|
||||
func (ms *MetricsServer) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
metrics := ms.metricsFunc()
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
if err := json.NewEncoder(w).Encode(metrics); err != nil {
|
||||
http.Error(w, "Failed to encode metrics", http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// handleHealth returns a simple health check response.
|
||||
func (ms *MetricsServer) handleHealth(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, `{"status":"healthy"}`)
|
||||
}
|
||||
|
||||
// IsRunning returns true if the server is running.
|
||||
func (ms *MetricsServer) IsRunning() bool {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
return ms.running
|
||||
}
|
||||
|
||||
// Addr returns the listening address.
|
||||
func (ms *MetricsServer) Addr() string {
|
||||
ms.mu.Lock()
|
||||
defer ms.mu.Unlock()
|
||||
if ms.listener == nil {
|
||||
return ""
|
||||
}
|
||||
return ms.listener.Addr().String()
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
package ports
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/antitbone/ja4/correlator/internal/domain"
|
||||
)
|
||||
|
||||
// EventSource defines the interface for log sources.
|
||||
type EventSource interface {
|
||||
// Start begins reading events and sending them to the channel.
|
||||
// Returns an error if the source cannot be started.
|
||||
Start(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) error
|
||||
|
||||
// Stop gracefully stops the source.
|
||||
Stop() error
|
||||
|
||||
// Name returns the source name.
|
||||
Name() string
|
||||
}
|
||||
|
||||
// CorrelatedLogSink defines the interface for correlated log destinations.
|
||||
type CorrelatedLogSink interface {
|
||||
// Write sends a correlated log to the sink.
|
||||
Write(ctx context.Context, log domain.CorrelatedLog) error
|
||||
|
||||
// Flush flushes any buffered logs.
|
||||
Flush(ctx context.Context) error
|
||||
|
||||
// Close closes the sink.
|
||||
Close() error
|
||||
|
||||
// Name returns the sink name.
|
||||
Name() string
|
||||
|
||||
// Reopen closes and reopens the sink (for log rotation on SIGHUP).
|
||||
// Optional: only FileSink implements this.
|
||||
Reopen() error
|
||||
}
|
||||
|
||||
// CorrelationProcessor defines the interface for the correlation service.
|
||||
// This allows for easier testing and alternative implementations.
|
||||
type CorrelationProcessor interface {
|
||||
// ProcessEvent processes an incoming event and returns correlated logs.
|
||||
ProcessEvent(event *domain.NormalizedEvent) []domain.CorrelatedLog
|
||||
|
||||
// Flush forces emission of remaining buffered events.
|
||||
Flush() []domain.CorrelatedLog
|
||||
|
||||
// EmitPendingOrphans emits orphan A events whose delay has expired.
|
||||
// Called periodically by the Orchestrator ticker so orphans are not blocked
|
||||
// waiting for the next incoming event.
|
||||
EmitPendingOrphans() []domain.CorrelatedLog
|
||||
|
||||
// GetBufferSizes returns the current buffer sizes for monitoring.
|
||||
GetBufferSizes() (int, int)
|
||||
}
|
||||
@ -1,34 +0,0 @@
|
||||
[Unit]
|
||||
Description=logcorrelator service
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=logcorrelator
|
||||
Group=logcorrelator
|
||||
ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml
|
||||
ExecReload=/bin/kill -HUP $MAINPID
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
|
||||
# Runtime directory: systemd crée /run/logcorrelator (= /var/run/logcorrelator)
|
||||
# avec le bon propriétaire (logcorrelator:logcorrelator) à chaque démarrage/restart,
|
||||
# ce qui évite que les sockets se retrouvent en root:root après un reboot (tmpfs vidé).
|
||||
RuntimeDirectory=logcorrelator
|
||||
RuntimeDirectoryMode=0755
|
||||
|
||||
# Security hardening
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
ReadWritePaths=/var/log/logcorrelator /etc/logcorrelator
|
||||
|
||||
# Resource limits
|
||||
LimitNOFILE=65536
|
||||
|
||||
# Systemd timeouts
|
||||
TimeoutStartSec=10
|
||||
TimeoutStopSec=30
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@ -1,366 +0,0 @@
|
||||
# logcorrelator RPM spec file
|
||||
# Compatible with CentOS 7, Rocky Linux 8, 9, 10
|
||||
# Built with rpmbuild (not FPM)
|
||||
|
||||
Name: logcorrelator
|
||||
Version: %{spec_version}
|
||||
Release: 1%{?dist}
|
||||
Summary: Log correlation service for HTTP and network events
|
||||
|
||||
License: MIT
|
||||
URL: https://github.com/logcorrelator/logcorrelator
|
||||
Vendor: logcorrelator <dev@example.com>
|
||||
Packager: logcorrelator <dev@example.com>
|
||||
|
||||
BuildArch: x86_64
|
||||
|
||||
# Version macro with safe fallback
|
||||
%if %{defined build_version}
|
||||
%define spec_version %{build_version}
|
||||
%else
|
||||
%define spec_version 1.1.22
|
||||
%endif
|
||||
|
||||
# Dependencies
|
||||
Requires: systemd
|
||||
Requires(pre): shadow-utils
|
||||
Requires(post): systemd
|
||||
Requires(preun): systemd
|
||||
Requires(postun): systemd
|
||||
BuildRequires: systemd-rpm-macros
|
||||
|
||||
%description
|
||||
logcorrelator est un service système écrit en Go qui reçoit deux flux de logs JSON
|
||||
via des sockets Unix, corrèle les événements HTTP applicatifs avec des événements
|
||||
réseau, et produit des logs corrélés en temps réel vers ClickHouse et/ou fichier local.
|
||||
|
||||
Notes de sécurité :
|
||||
- Le service s'exécute sous l'utilisateur logcorrelator (non-root)
|
||||
- Les sockets Unix sont créés avec des permissions 0666 (world read/write)
|
||||
- Les répertoires critiques sont protégés : /var/log (750), /var/lib (750), /etc (750)
|
||||
- /var/run/logcorrelator est en 755 pour permettre la création de sockets
|
||||
|
||||
%prep
|
||||
# Files are already in BUILD directory (copied by build-rpm.sh)
|
||||
# No extraction needed
|
||||
echo "Files available in BUILD directory:"
|
||||
ls -la %{_builddir}/
|
||||
|
||||
%install
|
||||
# Create directory structure in buildroot
|
||||
mkdir -p %{buildroot}/usr/bin
|
||||
mkdir -p %{buildroot}/etc/logcorrelator
|
||||
mkdir -p %{buildroot}/var/log/logcorrelator
|
||||
mkdir -p %{buildroot}/var/lib/logcorrelator
|
||||
mkdir -p %{buildroot}%{_unitdir}
|
||||
mkdir -p %{buildroot}/etc/logrotate.d
|
||||
|
||||
# Install binary (from BUILD directory)
|
||||
install -m 0755 %{_builddir}/usr/bin/logcorrelator %{buildroot}/usr/bin/logcorrelator
|
||||
|
||||
# Install config files
|
||||
install -m 0640 %{_builddir}/etc/logcorrelator/logcorrelator.yml %{buildroot}/etc/logcorrelator/logcorrelator.yml
|
||||
install -m 0640 %{_builddir}/etc/logcorrelator/logcorrelator.yml.example %{buildroot}/etc/logcorrelator/logcorrelator.yml.example
|
||||
|
||||
# Install systemd service — destination is %{_unitdir} (/usr/lib/systemd/system), never /etc/
|
||||
install -m 0644 %{_builddir}/etc/systemd/system/logcorrelator.service %{buildroot}%{_unitdir}/logcorrelator.service
|
||||
|
||||
# Install logrotate config
|
||||
install -m 0644 %{_builddir}/etc/logrotate.d/logcorrelator %{buildroot}/etc/logrotate.d/logcorrelator
|
||||
|
||||
%pre
|
||||
# Create service user and group before file installation so %attr() works correctly
|
||||
if ! getent group logcorrelator >/dev/null 2>&1; then
|
||||
groupadd --system logcorrelator
|
||||
fi
|
||||
if ! getent passwd logcorrelator >/dev/null 2>&1; then
|
||||
useradd --system \
|
||||
--gid logcorrelator \
|
||||
--home-dir /var/lib/logcorrelator \
|
||||
--no-create-home \
|
||||
--shell /usr/sbin/nologin \
|
||||
logcorrelator
|
||||
fi
|
||||
|
||||
%post
|
||||
# Use standard systemd RPM macros (daemon-reload, preset, no-op in containers)
|
||||
%systemd_post logcorrelator.service
|
||||
# Explicitly enable+start on fresh install
|
||||
if [ $1 -eq 1 ] && [ -x /usr/bin/systemctl ] && [ -d /run/systemd/system ]; then
|
||||
/usr/bin/systemctl enable logcorrelator.service 2>/dev/null || :
|
||||
/usr/bin/systemctl start logcorrelator.service 2>/dev/null || :
|
||||
fi
|
||||
|
||||
# Set ownership and permissions (idempotent — safe on both install and upgrade)
|
||||
chown -R logcorrelator:logcorrelator /var/lib/logcorrelator 2>/dev/null || true
|
||||
chown -R logcorrelator:logcorrelator /var/log/logcorrelator 2>/dev/null || true
|
||||
chown -R logcorrelator:logcorrelator /etc/logcorrelator 2>/dev/null || true
|
||||
chmod 750 /var/lib/logcorrelator 2>/dev/null || true
|
||||
chmod 750 /var/log/logcorrelator 2>/dev/null || true
|
||||
chmod 750 /etc/logcorrelator 2>/dev/null || true
|
||||
|
||||
# Copy default config if not exists
|
||||
if [ ! -f /etc/logcorrelator/logcorrelator.yml ]; then
|
||||
cp /etc/logcorrelator/logcorrelator.yml.example /etc/logcorrelator/logcorrelator.yml
|
||||
chown logcorrelator:logcorrelator /etc/logcorrelator/logcorrelator.yml
|
||||
chmod 640 /etc/logcorrelator/logcorrelator.yml
|
||||
fi
|
||||
|
||||
%preun
|
||||
%systemd_preun logcorrelator.service
|
||||
|
||||
%postun
|
||||
%systemd_postun_with_restart logcorrelator.service
|
||||
|
||||
%files
|
||||
/usr/bin/logcorrelator
|
||||
%config(noreplace) /etc/logcorrelator/logcorrelator.yml
|
||||
/etc/logcorrelator/logcorrelator.yml.example
|
||||
%dir /var/log/logcorrelator
|
||||
%dir /var/lib/logcorrelator
|
||||
%{_unitdir}/logcorrelator.service
|
||||
%config(noreplace) /etc/logrotate.d/logcorrelator
|
||||
|
||||
%changelog
|
||||
* Wed Mar 11 2026 logcorrelator <dev@example.com> - 1.1.22-1
|
||||
- Feat(outputs): file output enabled/disabled toggle
|
||||
Ajout du champ enabled: true/false dans outputs.file de la configuration.
|
||||
Le sink fichier n'est cree que si enabled: true ET path: defini.
|
||||
Permet de desactiver completement la sortie fichier tout en gardant stdout/clickhouse.
|
||||
Tests: TestValidate_FileOutputDisabled, TestLoadConfig_FileOutputDisabled
|
||||
|
||||
- Fix(systemd): arret immediat sans vidage de queue
|
||||
orchestrator.Stop() ne vide plus les buffers (events en transit perdus).
|
||||
Suppression de ShutdownTimeout et de la logique de flush/attente.
|
||||
systemd TimeoutStopSec=30 gere l'arret force si besoin.
|
||||
Simplification: cancel() + Close() uniquement.
|
||||
|
||||
- Feat(sql): TTL et compression ZSTD sur tables ClickHouse
|
||||
http_logs_raw: TTL 1 jour, compression ZSTD sur raw_json
|
||||
http_logs: TTL 7 jours, compression ZSTD sur champs texte volumineux
|
||||
Parametre ttl_only_drop_parts = 1 pour optimiser les suppressions
|
||||
|
||||
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.21-1
|
||||
- Update: vues ClickHouse et schema SQL
|
||||
Ajout de bots.sql pour l'identification des bots (User-Agent parsing)
|
||||
Ajout de tables.sql pour les tables de reference
|
||||
Mise a jour de mv1.sql (vue materialisee) avec nouvelle structure de correlation
|
||||
Documentation views.md enrichie avec exemples de requetes et schema complet
|
||||
|
||||
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.20-1
|
||||
- Fix(rpm): suppression de systemd-tmpfiles.conf redondant
|
||||
RuntimeDirectory=logcorrelator dans le service systemd gere deja /run/logcorrelator
|
||||
automatiquement. La commande systemd-tmpfiles --create causait des erreurs sur
|
||||
les systemes avec /var/lib/mysql existant (fichier au lieu de repertoire).
|
||||
Suppression de /usr/lib/tmpfiles.d/logcorrelator.conf et de systemd-tmpfiles --create.
|
||||
|
||||
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.19-1
|
||||
- Fix(systemd): stop/restart immediat sans attendre vidage queue
|
||||
L'arret du service ne vide plus les buffers (events en transit perdus).
|
||||
systemd TimeoutStopSec=30 gere deja l'arret force si besoin.
|
||||
Simplification de orchestrator.Stop() : cancel() + Close() uniquement.
|
||||
Suppression de ShutdownTimeout devenu inutile.
|
||||
|
||||
* Mon Mar 09 2026 logcorrelator <dev@example.com> - 1.1.18-1
|
||||
- Fix(outputs): file output enabled: false ne coupait pas l ecriture du fichier
|
||||
Le champ Enabled manquait dans FileOutputConfig. Le sink fichier etait cree
|
||||
meme avec enabled: false tant que path etait defini. Desormais, la condition
|
||||
verifie explicitement enabled && path != "" dans main.go et Validate().
|
||||
Test: TestValidate_FileOutputDisabled et TestLoadConfig_FileOutputDisabled ajoutes.
|
||||
|
||||
* Fri Mar 06 2026 logcorrelator <dev@example.com> - 1.1.17-1
|
||||
- Fix(correlation): champ keepalives non peuple dans ClickHouse
|
||||
Le champ KeepAliveSeq de NormalizedEvent n'etait pas transfere dans les Fields
|
||||
de CorrelatedLog. La vue materialisee ClickHouse extrayait keepalives du JSON
|
||||
mais trouvait toujours 0. Desormais, NewCorrelatedLog et NewCorrelatedLogFromEvent
|
||||
ajoutent explicitement keepalives = KeepAliveSeq dans les Fields.
|
||||
|
||||
* Fri Mar 06 2026 logcorrelator <dev@example.com> - 1.1.16-1
|
||||
- Feat(correlation): emettre les evenements A filtrés par include_dest_ports vers ClickHouse
|
||||
Quand un evenement A (HTTP) etait exclu par le filtre include_dest_ports, il etait
|
||||
silencieusement ignore. Desormais, si ApacheAlwaysEmit=true, l evenement est emis comme
|
||||
non-correle (orphan_side=A) afin d apparaitre dans ClickHouse. Les evenements B restent
|
||||
ignores. Test: TestCorrelationService_IncludeDestPorts_FilteredPort mis a jour +
|
||||
TestCorrelationService_IncludeDestPorts_FilteredPort_NoAlwaysEmit ajoute.
|
||||
|
||||
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.15-1
|
||||
- Fix(correlation/bug3): perte de donnees quand B expire avec des orphelins en attente
|
||||
cleanNetworkBufferByTTL supprimait les pendingOrphans sans les emettre (perte silencieuse).
|
||||
Desormais, les orphelins A sont retournes immediatement a l'appelant quand B expire,
|
||||
et cleanExpired/ProcessEvent propagent ces resultats vers le sink.
|
||||
Test: TestBTTLExpiry_PurgesPendingOrphans etendu pour verifier l'emission effective.
|
||||
|
||||
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.14-1
|
||||
- Fix(correlation/bug1): Keep-Alive sessions au-dela de TimeWindow ne correlent plus en orphelins
|
||||
Le matcher dans processSourceA utilisait eventsMatch (comparaison de timestamps) en mode
|
||||
one_to_many. Apres ~10s, B.Timestamp_original depasse la TimeWindow et toutes les requetes
|
||||
suivantes devenaient orphelines. Nouveau matcher bEventHasValidTTL : un B event est valide
|
||||
tant que son TTL n'a pas expire (le TTL est reset a chaque correlation Keep-Alive).
|
||||
- Fix(correlation/bug4): checkPendingOrphansForCorrelation utilisait eventsMatch (meme bug)
|
||||
En mode one_to_many, un B arrivant avec un vieux timestamp ne matchait plus les pending orphans
|
||||
pour la meme cle. Remplace par une verification de cle uniquement (meme cle = meme connexion).
|
||||
- Fix(correlation/bug3): pendingOrphans non purges quand le B expire (cleanNetworkBufferByTTL)
|
||||
Quand un B event expirait (TTL), les pending orphan A associes etaient bloques indefiniment.
|
||||
Ils sont desormais emis immediatement lors de l'expiration du B correspondant.
|
||||
- Fix(correlation/bug2): orphans emis uniquement sur reception d'evenement (pas de timer dedie)
|
||||
EmitPendingOrphans() est maintenant une methode publique thread-safe. L'Orchestrateur
|
||||
demarre un goroutine ticker (250ms) qui appelle EmitPendingOrphans() independamment du flux,
|
||||
garantissant l'emission meme en l'absence de nouveaux evenements.
|
||||
- Feat(ports): ajout de EmitPendingOrphans() dans l'interface CorrelationProcessor
|
||||
- Test: 4 nouveaux tests de non-regression (Bug #1, #2, #3, #4)
|
||||
|
||||
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.13-1
|
||||
- Fix: Unix sockets ne passent plus en root:root lors des restarts du service
|
||||
- Fix: Ajout de RuntimeDirectory=logcorrelator dans le service systemd (systemd gère /run/logcorrelator avec le bon propriétaire à chaque démarrage/restart)
|
||||
- Fix: Ajout de /usr/lib/tmpfiles.d/logcorrelator.conf pour recréer /run/logcorrelator au boot
|
||||
- Chore: Retrait de /var/run/logcorrelator du RPM %files (géré par tmpfiles.d)
|
||||
- Fix(correlation): emitPendingOrphans - corruption de slice lors de l expiration simultanée de plusieurs orphelins pour la même clé (slice aliasing bug, émissions en double)
|
||||
- Fix(correlation): rotateOldestA - l événement rotaté était perdu silencieusement même avec ApacheAlwaysEmit=true (retourne désormais le CorrelatedLog)
|
||||
- Fix(correlation): Keep-Alive cassé dans le chemin pending-orphan-then-B - le B event n était pas bufferisé en mode one_to_many, bloquant la corrélation des requêtes A2+ du même Keep-Alive
|
||||
- Chore(correlation): suppression du champ mort timer *time.Timer dans pendingOrphan
|
||||
- Feat(correlation): ajout de keepalive_seq dans les logs orphelins pour faciliter le debug (numéro de requête dans la connexion Keep-Alive, 1-based)
|
||||
- Test: 4 nouveaux tests de non-régression pour les bugs de corrélation
|
||||
|
||||
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.12-1
|
||||
- Feat: New config directive include_dest_ports - restrict correlation to specific destination ports
|
||||
- Feat: If include_dest_ports is non-empty, events on unlisted ports are silently ignored (not correlated, not emitted as orphan)
|
||||
- Feat: New metric failed_dest_port_filtered for monitoring filtered traffic
|
||||
- Feat: Debug log for filtered events: "event excluded by dest port filter: source=A dst_port=22"
|
||||
- Test: New unit tests for include_dest_ports (allowed port, filtered port, empty=all)
|
||||
- Docs: README.md updated with include_dest_ports section and current version references
|
||||
- Docs: architecture.yml updated with include_dest_ports
|
||||
- Fix: config.example.yml - removed obsolete stdout.level field
|
||||
|
||||
* Thu Mar 05 2026 logcorrelator <dev@example.com> - 1.1.11-1
|
||||
- Fix: StdoutSink no longer writes correlated/orphan JSON to stdout
|
||||
- Fix: stdout sink is now a no-op for data; operational logs go to stderr via logger
|
||||
- Fix: ClickHouse sink had no logger - all flush errors were silently discarded
|
||||
- Fix: Periodic, batch and final-close flush errors are now logged at ERROR level
|
||||
- Fix: Buffer overflow with DropOnOverflow=true is now logged at WARN level
|
||||
- Fix: Retry attempts are now logged at WARN level with attempt number, delay and error
|
||||
- Feat: ClickHouse connection success logged at INFO (table, batch_size, flush_interval_ms)
|
||||
- Feat: Successful batch sends logged at DEBUG (rows count, table)
|
||||
- Feat: SetLogger() method added to ClickHouseSink for external logger injection
|
||||
- Test: New unit tests for StdoutSink asserting stdout remains empty for all log types
|
||||
|
||||
* Wed Mar 04 2026 logcorrelator <dev@example.com> - 1.1.10-1
|
||||
- Feat: IP exclusion filter - exclude specific source IPs or CIDR ranges
|
||||
- Feat: Configuration exclude_source_ips supports single IPs and CIDR notation
|
||||
- Feat: Debug logging for excluded IPs
|
||||
- Feat: New metric failed_ip_excluded for monitoring filtered traffic
|
||||
- Feat: Architecture documentation updated with observability section
|
||||
- Use cases: exclude health checks, internal traffic, known bad actors
|
||||
- Docs: README.md updated with IP exclusion documentation
|
||||
- Docs: architecture.yml updated with metrics and troubleshooting guide
|
||||
|
||||
* Wed Mar 04 2026 logcorrelator <dev@example.com> - 1.1.9-1
|
||||
- Feat: Debug logging - detailed DEBUG logs for correlation troubleshooting
|
||||
- Feat: Correlation metrics server (HTTP endpoint /metrics and /health)
|
||||
- Feat: New metrics: events_received, correlations_success/failed, failure reasons
|
||||
- Feat: Failure reason tracking: no_match_key, time_window, buffer_eviction, ttl_expired
|
||||
- Feat: Buffer size monitoring (buffer_a_size, buffer_b_size)
|
||||
- Feat: Orphan tracking (orphans_emitted, orphans_pending, pending_orphan_match)
|
||||
- Feat: Keep-Alive reset counter for connection tracking
|
||||
- Feat: Test scripts added (test-correlation.sh, test-correlation-advanced.py)
|
||||
- Change: Config example updated with metrics section
|
||||
- Docs: README.md updated with debugging guide and troubleshooting table
|
||||
|
||||
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.8-1
|
||||
- Migrated from FPM to rpmbuild (native RPM build)
|
||||
- Reduced build image size by 200MB (-40%)
|
||||
- Removed FPM gem dependency (use rpmbuild directly)
|
||||
- Scripts post/preun/postun now inline in spec file
|
||||
- Build image: rockylinux:8 instead of ruby:3.2-bookworm
|
||||
|
||||
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.7-1
|
||||
- Fix: Critical Keep-Alive bug - network events evicted based on original timestamp instead of reset TTL
|
||||
- Fix: Correlation time window increased from 1s to 10s for HTTP Keep-Alive support
|
||||
- Fix: Network source now uses payload timestamp if available (fallback to reception time)
|
||||
- Change: Default network TTL increased from 30s to 120s for long Keep-Alive sessions
|
||||
- Test: Added comprehensive Keep-Alive tests (TTL reset, long session scenarios)
|
||||
|
||||
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.6-1
|
||||
- Docs: Update ClickHouse schema documentation (http_logs_raw + http_logs tables)
|
||||
- Fix: ClickHouse insertion uses single raw_json column (FORMAT JSONEachRow)
|
||||
- Fix: ClickHouse native API (clickhouse-go/v2 PrepareBatch + Append + Send)
|
||||
|
||||
* Tue Mar 03 2026 logcorrelator <dev@example.com> - 1.1.5-1
|
||||
- Fix: ClickHouse insertion using native clickhouse-go/v2 API (PrepareBatch + Append + Send)
|
||||
- Fix: Replaced database/sql wrapper with clickhouse.Open() and clickhouse.Conn
|
||||
- Fix: Proper batch sending to avoid ATTEMPT_TO_READ_AFTER_EOF errors
|
||||
- Fix: Set correct permissions (755) on /var/run/logcorrelator in RPM post-install
|
||||
|
||||
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.4-1
|
||||
- Fix: Log raw JSON data on parse errors for debugging
|
||||
|
||||
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.3-1
|
||||
- Refactor: Switch Unix sockets from STREAM to DGRAM mode (SOCK_DGRAM)
|
||||
- Test: Comprehensive tests added - coverage improved to 74.4%
|
||||
- Fix: Example config file installed to /etc/logcorrelator/logcorrelator.yml.example
|
||||
- Change: Default socket permissions from 0660 to 0666 (world read/write)
|
||||
|
||||
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.2-1
|
||||
- Fix: Example config file installed to /etc/logcorrelator/logcorrelator.yml.example
|
||||
- Change: Default socket permissions from 0660 to 0666 (world read/write)
|
||||
|
||||
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.1-1
|
||||
- Fix: Move logcorrelator.yml.example from /usr/share/logcorrelator/ to /etc/logcorrelator/
|
||||
|
||||
* Mon Mar 02 2026 logcorrelator <dev@example.com> - 1.1.0-1
|
||||
- Feat: Keep-Alive support (one-to-many correlation mode)
|
||||
- Feat: Dynamic TTL for network events (source B)
|
||||
- Feat: Separate buffer sizes for HTTP and network events
|
||||
- Feat: SIGHUP signal handling for log rotation
|
||||
- Feat: File sink Reopen() method for log rotation
|
||||
- Feat: logrotate configuration included
|
||||
- Feat: ExecReload added to systemd service
|
||||
- Feat: New YAML config structure (time_window, orphan_policy, matching, buffers, ttl)
|
||||
- Docs: Updated architecture.yml and config.example.yml
|
||||
|
||||
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.7-1
|
||||
- Added: Log levels DEBUG, INFO, WARN, ERROR configurable via log.level
|
||||
- Added: Warn and Warnf methods for warning messages
|
||||
- Added: Debug logs for events received from sockets and correlations
|
||||
- Added: Warning logs for orphan events and buffer overflow
|
||||
- Changed: Configuration log.enabled replaced by log.level
|
||||
- Changed: Orphan events and buffer overflow now logged as WARN instead of DEBUG
|
||||
|
||||
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.6-1
|
||||
- Changed: Configuration YAML simplified, removed service.name, service.language
|
||||
- Changed: Correlation config simplified, time_window_s instead of nested object
|
||||
- Changed: Orphan policy simplified to emit_orphans boolean
|
||||
- Changed: Apache socket renamed to http.socket
|
||||
- Added: socket_permissions option on unix sockets
|
||||
|
||||
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.5-1
|
||||
- Added: Systemd service auto-start after RPM installation
|
||||
- Added: Systemd service hardening (TimeoutStartSec, TimeoutStopSec, ReadWritePaths)
|
||||
- Fixed: Systemd service unit correct config path (.yml instead of .conf)
|
||||
- Fixed: CI workflow branch name main to master
|
||||
- Changed: RPM packaging generic el8/el9/el10 directory naming
|
||||
|
||||
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.4-1
|
||||
- Breaking: Flattened JSON output structure - removed apache and network subdivisions
|
||||
- All log fields now merged into single-level JSON structure
|
||||
- ClickHouse schema: replaced apache JSON and network JSON columns with fields JSON column
|
||||
- Custom MarshalJSON() implementation for flat output
|
||||
|
||||
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.3-1
|
||||
- Fix: Added missing ClickHouse driver dependency
|
||||
- Fix: Fixed race condition in orchestrator
|
||||
- Security: Added explicit source_type configuration for Unix socket sources
|
||||
- Added: Comprehensive test suite improvements
|
||||
- Added: Test coverage improved from 50.6% to 62.0%
|
||||
|
||||
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.2-1
|
||||
- Added: Initial RPM packaging support for Rocky Linux 8/9 and AlmaLinux 10
|
||||
- Added: Docker multi-stage build pipeline
|
||||
- Added: Hexagonal architecture implementation
|
||||
- Added: Unix socket input sources (JSON line protocol)
|
||||
- Added: File output sink (JSON lines)
|
||||
- Added: ClickHouse output sink with batching and retry logic
|
||||
- Added: Time-window based correlation on src_ip + src_port
|
||||
- Added: Graceful shutdown with signal handling (SIGINT, SIGTERM)
|
||||
|
||||
* Sat Feb 28 2026 logcorrelator <dev@example.com> - 1.0.1-1
|
||||
- Initial package for CentOS 7, Rocky Linux 8, 9, 10
|
||||
@ -1,13 +0,0 @@
|
||||
/var/log/logcorrelator/correlated.log {
|
||||
daily
|
||||
rotate 7
|
||||
compress
|
||||
delaycompress
|
||||
missingok
|
||||
notifempty
|
||||
create 0640 logcorrelator logcorrelator
|
||||
sharedscripts
|
||||
postrotate
|
||||
/bin/systemctl reload logcorrelator > /dev/null 2>&1 || true
|
||||
endscript
|
||||
}
|
||||
@ -1,271 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Test script for logcorrelator RPM package
|
||||
# Verifies installation, permissions, and service status
|
||||
#
|
||||
# Usage: ./packaging/test/test-rpm.sh [el8|el9|el10]
|
||||
#
|
||||
# This script tests the RPM package in a Docker container to ensure:
|
||||
# - Installation succeeds
|
||||
# - File permissions are correct
|
||||
# - Service starts properly
|
||||
# - Sockets are created with correct ownership
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")"
|
||||
RPM_DIR="${PROJECT_ROOT}/dist/rpm"
|
||||
|
||||
# Default to el8 if no argument provided
|
||||
DISTRO="${1:-el8}"
|
||||
|
||||
echo "========================================="
|
||||
echo "Testing logcorrelator RPM for ${DISTRO}"
|
||||
echo "========================================="
|
||||
|
||||
# Find the RPM file
|
||||
case "${DISTRO}" in
|
||||
el8|rocky8)
|
||||
RPM_PATH="${RPM_DIR}/el8"
|
||||
BASE_IMAGE="rockylinux:8"
|
||||
;;
|
||||
el9|rocky9)
|
||||
RPM_PATH="${RPM_DIR}/el9"
|
||||
BASE_IMAGE="rockylinux:9"
|
||||
;;
|
||||
el10|alma10)
|
||||
RPM_PATH="${RPM_DIR}/el10"
|
||||
BASE_IMAGE="almalinux:10"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown distribution: ${DISTRO}"
|
||||
echo "Valid options: el8, el9, el10"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Find the latest RPM file — build it first if missing
|
||||
RPM_FILE=$(ls -t "${RPM_PATH}"/logcorrelator-*.rpm 2>/dev/null | head -n 1)
|
||||
|
||||
if [ -z "${RPM_FILE}" ]; then
|
||||
echo "No RPM found in ${RPM_PATH} — building now via Dockerfile.package..."
|
||||
REPO_ROOT="$(dirname "$(dirname "$(dirname "$SCRIPT_DIR")")")"
|
||||
VERSION="${VERSION:-$(git -C "${REPO_ROOT}" describe --tags --abbrev=0 2>/dev/null | sed 's/^v//' || echo "0.0.0-dev")}"
|
||||
docker build \
|
||||
--build-arg VERSION="${VERSION}" \
|
||||
-f "${REPO_ROOT}/services/correlator/Dockerfile.package" \
|
||||
--target output \
|
||||
-o "${RPM_DIR}" \
|
||||
"${REPO_ROOT}"
|
||||
RPM_FILE=$(ls -t "${RPM_PATH}"/logcorrelator-*.rpm 2>/dev/null | head -n 1)
|
||||
fi
|
||||
|
||||
if [ -z "${RPM_FILE}" ]; then
|
||||
echo "ERROR: RPM build failed — no RPM found in ${RPM_PATH}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Testing RPM: ${RPM_FILE}"
|
||||
echo "Base image: ${BASE_IMAGE}"
|
||||
echo ""
|
||||
|
||||
# Create test script
|
||||
TEST_SCRIPT=$(cat <<'EOF'
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
echo "=== Installing logcorrelator RPM ==="
|
||||
rpm -ivh /tmp/logcorrelator.rpm
|
||||
|
||||
echo ""
|
||||
echo "=== Checking user and group ==="
|
||||
if ! getent group logcorrelator >/dev/null; then
|
||||
echo "FAIL: logcorrelator group not created"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: logcorrelator group exists"
|
||||
|
||||
if ! getent passwd logcorrelator >/dev/null; then
|
||||
echo "FAIL: logcorrelator user not created"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: logcorrelator user exists"
|
||||
|
||||
echo ""
|
||||
echo "=== Checking directory permissions ==="
|
||||
|
||||
# Check /var/run/logcorrelator
|
||||
DIR="/var/run/logcorrelator"
|
||||
if [ ! -d "$DIR" ]; then
|
||||
echo "FAIL: $DIR does not exist"
|
||||
exit 1
|
||||
fi
|
||||
OWNER=$(stat -c '%U:%G' "$DIR")
|
||||
PERMS=$(stat -c '%a' "$DIR")
|
||||
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
|
||||
echo "FAIL: $DIR owner is $OWNER (expected logcorrelator:logcorrelator)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PERMS" != "755" ]; then
|
||||
echo "FAIL: $DIR permissions are $PERMS (expected 755)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: $DIR - owner=$OWNER, permissions=$PERMS"
|
||||
|
||||
# Check /var/log/logcorrelator
|
||||
DIR="/var/log/logcorrelator"
|
||||
if [ ! -d "$DIR" ]; then
|
||||
echo "FAIL: $DIR does not exist"
|
||||
exit 1
|
||||
fi
|
||||
OWNER=$(stat -c '%U:%G' "$DIR")
|
||||
PERMS=$(stat -c '%a' "$DIR")
|
||||
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
|
||||
echo "FAIL: $DIR owner is $OWNER (expected logcorrelator:logcorrelator)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PERMS" != "750" ]; then
|
||||
echo "FAIL: $DIR permissions are $PERMS (expected 750)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: $DIR - owner=$OWNER, permissions=$PERMS"
|
||||
|
||||
# Check /var/lib/logcorrelator
|
||||
DIR="/var/lib/logcorrelator"
|
||||
if [ ! -d "$DIR" ]; then
|
||||
echo "FAIL: $DIR does not exist"
|
||||
exit 1
|
||||
fi
|
||||
OWNER=$(stat -c '%U:%G' "$DIR")
|
||||
PERMS=$(stat -c '%a' "$DIR")
|
||||
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
|
||||
echo "FAIL: $DIR owner is $OWNER (expected logcorrelator:logcorrelator)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PERMS" != "750" ]; then
|
||||
echo "FAIL: $DIR permissions are $PERMS (expected 750)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: $DIR - owner=$OWNER, permissions=$PERMS"
|
||||
|
||||
echo ""
|
||||
echo "=== Checking config files ==="
|
||||
|
||||
# Check config file exists and has correct permissions
|
||||
CONFIG="/etc/logcorrelator/logcorrelator.yml"
|
||||
if [ ! -f "$CONFIG" ]; then
|
||||
echo "FAIL: $CONFIG does not exist"
|
||||
exit 1
|
||||
fi
|
||||
OWNER=$(stat -c '%U:%G' "$CONFIG")
|
||||
PERMS=$(stat -c '%a' "$CONFIG")
|
||||
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
|
||||
echo "FAIL: $CONFIG owner is $OWNER (expected logcorrelator:logcorrelator)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PERMS" != "640" ]; then
|
||||
echo "FAIL: $CONFIG permissions are $PERMS (expected 640)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: $CONFIG - owner=$OWNER, permissions=$PERMS"
|
||||
|
||||
# Check example config file
|
||||
EXAMPLE_CONFIG="/etc/logcorrelator/logcorrelator.yml.example"
|
||||
if [ ! -f "$EXAMPLE_CONFIG" ]; then
|
||||
echo "FAIL: $EXAMPLE_CONFIG does not exist"
|
||||
exit 1
|
||||
fi
|
||||
OWNER=$(stat -c '%U:%G' "$EXAMPLE_CONFIG")
|
||||
PERMS=$(stat -c '%a' "$EXAMPLE_CONFIG")
|
||||
if [ "$OWNER" != "logcorrelator:logcorrelator" ]; then
|
||||
echo "FAIL: $EXAMPLE_CONFIG owner is $OWNER (expected logcorrelator:logcorrelator)"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PERMS" != "640" ]; then
|
||||
echo "FAIL: $EXAMPLE_CONFIG permissions are $PERMS (expected 640)"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: $EXAMPLE_CONFIG - owner=$OWNER, permissions=$PERMS"
|
||||
|
||||
echo ""
|
||||
echo "=== Checking systemd service ==="
|
||||
# Packaged units must be in /usr/lib/systemd/system, not /etc/systemd/system
|
||||
if [ ! -f /usr/lib/systemd/system/logcorrelator.service ]; then
|
||||
echo "FAIL: systemd service file not found at /usr/lib/systemd/system/logcorrelator.service"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: systemd service file exists at /usr/lib/systemd/system/"
|
||||
|
||||
echo ""
|
||||
echo "=== Checking logrotate config ==="
|
||||
if [ ! -f /etc/logrotate.d/logcorrelator ]; then
|
||||
echo "FAIL: logrotate config not found"
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: logrotate config exists"
|
||||
|
||||
echo ""
|
||||
echo "=== Testing service start ==="
|
||||
# Try to start the service (may fail in container without full systemd)
|
||||
if command -v systemctl >/dev/null 2>&1; then
|
||||
systemctl daemon-reload || true
|
||||
if systemctl start logcorrelator.service 2>/dev/null; then
|
||||
echo "OK: service started successfully"
|
||||
|
||||
# Wait for sockets to be created
|
||||
sleep 2
|
||||
|
||||
echo ""
|
||||
echo "=== Checking sockets ==="
|
||||
HTTP_SOCKET="/var/run/logcorrelator/http.socket"
|
||||
NETWORK_SOCKET="/var/run/logcorrelator/network.socket"
|
||||
|
||||
if [ -S "$HTTP_SOCKET" ]; then
|
||||
OWNER=$(stat -c '%U:%G' "$HTTP_SOCKET")
|
||||
PERMS=$(stat -c '%a' "$HTTP_SOCKET")
|
||||
echo "OK: $HTTP_SOCKET exists - owner=$OWNER, permissions=$PERMS"
|
||||
if [ "$PERMS" != "666" ]; then
|
||||
echo "WARN: socket permissions are $PERMS (expected 666)"
|
||||
fi
|
||||
else
|
||||
echo "WARN: $HTTP_SOCKET not found (service may not have started)"
|
||||
fi
|
||||
|
||||
if [ -S "$NETWORK_SOCKET" ]; then
|
||||
OWNER=$(stat -c '%U:%G' "$NETWORK_SOCKET")
|
||||
PERMS=$(stat -c '%a' "$NETWORK_SOCKET")
|
||||
echo "OK: $NETWORK_SOCKET exists - owner=$OWNER, permissions=$PERMS"
|
||||
if [ "$PERMS" != "666" ]; then
|
||||
echo "WARN: socket permissions are $PERMS (expected 666)"
|
||||
fi
|
||||
else
|
||||
echo "WARN: $NETWORK_SOCKET not found (service may not have started)"
|
||||
fi
|
||||
|
||||
systemctl stop logcorrelator.service || true
|
||||
else
|
||||
echo "WARN: service failed to start (expected in minimal container)"
|
||||
fi
|
||||
else
|
||||
echo "WARN: systemctl not available (minimal container)"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "All tests passed!"
|
||||
echo "========================================="
|
||||
EOF
|
||||
)
|
||||
|
||||
# Run test in Docker container
|
||||
echo "Running tests in Docker container..."
|
||||
echo ""
|
||||
|
||||
docker run --rm \
|
||||
-v "${RPM_FILE}:/tmp/logcorrelator.rpm:ro" \
|
||||
-v "${TEST_SCRIPT}:/test.sh:ro" \
|
||||
"${BASE_IMAGE}" \
|
||||
bash /test.sh
|
||||
|
||||
echo ""
|
||||
echo "Test completed successfully for ${DISTRO}"
|
||||
@ -1,118 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# audit-architecture.sh — Vérifie la conformité de l'architecture du correlateur
|
||||
#
|
||||
# Ce script valide que les composants implémentés (service systemd, packaging RPM,
|
||||
# configuration YAML, sockets Unix, sinks de sortie, logique de corrélation) sont
|
||||
# présents et correctement configurés, conformément aux spécifications d'architecture.
|
||||
#
|
||||
# Usage:
|
||||
# ./audit-architecture.sh
|
||||
# docker run --rm -v $(pwd):/src <image> /src/scripts/audit-architecture.sh
|
||||
#
|
||||
# Prérequis:
|
||||
# - Exécuté depuis le répertoire source /src du correlateur (ou monté en volume)
|
||||
# - Les sources Go doivent être présentes (les checks sont basés sur grep)
|
||||
#
|
||||
# Variables d'environnement: aucune
|
||||
# =============================================================================
|
||||
set -e
|
||||
|
||||
echo "=== AUDIT ARCHITECTURE COMPLIANCE ==="
|
||||
echo ""
|
||||
|
||||
# 1. Runtime - systemd service
|
||||
echo "1. RUNTIME - SYSTEMD SERVICE"
|
||||
if [ -f /src/logcorrelator.service ]; then
|
||||
echo "✅ logcorrelator.service exists"
|
||||
grep -q "ExecStart=/usr/bin/logcorrelator" /src/logcorrelator.service && echo " ✅ ExecStart correct" || echo " ❌ ExecStart incorrect"
|
||||
grep -q "ExecReload=" /src/logcorrelator.service && echo " ✅ ExecReload present" || echo " ❌ ExecReload missing"
|
||||
grep -q "Restart=on-failure" /src/logcorrelator.service && echo " ✅ Restart policy correct" || echo " ❌ Restart policy incorrect"
|
||||
else
|
||||
echo "❌ logcorrelator.service missing"
|
||||
fi
|
||||
|
||||
# Check signal handling in code
|
||||
echo ""
|
||||
grep -r "SIGINT\|SIGTERM\|SIGHUP" /src/cmd/logcorrelator/main.go > /dev/null && echo "✅ Signal handling (SIGINT/SIGTERM/SIGHUP) implemented" || echo "❌ Signal handling missing"
|
||||
|
||||
# 2. Packaging - RPM
|
||||
echo ""
|
||||
echo "2. PACKAGING - RPM"
|
||||
[ -f /src/packaging/rpm/logcorrelator.spec ] && echo "✅ RPM spec file exists" || echo "❌ RPM spec missing"
|
||||
grep -q "fpm" /src/Dockerfile.package && echo "✅ fpm tool used for packaging" || echo "❌ fpm not found"
|
||||
|
||||
# 3. Config - YAML
|
||||
echo ""
|
||||
echo "3. CONFIG - YAML"
|
||||
[ -f /src/config.example.yml ] && echo "✅ config.example.yml exists" || echo "❌ config.example.yml missing"
|
||||
grep -q "log:" /src/config.example.yml && echo " ✅ log section present" || echo " ❌ log section missing"
|
||||
grep -q "inputs:" /src/config.example.yml && echo " ✅ inputs section present" || echo " ❌ inputs section missing"
|
||||
grep -q "outputs:" /src/config.example.yml && echo " ✅ outputs section present" || echo " ❌ outputs section missing"
|
||||
grep -q "correlation:" /src/config.example.yml && echo " ✅ correlation section present" || echo " ❌ correlation section missing"
|
||||
|
||||
# 4. Inputs - Unix datagram sockets
|
||||
echo ""
|
||||
echo "4. INPUTS - UNIX DATAGRAM SOCKETS"
|
||||
grep -q "ListenUnixgram" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ Using ListenUnixgram (SOCK_DGRAM)" || echo "❌ Not using SOCK_DGRAM"
|
||||
grep -q "ReadFromUnix" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ Using ReadFromUnix for datagrams" || echo "❌ Not using ReadFromUnix"
|
||||
grep -q "MaxDatagramSize = 65535" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ max_datagram_bytes = 65535" || echo "❌ max_datagram_bytes incorrect"
|
||||
grep -q "0666" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ Default socket permissions 0666" || echo "❌ Socket permissions not 0666"
|
||||
|
||||
# Check socket paths in config
|
||||
grep -q "http.socket" /src/config.example.yml && echo " ✅ http.socket path configured" || echo " ❌ http.socket path missing"
|
||||
grep -q "network.socket" /src/config.example.yml && echo " ✅ network.socket path configured" || echo " ❌ network.socket path missing"
|
||||
|
||||
# 5. Outputs - Sinks
|
||||
echo ""
|
||||
echo "5. OUTPUTS - SINKS"
|
||||
[ -f /src/internal/adapters/outbound/file/sink.go ] && echo "✅ File sink exists" || echo "❌ File sink missing"
|
||||
[ -f /src/internal/adapters/outbound/clickhouse/sink.go ] && echo "✅ ClickHouse sink exists" || echo "❌ ClickHouse sink missing"
|
||||
[ -f /src/internal/adapters/outbound/multi/sink.go ] && echo "✅ MultiSink exists" || echo "❌ MultiSink missing"
|
||||
|
||||
# Check SIGHUP reopen in file sink
|
||||
grep -q "Reopen" /src/internal/adapters/outbound/file/sink.go && echo " ✅ FileSink.Reopen() for SIGHUP" || echo " ❌ FileSink.Reopen() missing"
|
||||
|
||||
# Check ClickHouse batching
|
||||
grep -q "batch" /src/internal/adapters/outbound/clickhouse/sink.go && echo " ✅ ClickHouse batching implemented" || echo " ❌ ClickHouse batching missing"
|
||||
grep -q "drop_on_overflow\|DropOnOverflow" /src/internal/adapters/outbound/clickhouse/sink.go && echo " ✅ drop_on_overflow implemented" || echo " ❌ drop_on_overflow missing"
|
||||
|
||||
# 6. Correlation
|
||||
echo ""
|
||||
echo "6. CORRELATION"
|
||||
grep -q "src_ip" /src/internal/domain/correlation_service.go && echo "✅ src_ip in correlation key" || echo "❌ src_ip missing"
|
||||
grep -q "src_port" /src/internal/domain/correlation_service.go && echo "✅ src_port in correlation key" || echo "❌ src_port missing"
|
||||
grep -q "MatchingMode" /src/internal/domain/correlation_service.go && echo "✅ MatchingMode (one_to_one/one_to_many) implemented" || echo "❌ MatchingMode missing"
|
||||
grep -q "ApacheAlwaysEmit" /src/internal/domain/correlation_service.go && echo "✅ apache_always_emit orphan policy" || echo "❌ apache_always_emit missing"
|
||||
grep -q "network_ttl\|NetworkTTLS" /src/internal/domain/correlation_service.go && echo "✅ TTL management for network events" || echo "❌ TTL management missing"
|
||||
grep -q "max_http_items\|maxHttpItems\|MaxHTTPItems" /src/internal/domain/correlation_service.go && echo "✅ Buffer limit max_http_items" || echo " ⚠️ Buffer limit naming may differ"
|
||||
grep -q "max_network_items\|maxNetworkItems\|MaxNetworkItems" /src/internal/domain/correlation_service.go && echo "✅ Buffer limit max_network_items" || echo " ⚠️ Buffer limit naming may differ"
|
||||
|
||||
# 7. Schema - Source A and B
|
||||
echo ""
|
||||
echo "7. SCHEMA - SOURCE A AND B"
|
||||
grep -q "timestamp" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ timestamp field for Source A" || echo "❌ timestamp missing for Source A"
|
||||
grep -q "SourceA\|SourceB" /src/internal/domain/event.go && echo "✅ EventSource enum (A/B)" || echo "❌ EventSource enum missing"
|
||||
grep -q "header_" /src/internal/adapters/inbound/unixsocket/source.go && echo "✅ header_* dynamic fields" || echo "❌ header_* fields missing"
|
||||
grep -q "Extra" /src/internal/domain/event.go && echo "✅ Extra fields map" || echo "❌ Extra fields missing"
|
||||
|
||||
# 8. Architecture modules
|
||||
echo ""
|
||||
echo "8. ARCHITECTURE MODULES"
|
||||
[ -d /src/internal/domain ] && echo "✅ internal/domain" || echo "❌ internal/domain missing"
|
||||
[ -d /src/internal/ports ] && echo "✅ internal/ports" || echo "❌ internal/ports missing"
|
||||
[ -d /src/internal/app ] && echo "✅ internal/app" || echo "❌ internal/app missing"
|
||||
[ -d /src/internal/adapters/inbound ] && echo "✅ internal/adapters/inbound" || echo "❌ internal/adapters/inbound missing"
|
||||
[ -d /src/internal/adapters/outbound ] && echo "✅ internal/adapters/outbound" || echo "❌ internal/adapters/outbound missing"
|
||||
[ -d /src/internal/config ] && echo "✅ internal/config" || echo "❌ internal/config missing"
|
||||
[ -d /src/internal/observability ] && echo "✅ internal/observability" || echo "❌ internal/observability missing"
|
||||
[ -d /src/cmd/logcorrelator ] && echo "✅ cmd/logcorrelator" || echo "❌ cmd/logcorrelator missing"
|
||||
|
||||
# 9. Testing
|
||||
echo ""
|
||||
echo "9. TESTING"
|
||||
echo "Running tests with coverage..."
|
||||
cd /src && go test ./... -cover 2>&1 | grep -E "^(ok|FAIL|\?)" || true
|
||||
|
||||
echo ""
|
||||
echo "=== AUDIT COMPLETE ==="
|
||||
@ -1,582 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
test-correlation-advanced.py - Advanced correlation testing tool
|
||||
|
||||
This script provides comprehensive testing for the logcorrelator service,
|
||||
including various scenarios to debug correlation issues.
|
||||
|
||||
Usage:
|
||||
python3 test-correlation-advanced.py [options]
|
||||
|
||||
Requirements:
|
||||
- Python 3.6+
|
||||
- requests library (for metrics): pip install requests
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import socket
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, Optional, Tuple
|
||||
|
||||
try:
|
||||
import requests
|
||||
HAS_REQUESTS = True
|
||||
except ImportError:
|
||||
HAS_REQUESTS = False
|
||||
|
||||
|
||||
class Colors:
|
||||
"""ANSI color codes for terminal output."""
|
||||
BLUE = '\033[0;34m'
|
||||
GREEN = '\033[0;32m'
|
||||
YELLOW = '\033[1;33m'
|
||||
RED = '\033[0;31m'
|
||||
NC = '\033[0m' # No Color
|
||||
BOLD = '\033[1m'
|
||||
|
||||
|
||||
def colorize(text: str, color: str) -> str:
|
||||
"""Wrap text with ANSI color codes."""
|
||||
return f"{color}{text}{Colors.NC}"
|
||||
|
||||
|
||||
def info(text: str):
|
||||
print(colorize(f"[INFO] ", Colors.BLUE) + text)
|
||||
|
||||
|
||||
def success(text: str):
|
||||
print(colorize(f"[OK] ", Colors.GREEN) + text)
|
||||
|
||||
|
||||
def warn(text: str):
|
||||
print(colorize(f"[WARN] ", Colors.YELLOW) + text)
|
||||
|
||||
|
||||
def error(text: str):
|
||||
print(colorize(f"[ERROR] ", Colors.RED) + text)
|
||||
|
||||
|
||||
def debug(text: str, verbose: bool = False):
|
||||
if verbose:
|
||||
print(colorize(f"[DEBUG] ", Colors.BLUE) + text)
|
||||
|
||||
|
||||
class CorrelationTester:
|
||||
"""Main test class for correlation testing."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
http_socket: str = "/var/run/logcorrelator/http.socket",
|
||||
network_socket: str = "/var/run/logcorrelator/network.socket",
|
||||
metrics_url: str = "http://localhost:8080/metrics",
|
||||
verbose: bool = False,
|
||||
skip_metrics: bool = False
|
||||
):
|
||||
self.http_socket = http_socket
|
||||
self.network_socket = network_socket
|
||||
self.metrics_url = metrics_url
|
||||
self.verbose = verbose
|
||||
self.skip_metrics = skip_metrics
|
||||
self.http_sock: Optional[socket.socket] = None
|
||||
self.network_sock: Optional[socket.socket] = None
|
||||
|
||||
def connect(self) -> bool:
|
||||
"""Connect to Unix sockets."""
|
||||
try:
|
||||
# HTTP socket
|
||||
self.http_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
self.http_sock.connect(self.http_socket)
|
||||
debug(f"Connected to HTTP socket: {self.http_socket}", self.verbose)
|
||||
|
||||
# Network socket
|
||||
self.network_sock = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM)
|
||||
self.network_sock.connect(self.network_socket)
|
||||
debug(f"Connected to Network socket: {self.network_socket}", self.verbose)
|
||||
|
||||
return True
|
||||
except FileNotFoundError as e:
|
||||
error(f"Socket not found: {e}")
|
||||
return False
|
||||
except Exception as e:
|
||||
error(f"Connection error: {e}")
|
||||
return False
|
||||
|
||||
def close(self):
|
||||
"""Close socket connections."""
|
||||
if self.http_sock:
|
||||
self.http_sock.close()
|
||||
if self.network_sock:
|
||||
self.network_sock.close()
|
||||
|
||||
def send_http_event(
|
||||
self,
|
||||
src_ip: str,
|
||||
src_port: int,
|
||||
timestamp: int,
|
||||
method: str = "GET",
|
||||
path: str = "/test",
|
||||
host: str = "example.com",
|
||||
extra_headers: Optional[Dict[str, str]] = None
|
||||
) -> Dict[str, Any]:
|
||||
"""Send an HTTP (source A) event."""
|
||||
event = {
|
||||
"src_ip": src_ip,
|
||||
"src_port": src_port,
|
||||
"dst_ip": "10.0.0.1",
|
||||
"dst_port": 443,
|
||||
"timestamp": timestamp,
|
||||
"method": method,
|
||||
"path": path,
|
||||
"host": host,
|
||||
"http_version": "HTTP/1.1",
|
||||
"header_user_agent": "TestAgent/1.0",
|
||||
"header_accept": "*/*"
|
||||
}
|
||||
|
||||
if extra_headers:
|
||||
for key, value in extra_headers.items():
|
||||
event[f"header_{key}"] = value
|
||||
|
||||
json_data = json.dumps(event)
|
||||
|
||||
if self.http_sock:
|
||||
self.http_sock.sendall(json_data.encode())
|
||||
debug(f"Sent HTTP event: {src_ip}:{src_port} ts={timestamp}", self.verbose)
|
||||
|
||||
return event
|
||||
|
||||
def send_network_event(
|
||||
self,
|
||||
src_ip: str,
|
||||
src_port: int,
|
||||
timestamp: int,
|
||||
ja3: str = "abc123",
|
||||
ja4: str = "def456",
|
||||
tls_version: str = "TLS1.3",
|
||||
tls_sni: str = "example.com"
|
||||
) -> Dict[str, Any]:
|
||||
"""Send a Network (source B) event."""
|
||||
event = {
|
||||
"src_ip": src_ip,
|
||||
"src_port": src_port,
|
||||
"dst_ip": "10.0.0.1",
|
||||
"dst_port": 443,
|
||||
"timestamp": timestamp,
|
||||
"ja3": ja3,
|
||||
"ja4": ja4,
|
||||
"tls_version": tls_version,
|
||||
"tls_sni": tls_sni
|
||||
}
|
||||
|
||||
json_data = json.dumps(event)
|
||||
|
||||
if self.network_sock:
|
||||
self.network_sock.sendall(json_data.encode())
|
||||
debug(f"Sent Network event: {src_ip}:{src_port} ts={timestamp}", self.verbose)
|
||||
|
||||
return event
|
||||
|
||||
def get_metrics(self) -> Dict[str, Any]:
|
||||
"""Fetch metrics from the metrics server."""
|
||||
if self.skip_metrics:
|
||||
return {}
|
||||
|
||||
if not HAS_REQUESTS:
|
||||
warn("requests library not installed, skipping metrics")
|
||||
return {}
|
||||
|
||||
try:
|
||||
response = requests.get(self.metrics_url, timeout=5)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
except Exception as e:
|
||||
warn(f"Failed to fetch metrics: {e}")
|
||||
return {}
|
||||
|
||||
def print_metrics(self, metrics: Dict[str, Any], title: str = "Metrics"):
|
||||
"""Print metrics in a formatted way."""
|
||||
if not metrics:
|
||||
return
|
||||
|
||||
print(f"\n{colorize(f'=== {title} ===', Colors.BOLD)}")
|
||||
|
||||
keys_to_show = [
|
||||
("events_received_a", "Events A"),
|
||||
("events_received_b", "Events B"),
|
||||
("correlations_success", "Correlations"),
|
||||
("correlations_failed", "Failures"),
|
||||
("failed_no_match_key", " - No match key"),
|
||||
("failed_time_window", " - Time window"),
|
||||
("failed_buffer_eviction", " - Buffer eviction"),
|
||||
("failed_ttl_expired", " - TTL expired"),
|
||||
("buffer_a_size", "Buffer A size"),
|
||||
("buffer_b_size", "Buffer B size"),
|
||||
("orphans_emitted_a", "Orphans A"),
|
||||
("orphans_emitted_b", "Orphans B"),
|
||||
("pending_orphan_match", "Pending orphan matches"),
|
||||
("keepalive_resets", "Keep-Alive resets"),
|
||||
]
|
||||
|
||||
for key, label in keys_to_show:
|
||||
if key in metrics:
|
||||
print(f" {label}: {metrics[key]}")
|
||||
|
||||
def check_sockets(self) -> bool:
|
||||
"""Check if sockets exist."""
|
||||
import os
|
||||
|
||||
errors = 0
|
||||
for name, path in [("HTTP", self.http_socket), ("Network", self.network_socket)]:
|
||||
if not os.path.exists(path):
|
||||
error(f"{name} socket not found: {path}")
|
||||
errors += 1
|
||||
elif not os.path.exists(path) or not os.path.stat(path).st_mode & 0o170000 == 0o140000:
|
||||
# Check if it's a socket
|
||||
try:
|
||||
if not socket.getaddrinfo(path, None, socket.AF_UNIX):
|
||||
error(f"{name} path exists but is not a socket: {path}")
|
||||
errors += 1
|
||||
except:
|
||||
pass
|
||||
else:
|
||||
debug(f"{name} socket found: {path}", self.verbose)
|
||||
|
||||
return errors == 0
|
||||
|
||||
def run_basic_test(self, count: int = 10, delay_ms: int = 100) -> Tuple[bool, Dict[str, int]]:
|
||||
"""
|
||||
Run basic correlation test.
|
||||
|
||||
Sends N pairs of A+B events with matching src_ip:src_port and timestamps.
|
||||
All should correlate successfully.
|
||||
"""
|
||||
info(f"Running basic correlation test with {count} pairs...")
|
||||
|
||||
# Get initial metrics
|
||||
initial_metrics = self.get_metrics()
|
||||
self.print_metrics(initial_metrics, "Initial Metrics")
|
||||
|
||||
initial_success = initial_metrics.get("correlations_success", 0)
|
||||
initial_failed = initial_metrics.get("correlations_failed", 0)
|
||||
initial_a = initial_metrics.get("events_received_a", 0)
|
||||
initial_b = initial_metrics.get("events_received_b", 0)
|
||||
|
||||
# Send test events
|
||||
print(f"\nSending {count} event pairs...")
|
||||
|
||||
base_timestamp = time.time_ns()
|
||||
sent = 0
|
||||
|
||||
for i in range(1, count + 1):
|
||||
src_ip = f"192.168.1.{(i % 254) + 1}"
|
||||
src_port = 8000 + i
|
||||
|
||||
# Same timestamp for perfect correlation
|
||||
timestamp = base_timestamp + (i * 1_000_000)
|
||||
|
||||
self.send_http_event(src_ip, src_port, timestamp)
|
||||
self.send_network_event(src_ip, src_port, timestamp)
|
||||
|
||||
sent += 1
|
||||
|
||||
if delay_ms > 0:
|
||||
time.sleep(delay_ms / 1000.0)
|
||||
|
||||
success(f"Sent {sent} event pairs")
|
||||
|
||||
# Wait for processing
|
||||
info("Waiting for processing (2 seconds)...")
|
||||
time.sleep(2)
|
||||
|
||||
# Get final metrics
|
||||
final_metrics = self.get_metrics()
|
||||
self.print_metrics(final_metrics, "Final Metrics")
|
||||
|
||||
# Calculate deltas
|
||||
delta_success = final_metrics.get("correlations_success", 0) - initial_success
|
||||
delta_failed = final_metrics.get("correlations_failed", 0) - initial_failed
|
||||
delta_a = final_metrics.get("events_received_a", 0) - initial_a
|
||||
delta_b = final_metrics.get("events_received_b", 0) - initial_b
|
||||
|
||||
results = {
|
||||
"sent": sent,
|
||||
"received_a": delta_a,
|
||||
"received_b": delta_b,
|
||||
"correlations": delta_success,
|
||||
"failures": delta_failed
|
||||
}
|
||||
|
||||
# Print results
|
||||
print(f"\n{colorize('=== Results ===', Colors.BOLD)}")
|
||||
print(f" Events A sent: {delta_a} (expected: {sent})")
|
||||
print(f" Events B sent: {delta_b} (expected: {sent})")
|
||||
print(f" Correlations: {delta_success}")
|
||||
print(f" Failures: {delta_failed}")
|
||||
|
||||
# Validation
|
||||
test_passed = True
|
||||
|
||||
if delta_a != sent:
|
||||
error(f"Event A count mismatch: got {delta_a}, expected {sent}")
|
||||
test_passed = False
|
||||
|
||||
if delta_b != sent:
|
||||
error(f"Event B count mismatch: got {delta_b}, expected {sent}")
|
||||
test_passed = False
|
||||
|
||||
if delta_success != sent:
|
||||
error(f"Correlation count mismatch: got {delta_success}, expected {sent}")
|
||||
test_passed = False
|
||||
|
||||
if delta_failed > 0:
|
||||
warn(f"Unexpected correlation failures: {delta_failed}")
|
||||
|
||||
if test_passed:
|
||||
success("All tests passed! Correlation is working correctly.")
|
||||
else:
|
||||
error("Some tests failed. Check logs for details.")
|
||||
|
||||
return test_passed, results
|
||||
|
||||
def run_time_window_test(self) -> bool:
|
||||
"""Test time window expiration."""
|
||||
info("Running time window test...")
|
||||
|
||||
src_ip = "192.168.100.1"
|
||||
src_port = 9999
|
||||
|
||||
# Send A event
|
||||
ts_a = time.time_ns()
|
||||
self.send_http_event(src_ip, src_port, ts_a)
|
||||
info(f"Sent A event at {ts_a}")
|
||||
|
||||
# Wait for time window to expire (default 10s)
|
||||
info("Waiting 11 seconds (time window should expire)...")
|
||||
time.sleep(11)
|
||||
|
||||
# Send B event
|
||||
ts_b = time.time_ns()
|
||||
self.send_network_event(src_ip, src_port, ts_b)
|
||||
info(f"Sent B event at {ts_b}")
|
||||
|
||||
time_diff_sec = (ts_b - ts_a) / 1_000_000_000
|
||||
info(f"Time difference: {time_diff_sec:.1f} seconds")
|
||||
info("Expected: time_window failure (check metrics)")
|
||||
|
||||
return True
|
||||
|
||||
def run_different_ip_test(self) -> bool:
|
||||
"""Test different IP (should not correlate)."""
|
||||
info("Running different IP test...")
|
||||
|
||||
ts = time.time_ns()
|
||||
|
||||
# Send A with IP 192.168.200.1
|
||||
self.send_http_event("192.168.200.1", 7777, ts)
|
||||
info("Sent A event from 192.168.200.1:7777")
|
||||
|
||||
# Send B with different IP
|
||||
self.send_network_event("192.168.200.2", 7777, ts)
|
||||
info("Sent B event from 192.168.200.2:7777 (different IP)")
|
||||
|
||||
info("Expected: no_match_key failure (different src_ip)")
|
||||
|
||||
return True
|
||||
|
||||
def run_keepalive_test(self, count: int = 5) -> bool:
|
||||
"""Test Keep-Alive mode (one B correlates with multiple A)."""
|
||||
info(f"Running Keep-Alive test with {count} HTTP requests on same connection...")
|
||||
|
||||
src_ip = "192.168.50.1"
|
||||
src_port = 6000
|
||||
|
||||
# Send one B event first (network/TCP connection)
|
||||
ts_b = time.time_ns()
|
||||
self.send_network_event(src_ip, src_port, ts_b)
|
||||
info(f"Sent B event (connection): {src_ip}:{src_port}")
|
||||
|
||||
# Send multiple A events (HTTP requests) on same connection
|
||||
for i in range(count):
|
||||
ts_a = time.time_ns() + (i * 100_000_000) # 100ms apart
|
||||
self.send_http_event(src_ip, src_port, ts_a, path=f"/request{i}")
|
||||
info(f"Sent A event (request {i}): {src_ip}:{src_port}")
|
||||
time.sleep(0.05) # 50ms delay
|
||||
|
||||
time.sleep(2) # Wait for processing
|
||||
|
||||
# Check metrics
|
||||
metrics = self.get_metrics()
|
||||
keepalive_resets = metrics.get("keepalive_resets", 0)
|
||||
|
||||
info(f"Keep-Alive resets: {keepalive_resets} (expected: {count - 1})")
|
||||
|
||||
if keepalive_resets >= count - 1:
|
||||
success("Keep-Alive test passed!")
|
||||
return True
|
||||
else:
|
||||
warn(f"Keep-Alive resets lower than expected. This may be normal depending on timing.")
|
||||
return True
|
||||
|
||||
def run_all_tests(self) -> bool:
|
||||
"""Run all test scenarios."""
|
||||
results = []
|
||||
|
||||
# Basic test
|
||||
passed, _ = self.run_basic_test(count=10)
|
||||
results.append(("Basic correlation", passed))
|
||||
|
||||
print("\n" + "=" * 50 + "\n")
|
||||
|
||||
# Time window test
|
||||
self.run_time_window_test()
|
||||
results.append(("Time window", True)) # Informational
|
||||
|
||||
print("\n" + "=" * 50 + "\n")
|
||||
|
||||
# Different IP test
|
||||
self.run_different_ip_test()
|
||||
results.append(("Different IP", True)) # Informational
|
||||
|
||||
print("\n" + "=" * 50 + "\n")
|
||||
|
||||
# Keep-Alive test
|
||||
self.run_keepalive_test()
|
||||
results.append(("Keep-Alive", True))
|
||||
|
||||
# Summary
|
||||
print(f"\n{colorize('=== Test Summary ===', Colors.BOLD)}")
|
||||
for name, passed in results:
|
||||
status = colorize("PASS", Colors.GREEN) if passed else colorize("FAIL", Colors.RED)
|
||||
print(f" {name}: {status}")
|
||||
|
||||
return all(r[1] for r in results)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Advanced correlation testing tool for logcorrelator",
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter,
|
||||
epilog="""
|
||||
Examples:
|
||||
# Run basic test with 20 pairs
|
||||
python3 test-correlation-advanced.py -c 20
|
||||
|
||||
# Run all tests with verbose output
|
||||
python3 test-correlation-advanced.py --all -v
|
||||
|
||||
# Test with custom socket paths
|
||||
python3 test-correlation-advanced.py -H /tmp/http.sock -N /tmp/network.sock
|
||||
|
||||
# Skip metrics check
|
||||
python3 test-correlation-advanced.py --skip-metrics
|
||||
"""
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"-H", "--http-socket",
|
||||
default="/var/run/logcorrelator/http.socket",
|
||||
help="Path to HTTP Unix socket (default: /var/run/logcorrelator/http.socket)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-N", "--network-socket",
|
||||
default="/var/run/logcorrelator/network.socket",
|
||||
help="Path to Network Unix socket (default: /var/run/logcorrelator/network.socket)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-m", "--metrics-url",
|
||||
default="http://localhost:8080/metrics",
|
||||
help="Metrics server URL (default: http://localhost:8080/metrics)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-c", "--count",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Number of test pairs to send (default: 10)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d", "--delay",
|
||||
type=int,
|
||||
default=100,
|
||||
help="Delay between pairs in milliseconds (default: 100)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-v", "--verbose",
|
||||
action="store_true",
|
||||
help="Enable verbose output"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--skip-metrics",
|
||||
action="store_true",
|
||||
help="Skip metrics check"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="Run all test scenarios"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--time-window",
|
||||
action="store_true",
|
||||
help="Run time window test only"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--different-ip",
|
||||
action="store_true",
|
||||
help="Run different IP test only"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--keepalive",
|
||||
action="store_true",
|
||||
help="Run Keep-Alive test only"
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# Create tester
|
||||
tester = CorrelationTester(
|
||||
http_socket=args.http_socket,
|
||||
network_socket=args.network_socket,
|
||||
metrics_url=args.metrics_url,
|
||||
verbose=args.verbose,
|
||||
skip_metrics=args.skip_metrics
|
||||
)
|
||||
|
||||
# Check sockets
|
||||
if not tester.check_sockets():
|
||||
error("Socket check failed. Is logcorrelator running?")
|
||||
sys.exit(1)
|
||||
|
||||
success("Socket check passed")
|
||||
|
||||
# Connect
|
||||
if not tester.connect():
|
||||
error("Failed to connect to sockets")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
if args.all:
|
||||
success = tester.run_all_tests()
|
||||
elif args.time_window:
|
||||
tester.run_time_window_test()
|
||||
success = True
|
||||
elif args.different_ip:
|
||||
tester.run_different_ip_test()
|
||||
success = True
|
||||
elif args.keepalive:
|
||||
tester.run_keepalive_test()
|
||||
success = True
|
||||
else:
|
||||
_, _ = tester.run_basic_test(count=args.count, delay_ms=args.delay)
|
||||
success = True
|
||||
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
finally:
|
||||
tester.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -1,404 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# test-correlation.sh - Test script for log correlation debugging
|
||||
#
|
||||
# This script sends test HTTP (A) and Network (B) events to the logcorrelator
|
||||
# Unix sockets and verifies that correlation is working correctly.
|
||||
#
|
||||
# Usage:
|
||||
# ./test-correlation.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# -h, --http-socket PATH Path to HTTP socket (default: /var/run/logcorrelator/http.socket)
|
||||
# -n, --network-socket PATH Path to Network socket (default: /var/run/logcorrelator/network.socket)
|
||||
# -c, --count NUM Number of test pairs to send (default: 10)
|
||||
# -d, --delay MS Delay between pairs in milliseconds (default: 100)
|
||||
# -v, --verbose Enable verbose output
|
||||
# -m, --metrics-url URL Metrics server URL (default: http://localhost:8080/metrics)
|
||||
# --skip-metrics Skip metrics check
|
||||
# --help Show this help message
|
||||
#
|
||||
|
||||
set -e
|
||||
|
||||
# Default values
|
||||
HTTP_SOCKET="/var/run/logcorrelator/http.socket"
|
||||
NETWORK_SOCKET="/var/run/logcorrelator/network.socket"
|
||||
COUNT=10
|
||||
DELAY_MS=100
|
||||
VERBOSE=false
|
||||
METRICS_URL="http://localhost:8080/metrics"
|
||||
SKIP_METRICS=false
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Print functions
|
||||
info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $1"
|
||||
}
|
||||
|
||||
success() {
|
||||
echo -e "${GREEN}[OK]${NC} $1"
|
||||
}
|
||||
|
||||
warn() {
|
||||
echo -e "${YELLOW}[WARN]${NC} $1"
|
||||
}
|
||||
|
||||
error() {
|
||||
echo -e "${RED}[ERROR]${NC} $1"
|
||||
}
|
||||
|
||||
verbose() {
|
||||
if [ "$VERBOSE" = true ]; then
|
||||
echo -e "${BLUE}[DEBUG]${NC} $1"
|
||||
fi
|
||||
}
|
||||
|
||||
# Show help
|
||||
show_help() {
|
||||
head -20 "$0" | tail -17 | sed 's/^#//' | sed 's/^ //'
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Parse arguments
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
-h|--http-socket)
|
||||
HTTP_SOCKET="$2"
|
||||
shift 2
|
||||
;;
|
||||
-n|--network-socket)
|
||||
NETWORK_SOCKET="$2"
|
||||
shift 2
|
||||
;;
|
||||
-c|--count)
|
||||
COUNT="$2"
|
||||
shift 2
|
||||
;;
|
||||
-d|--delay)
|
||||
DELAY_MS="$2"
|
||||
shift 2
|
||||
;;
|
||||
-v|--verbose)
|
||||
VERBOSE=true
|
||||
shift
|
||||
;;
|
||||
-m|--metrics-url)
|
||||
METRICS_URL="$2"
|
||||
shift 2
|
||||
;;
|
||||
--skip-metrics)
|
||||
SKIP_METRICS=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
show_help
|
||||
;;
|
||||
*)
|
||||
error "Unknown option: $1"
|
||||
echo "Use --help for usage information"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Check if socat or netcat is available
|
||||
if command -v socat &> /dev/null; then
|
||||
SEND_CMD="socat"
|
||||
elif command -v nc &> /dev/null; then
|
||||
SEND_CMD="nc"
|
||||
else
|
||||
error "Neither socat nor nc (netcat) found. Please install one of them."
|
||||
echo " Ubuntu/Debian: apt-get install socat OR apt-get install netcat-openbsd"
|
||||
echo " RHEL/CentOS: yum install socat OR yum install nc"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Function to send data to Unix socket
|
||||
send_to_socket() {
|
||||
local socket="$1"
|
||||
local data="$2"
|
||||
|
||||
if [ "$SEND_CMD" = "socat" ]; then
|
||||
echo "$data" | socat - "UNIX-SENDTO:$socket" 2>/dev/null
|
||||
else
|
||||
echo "$data" | nc -U -u "$socket" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to generate timestamp in nanoseconds
|
||||
get_timestamp_ns() {
|
||||
date +%s%N
|
||||
}
|
||||
|
||||
# Function to send HTTP (A) event
|
||||
send_http_event() {
|
||||
local src_ip="$1"
|
||||
local src_port="$2"
|
||||
local timestamp="$3"
|
||||
local method="${4:-GET}"
|
||||
local path="${5:-/test}"
|
||||
local host="${6:-example.com}"
|
||||
|
||||
local json=$(cat <<EOF
|
||||
{"src_ip":"$src_ip","src_port":$src_port,"dst_ip":"10.0.0.1","dst_port":443,"timestamp":$timestamp,"method":"$method","path":"$path","host":"$host","http_version":"HTTP/1.1","header_user_agent":"TestAgent/1.0","header_accept":"*/*"}
|
||||
EOF
|
||||
)
|
||||
|
||||
verbose "Sending HTTP event: $json"
|
||||
send_to_socket "$HTTP_SOCKET" "$json"
|
||||
}
|
||||
|
||||
# Function to send Network (B) event
|
||||
send_network_event() {
|
||||
local src_ip="$1"
|
||||
local src_port="$2"
|
||||
local timestamp="$3"
|
||||
local ja3="${4:-abc123}"
|
||||
local ja4="${5:-def456}"
|
||||
|
||||
local json=$(cat <<EOF
|
||||
{"src_ip":"$src_ip","src_port":$src_port,"dst_ip":"10.0.0.1","dst_port":443,"timestamp":$timestamp,"ja3":"$ja3","ja4":"$ja4","tls_version":"TLS1.3","tls_sni":"example.com"}
|
||||
EOF
|
||||
)
|
||||
|
||||
verbose "Sending Network event: $json"
|
||||
send_to_socket "$NETWORK_SOCKET" "$json"
|
||||
}
|
||||
|
||||
# Check sockets exist
|
||||
check_sockets() {
|
||||
local errors=0
|
||||
|
||||
if [ ! -S "$HTTP_SOCKET" ]; then
|
||||
error "HTTP socket not found: $HTTP_SOCKET"
|
||||
errors=$((errors + 1))
|
||||
else
|
||||
verbose "HTTP socket found: $HTTP_SOCKET"
|
||||
fi
|
||||
|
||||
if [ ! -S "$NETWORK_SOCKET" ]; then
|
||||
error "Network socket not found: $NETWORK_SOCKET"
|
||||
errors=$((errors + 1))
|
||||
else
|
||||
verbose "Network socket found: $NETWORK_SOCKET"
|
||||
fi
|
||||
|
||||
if [ $errors -gt 0 ]; then
|
||||
error "$errors socket(s) not found. Is logcorrelator running?"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
success "Sockets check passed"
|
||||
}
|
||||
|
||||
# Get metrics from server
|
||||
get_metrics() {
|
||||
if [ "$SKIP_METRICS" = true ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
if command -v curl &> /dev/null; then
|
||||
curl -s "$METRICS_URL" 2>/dev/null || echo "{}"
|
||||
elif command -v wget &> /dev/null; then
|
||||
wget -qO- "$METRICS_URL" 2>/dev/null || echo "{}"
|
||||
else
|
||||
warn "Neither curl nor wget found. Skipping metrics check."
|
||||
echo "{}"
|
||||
fi
|
||||
}
|
||||
|
||||
# Extract value from JSON (simple grep-based, requires jq for complex queries)
|
||||
get_json_value() {
|
||||
local json="$1"
|
||||
local key="$2"
|
||||
|
||||
if command -v jq &> /dev/null; then
|
||||
echo "$json" | jq -r ".$key // 0"
|
||||
else
|
||||
# Fallback: simple grep (works for flat JSON)
|
||||
echo "$json" | grep -o "\"$key\":[0-9]*" | cut -d: -f2 || echo "0"
|
||||
fi
|
||||
}
|
||||
|
||||
# Main test function
|
||||
run_test() {
|
||||
info "Starting correlation test..."
|
||||
info "Configuration:"
|
||||
echo " HTTP Socket: $HTTP_SOCKET"
|
||||
echo " Network Socket: $NETWORK_SOCKET"
|
||||
echo " Test pairs: $COUNT"
|
||||
echo " Delay between: ${DELAY_MS}ms"
|
||||
echo " Metrics URL: $METRICS_URL"
|
||||
echo " Send command: $SEND_CMD"
|
||||
echo ""
|
||||
|
||||
# Get initial metrics
|
||||
info "Fetching initial metrics..."
|
||||
local initial_metrics=$(get_metrics)
|
||||
local initial_success=$(get_json_value "$initial_metrics" "correlations_success")
|
||||
local initial_failed=$(get_json_value "$initial_metrics" "correlations_failed")
|
||||
local initial_a=$(get_json_value "$initial_metrics" "events_received_a")
|
||||
local initial_b=$(get_json_value "$initial_metrics" "events_received_b")
|
||||
|
||||
info "Initial metrics:"
|
||||
echo " Events A: $initial_a"
|
||||
echo " Events B: $initial_b"
|
||||
echo " Success: $initial_success"
|
||||
echo " Failed: $initial_failed"
|
||||
echo ""
|
||||
|
||||
# Send test events
|
||||
info "Sending $COUNT test event pairs..."
|
||||
|
||||
local base_timestamp=$(get_timestamp_ns)
|
||||
local sent=0
|
||||
local correlated=0
|
||||
|
||||
for i in $(seq 1 $COUNT); do
|
||||
local src_ip="192.168.1.$((i % 254 + 1))"
|
||||
local src_port=$((8000 + i))
|
||||
|
||||
# Send A and B with same timestamp (should correlate)
|
||||
local ts_a=$((base_timestamp + i * 1000000))
|
||||
local ts_b=$ts_a # Same timestamp for perfect correlation
|
||||
|
||||
send_http_event "$src_ip" "$src_port" "$ts_a"
|
||||
send_network_event "$src_ip" "$src_port" "$ts_b"
|
||||
|
||||
sent=$((sent + 1))
|
||||
verbose "Sent pair $i: $src_ip:$src_port"
|
||||
|
||||
if [ $DELAY_MS -gt 0 ]; then
|
||||
sleep $(echo "scale=3; $DELAY_MS / 1000" | bc)
|
||||
fi
|
||||
done
|
||||
|
||||
success "Sent $sent event pairs"
|
||||
echo ""
|
||||
|
||||
# Wait for processing
|
||||
info "Waiting for processing (2 seconds)..."
|
||||
sleep 2
|
||||
|
||||
# Get final metrics
|
||||
info "Fetching final metrics..."
|
||||
local final_metrics=$(get_metrics)
|
||||
local final_success=$(get_json_value "$final_metrics" "correlations_success")
|
||||
local final_failed=$(get_json_value "$final_metrics" "correlations_failed")
|
||||
local final_a=$(get_json_value "$final_metrics" "events_received_a")
|
||||
local final_b=$(get_json_value "$final_metrics" "events_received_b")
|
||||
|
||||
# Calculate deltas
|
||||
local delta_success=$((final_success - initial_success))
|
||||
local delta_failed=$((final_failed - initial_failed))
|
||||
local delta_a=$((final_a - initial_a))
|
||||
local delta_b=$((final_b - initial_b))
|
||||
|
||||
echo ""
|
||||
info "Results:"
|
||||
echo " Events A sent: $delta_a (expected: $sent)"
|
||||
echo " Events B sent: $delta_b (expected: $sent)"
|
||||
echo " Correlations: $delta_success"
|
||||
echo " Failures: $delta_failed"
|
||||
echo ""
|
||||
|
||||
# Validation
|
||||
local test_passed=true
|
||||
|
||||
if [ "$delta_a" -ne "$sent" ]; then
|
||||
error "Event A count mismatch: got $delta_a, expected $sent"
|
||||
test_passed=false
|
||||
fi
|
||||
|
||||
if [ "$delta_b" -ne "$sent" ]; then
|
||||
error "Event B count mismatch: got $delta_b, expected $sent"
|
||||
test_passed=false
|
||||
fi
|
||||
|
||||
if [ "$delta_success" -ne "$sent" ]; then
|
||||
error "Correlation count mismatch: got $delta_success, expected $sent"
|
||||
test_passed=false
|
||||
fi
|
||||
|
||||
if [ "$delta_failed" -ne 0 ]; then
|
||||
warn "Unexpected correlation failures: $delta_failed"
|
||||
fi
|
||||
|
||||
if [ "$test_passed" = true ]; then
|
||||
success "All tests passed! Correlation is working correctly."
|
||||
exit 0
|
||||
else
|
||||
error "Some tests failed. Check the logs for details."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Test with time window exceeded
|
||||
run_time_window_test() {
|
||||
info "Running time window test (B arrives after time window)..."
|
||||
|
||||
local src_ip="192.168.100.1"
|
||||
local src_port="9999"
|
||||
|
||||
# Send A event
|
||||
local ts_a=$(get_timestamp_ns)
|
||||
send_http_event "$src_ip" "$src_port" "$ts_a"
|
||||
info "Sent A event at timestamp $ts_a"
|
||||
|
||||
# Wait for time window to expire (default is 10s, we wait 11s)
|
||||
info "Waiting 11 seconds (time window should expire)..."
|
||||
sleep 11
|
||||
|
||||
# Send B event
|
||||
local ts_b=$(get_timestamp_ns)
|
||||
send_network_event "$src_ip" "$src_port" "$ts_b"
|
||||
info "Sent B event at timestamp $ts_b"
|
||||
|
||||
info "This should result in a time_window failure (check metrics)"
|
||||
}
|
||||
|
||||
# Test with different src_ip
|
||||
run_different_ip_test() {
|
||||
info "Running different IP test (should NOT correlate)..."
|
||||
|
||||
# Send A with IP 192.168.200.1
|
||||
local ts=$(get_timestamp_ns)
|
||||
send_http_event "192.168.200.1" "7777" "$ts"
|
||||
info "Sent A event from 192.168.200.1:7777"
|
||||
|
||||
# Send B with different IP
|
||||
send_network_event "192.168.200.2" "7777" "$ts"
|
||||
info "Sent B event from 192.168.200.2:7777 (different IP)"
|
||||
|
||||
info "These should NOT correlate (different src_ip)"
|
||||
}
|
||||
|
||||
# Run tests
|
||||
check_sockets
|
||||
echo ""
|
||||
|
||||
# Run main test
|
||||
run_test
|
||||
|
||||
echo ""
|
||||
info "Additional tests available:"
|
||||
echo " --test-time-window Test time window expiration"
|
||||
echo " --test-different-ip Test different IP (no correlation)"
|
||||
|
||||
# Check for additional test flags
|
||||
if [[ "$@" == *"--test-time-window"* ]]; then
|
||||
echo ""
|
||||
run_time_window_test
|
||||
fi
|
||||
|
||||
if [[ "$@" == *"--test-different-ip"* ]]; then
|
||||
echo ""
|
||||
run_different_ip_test
|
||||
fi
|
||||
@ -1,21 +0,0 @@
|
||||
DROP TABLE IF EXISTS ja4_processing.ref_bot_networks;
|
||||
|
||||
CREATE TABLE ja4_processing.ref_bot_networks (
|
||||
-- On utilise IPv6CIDR car il accepte aussi les IPv4 au format ::ffff:1.2.3.4/120
|
||||
network IPv6CIDR,
|
||||
bot_name LowCardinality(String),
|
||||
is_legitimate UInt8,
|
||||
last_update DateTime
|
||||
) ENGINE = ReplacingMergeTree(last_update)
|
||||
ORDER BY (network, bot_name);
|
||||
|
||||
|
||||
-- Création de la table lisant le fichier des IPs
|
||||
CREATE TABLE ja4_processing.bot_ip (
|
||||
ip String
|
||||
) ENGINE = File(CSV, 'bot_ip.csv');
|
||||
|
||||
-- Création de la table lisant le fichier des signatures JA4
|
||||
CREATE TABLE ja4_processing.bot_ja4 (
|
||||
ja4 String
|
||||
) ENGINE = File(CSV, 'bot_ja4.csv');
|
||||
@ -1,235 +0,0 @@
|
||||
-- =============================================================================
|
||||
-- logcorrelator - Initialisation ClickHouse
|
||||
-- =============================================================================
|
||||
-- Ce fichier crée la base de données, les tables, la vue matérialisée
|
||||
-- et les utilisateurs nécessaires au fonctionnement de logcorrelator.
|
||||
--
|
||||
-- Usage :
|
||||
-- clickhouse-client --multiquery < sql/init.sql
|
||||
-- =============================================================================
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Base de données
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE DATABASE IF NOT EXISTS ja4_logs;
|
||||
CREATE DATABASE IF NOT EXISTS ja4_processing;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Table brute : cible directe des inserts du service
|
||||
-- Le service n'insère que dans cette table (colonne raw_json).
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS ja4_logs.http_logs_raw
|
||||
(
|
||||
`raw_json` String CODEC(ZSTD(3)),
|
||||
`ingest_time` DateTime DEFAULT now()
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(ingest_time)
|
||||
ORDER BY ingest_time
|
||||
TTL ingest_time + INTERVAL 1 DAY
|
||||
SETTINGS
|
||||
index_granularity = 8192,
|
||||
ttl_only_drop_parts = 1;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Table parsée : alimentée automatiquement par la vue matérialisée
|
||||
-- -----------------------------------------------------------------------------
|
||||
|
||||
CREATE TABLE ja4_logs.http_logs
|
||||
(
|
||||
-- Temporel
|
||||
`time` DateTime,
|
||||
`log_date` Date DEFAULT toDate(time),
|
||||
|
||||
-- Réseau
|
||||
`src_ip` IPv4,
|
||||
`src_port` UInt16,
|
||||
`dst_ip` IPv4,
|
||||
`dst_port` UInt16,
|
||||
|
||||
-- Enrichissement IPLocate
|
||||
`src_asn` UInt32,
|
||||
`src_country_code` LowCardinality(String),
|
||||
`src_as_name` LowCardinality(String),
|
||||
`src_org` LowCardinality(String),
|
||||
`src_domain` LowCardinality(String),
|
||||
|
||||
-- HTTP
|
||||
`method` LowCardinality(String),
|
||||
`scheme` LowCardinality(String),
|
||||
`host` LowCardinality(String),
|
||||
`path` String CODEC(ZSTD(3)),
|
||||
`query` String CODEC(ZSTD(3)),
|
||||
`http_version` LowCardinality(String),
|
||||
|
||||
-- Corrélation
|
||||
`orphan_side` LowCardinality(String),
|
||||
`correlated` UInt8,
|
||||
`keepalives` UInt16,
|
||||
`a_timestamp` UInt64,
|
||||
`b_timestamp` UInt64,
|
||||
`conn_id` String CODEC(ZSTD(3)),
|
||||
|
||||
-- Métadonnées IP
|
||||
`ip_meta_df` UInt8,
|
||||
`ip_meta_id` UInt16,
|
||||
`ip_meta_total_length` UInt16,
|
||||
`ip_meta_ttl` UInt8,
|
||||
|
||||
-- Métadonnées TCP
|
||||
`tcp_meta_options` LowCardinality(String),
|
||||
`tcp_meta_window_size` UInt32,
|
||||
`tcp_meta_mss` UInt16,
|
||||
`tcp_meta_window_scale` UInt8,
|
||||
`syn_to_clienthello_ms` Int32,
|
||||
|
||||
-- TLS / fingerprint
|
||||
`tls_version` LowCardinality(String),
|
||||
`tls_sni` LowCardinality(String),
|
||||
`tls_alpn` LowCardinality(String),
|
||||
`ja3` String CODEC(ZSTD(3)),
|
||||
`ja3_hash` String CODEC(ZSTD(3)),
|
||||
`ja4` String CODEC(ZSTD(3)),
|
||||
|
||||
-- En-têtes HTTP
|
||||
`client_headers` String CODEC(ZSTD(3)),
|
||||
`header_user_agent` String CODEC(ZSTD(3)),
|
||||
`header_accept` String CODEC(ZSTD(3)),
|
||||
`header_accept_encoding` String CODEC(ZSTD(3)),
|
||||
`header_accept_language` String CODEC(ZSTD(3)),
|
||||
`header_content_type` String CODEC(ZSTD(3)),
|
||||
`header_x_request_id` String CODEC(ZSTD(3)),
|
||||
`header_x_trace_id` String CODEC(ZSTD(3)),
|
||||
`header_x_forwarded_for` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua_mobile` String CODEC(ZSTD(3)),
|
||||
`header_sec_ch_ua_platform` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_dest` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_mode` String CODEC(ZSTD(3)),
|
||||
`header_sec_fetch_site` String CODEC(ZSTD(3))
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY log_date
|
||||
ORDER BY (time, src_ip, dst_ip, ja4)
|
||||
TTL log_date + INTERVAL 7 DAY
|
||||
SETTINGS
|
||||
index_granularity = 8192,
|
||||
ttl_only_drop_parts = 1;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vue matérialisée : parse le JSON de http_logs_raw vers http_logs
|
||||
-- -----------------------------------------------------------------------------
|
||||
DROP VIEW IF EXISTS ja4_logs.mv_http_logs;
|
||||
|
||||
CREATE MATERIALIZED VIEW IF NOT EXISTS ja4_logs.mv_http_logs
|
||||
TO ja4_logs.http_logs
|
||||
AS
|
||||
SELECT
|
||||
parseDateTimeBestEffort(coalesce(JSONExtractString(raw_json, 'time'), '1970-01-01T00:00:00Z')) AS time,
|
||||
toDate(time) AS log_date,
|
||||
|
||||
toIPv4(coalesce(JSONExtractString(raw_json, 'src_ip'), '0.0.0.0')) AS src_ip,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'src_port'), 0)) AS src_port,
|
||||
toIPv4(coalesce(JSONExtractString(raw_json, 'dst_ip'), '0.0.0.0')) AS dst_ip,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'dst_port'), 0)) AS dst_port,
|
||||
|
||||
dictGetOrDefault(
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'asn',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
toUInt32(0)
|
||||
) AS src_asn,
|
||||
dictGetOrDefault(
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'country_code',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_country_code,
|
||||
dictGetOrDefault(
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'name',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_as_name,
|
||||
dictGetOrDefault(
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'org',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_org,
|
||||
dictGetOrDefault(
|
||||
'ja4_processing.dict_iplocate_asn',
|
||||
'domain',
|
||||
IPv4ToIPv6(IPv4StringToNum(toString(src_ip))),
|
||||
''
|
||||
) AS src_domain,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'method'), '') AS method,
|
||||
coalesce(JSONExtractString(raw_json, 'scheme'), '') AS scheme,
|
||||
coalesce(JSONExtractString(raw_json, 'host'), '') AS host,
|
||||
coalesce(JSONExtractString(raw_json, 'path'), '') AS path,
|
||||
coalesce(JSONExtractString(raw_json, 'query'), '') AS query,
|
||||
coalesce(JSONExtractString(raw_json, 'http_version'), '') AS http_version,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'orphan_side'), '') AS orphan_side,
|
||||
toUInt8(coalesce(JSONExtractBool(raw_json, 'correlated'), 0)) AS correlated,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'keepalives'), 0)) AS keepalives,
|
||||
coalesce(JSONExtractUInt(raw_json, 'a_timestamp'), 0) AS a_timestamp,
|
||||
coalesce(JSONExtractUInt(raw_json, 'b_timestamp'), 0) AS b_timestamp,
|
||||
coalesce(JSONExtractString(raw_json, 'conn_id'), '') AS conn_id,
|
||||
|
||||
toUInt8(coalesce(JSONExtractBool(raw_json, 'ip_meta_df'), 0)) AS ip_meta_df,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'ip_meta_id'), 0)) AS ip_meta_id,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'ip_meta_total_length'), 0)) AS ip_meta_total_length,
|
||||
toUInt8(coalesce(JSONExtractUInt(raw_json, 'ip_meta_ttl'), 0)) AS ip_meta_ttl,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'tcp_meta_options'), '') AS tcp_meta_options,
|
||||
toUInt32(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_window_size'), 0)) AS tcp_meta_window_size,
|
||||
toUInt16(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_mss'), 0)) AS tcp_meta_mss,
|
||||
toUInt8(coalesce(JSONExtractUInt(raw_json, 'tcp_meta_window_scale'), 0)) AS tcp_meta_window_scale,
|
||||
toInt32(coalesce(JSONExtractInt(raw_json, 'syn_to_clienthello_ms'), 0)) AS syn_to_clienthello_ms,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'tls_version'), '') AS tls_version,
|
||||
coalesce(JSONExtractString(raw_json, 'tls_sni'), '') AS tls_sni,
|
||||
coalesce(JSONExtractString(raw_json, 'tls_alpn'), '') AS tls_alpn,
|
||||
coalesce(JSONExtractString(raw_json, 'ja3'), '') AS ja3,
|
||||
coalesce(JSONExtractString(raw_json, 'ja3_hash'), '') AS ja3_hash,
|
||||
coalesce(JSONExtractString(raw_json, 'ja4'), '') AS ja4,
|
||||
|
||||
coalesce(JSONExtractString(raw_json, 'client_headers'), '') AS client_headers,
|
||||
coalesce(JSONExtractString(raw_json, 'header_User-Agent'), '') AS header_user_agent,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Accept'), '') AS header_accept,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Accept-Encoding'), '') AS header_accept_encoding,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Accept-Language'), '') AS header_accept_language,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Content-Type'), '') AS header_content_type,
|
||||
coalesce(JSONExtractString(raw_json, 'header_X-Request-Id'), '') AS header_x_request_id,
|
||||
coalesce(JSONExtractString(raw_json, 'header_X-Trace-Id'), '') AS header_x_trace_id,
|
||||
coalesce(JSONExtractString(raw_json, 'header_X-Forwarded-For'), '') AS header_x_forwarded_for,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA'), '') AS header_sec_ch_ua,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA-Mobile'), '') AS header_sec_ch_ua_mobile,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-CH-UA-Platform'), '') AS header_sec_ch_ua_platform,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Dest'), '') AS header_sec_fetch_dest,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Mode'), '') AS header_sec_fetch_mode,
|
||||
coalesce(JSONExtractString(raw_json, 'header_Sec-Fetch-Site'), '') AS header_sec_fetch_site
|
||||
|
||||
FROM ja4_logs.http_logs_raw;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Utilisateurs et permissions
|
||||
-- -----------------------------------------------------------------------------
|
||||
CREATE USER IF NOT EXISTS data_writer IDENTIFIED WITH plaintext_password BY 'ChangeMe';
|
||||
CREATE USER IF NOT EXISTS analyst IDENTIFIED WITH plaintext_password BY 'ChangeMe';
|
||||
|
||||
-- data_writer : INSERT uniquement sur la table brute
|
||||
GRANT INSERT ON ja4_logs.http_logs_raw TO data_writer;
|
||||
GRANT SELECT ON ja4_logs.http_logs_raw TO data_writer;
|
||||
|
||||
-- analyst : lecture sur la table parsée
|
||||
GRANT SELECT ON ja4_logs.http_logs TO analyst;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vérifications post-installation
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- SELECT count(*), min(ingest_time), max(ingest_time) FROM ja4_logs.http_logs_raw;
|
||||
-- SELECT count(*), min(time), max(time) FROM ja4_logs.http_logs;
|
||||
-- SELECT time, src_ip, dst_ip, method, host, path, ja4 FROM ja4_logs.http_logs ORDER BY time DESC LIMIT 10;
|
||||
@ -1,90 +0,0 @@
|
||||
-- =============================================================================
|
||||
-- 01_ttl_adjustments.sql — Ajustement des TTL de rétention des logs
|
||||
--
|
||||
-- Objectif :
|
||||
-- - http_logs : conserver 30 jours (était 7 jours)
|
||||
-- - http_logs_raw : supprimer au plus vite (2 heures, était 1 jour)
|
||||
--
|
||||
-- Usage (déploiements existants) :
|
||||
-- clickhouse-client --multiquery < sql/migrations/01_ttl_adjustments.sql
|
||||
--
|
||||
-- IMPORTANT — http_logs_raw (partition) :
|
||||
-- La modification du TTL ci-dessous s'applique immédiatement aux nouvelles
|
||||
-- insertions. Toutefois, la PARTITION BY (toDate → toStartOfHour) ne peut
|
||||
-- PAS être modifiée via ALTER TABLE — elle est fixée à la création.
|
||||
--
|
||||
-- Pour adopter la partition horaire sur un déploiement existant, procéder
|
||||
-- manuellement :
|
||||
-- 1. CREATE TABLE ja4_logs.http_logs_raw_new ( ... identique à init.sql ... )
|
||||
-- 2. Attendre que http_logs_raw soit vide (ou accepter la perte des raw non traités)
|
||||
-- 3. DROP VIEW IF EXISTS ja4_logs.mv_http_logs
|
||||
-- 4. DROP TABLE ja4_logs.http_logs_raw
|
||||
-- 5. RENAME TABLE ja4_logs.http_logs_raw_new TO ja4_logs.http_logs_raw
|
||||
-- 6. Recréer mv_http_logs (cf. init.sql)
|
||||
--
|
||||
-- Si la migration complète n'est pas possible, la modification du TTL seul
|
||||
-- (ci-dessous) ramène la suppression à : fin de la partition quotidienne + 2h,
|
||||
-- soit au plus 26h au lieu de 48h — amélioration immédiate sans downtime.
|
||||
-- =============================================================================
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 1. http_logs — rétention étendue à 30 jours
|
||||
-- -----------------------------------------------------------------------------
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
MODIFY TTL log_date + INTERVAL 30 DAY;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- 2. http_logs_raw — TTL réduit à 2 heures
|
||||
-- (la MV traite les données de façon synchrone à l'INSERT, donc les raw
|
||||
-- sont redondants dès leur insertion)
|
||||
-- -----------------------------------------------------------------------------
|
||||
ALTER TABLE ja4_logs.http_logs_raw
|
||||
MODIFY TTL ingest_time + INTERVAL 2 HOUR;
|
||||
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- Vérifications post-migration
|
||||
-- -----------------------------------------------------------------------------
|
||||
-- SELECT engine_full FROM system.tables
|
||||
-- WHERE database IN ('ja4_logs', 'ja4_processing') AND name IN ('http_logs', 'http_logs_raw');
|
||||
|
||||
-- =============================================================================
|
||||
-- Ajustements TTL des tables de MVs (bot_detector) — 7 jours max
|
||||
-- =============================================================================
|
||||
|
||||
-- agg_host_ip_ja4_1h — TTL 7 jours (était : aucun)
|
||||
-- Note : PARTITION BY ne peut pas être ajouté via ALTER TABLE.
|
||||
-- Pour la partition quotidienne (recommandée), recréer la table via deploy_views.sql.
|
||||
-- Sans partition explicite, ClickHouse applique le TTL par granule (moins efficace).
|
||||
ALTER TABLE ja4_processing.agg_host_ip_ja4_1h
|
||||
MODIFY TTL window_start + INTERVAL 7 DAY;
|
||||
|
||||
-- agg_header_fingerprint_1h — TTL 7 jours (était : aucun)
|
||||
ALTER TABLE ja4_processing.agg_header_fingerprint_1h
|
||||
MODIFY TTL window_start + INTERVAL 7 DAY;
|
||||
|
||||
-- ml_detected_anomalies — TTL 7 jours (était 30 jours, déjà corrigé en 7j pour certains)
|
||||
-- Note : MODIFY ORDER BY (src_ip, ja4, host, model_name) ne peut PAS être appliqué
|
||||
-- sur ClickHouse 24.8 car ja4/host sont des colonnes existantes (erreur BAD_ARGUMENTS 36).
|
||||
-- Le correctif est dans le schéma de base (06_ml_tables.sql) — ORDER BY corrigé à la
|
||||
-- création. Pour les déploiements existants avec l'ancien ORDER BY, recréer la table.
|
||||
ALTER TABLE ja4_processing.ml_detected_anomalies
|
||||
MODIFY TTL detected_at + INTERVAL 7 DAY;
|
||||
|
||||
-- ml_all_scores — TTL 7 jours (était 3 jours)
|
||||
-- Note : PARTITION BY ne peut pas être ajouté via ALTER TABLE.
|
||||
-- Pour la partition quotidienne (recommandée), recréer la table via deploy_views.sql.
|
||||
ALTER TABLE ja4_processing.ml_all_scores
|
||||
MODIFY TTL window_start + INTERVAL 7 DAY;
|
||||
|
||||
-- =============================================================================
|
||||
-- Vues dashboard — suppression des vues inutilisées, ajout des vues manquantes
|
||||
-- Ces vues n'ont pas d'état persistant : les DROP/CREATE sont idempotents.
|
||||
-- =============================================================================
|
||||
-- Supprimer les vues inutilisées (non référencées par les routes Python)
|
||||
DROP VIEW IF EXISTS ja4_processing.view_dashboard_summary;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_dashboard_timeseries;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_dashboard_threat_dist;
|
||||
DROP VIEW IF EXISTS ja4_processing.view_dashboard_variability;
|
||||
-- Les nouvelles vues (view_dashboard_entities, view_dashboard_user_agents,
|
||||
-- view_ai_features_24h) sont créées par deploy_views.sql (section finale).
|
||||
-- Les exécuter ici revient à rejouer cette section de deploy_views.sql.
|
||||
@ -1,17 +0,0 @@
|
||||
-- =============================================================================
|
||||
-- 02_detection_features.sql — Ajout des features de détection P0+P1
|
||||
-- Colonnes supplémentaires dans agg_host_ip_ja4_1h et agg_header_fingerprint_1h
|
||||
-- NOTE : les MVs doivent être recréées (DROP + CREATE) car ALTER VIEW n'existe pas.
|
||||
-- Exécuter deploy_schema.sh pour recréer les MVs, ou relancer le schema complet.
|
||||
-- =============================================================================
|
||||
|
||||
-- agg_host_ip_ja4_1h : nouvelles colonnes de comptage
|
||||
ALTER TABLE ja4_processing.agg_host_ip_ja4_1h
|
||||
ADD COLUMN IF NOT EXISTS count_xff SimpleAggregateFunction(sum, UInt64) AFTER count_http_scheme,
|
||||
ADD COLUMN IF NOT EXISTS count_unusual_ct SimpleAggregateFunction(sum, UInt64) AFTER count_xff,
|
||||
ADD COLUMN IF NOT EXISTS count_non_std_port SimpleAggregateFunction(sum, UInt64) AFTER count_unusual_ct,
|
||||
ADD COLUMN IF NOT EXISTS count_login_post SimpleAggregateFunction(sum, UInt64) AFTER count_non_std_port;
|
||||
|
||||
-- agg_header_fingerprint_1h : mismatch mobile Sec-CH-UA
|
||||
ALTER TABLE ja4_processing.agg_header_fingerprint_1h
|
||||
ADD COLUMN IF NOT EXISTS sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8) AFTER ua_ch_mismatch;
|
||||
@ -1,45 +0,0 @@
|
||||
-- === 03_remove_ua_browser_detection.sql — Suppression dépendance User-Agent ===
|
||||
--
|
||||
-- Contexte : l'identification navigateur ne doit PAS se baser sur le User-Agent
|
||||
-- (trivalement falsifiable). Cette migration :
|
||||
-- 1. Ajoute has_sec_ch_ua à agg_header_fingerprint_1h
|
||||
-- 2. Recréé la MV avec modern_browser_score basé sur Client Hints + Sec-Fetch
|
||||
-- 3. Recréé sec_ch_mobile_mismatch sans UA (Client Hints only)
|
||||
-- 4. Ajoute has_sec_ch_ua à ml_all_scores
|
||||
-- ---------------------------------------------------------------------------
|
||||
|
||||
-- 1. Nouvelle colonne has_sec_ch_ua
|
||||
ALTER TABLE ja4_processing.agg_header_fingerprint_1h
|
||||
ADD COLUMN IF NOT EXISTS has_sec_ch_ua SimpleAggregateFunction(max, UInt8)
|
||||
AFTER modern_browser_score;
|
||||
|
||||
-- 2. Recréer la MV sans dépendance UA
|
||||
DROP VIEW IF EXISTS ja4_processing.mv_agg_header_fingerprint_1h;
|
||||
|
||||
CREATE MATERIALIZED VIEW ja4_processing.mv_agg_header_fingerprint_1h
|
||||
TO ja4_processing.agg_header_fingerprint_1h AS
|
||||
SELECT
|
||||
toStartOfHour(src.time) AS window_start,
|
||||
toIPv6(src.src_ip) AS src_ip,
|
||||
any(toString(cityHash64(src.client_headers))) AS header_order_hash,
|
||||
max(toUInt16(length(src.client_headers) - length(replaceAll(src.client_headers, ',', '')) + 1)) AS header_count,
|
||||
max(toUInt8(if(position(src.client_headers, 'Accept-Language') > 0, 1, 0))) AS has_accept_language,
|
||||
max(toUInt8(if(position(src.client_headers, 'Cookie') > 0, 1, 0))) AS has_cookie,
|
||||
max(toUInt8(if(position(src.client_headers, 'Referer') > 0, 1, 0))) AS has_referer,
|
||||
-- modern_browser_score : sec-ch-ua → 100, Sec-Fetch → 70, sinon → 0 (PAS de UA)
|
||||
max(toUInt8(if(length(src.header_sec_ch_ua) > 0, 100, if(length(src.header_sec_fetch_site) > 0, 70, 0)))) AS modern_browser_score,
|
||||
max(toUInt8(if(length(src.header_sec_ch_ua) > 0, 1, 0))) AS has_sec_ch_ua,
|
||||
max(toUInt8(if((position(src.header_user_agent, 'Windows') > 0 AND position(src.header_sec_ch_ua_platform, 'Windows') == 0) OR (position(src.header_user_agent, 'iPhone') > 0 AND position(src.header_sec_ch_ua_platform, 'iOS') == 0), 1, 0))) AS ua_ch_mismatch,
|
||||
-- sec_ch_mobile_mismatch : incohérence interne Client Hints (pas de UA)
|
||||
max(toUInt8(if(
|
||||
(src.header_sec_ch_ua_mobile = '?1' AND position(src.header_sec_ch_ua_platform, 'Windows') > 0)
|
||||
OR (src.header_sec_ch_ua_mobile = '?0' AND position(src.header_sec_ch_ua_platform, 'Android') > 0),
|
||||
1, 0))) AS sec_ch_mobile_mismatch,
|
||||
any(src.header_sec_fetch_mode) AS sec_fetch_mode,
|
||||
any(src.header_sec_fetch_dest) AS sec_fetch_dest
|
||||
FROM ja4_logs.http_logs AS src
|
||||
GROUP BY window_start, src.src_ip;
|
||||
|
||||
-- 3. Ajouter has_sec_ch_ua à ml_all_scores
|
||||
ALTER TABLE ja4_processing.ml_all_scores
|
||||
ADD COLUMN IF NOT EXISTS has_sec_ch_ua UInt8 DEFAULT 0;
|
||||
@ -1,52 +0,0 @@
|
||||
-- === 04_http2_fields.sql — Ajout des colonnes HTTP/2 à http_logs ===
|
||||
--
|
||||
-- Migration pour les déploiements existants : ajoute les colonnes de
|
||||
-- fingerprint HTTP/2 passif extraites par mod_reqin_log via son hook
|
||||
-- process_connection (APR_HOOK_FIRST, AP_MODE_SPECULATIVE).
|
||||
--
|
||||
-- Format du fingerprint Akamai (h2_fingerprint) :
|
||||
-- Chrome : "1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p"
|
||||
-- Firefox : "1:65536,4:131072,5:16384|12517377|0|m,p,s,a"
|
||||
-- Safari : "1:4096,3:100,4:65535|10485760|0|m,a,s,p"
|
||||
--
|
||||
-- Appliquer avec :
|
||||
-- clickhouse-client --multiquery < 04_http2_fields.sql
|
||||
|
||||
-- Champs composites (fingerprint global + valeurs agrégées)
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_fingerprint` String DEFAULT '' CODEC(ZSTD(3));
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_settings_fp` String DEFAULT '' CODEC(ZSTD(3));
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_window_update` UInt32 DEFAULT 0;
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_pseudo_order` LowCardinality(String) DEFAULT '';
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_has_priority` UInt8 DEFAULT 0;
|
||||
|
||||
-- Paramètres SETTINGS individuels (RFC 9113 §6.5.2)
|
||||
-- Valeur -1 = paramètre absent du preface client (non envoyé)
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_header_table_size` Int32 DEFAULT -1;
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_enable_push` Int32 DEFAULT -1;
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_max_concurrent_streams` Int32 DEFAULT -1;
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_initial_window_size` Int64 DEFAULT -1;
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_max_frame_size` Int32 DEFAULT -1;
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_max_header_list_size` Int32 DEFAULT -1;
|
||||
|
||||
ALTER TABLE ja4_logs.http_logs
|
||||
ADD COLUMN IF NOT EXISTS `h2_enable_connect_protocol` Int32 DEFAULT -1;
|
||||
@ -1,55 +0,0 @@
|
||||
-- === 05_fleet_metrics_tables.sql — Tables fleet_detections et ml_performance_metrics ===
|
||||
--
|
||||
-- fleet_detections : résultats du détecteur de flottes §5.2 (JA4×ASN bipartite graph)
|
||||
-- ml_performance_metrics : métriques de performance du pipeline ML par cycle
|
||||
--
|
||||
-- Appliquer avec :
|
||||
-- clickhouse-client --multiquery < 05_fleet_metrics_tables.sql
|
||||
|
||||
-- --- fleet_detections ---
|
||||
CREATE TABLE IF NOT EXISTS ja4_processing.fleet_detections
|
||||
(
|
||||
detected_at DateTime,
|
||||
community_id UInt32,
|
||||
ja4_set Array(String),
|
||||
asn_set Array(String),
|
||||
fleet_score Float32,
|
||||
n_ips UInt32,
|
||||
ip_sample Array(String), -- échantillon des 20 premières IPs
|
||||
model_name LowCardinality(String) DEFAULT ''
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(detected_at)
|
||||
ORDER BY (detected_at, community_id)
|
||||
TTL detected_at + INTERVAL 7 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
|
||||
-- --- ml_performance_metrics ---
|
||||
CREATE TABLE IF NOT EXISTS ja4_processing.ml_performance_metrics
|
||||
(
|
||||
cycle_at DateTime,
|
||||
model_name LowCardinality(String),
|
||||
total_sessions UInt64,
|
||||
correlated_rate Float32,
|
||||
anomaly_rate Float32,
|
||||
critical_count UInt32,
|
||||
high_count UInt32,
|
||||
medium_count UInt32,
|
||||
low_count UInt32,
|
||||
known_bot_count UInt32,
|
||||
anubis_deny_count UInt32,
|
||||
legit_browser_count UInt32,
|
||||
drift_rate Float32,
|
||||
drift_alert UInt8,
|
||||
cycle_latency_ms UInt32,
|
||||
features_valid UInt16,
|
||||
features_total UInt16,
|
||||
baseline_size UInt32,
|
||||
threshold Float32,
|
||||
meta_learner_active UInt8 DEFAULT 0
|
||||
)
|
||||
ENGINE = MergeTree
|
||||
PARTITION BY toDate(cycle_at)
|
||||
ORDER BY (cycle_at, model_name)
|
||||
TTL cycle_at + INTERVAL 90 DAY
|
||||
SETTINGS ttl_only_drop_parts = 1;
|
||||
@ -1,72 +0,0 @@
|
||||
-- === 06_browser_signature_detection.sql ===
|
||||
-- Migration : Browser Signature Detection
|
||||
--
|
||||
-- 1. Création de browser_h2_signatures (table source structurée)
|
||||
-- 2. Peuplement des fingerprints Akamai de référence
|
||||
-- 3. Recréation de dict_browser_h2 avec le champ confidence
|
||||
-- (anciennement chargé depuis CSV — remplacé par la table ClickHouse)
|
||||
--
|
||||
-- Appliquer avec :
|
||||
-- clickhouse-client --multiquery < 06_browser_signature_detection.sql
|
||||
|
||||
-- --- Table source des signatures H2 ---
|
||||
CREATE TABLE IF NOT EXISTS ja4_processing.browser_h2_signatures
|
||||
(
|
||||
h2_fingerprint String,
|
||||
browser_family LowCardinality(String),
|
||||
confidence Float32,
|
||||
notes String
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY h2_fingerprint
|
||||
SETTINGS index_granularity = 128;
|
||||
|
||||
-- Peuplement initial depuis les données historiques (correspondance exacte CSV)
|
||||
-- Format Akamai : SETTINGS|WINDOW_UPDATE|PRIORITY|PSEUDO_HEADER_ORDER
|
||||
-- SETTINGS : paires clé:valeur séparées par des virgules
|
||||
INSERT INTO ja4_processing.browser_h2_signatures VALUES
|
||||
-- Chrome (Blink + BoringSSL, INITIAL_WINDOW_SIZE=6291456)
|
||||
('1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p',
|
||||
'Chrome', 1.0, 'Chrome 106–142 stable'),
|
||||
('1:65536,3:1000,4:6291456,6:262144|15663105|0|m,a,s,p',
|
||||
'Chrome', 0.95, 'Chrome avec MAX_CONCURRENT_STREAMS=1000 (certains proxies)'),
|
||||
('1:65536,2:0,3:100,4:6291456,6:262144|15663105|0|m,a,s,p',
|
||||
'Chrome', 0.90, 'Chrome avec MAX_CONCURRENT_STREAMS=100'),
|
||||
-- Firefox (Gecko + NSS, INITIAL_WINDOW_SIZE=131072, MAX_FRAME_SIZE=16384)
|
||||
('1:65536,4:131072,5:16384|12517377|0|m,p,s,a',
|
||||
'Firefox', 1.0, 'Firefox 90–127 stable'),
|
||||
('1:65536,4:131072|12517377|0|m,p,s,a',
|
||||
'Firefox', 0.95, 'Firefox sans MAX_FRAME_SIZE explicite'),
|
||||
('1:65536,3:100,4:131072,5:16384|12517377|0|m,p,s,a',
|
||||
'Firefox', 0.90, 'Firefox avec MAX_CONCURRENT_STREAMS=100'),
|
||||
-- Safari (WebKit, INITIAL_WINDOW_SIZE=65535)
|
||||
('1:4096,3:100,4:65535|10485760|0|m,a,s,p',
|
||||
'Safari', 1.0, 'Safari 15+ macOS et iOS'),
|
||||
('1:4096,3:100,4:65535,5:16384|10485760|0|m,a,s,p',
|
||||
'Safari', 0.95, 'Safari avec MAX_FRAME_SIZE=16384'),
|
||||
('1:4096,3:100,4:65535,6:16384|10485760|0|m,a,s,p',
|
||||
'Safari', 0.95, 'Safari avec MAX_HEADER_LIST_SIZE=16384'),
|
||||
-- Edge Chromium (fingerprint H2 identique à Chrome)
|
||||
('1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p',
|
||||
'Edge', 0.9, 'Edge Chromium — fingerprint H2 identique à Chrome (dédoublonné)');
|
||||
|
||||
-- Recréation du dictionnaire avec le champ confidence
|
||||
-- (remplace la version CSV-backed sans confidence)
|
||||
DROP DICTIONARY IF EXISTS ja4_processing.dict_browser_h2;
|
||||
|
||||
CREATE DICTIONARY ja4_processing.dict_browser_h2
|
||||
(
|
||||
h2_fingerprint String,
|
||||
browser_family String,
|
||||
confidence Float32
|
||||
)
|
||||
PRIMARY KEY h2_fingerprint
|
||||
SOURCE(CLICKHOUSE(query '
|
||||
SELECT h2_fingerprint,
|
||||
argMax(browser_family, confidence) AS browser_family,
|
||||
max(confidence) AS confidence
|
||||
FROM ja4_processing.browser_h2_signatures
|
||||
GROUP BY h2_fingerprint
|
||||
'))
|
||||
LAYOUT(COMPLEX_KEY_HASHED())
|
||||
LIFETIME(MIN 300 MAX 600);
|
||||
@ -1,29 +0,0 @@
|
||||
DROP DICTIONARY IF EXISTS ja4_processing.dict_iplocate_asn;
|
||||
|
||||
CREATE DICTIONARY IF NOT EXISTS ja4_processing.dict_iplocate_asn
|
||||
(
|
||||
network String,
|
||||
asn UInt32,
|
||||
country_code String,
|
||||
name String,
|
||||
org String,
|
||||
domain String
|
||||
)
|
||||
PRIMARY KEY network
|
||||
SOURCE(FILE(path '/var/lib/clickhouse/user_files/iplocate-ip-to-asn.csv' format 'CSVWithNames'))
|
||||
LAYOUT(IP_TRIE())
|
||||
LIFETIME(MIN 3600 MAX 7200);
|
||||
|
||||
|
||||
|
||||
-- Suppression si existe pour reconfiguration
|
||||
DROP TABLE IF EXISTS ja4_processing.ref_bot_networks;
|
||||
|
||||
-- Table optimisée pour le filtrage binaire de CIDR
|
||||
CREATE TABLE ja4_processing.ref_bot_networks (
|
||||
network IPv6CIDR, -- Gère nativement '1.2.3.0/24' et '2001:db8::/32'
|
||||
bot_name LowCardinality(String),
|
||||
is_legitimate UInt8, -- 1 = Whitelist, 0 = Blacklist
|
||||
last_update DateTime
|
||||
) ENGINE = ReplacingMergeTree(last_update)
|
||||
ORDER BY (network, bot_name)
|
||||
35
old/services/mod-reqin-log/.gitignore
vendored
35
old/services/mod-reqin-log/.gitignore
vendored
@ -1,35 +0,0 @@
|
||||
# Build artifacts
|
||||
*.o
|
||||
*.so
|
||||
*.a
|
||||
*.la
|
||||
.deps
|
||||
.libs
|
||||
|
||||
# Build directories
|
||||
build/
|
||||
cmake-build-*/
|
||||
dist/
|
||||
bin/
|
||||
obj/
|
||||
|
||||
# IDE
|
||||
.vscode/
|
||||
.idea/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Testing
|
||||
coverage/
|
||||
*.gcno
|
||||
*.gcda
|
||||
|
||||
# Packaging
|
||||
*.rpm
|
||||
*.deb
|
||||
*.tar.gz
|
||||
.aider*
|
||||
@ -1,94 +0,0 @@
|
||||
# GitLab CI/CD configuration for mod_reqin_log
|
||||
# Uses Docker-in-Docker (dind) for building and testing
|
||||
|
||||
stages:
|
||||
- build
|
||||
- test
|
||||
- package
|
||||
- verify
|
||||
|
||||
# =============================================================================
|
||||
# Variables
|
||||
# =============================================================================
|
||||
variables:
|
||||
DOCKER_TLS_CERTDIR: "/certs"
|
||||
DOCKER_DRIVER: overlay2
|
||||
VERSION: "1.0.2"
|
||||
|
||||
# =============================================================================
|
||||
# Build Stage - Compile all RPM packages
|
||||
# =============================================================================
|
||||
|
||||
build-packages:
|
||||
stage: build
|
||||
image: docker:24
|
||||
services:
|
||||
- docker:24-dind
|
||||
script:
|
||||
# Build all RPM packages (el8, el9, el10)
|
||||
- docker build -f Dockerfile.package --target output --build-arg VERSION=$VERSION -t mod_reqin_log:packages .
|
||||
|
||||
# Create output directories
|
||||
- mkdir -p dist/rpm
|
||||
|
||||
# Extract packages from Docker image
|
||||
- docker run --rm -v $(pwd)/dist:/output mod_reqin_log:packages sh -c 'cp -r /packages/rpm/* /output/rpm/'
|
||||
|
||||
# List built packages
|
||||
- echo "=== RPM Packages ==="
|
||||
- ls -la dist/rpm/
|
||||
artifacts:
|
||||
paths:
|
||||
- dist/rpm/
|
||||
expire_in: 30 days
|
||||
|
||||
# =============================================================================
|
||||
# Test Stage - Unit tests
|
||||
# =============================================================================
|
||||
|
||||
unit-tests:
|
||||
stage: test
|
||||
image: docker:24
|
||||
services:
|
||||
- docker:24-dind
|
||||
script:
|
||||
# Build test image
|
||||
- docker build -f Dockerfile.tests -t mod_reqin_log:tests .
|
||||
|
||||
# Run unit tests
|
||||
- docker run --rm mod_reqin_log:tests ctest --output-on-failure
|
||||
|
||||
# =============================================================================
|
||||
# Package Stage - Already done in build-packages
|
||||
# =============================================================================
|
||||
|
||||
# =============================================================================
|
||||
# Verify Stage - Test RPM package installation on each target distribution
|
||||
# =============================================================================
|
||||
|
||||
verify-rpm-el8:
|
||||
stage: verify
|
||||
image: docker:24
|
||||
services:
|
||||
- docker:24-dind
|
||||
needs: [build-packages]
|
||||
script:
|
||||
- docker run --rm -v $(pwd)/dist:/packages rockylinux:8 sh -c "dnf install -y /packages/rpm/*.el8.*.rpm && httpd -M 2>&1 | grep reqin_log && echo 'RPM el8 verification OK'"
|
||||
|
||||
verify-rpm-el9:
|
||||
stage: verify
|
||||
image: docker:24
|
||||
services:
|
||||
- docker:24-dind
|
||||
needs: [build-packages]
|
||||
script:
|
||||
- docker run --rm -v $(pwd)/dist:/packages rockylinux:9 sh -c "dnf install -y /packages/rpm/*.el9.*.rpm && httpd -M 2>&1 | grep reqin_log && echo 'RPM el9 verification OK'"
|
||||
|
||||
verify-rpm-el10:
|
||||
stage: verify
|
||||
image: docker:24
|
||||
services:
|
||||
- docker:24-dind
|
||||
needs: [build-packages]
|
||||
script:
|
||||
- docker run --rm -v $(pwd)/dist:/packages almalinux:10 sh -c "dnf install -y /packages/rpm/*.el10.*.rpm && httpd -M 2>&1 | grep reqin_log && echo 'RPM el10 verification OK'"
|
||||
@ -1,47 +0,0 @@
|
||||
cmake_minimum_required(VERSION 3.10)
|
||||
project(mod_reqin_log_tests C)
|
||||
|
||||
set(CMAKE_C_STANDARD 99)
|
||||
set(CMAKE_C_STANDARD_REQUIRED ON)
|
||||
|
||||
# Find required packages
|
||||
find_package(PkgConfig REQUIRED)
|
||||
pkg_check_modules(CMOCKA REQUIRED cmocka)
|
||||
pkg_check_modules(APR REQUIRED apr-1)
|
||||
pkg_check_modules(APRUTIL REQUIRED apr-util-1)
|
||||
|
||||
# Include directories
|
||||
include_directories(${CMOCKA_INCLUDE_DIRS})
|
||||
include_directories(${APR_INCLUDE_DIRS})
|
||||
include_directories(${APRUTIL_INCLUDE_DIRS})
|
||||
include_directories(/usr/include/httpd)
|
||||
|
||||
# Test executable - Real module tests (testing actual implementation)
|
||||
add_executable(test_module_real tests/unit/test_module_real.c)
|
||||
target_link_libraries(test_module_real ${CMOCKA_LIBRARIES} ${APR_LIBRARIES} ${APRUTIL_LIBRARIES} m)
|
||||
|
||||
add_executable(test_config_parsing tests/unit/test_config_parsing.c)
|
||||
target_link_libraries(test_config_parsing ${CMOCKA_LIBRARIES})
|
||||
|
||||
add_executable(test_header_handling tests/unit/test_header_handling.c)
|
||||
target_link_libraries(test_header_handling ${CMOCKA_LIBRARIES} ${APR_LIBRARIES})
|
||||
|
||||
add_executable(test_json_serialization tests/unit/test_json_serialization.c)
|
||||
target_link_libraries(test_json_serialization ${CMOCKA_LIBRARIES} ${APR_LIBRARIES})
|
||||
|
||||
add_executable(test_h2_parsing tests/unit/test_h2_parsing.c)
|
||||
target_link_libraries(test_h2_parsing ${CMOCKA_LIBRARIES})
|
||||
|
||||
# Enable testing
|
||||
enable_testing()
|
||||
add_test(NAME RealModuleTest COMMAND test_module_real)
|
||||
add_test(NAME ConfigParsingTest COMMAND test_config_parsing)
|
||||
add_test(NAME HeaderHandlingTest COMMAND test_header_handling)
|
||||
add_test(NAME JsonSerializationTest COMMAND test_json_serialization)
|
||||
add_test(NAME H2ParsingTest COMMAND test_h2_parsing)
|
||||
|
||||
# Custom target for running tests
|
||||
add_custom_target(run_tests
|
||||
COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure
|
||||
DEPENDS test_module_real test_config_parsing test_header_handling test_json_serialization test_h2_parsing
|
||||
)
|
||||
@ -1,176 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# mod_reqin_log - Dockerfile de packaging RPM
|
||||
# Builds RPMs for multiple RHEL-compatible versions:
|
||||
# - Rocky Linux 8 (el8) - RHEL 8 compatible
|
||||
# - Rocky Linux 9 (el9) - RHEL 9 compatible
|
||||
# - AlmaLinux 10 (el10) - RHEL 10 compatible
|
||||
# =============================================================================
|
||||
|
||||
# =============================================================================
|
||||
# Stage 1a: Builder Rocky Linux 8
|
||||
# =============================================================================
|
||||
FROM rockylinux:8 AS builder-el8
|
||||
|
||||
RUN dnf install -y epel-release && \
|
||||
dnf install -y --allowerasing \
|
||||
gcc \
|
||||
make \
|
||||
httpd \
|
||||
httpd-devel \
|
||||
apr-devel \
|
||||
apr-util-devel \
|
||||
python3 \
|
||||
curl \
|
||||
redhat-rpm-config \
|
||||
&& dnf clean all
|
||||
|
||||
WORKDIR /build
|
||||
COPY services/mod-reqin-log/src/ src/
|
||||
COPY services/mod-reqin-log/Makefile Makefile
|
||||
COPY services/mod-reqin-log/conf/ conf/
|
||||
RUN make APXS=/usr/bin/apxs
|
||||
RUN ls -la modules/mod_reqin_log.so
|
||||
|
||||
# =============================================================================
|
||||
# Stage 1b: Builder Rocky Linux 9
|
||||
# =============================================================================
|
||||
FROM rockylinux:9 AS builder-el9
|
||||
|
||||
RUN dnf install -y epel-release && \
|
||||
dnf install -y --allowerasing \
|
||||
gcc \
|
||||
make \
|
||||
httpd \
|
||||
httpd-devel \
|
||||
apr-devel \
|
||||
apr-util-devel \
|
||||
python3 \
|
||||
curl \
|
||||
redhat-rpm-config \
|
||||
&& dnf clean all
|
||||
|
||||
WORKDIR /build
|
||||
COPY services/mod-reqin-log/src/ src/
|
||||
COPY services/mod-reqin-log/Makefile Makefile
|
||||
COPY services/mod-reqin-log/conf/ conf/
|
||||
RUN make APXS=/usr/bin/apxs
|
||||
RUN ls -la modules/mod_reqin_log.so
|
||||
|
||||
# =============================================================================
|
||||
# Stage 1c: Builder AlmaLinux 10 (RHEL 10 compatible)
|
||||
# =============================================================================
|
||||
FROM almalinux:10 AS builder-el10
|
||||
|
||||
RUN dnf install -y epel-release && \
|
||||
dnf install -y --allowerasing \
|
||||
gcc \
|
||||
make \
|
||||
httpd \
|
||||
httpd-devel \
|
||||
apr-devel \
|
||||
apr-util-devel \
|
||||
python3 \
|
||||
curl \
|
||||
redhat-rpm-config \
|
||||
&& dnf clean all
|
||||
|
||||
WORKDIR /build
|
||||
COPY services/mod-reqin-log/src/ src/
|
||||
COPY services/mod-reqin-log/Makefile Makefile
|
||||
COPY services/mod-reqin-log/conf/ conf/
|
||||
RUN make APXS=/usr/bin/apxs
|
||||
RUN ls -la modules/mod_reqin_log.so
|
||||
|
||||
# =============================================================================
|
||||
# Stage 2: Package builder - rpmbuild pour RPM
|
||||
# =============================================================================
|
||||
FROM rockylinux:9 AS package-builder
|
||||
|
||||
WORKDIR /package
|
||||
|
||||
# Install rpm-build and dependencies
|
||||
RUN dnf install -y rpm-build rpmdevtools && \
|
||||
dnf clean all
|
||||
|
||||
# Create rpmbuild directory structure
|
||||
RUN rpmdev-setuptree
|
||||
|
||||
# =============================================================================
|
||||
# Copy spec file and source files
|
||||
# =============================================================================
|
||||
COPY services/mod-reqin-log/mod_reqin_log.spec /package/mod_reqin_log.spec
|
||||
|
||||
# =============================================================================
|
||||
# Copy binaries from each builder stage into pkgroot directories
|
||||
# =============================================================================
|
||||
|
||||
# Rocky Linux 8 (el8)
|
||||
COPY --from=builder-el8 /build/modules/mod_reqin_log.so /tmp/pkgroot-el8/usr/lib64/httpd/modules/mod_reqin_log.so
|
||||
COPY --from=builder-el8 /build/conf/mod_reqin_log.conf /tmp/pkgroot-el8/etc/httpd/conf.d/mod_reqin_log.conf
|
||||
RUN chmod 755 /tmp/pkgroot-el8/usr/lib64/httpd/modules/mod_reqin_log.so && \
|
||||
chmod 644 /tmp/pkgroot-el8/etc/httpd/conf.d/mod_reqin_log.conf
|
||||
|
||||
# Rocky Linux 9 (el9)
|
||||
COPY --from=builder-el9 /build/modules/mod_reqin_log.so /tmp/pkgroot-el9/usr/lib64/httpd/modules/mod_reqin_log.so
|
||||
COPY --from=builder-el9 /build/conf/mod_reqin_log.conf /tmp/pkgroot-el9/etc/httpd/conf.d/mod_reqin_log.conf
|
||||
RUN chmod 755 /tmp/pkgroot-el9/usr/lib64/httpd/modules/mod_reqin_log.so && \
|
||||
chmod 644 /tmp/pkgroot-el9/etc/httpd/conf.d/mod_reqin_log.conf
|
||||
|
||||
# AlmaLinux 10 (el10)
|
||||
COPY --from=builder-el10 /build/modules/mod_reqin_log.so /tmp/pkgroot-el10/usr/lib64/httpd/modules/mod_reqin_log.so
|
||||
COPY --from=builder-el10 /build/conf/mod_reqin_log.conf /tmp/pkgroot-el10/etc/httpd/conf.d/mod_reqin_log.conf
|
||||
RUN chmod 755 /tmp/pkgroot-el10/usr/lib64/httpd/modules/mod_reqin_log.so && \
|
||||
chmod 644 /tmp/pkgroot-el10/etc/httpd/conf.d/mod_reqin_log.conf
|
||||
|
||||
# =============================================================================
|
||||
# Build RPM packages for each distribution using rpmbuild
|
||||
# =============================================================================
|
||||
|
||||
# Create packages directory
|
||||
RUN mkdir -p /tmp/packages/el8 /tmp/packages/el9 /tmp/packages/el10
|
||||
|
||||
# Build for el8
|
||||
RUN VERSION=$(grep "^Version:" /package/mod_reqin_log.spec | awk '{print $2}') && \
|
||||
mkdir -p /tmp/pkgroot-el8-rpm/usr/lib64/httpd/modules /tmp/pkgroot-el8-rpm/etc/httpd/conf.d && \
|
||||
cp /tmp/pkgroot-el8/usr/lib64/httpd/modules/mod_reqin_log.so /tmp/pkgroot-el8-rpm/usr/lib64/httpd/modules/ && \
|
||||
cp /tmp/pkgroot-el8/etc/httpd/conf.d/mod_reqin_log.conf /tmp/pkgroot-el8-rpm/etc/httpd/conf.d/ && \
|
||||
rpmbuild -bb /package/mod_reqin_log.spec \
|
||||
--define "_topdir /tmp/rpmbuild-el8" \
|
||||
--define "_pkgroot /tmp/pkgroot-el8-rpm" \
|
||||
--define "dist .el8" && \
|
||||
cp /tmp/rpmbuild-el8/RPMS/x86_64/*.rpm /tmp/packages/el8/
|
||||
|
||||
# Build for el9
|
||||
RUN VERSION=$(grep "^Version:" /package/mod_reqin_log.spec | awk '{print $2}') && \
|
||||
mkdir -p /tmp/pkgroot-el9-rpm/usr/lib64/httpd/modules /tmp/pkgroot-el9-rpm/etc/httpd/conf.d && \
|
||||
cp /tmp/pkgroot-el9/usr/lib64/httpd/modules/mod_reqin_log.so /tmp/pkgroot-el9-rpm/usr/lib64/httpd/modules/ && \
|
||||
cp /tmp/pkgroot-el9/etc/httpd/conf.d/mod_reqin_log.conf /tmp/pkgroot-el9-rpm/etc/httpd/conf.d/ && \
|
||||
rpmbuild -bb /package/mod_reqin_log.spec \
|
||||
--define "_topdir /tmp/rpmbuild-el9" \
|
||||
--define "_pkgroot /tmp/pkgroot-el9-rpm" \
|
||||
--define "dist .el9" && \
|
||||
cp /tmp/rpmbuild-el9/RPMS/x86_64/*.rpm /tmp/packages/el9/
|
||||
|
||||
# Build for el10
|
||||
RUN VERSION=$(grep "^Version:" /package/mod_reqin_log.spec | awk '{print $2}') && \
|
||||
mkdir -p /tmp/pkgroot-el10-rpm/usr/lib64/httpd/modules /tmp/pkgroot-el10-rpm/etc/httpd/conf.d && \
|
||||
cp /tmp/pkgroot-el10/usr/lib64/httpd/modules/mod_reqin_log.so /tmp/pkgroot-el10-rpm/usr/lib64/httpd/modules/ && \
|
||||
cp /tmp/pkgroot-el10/etc/httpd/conf.d/mod_reqin_log.conf /tmp/pkgroot-el10-rpm/etc/httpd/conf.d/ && \
|
||||
rpmbuild -bb /package/mod_reqin_log.spec \
|
||||
--define "_topdir /tmp/rpmbuild-el10" \
|
||||
--define "_pkgroot /tmp/pkgroot-el10-rpm" \
|
||||
--define "dist .el10" && \
|
||||
cp /tmp/rpmbuild-el10/RPMS/x86_64/*.rpm /tmp/packages/el10/
|
||||
|
||||
# =============================================================================
|
||||
# Stage 3: Output - Image finale avec les packages RPM
|
||||
# =============================================================================
|
||||
FROM alpine:latest AS output
|
||||
|
||||
WORKDIR /packages
|
||||
COPY --from=package-builder /tmp/packages/el8/*.rpm /packages/rpm/el8/
|
||||
COPY --from=package-builder /tmp/packages/el9/*.rpm /packages/rpm/el9/
|
||||
COPY --from=package-builder /tmp/packages/el10/*.rpm /packages/rpm/el10/
|
||||
|
||||
CMD ["sh", "-c", "echo '=== RPM Packages (el8) ===' && ls -la /packages/rpm/el8/ && echo '' && echo '=== RPM Packages (el9) ===' && ls -la /packages/rpm/el9/ && echo '' && echo '=== RPM Packages (el10) ===' && ls -la /packages/rpm/el10/"]
|
||||
@ -1,43 +0,0 @@
|
||||
# Dockerfile for running unit tests (monorepo root build context)
|
||||
FROM rockylinux:8
|
||||
|
||||
# Install build and test dependencies
|
||||
RUN dnf install -y epel-release && \
|
||||
dnf install -y \
|
||||
gcc \
|
||||
make \
|
||||
httpd \
|
||||
httpd-devel \
|
||||
apr-devel \
|
||||
apr-util-devel \
|
||||
cmake \
|
||||
python3 \
|
||||
curl \
|
||||
git \
|
||||
pkgconfig \
|
||||
libxml2-devel \
|
||||
&& dnf clean all
|
||||
|
||||
# Build and install cmocka from source
|
||||
RUN cd /tmp && \
|
||||
git clone https://git.cryptomilk.org/projects/cmocka.git && \
|
||||
cd cmocka && \
|
||||
git checkout cmocka-1.1.5 && \
|
||||
mkdir build && cd build && \
|
||||
cmake .. -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_BUILD_TYPE=Release && \
|
||||
make && \
|
||||
make install && \
|
||||
ldconfig && \
|
||||
cd / && \
|
||||
rm -rf /tmp/cmocka
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
COPY services/mod-reqin-log/src/ src/
|
||||
COPY services/mod-reqin-log/tests/ tests/
|
||||
COPY services/mod-reqin-log/CMakeLists.txt CMakeLists.txt
|
||||
COPY services/mod-reqin-log/Makefile Makefile
|
||||
|
||||
RUN mkdir -p build/tests && cd build/tests && cmake ../../ && make
|
||||
|
||||
CMD ["ctest", "--test-dir", "build/tests", "--output-on-failure"]
|
||||
@ -1,107 +0,0 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction, and
|
||||
distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by the
|
||||
copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all other
|
||||
entities that control, are controlled by, or are under common control with
|
||||
that entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity exercising
|
||||
permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation source,
|
||||
and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical transformation
|
||||
or translation of a Source form, including but not limited to compiled
|
||||
object code, generated documentation, and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or Object form,
|
||||
made available under the License.
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object form,
|
||||
that is based on (or derived from) the Work.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including the original
|
||||
version of the Work and any modifications or additions to that Work or
|
||||
Derivative Works thereof.
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity on
|
||||
behalf of whom a Contribution has been received by Licensor.
|
||||
|
||||
2. Grant of Copyright License.
|
||||
|
||||
Subject to the terms and conditions of this License, each Contributor hereby
|
||||
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
|
||||
irrevocable copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the Work and
|
||||
such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License.
|
||||
|
||||
Subject to the terms and conditions of this License, each Contributor hereby
|
||||
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
|
||||
irrevocable (except as stated in this section) patent license to make, have
|
||||
made, use, offer to sell, sell, import, and otherwise transfer the Work.
|
||||
|
||||
4. Redistribution.
|
||||
|
||||
You may reproduce and distribute copies of the Work or Derivative Works
|
||||
thereof in any medium, with or without modifications, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or Derivative Works a
|
||||
copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices stating
|
||||
that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works that You
|
||||
distribute, all copyright, patent, trademark, and attribution notices
|
||||
from the Source form of the Work; and
|
||||
|
||||
(d) You may add Your own copyright statement to Your modifications and may
|
||||
provide additional or different license terms and conditions for use,
|
||||
reproduction, or distribution of Your modifications.
|
||||
|
||||
5. Submission of Contributions.
|
||||
|
||||
Unless You explicitly state otherwise, any Contribution intentionally
|
||||
submitted for inclusion in the Work by You shall be under the terms and
|
||||
conditions of this License.
|
||||
|
||||
6. Trademarks.
|
||||
|
||||
This License does not grant permission to use the trade names, trademarks,
|
||||
service marks, or product names of the Licensor.
|
||||
|
||||
7. Disclaimer of Warranty.
|
||||
|
||||
Unless required by applicable law or agreed to in writing, Licensor provides
|
||||
the Work on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
|
||||
either express or implied.
|
||||
|
||||
8. Limitation of Liability.
|
||||
|
||||
In no event shall any Contributor be liable to You for damages, including
|
||||
any direct, indirect, special, incidental, or consequential damages arising
|
||||
out of the use or inability to use the Work.
|
||||
|
||||
9. Accepting Warranty or Additional Liability.
|
||||
|
||||
While redistributing the Work or Derivative Works thereof, You may choose
|
||||
to offer, and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this License.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
@ -1,153 +0,0 @@
|
||||
# Makefile for mod_reqin_log
|
||||
# Apache HTTPD module for logging HTTP requests as JSON to Unix socket
|
||||
|
||||
# APXS tool path (can be overridden)
|
||||
APXS ?= apxs
|
||||
|
||||
# Compiler settings
|
||||
CC ?= gcc
|
||||
CFLAGS ?= -Wall -Wextra -O2 -std=gnu11 -x c -Wno-error=format-security
|
||||
|
||||
# Directories
|
||||
SRC_DIR = src
|
||||
BUILD_DIR = build
|
||||
INSTALL_DIR = modules
|
||||
DIST_DIR = dist
|
||||
|
||||
# Source files
|
||||
SRCS = $(SRC_DIR)/mod_reqin_log.c
|
||||
|
||||
# Module name
|
||||
MODULE_NAME = mod_reqin_log
|
||||
|
||||
# Package version
|
||||
VERSION ?= 1.0.7
|
||||
|
||||
.PHONY: all clean install uninstall test package package-deb package-rpm
|
||||
|
||||
all: $(MODULE_NAME).so
|
||||
|
||||
# Build the module using apxs
|
||||
# Note: Use -Wc to pass flags to the C compiler through apxs
|
||||
$(MODULE_NAME).so: $(SRCS)
|
||||
@mkdir -p $(BUILD_DIR)
|
||||
$(APXS) -c -Wc,"$(CFLAGS)" -o $(BUILD_DIR)/$(MODULE_NAME).so $(SRCS)
|
||||
@mkdir -p $(INSTALL_DIR)
|
||||
@if [ -f $(BUILD_DIR)/.libs/$(MODULE_NAME).so ]; then \
|
||||
cp $(BUILD_DIR)/.libs/$(MODULE_NAME).so $(INSTALL_DIR)/; \
|
||||
elif [ -f $(BUILD_DIR)/$(MODULE_NAME).so ]; then \
|
||||
cp $(BUILD_DIR)/$(MODULE_NAME).so $(INSTALL_DIR)/; \
|
||||
fi
|
||||
|
||||
# Install the module
|
||||
install: $(MODULE_NAME).so
|
||||
@echo "Installing $(MODULE_NAME).so..."
|
||||
@mkdir -p $(DESTDIR)/usr/lib/apache2/modules
|
||||
@if [ -f $(INSTALL_DIR)/$(MODULE_NAME).so ]; then \
|
||||
cp $(INSTALL_DIR)/$(MODULE_NAME).so $(DESTDIR)/usr/lib/apache2/modules/; \
|
||||
elif [ -f $(BUILD_DIR)/.libs/$(MODULE_NAME).so ]; then \
|
||||
cp $(BUILD_DIR)/.libs/$(MODULE_NAME).so $(DESTDIR)/usr/lib/apache2/modules/; \
|
||||
elif [ -f $(BUILD_DIR)/$(MODULE_NAME).so ]; then \
|
||||
cp $(BUILD_DIR)/$(MODULE_NAME).so $(DESTDIR)/usr/lib/apache2/modules/; \
|
||||
else \
|
||||
echo "Error: $(MODULE_NAME).so not found"; \
|
||||
exit 1; \
|
||||
fi
|
||||
@echo "Installation complete."
|
||||
@echo "Enable the module by adding to your httpd.conf:"
|
||||
@echo " LoadModule reqin_log_module modules/mod_reqin_log.so"
|
||||
|
||||
# Uninstall the module
|
||||
uninstall:
|
||||
rm -f $(DESTDIR)/usr/lib/apache2/modules/$(MODULE_NAME).so
|
||||
@echo "Uninstallation complete."
|
||||
|
||||
# Clean build artifacts
|
||||
clean:
|
||||
rm -rf $(BUILD_DIR) $(INSTALL_DIR)
|
||||
rm -f .libs/*.o .libs/*.la .libs/*.so
|
||||
rm -f *.o *.la *.lo
|
||||
rm -rf .libs
|
||||
|
||||
# Run unit tests (requires cmocka)
|
||||
test:
|
||||
@mkdir -p build/tests
|
||||
cd build/tests && cmake ../../ -DCMAKE_BUILD_TYPE=Debug
|
||||
$(MAKE) -C build/tests run_tests
|
||||
|
||||
# Build with debug symbols
|
||||
debug: CFLAGS += -g -DDEBUG
|
||||
debug: clean all
|
||||
|
||||
# =============================================================================
|
||||
# Packaging (RPM with Docker + fpm)
|
||||
# Dockerfile.package builds RPMs in a single multi-stage build:
|
||||
# - 3 RPM packages (el8, el9, el10 for RHEL/Rocky/AlmaLinux compatibility)
|
||||
# =============================================================================
|
||||
|
||||
## package: Build all RPM packages (el8, el9, el10)
|
||||
package:
|
||||
mkdir -p $(DIST_DIR)/rpm/el8 $(DIST_DIR)/rpm/el9 $(DIST_DIR)/rpm/el10
|
||||
docker build --target output -t mod_reqin_log:packager \
|
||||
--build-arg VERSION=$(VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Extracting packages from Docker image..."
|
||||
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el8:/output/el8 \
|
||||
-v $(PWD)/$(DIST_DIR)/rpm/el9:/output/el9 \
|
||||
-v $(PWD)/$(DIST_DIR)/rpm/el10:/output/el10 \
|
||||
mod_reqin_log:packager \
|
||||
sh -c 'cp /packages/rpm/el8/*.rpm /output/el8/ && cp /packages/rpm/el9/*.rpm /output/el9/ && cp /packages/rpm/el10/*.rpm /output/el10/'
|
||||
@echo "Packages created:"
|
||||
@echo " RPM (el8, el9, el10):"
|
||||
@ls -la $(DIST_DIR)/rpm/el8/
|
||||
@ls -la $(DIST_DIR)/rpm/el9/
|
||||
@ls -la $(DIST_DIR)/rpm/el10/
|
||||
|
||||
## package-rpm: Build RPM packages (el8, el9, el10)
|
||||
package-rpm: package
|
||||
@echo "RPM packages built in Dockerfile.package"
|
||||
|
||||
## test-package-rpm: Test RPM package installation in Docker (tests el9 by default)
|
||||
test-package-rpm: package
|
||||
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el9:/packages:ro rockylinux:9 \
|
||||
sh -c "dnf install -y /packages/*.el9.*.rpm && echo 'RPM el9 install OK'"
|
||||
|
||||
## test-package-rpm-el8: Test el8 RPM installation
|
||||
test-package-rpm-el8: package
|
||||
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el8:/packages:ro rockylinux:8 \
|
||||
sh -c "dnf install -y /packages/*.el8.*.rpm && echo 'RPM el8 install OK'"
|
||||
|
||||
## test-package-rpm-el9: Test el9 RPM installation
|
||||
test-package-rpm-el9: package
|
||||
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el9:/packages:ro rockylinux:9 \
|
||||
sh -c "dnf install -y /packages/*.el9.*.rpm && echo 'RPM el9 install OK'"
|
||||
|
||||
## test-package-rpm-el10: Test el10 RPM installation
|
||||
test-package-rpm-el10: package
|
||||
docker run --rm -v $(PWD)/$(DIST_DIR)/rpm/el10:/packages:ro almalinux:10 \
|
||||
sh -c "dnf install -y /packages/*.el10.*.rpm && echo 'RPM el10 install OK'"
|
||||
|
||||
## test-package: Test all RPM packages installation
|
||||
test-package: test-package-rpm-el8 test-package-rpm-el9 test-package-rpm-el10
|
||||
|
||||
# Help target
|
||||
help:
|
||||
@echo "mod_reqin_log Makefile"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@echo " all - Build the module (default)"
|
||||
@echo " install - Install the module to DESTDIR"
|
||||
@echo " uninstall - Remove the module from DESTDIR"
|
||||
@echo " clean - Remove build artifacts"
|
||||
@echo " test - Run unit tests"
|
||||
@echo " debug - Build with debug symbols"
|
||||
@echo " package - Build all RPM packages (el8, el9, el10)"
|
||||
@echo " package-rpm - Build RPM packages"
|
||||
@echo " test-package - Test RPM package installation"
|
||||
@echo ""
|
||||
@echo "Variables:"
|
||||
@echo " APXS - Path to apxs tool (default: apxs)"
|
||||
@echo " CC - C compiler (default: gcc)"
|
||||
@echo " CFLAGS - Compiler flags (default: -Wall -Wextra -O2)"
|
||||
@echo " DESTDIR - Installation destination (default: /)"
|
||||
@echo " VERSION - Package version (default: 1.0.4)"
|
||||
@ -1,284 +0,0 @@
|
||||
# mod_reqin_log
|
||||
|
||||
Apache HTTPD 2.4 module for logging all incoming HTTP requests as JSON lines to a Unix domain socket.
|
||||
|
||||
## Features
|
||||
|
||||
- **Non-blocking I/O**: Logging never blocks worker processes
|
||||
- **Request-time logging**: Logs at `post_read_request` phase, capturing request data before application processing
|
||||
- **Configurable headers**: Select which HTTP headers to include in logs
|
||||
- **Header truncation**: Limit header value length to protect against oversized logs
|
||||
- **Automatic reconnection**: Reconnects to Unix socket on failure with configurable backoff
|
||||
- **Throttled error reporting**: Prevents error_log flooding on persistent failures
|
||||
- **MPM compatible**: Works with prefork, worker, and event MPMs
|
||||
- **Built-in security**: Sensitive headers (Authorization, Cookie, etc.) are automatically excluded
|
||||
- **RPM packaging**: Standard RPM packages for Rocky Linux 8/9 and AlmaLinux 10
|
||||
|
||||
## Requirements
|
||||
|
||||
### Runtime
|
||||
- Apache HTTPD 2.4+
|
||||
- GCC compiler
|
||||
- APR development libraries
|
||||
- Apache development headers (`httpd-devel` or `apache2-dev`)
|
||||
|
||||
### Packaging (RPM)
|
||||
- Docker (for reproducible builds)
|
||||
- rpmbuild (inside Docker)
|
||||
|
||||
## Installation
|
||||
|
||||
### Using Docker (recommended)
|
||||
|
||||
```bash
|
||||
# Build all RPM packages (el8, el9, el10)
|
||||
make package
|
||||
|
||||
# Test RPM package installation
|
||||
make test-package-rpm-el8 # Test el8 RPM (Rocky 8/RHEL 8)
|
||||
make test-package-rpm-el9 # Test el9 RPM (Rocky 9/RHEL 9)
|
||||
make test-package-rpm-el10 # Test el10 RPM (AlmaLinux 10/RHEL 10)
|
||||
make test-package # Test all RPM packages
|
||||
```
|
||||
|
||||
### Build from Source
|
||||
|
||||
```bash
|
||||
# Clone or extract the source
|
||||
cd mod_reqin_log
|
||||
|
||||
# Build the module
|
||||
make
|
||||
|
||||
# Install (requires root privileges)
|
||||
sudo make install
|
||||
```
|
||||
|
||||
## Configuration
|
||||
|
||||
Load the module and configure in your Apache configuration:
|
||||
|
||||
```apache
|
||||
# Load the module
|
||||
LoadModule reqin_log_module modules/mod_reqin_log.so
|
||||
|
||||
# Enable logging
|
||||
JsonSockLogEnabled On
|
||||
|
||||
# Unix socket path
|
||||
JsonSockLogSocket "/var/run/logcorrelator/http.socket"
|
||||
|
||||
# Headers to log (be careful not to log sensitive data)
|
||||
JsonSockLogHeaders X-Request-Id X-Trace-Id User-Agent Referer
|
||||
|
||||
# Maximum headers to log
|
||||
JsonSockLogMaxHeaders 10
|
||||
|
||||
# Maximum header value length
|
||||
JsonSockLogMaxHeaderValueLen 256
|
||||
|
||||
# Reconnect interval (seconds)
|
||||
JsonSockLogReconnectInterval 10
|
||||
|
||||
# Error report interval (seconds)
|
||||
JsonSockLogErrorReportInterval 10
|
||||
```
|
||||
|
||||
> **Important startup validation:** if `JsonSockLogEnabled On` is set without a valid `JsonSockLogSocket`, Apache startup fails with a configuration error.
|
||||
|
||||
### Configuration Directives
|
||||
|
||||
| Directive | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `JsonSockLogEnabled` | On/Off | Off | Enable or disable logging |
|
||||
| `JsonSockLogSocket` | String | - | Unix domain socket path |
|
||||
| `JsonSockLogHeaders` | List | - | HTTP headers to include |
|
||||
| `JsonSockLogMaxHeaders` | Integer | 10 | Max headers to log |
|
||||
| `JsonSockLogMaxHeaderValueLen` | Integer | 256 | Max header value length |
|
||||
| `JsonSockLogReconnectInterval` | Integer | 10 | Reconnect delay (seconds) |
|
||||
| `JsonSockLogErrorReportInterval` | Integer | 10 | Error log throttle (seconds) |
|
||||
|
||||
## JSON Log Format
|
||||
|
||||
Each log entry is a single-line JSON object with a flat structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"time": "2026-02-26T11:59:30Z",
|
||||
"timestamp": 1708948770000000000,
|
||||
"src_ip": "192.0.2.10",
|
||||
"src_port": 45678,
|
||||
"dst_ip": "198.51.100.5",
|
||||
"dst_port": 443,
|
||||
"method": "GET",
|
||||
"path": "/api/users",
|
||||
"host": "example.com",
|
||||
"http_version": "HTTP/1.1",
|
||||
"header_X-Request-Id": "abcd-1234",
|
||||
"header_User-Agent": "curl/7.70.0"
|
||||
}
|
||||
```
|
||||
|
||||
### Fields
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `time` | String | ISO8601 timestamp with timezone |
|
||||
| `timestamp` | Integer | Microseconds since epoch (expressed as nanoseconds for compatibility) |
|
||||
| `src_ip` | String | Client IP address |
|
||||
| `src_port` | Integer | Client port |
|
||||
| `dst_ip` | String | Server IP address |
|
||||
| `dst_port` | Integer | Server port |
|
||||
| `method` | String | HTTP method |
|
||||
| `path` | String | Request path |
|
||||
| `host` | String | Host header value |
|
||||
| `http_version` | String | HTTP protocol version |
|
||||
| `header_<Name>` | String | Flattened HTTP headers (e.g., `header_X-Request-Id`) |
|
||||
|
||||
**Note:** Headers are logged as flat fields at the root level (not nested). Sensitive headers are automatically excluded. The `timestamp` field has microsecond precision (APR's `apr_time_now()` returns microseconds, multiplied by 1000 for nanosecond representation).
|
||||
|
||||
## Unix Socket Consumer
|
||||
|
||||
Create a Unix socket listener to receive log entries:
|
||||
|
||||
```python
|
||||
#!/usr/bin/env python3
|
||||
import socket
|
||||
import os
|
||||
import json
|
||||
|
||||
SOCKET_PATH = os.environ.get("MOD_REQIN_LOG_SOCKET", "/var/run/logcorrelator/http.socket")
|
||||
|
||||
# Remove existing socket file
|
||||
if os.path.exists(SOCKET_PATH):
|
||||
os.remove(SOCKET_PATH)
|
||||
|
||||
# Create Unix socket server
|
||||
server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
server.bind(SOCKET_PATH)
|
||||
server.listen(5)
|
||||
# Set secure permissions: owner and group only (not world-writable)
|
||||
os.chmod(SOCKET_PATH, 0o660)
|
||||
|
||||
print(f"Listening on {SOCKET_PATH}")
|
||||
|
||||
while True:
|
||||
conn, addr = server.accept()
|
||||
data = b""
|
||||
while True:
|
||||
chunk = conn.recv(4096)
|
||||
if not chunk:
|
||||
break
|
||||
data += chunk
|
||||
if b"\n" in data:
|
||||
for line in data.decode().strip().split("\n"):
|
||||
if line:
|
||||
log_entry = json.loads(line)
|
||||
print(json.dumps(log_entry, indent=2))
|
||||
data = b""
|
||||
conn.close()
|
||||
```
|
||||
|
||||
**Note:** Ensure the Apache user is in the socket file's group to allow connections.
|
||||
|
||||
## Security Considerations
|
||||
|
||||
### Built-in Sensitive Headers Blacklist
|
||||
|
||||
⚠️ **The module automatically blocks logging of sensitive headers:**
|
||||
|
||||
The following headers are **always excluded** from logs to prevent credential leakage:
|
||||
- `Authorization`
|
||||
- `Cookie`, `Set-Cookie`
|
||||
- `X-Api-Key`, `X-Auth-Token`
|
||||
- `Proxy-Authorization`
|
||||
- `WWW-Authenticate`
|
||||
|
||||
These headers are silently skipped (logged at DEBUG level only).
|
||||
|
||||
### Socket Security
|
||||
|
||||
- **Socket permissions**: Default to `0o660` (owner and group only)
|
||||
- **Recommended path**: `/var/run/logcorrelator/http.socket` (not `/tmp`)
|
||||
- **Environment variable**: Use `MOD_REQIN_LOG_SOCKET` to configure path
|
||||
- **Group membership**: Ensure Apache user is in the socket's group
|
||||
|
||||
### Additional Hardening
|
||||
|
||||
- **Socket path length**: Validated against system limit (108 bytes max)
|
||||
- **JSON size limit**: 64KB max per log line (prevents memory DoS)
|
||||
- **NULL pointer checks**: All connection/request fields validated
|
||||
- **Thread safety**: Mutex protects socket FD in worker/event MPMs
|
||||
- **Error logging**: Generic messages in error_log, details at DEBUG level
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Module not loading
|
||||
|
||||
```
|
||||
AH00534: mod_reqin_log: Unable to load module
|
||||
```
|
||||
|
||||
Ensure the module path is correct and the file exists:
|
||||
```bash
|
||||
ls -la /usr/lib/apache2/modules/mod_reqin_log.so
|
||||
```
|
||||
|
||||
### Socket connection failures
|
||||
|
||||
```
|
||||
[mod_reqin_log] Unix socket connect failed: /var/run/logcorrelator/http.socket
|
||||
```
|
||||
|
||||
- Ensure the socket consumer is running
|
||||
- Check socket file permissions
|
||||
- Verify SELinux/AppArmor policies if applicable
|
||||
|
||||
### No logs appearing
|
||||
|
||||
1. Verify `JsonSockLogEnabled On` is set
|
||||
2. Verify `JsonSockLogSocket` path is configured
|
||||
3. Check Apache error log for module errors
|
||||
4. Ensure socket consumer is listening
|
||||
|
||||
## Testing
|
||||
|
||||
### Run Unit Tests
|
||||
|
||||
```bash
|
||||
# Using Docker (recommended)
|
||||
docker build -f Dockerfile.tests -t mod_reqin_log:tests .
|
||||
docker run --rm mod_reqin_log:tests ctest --output-on-failure
|
||||
|
||||
# Or locally with cmocka
|
||||
sudo dnf install cmocka-devel # Rocky Linux
|
||||
sudo apt install libcmocka-dev # Debian/Ubuntu
|
||||
mkdir build && cd build
|
||||
cmake ..
|
||||
make test
|
||||
```
|
||||
|
||||
### Build and Test Packages
|
||||
|
||||
```bash
|
||||
# Build all RPM packages (el8, el9, el10)
|
||||
make package
|
||||
|
||||
# Test RPM package installation
|
||||
make test-package-rpm-el8 # Test el8 RPM in Docker
|
||||
make test-package-rpm-el9 # Test el9 RPM in Docker
|
||||
make test-package-rpm-el10 # Test el10 RPM in Docker
|
||||
make test-package # Test all RPM packages
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
Apache License 2.0
|
||||
|
||||
## Contributing
|
||||
|
||||
1. Fork the repository
|
||||
2. Create a feature branch
|
||||
3. Make your changes
|
||||
4. Run tests
|
||||
5. Submit a pull request
|
||||
@ -1,443 +0,0 @@
|
||||
project:
|
||||
name: mod_reqin_log
|
||||
description: >
|
||||
Apache HTTPD 2.4 module logging all incoming HTTP requests as JSON lines
|
||||
to a Unix domain socket at request reception time (no processing time).
|
||||
language: c
|
||||
author:
|
||||
name: Jacquin Antoine
|
||||
email: rpm@arkel.fr
|
||||
target:
|
||||
server: apache-httpd
|
||||
version: "2.4"
|
||||
os: rocky-linux-8+, almalinux-10+
|
||||
build:
|
||||
toolchain: gcc
|
||||
apache_dev: httpd-devel (apxs)
|
||||
artifacts:
|
||||
- mod_reqin_log.so
|
||||
|
||||
context:
|
||||
architecture:
|
||||
pattern: native-apache-module
|
||||
scope: global
|
||||
mpm_compatibility:
|
||||
- prefork
|
||||
- worker
|
||||
- event
|
||||
request_phase:
|
||||
hook: post_read_request
|
||||
rationale: >
|
||||
Log as soon as the HTTP request is fully read to capture input-side data
|
||||
(client/server addresses, request line, headers) without waiting for
|
||||
application processing.
|
||||
filters:
|
||||
- Subrequests (r->main != NULL) are skipped.
|
||||
- Internal redirects (r->prev != NULL) are skipped.
|
||||
- Only the original client request is logged.
|
||||
logging_scope:
|
||||
coverage: all-traffic
|
||||
description: >
|
||||
Every HTTP request handled by the Apache instance is considered for logging
|
||||
when the module is enabled and the Unix socket is configured.
|
||||
|
||||
module:
|
||||
name: mod_reqin_log
|
||||
files:
|
||||
source:
|
||||
- src/mod_reqin_log.c
|
||||
- src/mod_reqin_log.h
|
||||
packaging:
|
||||
- mod_reqin_log.spec
|
||||
tests:
|
||||
- tests/unit/test_module_real.c
|
||||
- tests/unit/test_config_parsing.c
|
||||
- tests/unit/test_header_handling.c
|
||||
- tests/unit/test_json_serialization.c
|
||||
hooks:
|
||||
- name: register_hooks
|
||||
responsibilities:
|
||||
- Register post_read_request hook for logging at request reception.
|
||||
- Register child_init hook for per-process state initialization.
|
||||
- Initialize per-process server configuration structure.
|
||||
- name: child_init
|
||||
responsibilities:
|
||||
- Initialize module state for each Apache child process.
|
||||
- Reset per-process socket state (fd, timers, error flags).
|
||||
- Attempt initial non-blocking connection to Unix socket if configured.
|
||||
- name: post_read_request
|
||||
responsibilities:
|
||||
- Retrieve per-process server configuration (thread-safe).
|
||||
- Ensure Unix socket is connected (with periodic reconnect).
|
||||
- Build JSON log document for the request.
|
||||
- Write JSON line to Unix socket using non-blocking I/O.
|
||||
- Handle errors by dropping the current log line and rate-limiting
|
||||
error reports into Apache error_log.
|
||||
thread_safety:
|
||||
model: per-process-state
|
||||
description: >
|
||||
Each Apache child process maintains its own socket state stored in the
|
||||
server configuration structure (reqin_log_server_conf_t). This avoids
|
||||
race conditions in worker and event MPMs where multiple threads share
|
||||
a process.
|
||||
implementation:
|
||||
- State stored via ap_get_module_config(s->module_config)
|
||||
- No global variables for socket state
|
||||
- Each process has independent: socket_fd, connect timers, error timers
|
||||
|
||||
data_model:
|
||||
json_line:
|
||||
description: >
|
||||
One JSON object per HTTP request, serialized on a single line and
|
||||
terminated by "\n". Uses flat structure with header fields at root level.
|
||||
structure: flat
|
||||
fields:
|
||||
- name: time
|
||||
type: string
|
||||
format: iso8601-with-timezone
|
||||
example: "2026-02-26T11:59:30Z"
|
||||
- name: timestamp
|
||||
type: integer
|
||||
unit: microseconds (expressed as nanoseconds)
|
||||
description: >
|
||||
Wall-clock timestamp in microseconds since Unix epoch, expressed
|
||||
as nanoseconds for compatibility (multiplied by 1000).
|
||||
Uses r->request_time (set by Apache at request reception).
|
||||
The nanosecond representation is for API compatibility only.
|
||||
example: 1708948770000000000
|
||||
- name: scheme
|
||||
type: string
|
||||
description: Connection scheme evaluated via ap_http_scheme(r).
|
||||
example: "https"
|
||||
- name: src_ip
|
||||
type: string
|
||||
example: "192.0.2.10"
|
||||
- name: src_port
|
||||
type: integer
|
||||
example: 45678
|
||||
- name: dst_ip
|
||||
type: string
|
||||
example: "198.51.100.5"
|
||||
- name: dst_port
|
||||
type: integer
|
||||
example: 443
|
||||
- name: method
|
||||
type: string
|
||||
example: "GET"
|
||||
- name: path
|
||||
type: string
|
||||
description: Cleaned and normalized path (r->parsed_uri.path).
|
||||
example: "/api/users"
|
||||
- name: query
|
||||
type: string
|
||||
description: >
|
||||
Query string component from the parsed URI (r->parsed_uri.query).
|
||||
Does not include the leading '?'. Allows detection of payloads like
|
||||
SQLi or XSS passed in GET requests.
|
||||
example: "id=1%20UNION%20SELECT"
|
||||
- name: host
|
||||
type: string
|
||||
example: "example.com"
|
||||
- name: http_version
|
||||
type: string
|
||||
example: "HTTP/1.1"
|
||||
- name: keepalives
|
||||
type: integer
|
||||
description: >
|
||||
Number of requests served over the current connection (r->connection->keepalives).
|
||||
If 0, it indicates a newly established TCP connection.
|
||||
If > 0, it confirms an active Keep-Alive session.
|
||||
example: 2
|
||||
- name: client_headers
|
||||
type: array of strings
|
||||
description: >
|
||||
Ordered list of all HTTP header names as received from the client
|
||||
(r->headers_in), preserving original order and case.
|
||||
Useful for browser/bot fingerprinting (header order is client-specific).
|
||||
example: ["Host", "User-Agent", "Accept", "Accept-Language", "Accept-Encoding"]
|
||||
- name: content_length
|
||||
type: integer
|
||||
description: >
|
||||
Declared size of the request body (POST payload),
|
||||
extracted directly from the 'Content-Length' header.
|
||||
example: 1048576
|
||||
- name: header_<HeaderName>
|
||||
type: string
|
||||
description: >
|
||||
Flattened header fields at root level. For each configured header <H>,
|
||||
a field 'header_<H>' is added directly to the JSON root object.
|
||||
Headers are only included if present in the request.
|
||||
key_pattern: "header_<configured_header_name>"
|
||||
optional: true
|
||||
example:
|
||||
header_X-Request-Id: "abcd-1234"
|
||||
header_User-Agent: "curl/7.70.0"
|
||||
example_full: |
|
||||
{"time":"2026-02-26T11:59:30Z","timestamp":1708948770000000000,"scheme":"https","src_ip":"192.0.2.10","src_port":45678,"dst_ip":"198.51.100.5","dst_port":443,"method":"GET","path":"/api/users","query":"id=1","host":"example.com","http_version":"HTTP/1.1","keepalives":0,"client_headers":["Host","User-Agent","Accept","Accept-Language","Accept-Encoding","X-Request-Id"],"content_length":0,"header_X-Request-Id":"abcd-1234","header_User-Agent":"curl/7.70.0"}
|
||||
|
||||
configuration:
|
||||
scope: global
|
||||
directives:
|
||||
- name: JsonSockLogEnabled
|
||||
type: flag
|
||||
context: server-config
|
||||
default: "Off"
|
||||
description: >
|
||||
Enable or disable mod_reqin_log logging globally. Logging only occurs
|
||||
when this directive is On and JsonSockLogSocket is set.
|
||||
- name: JsonSockLogSocket
|
||||
type: string
|
||||
context: server-config
|
||||
required_when_enabled: true
|
||||
example: "/var/run/logcorrelator/http.socket"
|
||||
description: >
|
||||
Filesystem path of the Unix domain socket to which JSON log lines
|
||||
will be written.
|
||||
- name: JsonSockLogHeaders
|
||||
type: list
|
||||
context: server-config
|
||||
value_example: ["X-Request-Id", "X-Trace-Id", "User-Agent", "Referer",
|
||||
"X-Forwarded-For", "Sec-CH-UA", "Sec-CH-UA-Mobile", "Sec-CH-UA-Platform",
|
||||
"Sec-Fetch-Dest", "Sec-Fetch-Mode", "Sec-Fetch-Site",
|
||||
"Accept", "Accept-Language", "Accept-Encoding"]
|
||||
description: >
|
||||
List of HTTP header names to log. For each configured header <H>,
|
||||
the module adds a JSON field 'header_<H>' at the root level of the
|
||||
JSON log entry (flat structure). Order matters for applying the
|
||||
JsonSockLogMaxHeaders limit.
|
||||
- name: JsonSockLogMaxHeaders
|
||||
type: integer
|
||||
context: server-config
|
||||
default: 25
|
||||
min: 0
|
||||
description: >
|
||||
Maximum number of headers from JsonSockLogHeaders to actually log.
|
||||
If more headers are configured, only the first N are considered.
|
||||
- name: JsonSockLogMaxHeaderValueLen
|
||||
type: integer
|
||||
context: server-config
|
||||
default: 256
|
||||
min: 1
|
||||
description: >
|
||||
Maximum length in characters for each logged header value.
|
||||
Values longer than this limit are truncated before JSON encoding.
|
||||
- name: JsonSockLogReconnectInterval
|
||||
type: integer
|
||||
context: server-config
|
||||
default: 10
|
||||
unit: seconds
|
||||
description: >
|
||||
Minimal delay between two connection attempts to the Unix socket after
|
||||
a failure. Used to avoid reconnect attempts on every request.
|
||||
- name: JsonSockLogErrorReportInterval
|
||||
type: integer
|
||||
context: server-config
|
||||
default: 10
|
||||
unit: seconds
|
||||
description: >
|
||||
Minimal delay between two error messages emitted into Apache error_log
|
||||
for repeated I/O or connection errors on the Unix socket.
|
||||
|
||||
behavior:
|
||||
enabling_rules:
|
||||
- JsonSockLogEnabled must be On.
|
||||
- JsonSockLogSocket must be set to a non-empty path.
|
||||
header_handling:
|
||||
- Built-in blacklist prevents logging of sensitive headers by default.
|
||||
- Blacklisted headers: Authorization, Cookie, Set-Cookie, X-Api-Key,
|
||||
X-Auth-Token, Proxy-Authorization, WWW-Authenticate.
|
||||
- Blacklisted headers are silently skipped (logged at DEBUG level only).
|
||||
- If a configured header is absent in a request, the corresponding
|
||||
JSON key is omitted from the log entry.
|
||||
- Header values are truncated to JsonSockLogMaxHeaderValueLen characters.
|
||||
|
||||
io:
|
||||
socket:
|
||||
type: unix-domain
|
||||
protocol: SOCK_DGRAM
|
||||
mode: client
|
||||
path_source: JsonSockLogSocket
|
||||
connection:
|
||||
persistence: false
|
||||
non_blocking: true
|
||||
lifecycle:
|
||||
open:
|
||||
- Create DGRAM socket and set default destination address via connect()
|
||||
during child_init if enabled.
|
||||
- Re-attempt addressing after reconnect interval expiry if target
|
||||
was previously unavailable.
|
||||
failure:
|
||||
- On missing target socket (ECONNREFUSED/ENOENT), mark target as unavailable.
|
||||
- Do not block the worker process.
|
||||
reconnect:
|
||||
strategy: time-based
|
||||
interval_seconds: config.JsonSockLogReconnectInterval
|
||||
trigger: >
|
||||
When a request arrives and the last target resolution attempt time is older
|
||||
than reconnect interval, a new attempt to address the socket is made.
|
||||
write:
|
||||
format: json_object
|
||||
mode: non-blocking
|
||||
atomicity: >
|
||||
Full JSON line is sent as a single datagram. Message size must not exceed
|
||||
system DGRAM limits or MAX_JSON_SIZE (64KB).
|
||||
error_handling:
|
||||
on_eagain_or_ewouldblock:
|
||||
action: drop-current-log-line
|
||||
note: "OS buffer full (receiver is too slow). Do not retry, do not spam error_log."
|
||||
on_econnrefused_or_enoent:
|
||||
action:
|
||||
- close_socket
|
||||
- mark_target_unavailable
|
||||
- schedule_reconnect
|
||||
note: "Target socket closed or deleted by log receiver."
|
||||
generic_errors:
|
||||
action: drop-current-log-line
|
||||
drop_policy:
|
||||
description: >
|
||||
Logging errors never impact client response. The current log line is
|
||||
silently dropped except for throttled error_log reporting.
|
||||
|
||||
error_handling:
|
||||
apache_error_log_reporting:
|
||||
enabled: true
|
||||
throttle_interval_seconds: config.JsonSockLogErrorReportInterval
|
||||
events:
|
||||
- type: connect_failure
|
||||
message_template: "mod_reqin_log: Unix socket connect failed: [errno_detail]"
|
||||
- type: write_failure
|
||||
message_template: "mod_reqin_log: Unix socket write failed: [errno_detail]"
|
||||
fatal_conditions:
|
||||
- description: >
|
||||
Misconfiguration (JsonSockLogEnabled On but missing JsonSockLogSocket)
|
||||
should be reported at startup as a configuration error.
|
||||
- description: >
|
||||
Any internal JSON-encoding failure should be treated as non-fatal
|
||||
(drop current log and optionally emit a throttled error_log entry).
|
||||
|
||||
constraints:
|
||||
performance:
|
||||
objectives:
|
||||
- Logging overhead per request should be minimal and non-blocking.
|
||||
- No dynamic allocations in hot path beyond what is strictly necessary
|
||||
(prefer APR pools where possible).
|
||||
design_choices:
|
||||
- Single JSON serialization pass per request.
|
||||
- Use non-blocking I/O to avoid stalling worker threads/processes.
|
||||
- Avoid reconnect attempts on every request via time-based backoff.
|
||||
security:
|
||||
notes:
|
||||
- Module includes built-in blacklist of sensitive headers to prevent
|
||||
accidental credential leakage (Authorization, Cookie, X-Api-Key, etc.).
|
||||
- Socket permissions default to 0o660 (owner/group only) for security.
|
||||
- Recommended socket path: /var/run/logcorrelator/http.socket (not /tmp).
|
||||
- Use environment variable MOD_REQINLOG_SOCKET to configure socket path.
|
||||
- Module does not anonymize IPs (data protection is delegated to configuration).
|
||||
- No requests are rejected due to logging failures.
|
||||
hardening:
|
||||
- Socket path length validated against system limit (108 bytes).
|
||||
- JSON log line size limited to 64KB to prevent memory exhaustion DoS.
|
||||
- NULL pointer checks on all connection/request fields.
|
||||
- Thread-safe socket FD access via mutex (worker/event MPMs).
|
||||
- Error logging reduced to prevent information disclosure.
|
||||
robustness:
|
||||
requirements:
|
||||
- Logging failures must not crash Apache worker processes.
|
||||
- Module must behave correctly under high traffic, socket disappearance,
|
||||
and repeated connect failures.
|
||||
|
||||
testing_strategy:
|
||||
unit_tests:
|
||||
framework: cmocka
|
||||
location: tests/unit/test_module_real.c
|
||||
focus:
|
||||
- JSON serialization with header truncation and header count limits.
|
||||
- Dynamic buffer operations (dynbuf_t) with resize handling.
|
||||
- ISO8601 timestamp formatting.
|
||||
- Header value truncation to JsonSockLogMaxHeaderValueLen.
|
||||
- Control character escaping in JSON strings.
|
||||
execution:
|
||||
- docker build -f Dockerfile.tests .
|
||||
- docker run --rm <image> ctest --output-on-failure
|
||||
|
||||
ci_strategy:
|
||||
description: >
|
||||
All builds, tests and packaging are executed inside Docker containers
|
||||
using GitLab CI with Docker-in-Docker (dind). No RPM build or test is
|
||||
allowed on bare-metal or shared CI runners.
|
||||
tools:
|
||||
orchestrator: GitLab CI
|
||||
container_engine: docker
|
||||
dind: true
|
||||
workflow_file: .gitlab-ci.yml
|
||||
constraints:
|
||||
no_host_builds: true
|
||||
description: >
|
||||
It is forbidden to run rpmbuild, unit tests or package verification
|
||||
directly on the CI host. All steps MUST run inside Docker containers
|
||||
defined by project Dockerfiles.
|
||||
rpm_strategy: >
|
||||
Separate RPMs are built for each major RHEL/CentOS/Rocky/AlmaLinux version
|
||||
(el8, el9, el10) due to glibc and httpd-devel incompatibilities across
|
||||
major versions. A single RPM cannot work across all versions.
|
||||
RPM packages are built using rpmbuild with mod_reqin_log.spec file.
|
||||
rpm_changelog:
|
||||
policy: mandatory
|
||||
description: >
|
||||
For every version or release bump of the RPM (Version or Release tag
|
||||
in mod_reqin_log.spec), the %changelog section MUST be updated with:
|
||||
- date, packager, new version-release
|
||||
- brief description of the changes.
|
||||
validation:
|
||||
- A CI job MUST fail if Version/Release changed and no new %changelog
|
||||
entry is present.
|
||||
- Changelog is the single source of truth for packaged changes.
|
||||
stages:
|
||||
- name: validate-spec
|
||||
description: >
|
||||
Ensure that any change to Version/Release in mod_reqin_log.spec
|
||||
is accompanied by a new %changelog entry.
|
||||
containerized: true
|
||||
dockerfile: Dockerfile.tools
|
||||
checks:
|
||||
- script: scripts/check_spec_changelog.sh mod_reqin_log.spec
|
||||
fail_on_missing_changelog: true
|
||||
- name: build
|
||||
description: >
|
||||
Build all RPM packages (el8, el9, el10) using Dockerfile.package
|
||||
with multi-stage build, entirely inside a Docker container.
|
||||
dockerfile: Dockerfile.package
|
||||
containerized: true
|
||||
artifacts:
|
||||
- dist/rpm/*.el8.*.rpm
|
||||
- dist/rpm/*.el9.*.rpm
|
||||
- dist/rpm/*.el10.*.rpm
|
||||
- name: test
|
||||
description: >
|
||||
Run unit tests (C with cmocka) inside Docker containers, using
|
||||
Dockerfile.tests as the only execution environment.
|
||||
dockerfile: Dockerfile.tests
|
||||
containerized: true
|
||||
execution: ctest --output-on-failure
|
||||
- name: verify
|
||||
description: >
|
||||
Verify RPM installation and module loading on each target distribution
|
||||
by running containers for each OS.
|
||||
containerized: true
|
||||
jobs:
|
||||
- name: verify-rpm-el8
|
||||
image: rockylinux:8
|
||||
steps:
|
||||
- rpm -qi mod_reqin_log
|
||||
- httpd -M | grep reqin_log
|
||||
- name: verify-rpm-el9
|
||||
image: rockylinux:9
|
||||
steps:
|
||||
- rpm -qi mod_reqin_log
|
||||
- httpd -M | grep reqin_log
|
||||
- name: verify-rpm-el10
|
||||
image: almalinux:10
|
||||
steps:
|
||||
- rpm -qi mod_reqin_log
|
||||
- httpd -M | grep reqin_log
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
# mod_reqin_log example configuration
|
||||
# Load this configuration in your Apache httpd.conf or a separate included file
|
||||
|
||||
# Load the module (adjust path as needed)
|
||||
LoadModule reqin_log_module modules/mod_reqin_log.so
|
||||
|
||||
# Enable mod_reqin_log
|
||||
JsonSockLogEnabled On
|
||||
|
||||
# Unix domain socket path for JSON log output
|
||||
# Important: if JsonSockLogEnabled is On and this directive is missing/empty,
|
||||
# Apache startup fails due to strict configuration validation.
|
||||
JsonSockLogSocket "/var/run/logcorrelator/http.socket"
|
||||
|
||||
# HTTP headers to include in the JSON log
|
||||
# Warning: Be careful not to log sensitive headers like Authorization, Cookie, etc.
|
||||
JsonSockLogHeaders X-Request-Id X-Trace-Id User-Agent Referer X-Forwarded-For \
|
||||
Sec-CH-UA Sec-CH-UA-Mobile Sec-CH-UA-Platform \
|
||||
Sec-Fetch-Dest Sec-Fetch-Mode Sec-Fetch-Site \
|
||||
Accept Accept-Language Accept-Encoding Content-Type
|
||||
|
||||
# Maximum number of headers to log (from the configured list)
|
||||
JsonSockLogMaxHeaders 25
|
||||
|
||||
# Maximum length of each header value (longer values are truncated)
|
||||
JsonSockLogMaxHeaderValueLen 256
|
||||
|
||||
# Minimum delay between reconnect attempts to the Unix socket (seconds)
|
||||
JsonSockLogReconnectInterval 10
|
||||
|
||||
# Minimum delay between error messages to Apache error_log (seconds)
|
||||
JsonSockLogErrorReportInterval 10
|
||||
|
||||
# Log level for module messages: DEBUG, INFO, WARNING, ERROR, EMERG (default: WARNING)
|
||||
# DEBUG: Log all messages including header skipping and buffer truncation
|
||||
# INFO: Log informational messages
|
||||
# WARNING: Log warnings (default)
|
||||
# ERROR: Log only errors
|
||||
# EMERG: Log only emergency messages
|
||||
JsonSockLogLevel WARNING
|
||||
@ -1,123 +0,0 @@
|
||||
%global spec_version 1.0.19
|
||||
|
||||
Name: mod_reqin_log
|
||||
Version: %{spec_version}
|
||||
Release: 1%{?dist}
|
||||
Summary: Apache HTTPD module for logging HTTP requests as JSON to Unix socket
|
||||
|
||||
License: Apache-2.0
|
||||
URL: https://github.com/example/mod_reqin_log
|
||||
Vendor: Developer <dev@example.com>
|
||||
BuildArch: x86_64
|
||||
|
||||
Requires: httpd
|
||||
|
||||
%description
|
||||
Apache HTTPD module for logging HTTP requests as JSON to Unix socket.
|
||||
Features non-blocking I/O with automatic reconnection, configurable headers
|
||||
with truncation support, and built-in sensitive headers blacklist.
|
||||
|
||||
%prep
|
||||
# No source extraction needed - binaries are pre-built
|
||||
|
||||
%build
|
||||
# No build needed - binaries are pre-built
|
||||
|
||||
%install
|
||||
mkdir -p %{buildroot}/%{_libdir}/httpd/modules
|
||||
mkdir -p %{buildroot}/%{_sysconfdir}/httpd/conf.d
|
||||
mkdir -p %{buildroot}/%{_docdir}/%{name}
|
||||
|
||||
install -m 755 %{_pkgroot}/%{_libdir}/httpd/modules/mod_reqin_log.so %{buildroot}/%{_libdir}/httpd/modules/
|
||||
install -m 644 %{_pkgroot}/%{_sysconfdir}/httpd/conf.d/mod_reqin_log.conf %{buildroot}/%{_sysconfdir}/httpd/conf.d/
|
||||
|
||||
%files
|
||||
%{_libdir}/httpd/modules/mod_reqin_log.so
|
||||
%config(noreplace) %{_sysconfdir}/httpd/conf.d/mod_reqin_log.conf
|
||||
%doc %{_docdir}/%{name}
|
||||
|
||||
%changelog
|
||||
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.19
|
||||
- FEATURE: Add client_headers JSON field - ordered list of all header names
|
||||
as received from the client, preserving original order and case
|
||||
- DOC: Update architecture.yml with client_headers field and example_full
|
||||
|
||||
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.18
|
||||
- FIX: JsonSockLogMaxHeaders now counts configured headers (by position in list)
|
||||
regardless of their presence in the request, matching the documented behavior
|
||||
|
||||
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.17
|
||||
- CONFIG: Extend default JsonSockLogHeaders list (User-Agent, Referer, X-Forwarded-For,
|
||||
Sec-CH-UA*, Sec-Fetch-*, Accept, Accept-Language, Accept-Encoding)
|
||||
- CONFIG: Raise DEFAULT_MAX_HEADERS from 10 to 25
|
||||
- DOC: Update architecture.yml and conf/mod_reqin_log.conf accordingly
|
||||
|
||||
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.16
|
||||
- FIX: Skip subrequests and internal redirects to log only the original client request
|
||||
- DOC: Document subrequest/redirect filtering in architecture.yml
|
||||
|
||||
* Thu Mar 05 2026 Developer <dev@example.com> - 1.0.15
|
||||
- FIX: timestamp field now uses r->request_time (request reception time) instead of apr_time_now()
|
||||
- DOC: Remove unparsed_uri and fragment fields from architecture.yml (not logged)
|
||||
- DOC: Update timestamp description and example_full in architecture.yml
|
||||
|
||||
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.14
|
||||
- REFACTOR: Harmonize JSON field construction - all fields now end with comma
|
||||
- FIX: Remove duplicate comma between query and host fields
|
||||
- FIX: Fix buffer corruption in dynbuf_append (copy null terminator)
|
||||
- PACKAGING: Config file marked as %config(noreplace)
|
||||
- CHANGE: Remove unparsed_uri, fragment, content_length fields
|
||||
|
||||
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.13
|
||||
- FIX: Correct JSON string length parameters for query and fragment fields
|
||||
- FIX: Add null-termination after buffer reallocation in dynbuf_append
|
||||
- CHANGE: Remove unparsed_uri, fragment, and content_length fields from JSON output
|
||||
- TEST: Update unit tests to match dynbuf_append fix
|
||||
|
||||
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.9
|
||||
- CHANGE: Remove req_id field from JSON output
|
||||
- FEATURE: Add query and fragment fields (URI components)
|
||||
|
||||
* Mon Mar 02 2026 Developer <dev@example.com> - 1.0.8
|
||||
- FEATURE: Add req_id, scheme, unparsed_uri, args, keepalives, content_length fields to JSON output
|
||||
- FIX: Change socket type from SOCK_STREAM to SOCK_DGRAM per architecture.yml
|
||||
|
||||
* Sun Mar 01 2026 Developer <dev@example.com> - 1.0.6
|
||||
- BUILD: Fix RPM package paths in Dockerfile.package (el8, el9, el10 directories)
|
||||
- BUILD: Fix Makefile RPM extraction with separate volume mounts
|
||||
- BUILD: Remove unused scripts (build.sh, test.sh)
|
||||
- BUILD: Remove Python integration tests (not automated in CI)
|
||||
- DOCS: Update README.md and architecture.yml for RPM-only packaging
|
||||
- CLEANUP: Remove DEB and el7 references
|
||||
|
||||
* Sat Feb 28 2026 Developer <dev@example.com> - 1.0.2
|
||||
- SECURITY: Add input sanitization for method, path, host, and http_version fields
|
||||
- SECURITY: Add Host header truncation (256 chars max) to prevent log injection
|
||||
- IMPROVEMENT: Add LOG_THROTTLED macro for consistent error reporting
|
||||
- IMPROVEMENT: Improve socket state double-check pattern
|
||||
- IMPROVEMENT: Fix const qualifier warnings in get_header() function
|
||||
- IMPROVEMENT: Add flags field to module definition
|
||||
- IMPROVEMENT: Add -Wno-error=format-security to Makefile
|
||||
- TEST: Add 4 new unit tests for input sanitization
|
||||
- DOC: Clarify timestamp precision
|
||||
- DOC: Update README and architecture.yml
|
||||
- BUILD: Update package version to 1.0.2
|
||||
|
||||
* Fri Feb 27 2026 Developer <dev@example.com> - 1.0.1
|
||||
- FIX: Fix socket reconnection logic
|
||||
- FIX: Improve error logging to prevent error_log flooding
|
||||
- IMPROVEMENT: Add built-in sensitive headers blacklist
|
||||
- IMPROVEMENT: Add thread-safe socket FD access via mutex
|
||||
- TEST: Add comprehensive unit tests
|
||||
- TEST: Add integration tests for socket loss and recovery
|
||||
- DOC: Add comprehensive README with configuration examples
|
||||
- DOC: Add architecture.yml documenting module design decisions
|
||||
|
||||
* Thu Feb 26 2026 Developer <dev@example.com> - 1.0.0
|
||||
- Initial release
|
||||
- Apache HTTPD 2.4 module for logging HTTP requests as JSON to Unix socket
|
||||
- Non-blocking I/O with automatic reconnection
|
||||
- Configurable headers with truncation support
|
||||
- Compatible with prefork, worker, and event MPMs
|
||||
- Built-in sensitive headers blacklist
|
||||
- Throttled error reporting to prevent log flooding
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,56 +0,0 @@
|
||||
/*
|
||||
* mod_reqin_log.h - Apache HTTPD module for logging HTTP requests as JSON to Unix socket
|
||||
*
|
||||
* Copyright (c) 2026. All rights reserved.
|
||||
*/
|
||||
|
||||
#ifndef MOD_REQIN_LOG_H
|
||||
#define MOD_REQIN_LOG_H
|
||||
|
||||
#include "httpd.h"
|
||||
#include "http_config.h"
|
||||
#include "apr_tables.h"
|
||||
|
||||
/* Module name */
|
||||
#define MOD_REQIN_LOG_NAME "mod_reqin_log"
|
||||
|
||||
/* Default configuration values */
|
||||
#define DEFAULT_MAX_HEADERS 25
|
||||
#define DEFAULT_MAX_HEADER_VALUE_LEN 256
|
||||
#define DEFAULT_RECONNECT_INTERVAL 10
|
||||
#define DEFAULT_ERROR_REPORT_INTERVAL 10
|
||||
|
||||
/* Module configuration structure */
|
||||
typedef struct {
|
||||
int enabled;
|
||||
const char *socket_path;
|
||||
apr_array_header_t *headers;
|
||||
int max_headers;
|
||||
int max_header_value_len;
|
||||
int reconnect_interval;
|
||||
int error_report_interval;
|
||||
} reqin_log_config_t;
|
||||
|
||||
/* External module declaration */
|
||||
extern module AP_MODULE_DECLARE_DATA reqin_log_module;
|
||||
|
||||
/* ====== Fingerprinting HTTP/2 passif ====== */
|
||||
|
||||
/* Clés des notes de connexion stockant le fingerprint HTTP/2 parsé */
|
||||
#define H2_NOTE_FINGERPRINT "reqin_h2_fp" /* Fingerprint Akamai complet */
|
||||
#define H2_NOTE_SETTINGS "reqin_h2_set" /* Entrées SETTINGS brutes */
|
||||
#define H2_NOTE_WUPDATE "reqin_h2_wu" /* Incrément WINDOW_UPDATE */
|
||||
#define H2_NOTE_PSEUDO_ORDER "reqin_h2_ps" /* Ordre pseudo-headers */
|
||||
#define H2_NOTE_HAS_PRIORITY "reqin_h2_pri" /* Flag PRIORITY présent */
|
||||
#define H2_NOTE_PARSED "reqin_h2_done" /* Marqueur "déjà parsé" */
|
||||
|
||||
/* Clés des notes pour chaque paramètre SETTINGS individuel (RFC 9113 §6.5.2) */
|
||||
#define H2_NOTE_SET_HEADER_TABLE_SIZE "reqin_h2_s1" /* ID 1 */
|
||||
#define H2_NOTE_SET_ENABLE_PUSH "reqin_h2_s2" /* ID 2 */
|
||||
#define H2_NOTE_SET_MAX_CONCURRENT_STREAMS "reqin_h2_s3" /* ID 3 */
|
||||
#define H2_NOTE_SET_INITIAL_WINDOW_SIZE "reqin_h2_s4" /* ID 4 */
|
||||
#define H2_NOTE_SET_MAX_FRAME_SIZE "reqin_h2_s5" /* ID 5 */
|
||||
#define H2_NOTE_SET_MAX_HEADER_LIST_SIZE "reqin_h2_s6" /* ID 6 */
|
||||
#define H2_NOTE_SET_ENABLE_CONNECT "reqin_h2_s8" /* ID 8 */
|
||||
|
||||
#endif /* MOD_REQIN_LOG_H */
|
||||
@ -1,333 +0,0 @@
|
||||
/*
|
||||
* test_config_parsing.c - Unit tests for configuration parsing
|
||||
*/
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <setjmp.h>
|
||||
#include <cmocka.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <errno.h>
|
||||
#include <limits.h>
|
||||
|
||||
/* Default configuration values */
|
||||
#define DEFAULT_MAX_HEADERS 10
|
||||
#define DEFAULT_MAX_HEADER_VALUE_LEN 256
|
||||
#define DEFAULT_RECONNECT_INTERVAL 10
|
||||
#define DEFAULT_ERROR_REPORT_INTERVAL 10
|
||||
#define MAX_SOCKET_PATH_LEN 108
|
||||
|
||||
/* Mock configuration structure */
|
||||
typedef struct {
|
||||
int enabled;
|
||||
const char *socket_path;
|
||||
int max_headers;
|
||||
int max_header_value_len;
|
||||
int reconnect_interval;
|
||||
int error_report_interval;
|
||||
} mock_config_t;
|
||||
|
||||
/* Mock parsing functions */
|
||||
static int parse_enabled(const char *value)
|
||||
{
|
||||
if (strcasecmp(value, "on") == 0 || strcmp(value, "1") == 0) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const char *parse_socket_path(const char *value)
|
||||
{
|
||||
if (value == NULL || strlen(value) == 0) {
|
||||
return NULL;
|
||||
}
|
||||
if (strlen(value) >= MAX_SOCKET_PATH_LEN) {
|
||||
return NULL;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
static int parse_int_strict(const char *value, int *result)
|
||||
{
|
||||
char *endptr = NULL;
|
||||
long val;
|
||||
|
||||
if (value == NULL || *value == '\0' || result == NULL) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
errno = 0;
|
||||
val = strtol(value, &endptr, 10);
|
||||
if (errno != 0 || endptr == value || *endptr != '\0' || val < INT_MIN || val > INT_MAX) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
*result = (int)val;
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int parse_max_headers(const char *value, int *result)
|
||||
{
|
||||
if (parse_int_strict(value, result) != 0 || *result < 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int parse_interval(const char *value, int *result)
|
||||
{
|
||||
if (parse_int_strict(value, result) != 0 || *result < 0) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int parse_max_header_value_len(const char *value, int *result)
|
||||
{
|
||||
if (parse_int_strict(value, result) != 0 || *result < 1) {
|
||||
return -1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Test: Parse enabled On */
|
||||
static void test_parse_enabled_on(void **state)
|
||||
{
|
||||
(void)state;
|
||||
assert_int_equal(parse_enabled("On"), 1);
|
||||
assert_int_equal(parse_enabled("on"), 1);
|
||||
assert_int_equal(parse_enabled("ON"), 1);
|
||||
assert_int_equal(parse_enabled("1"), 1);
|
||||
}
|
||||
|
||||
/* Test: Parse enabled Off */
|
||||
static void test_parse_enabled_off(void **state)
|
||||
{
|
||||
(void)state;
|
||||
assert_int_equal(parse_enabled("Off"), 0);
|
||||
assert_int_equal(parse_enabled("off"), 0);
|
||||
assert_int_equal(parse_enabled("OFF"), 0);
|
||||
assert_int_equal(parse_enabled("0"), 0);
|
||||
}
|
||||
|
||||
/* Test: Parse socket path valid */
|
||||
static void test_parse_socket_path_valid(void **state)
|
||||
{
|
||||
(void)state;
|
||||
const char *result = parse_socket_path("/var/run/logcorrelator/http.socket");
|
||||
assert_string_equal(result, "/var/run/logcorrelator/http.socket");
|
||||
}
|
||||
|
||||
/* Test: Parse socket path empty */
|
||||
static void test_parse_socket_path_empty(void **state)
|
||||
{
|
||||
(void)state;
|
||||
const char *result = parse_socket_path("");
|
||||
assert_null(result);
|
||||
}
|
||||
|
||||
/* Test: Parse socket path NULL */
|
||||
static void test_parse_socket_path_null(void **state)
|
||||
{
|
||||
(void)state;
|
||||
const char *result = parse_socket_path(NULL);
|
||||
assert_null(result);
|
||||
}
|
||||
|
||||
/* Test: Parse socket path max length valid */
|
||||
static void test_parse_socket_path_max_len_valid(void **state)
|
||||
{
|
||||
(void)state;
|
||||
char path[MAX_SOCKET_PATH_LEN];
|
||||
memset(path, 'a', MAX_SOCKET_PATH_LEN - 1);
|
||||
path[MAX_SOCKET_PATH_LEN - 1] = '\0';
|
||||
|
||||
assert_non_null(parse_socket_path(path));
|
||||
}
|
||||
|
||||
/* Test: Parse socket path max length invalid */
|
||||
static void test_parse_socket_path_max_len_invalid(void **state)
|
||||
{
|
||||
(void)state;
|
||||
char path[MAX_SOCKET_PATH_LEN + 1];
|
||||
memset(path, 'b', MAX_SOCKET_PATH_LEN);
|
||||
path[MAX_SOCKET_PATH_LEN] = '\0';
|
||||
|
||||
assert_null(parse_socket_path(path));
|
||||
}
|
||||
|
||||
/* Test: Parse max headers valid */
|
||||
static void test_parse_max_headers_valid(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_max_headers("10", &result), 0);
|
||||
assert_int_equal(result, 10);
|
||||
|
||||
assert_int_equal(parse_max_headers("0", &result), 0);
|
||||
assert_int_equal(result, 0);
|
||||
|
||||
assert_int_equal(parse_max_headers("100", &result), 0);
|
||||
assert_int_equal(result, 100);
|
||||
}
|
||||
|
||||
/* Test: Parse max headers invalid */
|
||||
static void test_parse_max_headers_invalid(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_max_headers("-1", &result), -1);
|
||||
assert_int_equal(parse_max_headers("abc", &result), -1);
|
||||
assert_int_equal(parse_max_headers("10abc", &result), -1);
|
||||
}
|
||||
|
||||
/* Test: Parse reconnect interval valid */
|
||||
static void test_parse_reconnect_interval_valid(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_interval("10", &result), 0);
|
||||
assert_int_equal(result, 10);
|
||||
|
||||
assert_int_equal(parse_interval("0", &result), 0);
|
||||
assert_int_equal(result, 0);
|
||||
|
||||
assert_int_equal(parse_interval("60", &result), 0);
|
||||
assert_int_equal(result, 60);
|
||||
}
|
||||
|
||||
/* Test: Parse reconnect interval invalid */
|
||||
static void test_parse_reconnect_interval_invalid(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_interval("-5", &result), -1);
|
||||
assert_int_equal(parse_interval("abc", &result), -1);
|
||||
assert_int_equal(parse_interval("10abc", &result), -1);
|
||||
}
|
||||
|
||||
/* Test: Parse max header value length valid */
|
||||
static void test_parse_max_header_value_len_valid(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_max_header_value_len("1", &result), 0);
|
||||
assert_int_equal(result, 1);
|
||||
|
||||
assert_int_equal(parse_max_header_value_len("256", &result), 0);
|
||||
assert_int_equal(result, 256);
|
||||
}
|
||||
|
||||
/* Test: Parse max header value length invalid */
|
||||
static void test_parse_max_header_value_len_invalid(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_max_header_value_len("0", &result), -1);
|
||||
assert_int_equal(parse_max_header_value_len("-1", &result), -1);
|
||||
assert_int_equal(parse_max_header_value_len("10abc", &result), -1);
|
||||
}
|
||||
|
||||
/* Test: strict numeric parsing invalid suffix for all int directives */
|
||||
static void test_strict_numeric_invalid_suffix_all(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_max_headers("10abc", &result), -1);
|
||||
assert_int_equal(parse_interval("10abc", &result), -1);
|
||||
assert_int_equal(parse_max_header_value_len("10abc", &result), -1);
|
||||
}
|
||||
|
||||
/* Test: Default configuration values */
|
||||
static void test_default_config_values(void **state)
|
||||
{
|
||||
(void)state;
|
||||
assert_int_equal(DEFAULT_MAX_HEADERS, 10);
|
||||
assert_int_equal(DEFAULT_MAX_HEADER_VALUE_LEN, 256);
|
||||
assert_int_equal(DEFAULT_RECONNECT_INTERVAL, 10);
|
||||
assert_int_equal(DEFAULT_ERROR_REPORT_INTERVAL, 10);
|
||||
}
|
||||
|
||||
/* Test: Configuration validation - enabled requires socket */
|
||||
static void test_config_validation_enabled_requires_socket(void **state)
|
||||
{
|
||||
int enabled = 1;
|
||||
const char *socket = "/var/run/socket";
|
||||
(void)state;
|
||||
|
||||
assert_true(enabled == 0 || socket != NULL);
|
||||
|
||||
socket = NULL;
|
||||
assert_false(enabled == 0 || socket != NULL);
|
||||
}
|
||||
|
||||
/* Test: Configuration validation - enabled with empty socket is invalid */
|
||||
static void test_config_validation_enabled_with_empty_socket(void **state)
|
||||
{
|
||||
int enabled = 1;
|
||||
const char *socket = parse_socket_path("");
|
||||
(void)state;
|
||||
|
||||
assert_false(enabled == 0 || socket != NULL);
|
||||
}
|
||||
|
||||
/* Test: Header value length validation */
|
||||
static void test_header_value_len_validation(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_max_header_value_len("1", &result), 0);
|
||||
assert_true(result >= 1);
|
||||
|
||||
assert_int_equal(parse_max_header_value_len("0", &result), -1);
|
||||
}
|
||||
|
||||
/* Test: Large but valid values */
|
||||
static void test_large_valid_values(void **state)
|
||||
{
|
||||
int result;
|
||||
(void)state;
|
||||
|
||||
assert_int_equal(parse_max_headers("1000000", &result), 0);
|
||||
assert_int_equal(result, 1000000);
|
||||
|
||||
assert_int_equal(parse_interval("86400", &result), 0);
|
||||
assert_int_equal(result, 86400);
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
const struct CMUnitTest tests[] = {
|
||||
cmocka_unit_test(test_parse_enabled_on),
|
||||
cmocka_unit_test(test_parse_enabled_off),
|
||||
cmocka_unit_test(test_parse_socket_path_valid),
|
||||
cmocka_unit_test(test_parse_socket_path_empty),
|
||||
cmocka_unit_test(test_parse_socket_path_null),
|
||||
cmocka_unit_test(test_parse_socket_path_max_len_valid),
|
||||
cmocka_unit_test(test_parse_socket_path_max_len_invalid),
|
||||
cmocka_unit_test(test_parse_max_headers_valid),
|
||||
cmocka_unit_test(test_parse_max_headers_invalid),
|
||||
cmocka_unit_test(test_parse_reconnect_interval_valid),
|
||||
cmocka_unit_test(test_parse_reconnect_interval_invalid),
|
||||
cmocka_unit_test(test_parse_max_header_value_len_valid),
|
||||
cmocka_unit_test(test_parse_max_header_value_len_invalid),
|
||||
cmocka_unit_test(test_strict_numeric_invalid_suffix_all),
|
||||
cmocka_unit_test(test_default_config_values),
|
||||
cmocka_unit_test(test_config_validation_enabled_requires_socket),
|
||||
cmocka_unit_test(test_config_validation_enabled_with_empty_socket),
|
||||
cmocka_unit_test(test_header_value_len_validation),
|
||||
cmocka_unit_test(test_large_valid_values),
|
||||
};
|
||||
|
||||
return cmocka_run_group_tests(tests, NULL, NULL);
|
||||
}
|
||||
@ -1,458 +0,0 @@
|
||||
/*
|
||||
* test_h2_parsing.c — Tests unitaires du fingerprinting HTTP/2 passif.
|
||||
*
|
||||
* Les fonctions testées (hpack_int_decode, h2_extract_pseudo_order,
|
||||
* h2_parse_preface_buf) sont réimplimentées localement pour éviter les
|
||||
* dépendances Apache/APR. La logique est identique à mod_reqin_log.c.
|
||||
*/
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <setjmp.h>
|
||||
#include <cmocka.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* ====== Réimplémentation locale des fonctions H2 ====== */
|
||||
|
||||
static int hpack_int_decode(const unsigned char *buf, size_t len, int prefix,
|
||||
size_t *pos, unsigned int *out)
|
||||
{
|
||||
unsigned int mask = (1u << prefix) - 1u;
|
||||
unsigned int b, m;
|
||||
|
||||
if (*pos >= len) return 0;
|
||||
*out = buf[(*pos)++] & mask;
|
||||
if (*out < mask) return 1;
|
||||
|
||||
m = 0;
|
||||
while (*pos < len) {
|
||||
b = buf[(*pos)++];
|
||||
*out += (b & 0x7fu) << m;
|
||||
m += 7;
|
||||
if (!(b & 0x80u)) return 1;
|
||||
if (m > 28) return 0;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
static char h2_hpack_pseudo(unsigned int index)
|
||||
{
|
||||
switch (index) {
|
||||
case 1: return 'a';
|
||||
case 2: case 3: return 'm';
|
||||
case 4: case 5: return 'p';
|
||||
case 6: case 7: return 's';
|
||||
default: return 0;
|
||||
}
|
||||
}
|
||||
|
||||
static void h2_extract_pseudo_order(const unsigned char *hpack, size_t len, char *out)
|
||||
{
|
||||
size_t pos = 0;
|
||||
int out_pos = 0;
|
||||
int first = 1;
|
||||
|
||||
while (pos < len && out_pos < 7) {
|
||||
unsigned char byte = hpack[pos];
|
||||
|
||||
if (byte & 0x80u) {
|
||||
unsigned int idx = 0;
|
||||
if (!hpack_int_decode(hpack, len, 7, &pos, &idx)) break;
|
||||
if (idx == 0) break;
|
||||
|
||||
char c = h2_hpack_pseudo(idx);
|
||||
if (!c) break;
|
||||
|
||||
if (!first) out[out_pos++] = ',';
|
||||
out[out_pos++] = c;
|
||||
first = 0;
|
||||
|
||||
} else if ((byte & 0xe0u) == 0x20u) {
|
||||
unsigned int sz = 0;
|
||||
if (!hpack_int_decode(hpack, len, 5, &pos, &sz)) break;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
out[out_pos] = '\0';
|
||||
}
|
||||
|
||||
/* Résultat de h2_parse_preface_buf — version allégée (pas d'APR) */
|
||||
typedef struct {
|
||||
char settings[256];
|
||||
char wupdate[16];
|
||||
char pseudo[16];
|
||||
char fingerprint[512];
|
||||
int has_priority;
|
||||
int is_h2;
|
||||
} h2_result_t;
|
||||
|
||||
static void h2_parse_preface_buf(const char *buf, size_t len, h2_result_t *res)
|
||||
{
|
||||
static const char H2_MAGIC[] = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n";
|
||||
const size_t MAGIC_LEN = 24u;
|
||||
const size_t FRAME_HDR = 9u;
|
||||
|
||||
memset(res, 0, sizeof(*res));
|
||||
strcpy(res->wupdate, "0");
|
||||
|
||||
if (len < MAGIC_LEN || memcmp(buf, H2_MAGIC, MAGIC_LEN) != 0) return;
|
||||
|
||||
int settings_out = 0;
|
||||
size_t pos = MAGIC_LEN;
|
||||
|
||||
while (pos + FRAME_HDR <= len) {
|
||||
size_t frame_len = ((unsigned char)buf[pos] << 16)
|
||||
| ((unsigned char)buf[pos+1] << 8)
|
||||
| (unsigned char)buf[pos+2];
|
||||
unsigned char type = (unsigned char)buf[pos+3];
|
||||
unsigned char flags = (unsigned char)buf[pos+4];
|
||||
uint32_t stream_id = (((unsigned char)buf[pos+5] & 0x7fu) << 24)
|
||||
| ((unsigned char)buf[pos+6] << 16)
|
||||
| ((unsigned char)buf[pos+7] << 8)
|
||||
| (unsigned char)buf[pos+8];
|
||||
|
||||
pos += FRAME_HDR;
|
||||
if (pos + frame_len > len) break;
|
||||
|
||||
if (type == 0x04u && stream_id == 0 && !(flags & 0x01u)) {
|
||||
size_t sp = 0;
|
||||
while (sp + 6 <= frame_len &&
|
||||
settings_out < (int)sizeof(res->settings) - 24) {
|
||||
uint16_t id = ((unsigned char)buf[pos + sp] << 8)
|
||||
| (unsigned char)buf[pos + sp + 1];
|
||||
uint32_t val = ((unsigned char)buf[pos + sp + 2] << 24)
|
||||
| ((unsigned char)buf[pos + sp + 3] << 16)
|
||||
| ((unsigned char)buf[pos + sp + 4] << 8)
|
||||
| (unsigned char)buf[pos + sp + 5];
|
||||
sp += 6;
|
||||
if (settings_out > 0)
|
||||
res->settings[settings_out++] = ',';
|
||||
settings_out += snprintf(res->settings + settings_out,
|
||||
(int)sizeof(res->settings) - settings_out,
|
||||
"%u:%u", id, val);
|
||||
}
|
||||
|
||||
} else if (type == 0x08u && stream_id == 0) {
|
||||
if (frame_len >= 4) {
|
||||
uint32_t inc = (((unsigned char)buf[pos] & 0x7fu) << 24)
|
||||
| ((unsigned char)buf[pos+1] << 16)
|
||||
| ((unsigned char)buf[pos+2] << 8)
|
||||
| (unsigned char)buf[pos+3];
|
||||
snprintf(res->wupdate, sizeof(res->wupdate), "%u", inc);
|
||||
}
|
||||
|
||||
} else if (type == 0x01u && stream_id > 0) {
|
||||
size_t hpack_start = 0;
|
||||
int parse_ok = 1;
|
||||
|
||||
if ((flags & 0x08u) && parse_ok) {
|
||||
if (hpack_start >= frame_len) {
|
||||
parse_ok = 0;
|
||||
} else {
|
||||
unsigned char pad_len = (unsigned char)buf[pos + hpack_start++];
|
||||
if (frame_len < hpack_start + (size_t)pad_len)
|
||||
parse_ok = 0;
|
||||
else
|
||||
frame_len -= (size_t)pad_len;
|
||||
}
|
||||
}
|
||||
|
||||
if ((flags & 0x20u) && parse_ok) {
|
||||
if (hpack_start + 5u > frame_len) {
|
||||
parse_ok = 0;
|
||||
} else {
|
||||
hpack_start += 5u;
|
||||
res->has_priority = 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (parse_ok && hpack_start < frame_len) {
|
||||
h2_extract_pseudo_order(
|
||||
(const unsigned char *)(buf + pos + hpack_start),
|
||||
frame_len - hpack_start,
|
||||
res->pseudo
|
||||
);
|
||||
}
|
||||
pos += frame_len;
|
||||
break;
|
||||
}
|
||||
pos += frame_len;
|
||||
}
|
||||
|
||||
if (res->settings[0] != '\0') {
|
||||
res->is_h2 = 1;
|
||||
snprintf(res->fingerprint, sizeof(res->fingerprint), "%s|%s|%d|%s",
|
||||
res->settings, res->wupdate, res->has_priority, res->pseudo);
|
||||
}
|
||||
}
|
||||
|
||||
/* ====== Données de test : preface Chrome 120 ====== */
|
||||
|
||||
/*
|
||||
* Preface HTTP/2 Chrome 120 (capturée) :
|
||||
* Magic (24 octets)
|
||||
* SETTINGS frame : HEADER_TABLE_SIZE=65536, ENABLE_PUSH=0,
|
||||
* INITIAL_WINDOW_SIZE=6291456, MAX_HEADER_LIST_SIZE=262144
|
||||
* WINDOW_UPDATE : incrément 15663105
|
||||
* HEADERS stream 1 : :method GET, :authority, :scheme https, :path /
|
||||
* → ordre HPACK indexé : 0x82(GET), 0x81(:auth), 0x87(https), 0x84(/)
|
||||
*/
|
||||
static const unsigned char CHROME_PREFACE[] = {
|
||||
/* Magic */
|
||||
'P','R','I',' ','*',' ','H','T','T','P','/','2','.','0','\r','\n',
|
||||
'\r','\n','S','M','\r','\n','\r','\n',
|
||||
/* SETTINGS frame : length=24, type=0x04, flags=0x00, stream=0 */
|
||||
0x00, 0x00, 0x18, /* length = 24 = 4×6 */
|
||||
0x04, /* type SETTINGS */
|
||||
0x00, /* flags = 0 */
|
||||
0x00, 0x00, 0x00, 0x00, /* stream 0 */
|
||||
/* Entry 1: HEADER_TABLE_SIZE (1) = 65536 = 0x00010000 */
|
||||
0x00, 0x01, 0x00, 0x01, 0x00, 0x00,
|
||||
/* Entry 2: ENABLE_PUSH (2) = 0 */
|
||||
0x00, 0x02, 0x00, 0x00, 0x00, 0x00,
|
||||
/* Entry 3: INITIAL_WINDOW_SIZE (4) = 6291456 = 0x00600000 */
|
||||
0x00, 0x04, 0x00, 0x60, 0x00, 0x00,
|
||||
/* Entry 4: MAX_HEADER_LIST_SIZE (6) = 262144 = 0x00040000 */
|
||||
0x00, 0x06, 0x00, 0x04, 0x00, 0x00,
|
||||
/* WINDOW_UPDATE frame : length=4, type=0x08, flags=0, stream=0 */
|
||||
0x00, 0x00, 0x04,
|
||||
0x08,
|
||||
0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
/* increment = 15663105 = 0x00EF0001 */
|
||||
0x00, 0xEF, 0x00, 0x01,
|
||||
/* HEADERS frame : length=14, type=0x01, flags=0x05 (END_STREAM|END_HEADERS), stream=1 */
|
||||
0x00, 0x00, 0x0E,
|
||||
0x01,
|
||||
0x05,
|
||||
0x00, 0x00, 0x00, 0x01,
|
||||
/* HPACK : :method GET (0x82), :authority (0x81), :scheme https (0x87), :path / (0x84) */
|
||||
/* → ordre Chrome : m,a,s,p */
|
||||
0x82, 0x81, 0x87, 0x84,
|
||||
/* + quelques headers supplémentaires (indices statiques) */
|
||||
0x86, /* :scheme http (index 6, régulier → stop après pseudo) */
|
||||
0x53, /* accept (sans valeur — littéral, arrête le scan) */
|
||||
0x00, 0x05, 0x74, 0x65, 0x78, 0x74, 0x2F, 0x68, 0x74, 0x6D, 0x6C
|
||||
};
|
||||
|
||||
/* ====== Données de test : preface Firefox 120 ====== */
|
||||
|
||||
/*
|
||||
* Preface HTTP/2 Firefox 120 :
|
||||
* SETTINGS: HEADER_TABLE_SIZE=65536, INITIAL_WINDOW_SIZE=131072, MAX_FRAME_SIZE=16384
|
||||
* WINDOW_UPDATE: 12517377
|
||||
* HEADERS: :method GET (0x82), :path / (0x84), :scheme https (0x87), :authority (0x81)
|
||||
* → ordre Firefox : m,p,s,a
|
||||
*/
|
||||
static const unsigned char FIREFOX_PREFACE[] = {
|
||||
/* Magic */
|
||||
'P','R','I',' ','*',' ','H','T','T','P','/','2','.','0','\r','\n',
|
||||
'\r','\n','S','M','\r','\n','\r','\n',
|
||||
/* SETTINGS frame : length=18, type=0x04, flags=0x00, stream=0 */
|
||||
0x00, 0x00, 0x12,
|
||||
0x04,
|
||||
0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
/* HEADER_TABLE_SIZE (1) = 65536 */
|
||||
0x00, 0x01, 0x00, 0x01, 0x00, 0x00,
|
||||
/* INITIAL_WINDOW_SIZE (4) = 131072 = 0x00020000 */
|
||||
0x00, 0x04, 0x00, 0x02, 0x00, 0x00,
|
||||
/* MAX_FRAME_SIZE (5) = 16384 = 0x00004000 */
|
||||
0x00, 0x05, 0x00, 0x00, 0x40, 0x00,
|
||||
/* WINDOW_UPDATE : increment = 12517377 = 0x00BF0001 */
|
||||
0x00, 0x00, 0x04,
|
||||
0x08,
|
||||
0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0xBF, 0x00, 0x01,
|
||||
/* HEADERS frame : length=4, type=0x01, flags=0x05, stream=1 */
|
||||
0x00, 0x00, 0x04,
|
||||
0x01,
|
||||
0x05,
|
||||
0x00, 0x00, 0x00, 0x01,
|
||||
/* HPACK : :method GET (0x82), :path / (0x84), :scheme https (0x87), :authority (0x81) */
|
||||
/* → ordre Firefox : m,p,s,a */
|
||||
0x82, 0x84, 0x87, 0x81
|
||||
};
|
||||
|
||||
/* ====== Données de test : flux HTTP/1.1 (ne doit pas matcher) ====== */
|
||||
static const char HTTP1_DATA[] =
|
||||
"GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
|
||||
|
||||
/* ====== Tests ====== */
|
||||
|
||||
static void test_chrome_settings_parsed(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
|
||||
|
||||
assert_int_equal(res.is_h2, 1);
|
||||
/* SETTINGS attendus : 1:65536,2:0,4:6291456,6:262144 */
|
||||
assert_string_equal(res.settings, "1:65536,2:0,4:6291456,6:262144");
|
||||
}
|
||||
|
||||
static void test_chrome_window_update(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
|
||||
|
||||
assert_string_equal(res.wupdate, "15663105");
|
||||
}
|
||||
|
||||
static void test_chrome_pseudo_order(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
|
||||
|
||||
/* Chrome : :method(m), :authority(a), :scheme(s), :path(p) */
|
||||
assert_string_equal(res.pseudo, "m,a,s,p");
|
||||
}
|
||||
|
||||
static void test_chrome_fingerprint_akamai(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf((const char *)CHROME_PREFACE, sizeof(CHROME_PREFACE), &res);
|
||||
|
||||
assert_string_equal(res.fingerprint,
|
||||
"1:65536,2:0,4:6291456,6:262144|15663105|0|m,a,s,p");
|
||||
}
|
||||
|
||||
static void test_firefox_settings_parsed(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf((const char *)FIREFOX_PREFACE, sizeof(FIREFOX_PREFACE), &res);
|
||||
|
||||
assert_int_equal(res.is_h2, 1);
|
||||
assert_string_equal(res.settings, "1:65536,4:131072,5:16384");
|
||||
}
|
||||
|
||||
static void test_firefox_pseudo_order(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf((const char *)FIREFOX_PREFACE, sizeof(FIREFOX_PREFACE), &res);
|
||||
|
||||
/* Firefox : :method(m), :path(p), :scheme(s), :authority(a) */
|
||||
assert_string_equal(res.pseudo, "m,p,s,a");
|
||||
}
|
||||
|
||||
static void test_firefox_fingerprint_akamai(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf((const char *)FIREFOX_PREFACE, sizeof(FIREFOX_PREFACE), &res);
|
||||
|
||||
assert_string_equal(res.fingerprint,
|
||||
"1:65536,4:131072,5:16384|12517377|0|m,p,s,a");
|
||||
}
|
||||
|
||||
static void test_http1_not_detected(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf(HTTP1_DATA, strlen(HTTP1_DATA), &res);
|
||||
|
||||
assert_int_equal(res.is_h2, 0);
|
||||
assert_string_equal(res.settings, "");
|
||||
assert_string_equal(res.fingerprint, "");
|
||||
}
|
||||
|
||||
static void test_empty_buffer_not_detected(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
h2_parse_preface_buf("", 0, &res);
|
||||
|
||||
assert_int_equal(res.is_h2, 0);
|
||||
}
|
||||
|
||||
static void test_truncated_preface_no_crash(void **state)
|
||||
{
|
||||
(void)state;
|
||||
h2_result_t res;
|
||||
/* Magic complet mais frame tronquée */
|
||||
h2_parse_preface_buf((const char *)CHROME_PREFACE, 30, &res);
|
||||
|
||||
assert_int_equal(res.is_h2, 0); /* SETTINGS incomplet → pas de fingerprint */
|
||||
}
|
||||
|
||||
static void test_hpack_int_single_byte(void **state)
|
||||
{
|
||||
(void)state;
|
||||
/* Entier 7-bit < 127 → encodé sur 1 octet */
|
||||
unsigned char buf[] = { 0x82 }; /* 0x80 | 2 → index=2 */
|
||||
size_t pos = 0;
|
||||
unsigned int out = 0;
|
||||
int ok = hpack_int_decode(buf, 1, 7, &pos, &out);
|
||||
|
||||
assert_int_equal(ok, 1);
|
||||
assert_int_equal(out, 2);
|
||||
assert_int_equal(pos, 1);
|
||||
}
|
||||
|
||||
static void test_hpack_pseudo_table(void **state)
|
||||
{
|
||||
(void)state;
|
||||
assert_int_equal(h2_hpack_pseudo(1), 'a');
|
||||
assert_int_equal(h2_hpack_pseudo(2), 'm');
|
||||
assert_int_equal(h2_hpack_pseudo(3), 'm');
|
||||
assert_int_equal(h2_hpack_pseudo(4), 'p');
|
||||
assert_int_equal(h2_hpack_pseudo(5), 'p');
|
||||
assert_int_equal(h2_hpack_pseudo(6), 's');
|
||||
assert_int_equal(h2_hpack_pseudo(7), 's');
|
||||
assert_int_equal(h2_hpack_pseudo(8), 0); /* header régulier */
|
||||
assert_int_equal(h2_hpack_pseudo(62), 0);
|
||||
}
|
||||
|
||||
static void test_pseudo_order_extraction_direct(void **state)
|
||||
{
|
||||
(void)state;
|
||||
/* HPACK block : :method(0x82), :path(0x84), :scheme(0x87), :authority(0x81) */
|
||||
unsigned char hpack[] = { 0x82, 0x84, 0x87, 0x81 };
|
||||
char out[16];
|
||||
h2_extract_pseudo_order(hpack, sizeof(hpack), out);
|
||||
|
||||
assert_string_equal(out, "m,p,s,a");
|
||||
}
|
||||
|
||||
static void test_pseudo_order_stops_at_regular_header(void **state)
|
||||
{
|
||||
(void)state;
|
||||
/* :method(0x82), puis header régulier (0x88 = index 8) */
|
||||
unsigned char hpack[] = { 0x82, 0x88 };
|
||||
char out[16];
|
||||
h2_extract_pseudo_order(hpack, sizeof(hpack), out);
|
||||
|
||||
assert_string_equal(out, "m");
|
||||
}
|
||||
|
||||
/* ====== main ====== */
|
||||
|
||||
int main(void)
|
||||
{
|
||||
const struct CMUnitTest tests[] = {
|
||||
cmocka_unit_test(test_chrome_settings_parsed),
|
||||
cmocka_unit_test(test_chrome_window_update),
|
||||
cmocka_unit_test(test_chrome_pseudo_order),
|
||||
cmocka_unit_test(test_chrome_fingerprint_akamai),
|
||||
cmocka_unit_test(test_firefox_settings_parsed),
|
||||
cmocka_unit_test(test_firefox_pseudo_order),
|
||||
cmocka_unit_test(test_firefox_fingerprint_akamai),
|
||||
cmocka_unit_test(test_http1_not_detected),
|
||||
cmocka_unit_test(test_empty_buffer_not_detected),
|
||||
cmocka_unit_test(test_truncated_preface_no_crash),
|
||||
cmocka_unit_test(test_hpack_int_single_byte),
|
||||
cmocka_unit_test(test_hpack_pseudo_table),
|
||||
cmocka_unit_test(test_pseudo_order_extraction_direct),
|
||||
cmocka_unit_test(test_pseudo_order_stops_at_regular_header),
|
||||
};
|
||||
return cmocka_run_group_tests(tests, NULL, NULL);
|
||||
}
|
||||
@ -1,226 +0,0 @@
|
||||
/*
|
||||
* test_header_handling.c - Unit tests for header handling (truncation and limits)
|
||||
*/
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <setjmp.h>
|
||||
#include <cmocka.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <apr_strings.h>
|
||||
#include <apr_tables.h>
|
||||
#include <apr_pools.h>
|
||||
#include <apr_general.h>
|
||||
|
||||
/* Mock header truncation function */
|
||||
static char *truncate_header_value(apr_pool_t *pool, const char *value, int max_len)
|
||||
{
|
||||
if (value == NULL) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
size_t len = strlen(value);
|
||||
if ((int)len > max_len) {
|
||||
return apr_pstrmemdup(pool, value, max_len);
|
||||
}
|
||||
return apr_pstrdup(pool, value);
|
||||
}
|
||||
|
||||
/* Mock header matching function */
|
||||
static int header_name_matches(const char *configured, const char *actual)
|
||||
{
|
||||
return strcasecmp(configured, actual) == 0;
|
||||
}
|
||||
|
||||
/* Test: Header value within limit */
|
||||
static void test_header_truncation_within_limit(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
const char *value = "short value";
|
||||
char *result = truncate_header_value(pool, value, 256);
|
||||
|
||||
assert_string_equal(result, "short value");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Header value exactly at limit */
|
||||
static void test_header_truncation_exact_limit(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
const char *value = "exactly10c";
|
||||
char *result = truncate_header_value(pool, value, 10);
|
||||
|
||||
assert_string_equal(result, "exactly10c");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Header value exceeds limit */
|
||||
static void test_header_truncation_exceeds_limit(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
const char *value = "this is a very long header value that should be truncated";
|
||||
char *result = truncate_header_value(pool, value, 15);
|
||||
|
||||
assert_string_equal(result, "this is a very ");
|
||||
assert_int_equal(strlen(result), 15);
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Header value with limit of 1 */
|
||||
static void test_header_truncation_limit_one(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
const char *value = "abc";
|
||||
char *result = truncate_header_value(pool, value, 1);
|
||||
|
||||
assert_string_equal(result, "a");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: NULL header value */
|
||||
static void test_header_truncation_null(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
char *result = truncate_header_value(pool, NULL, 256);
|
||||
|
||||
assert_null(result);
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Empty header value */
|
||||
static void test_header_truncation_empty(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
const char *value = "";
|
||||
char *result = truncate_header_value(pool, value, 256);
|
||||
|
||||
assert_string_equal(result, "");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Header name matching (case-insensitive) */
|
||||
static void test_header_name_matching_case_insensitive(void **state)
|
||||
{
|
||||
assert_true(header_name_matches("X-Request-Id", "x-request-id"));
|
||||
assert_true(header_name_matches("user-agent", "User-Agent"));
|
||||
assert_true(header_name_matches("HOST", "host"));
|
||||
}
|
||||
|
||||
/* Test: Header name matching (different headers) */
|
||||
static void test_header_name_matching_different(void **state)
|
||||
{
|
||||
assert_false(header_name_matches("X-Request-Id", "X-Trace-Id"));
|
||||
assert_false(header_name_matches("Host", "User-Agent"));
|
||||
}
|
||||
|
||||
/* Test: Multiple headers with limit */
|
||||
static void test_header_count_limit(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
/* Simulate configured headers */
|
||||
const char *configured[] = {"X-Request-Id", "X-Trace-Id", "User-Agent", "Referer"};
|
||||
int configured_count = 4;
|
||||
int max_headers = 2;
|
||||
|
||||
/* Simulate present headers */
|
||||
const char *present[] = {"X-Request-Id", "User-Agent", "Referer"};
|
||||
int present_count = 3;
|
||||
|
||||
int logged_count = 0;
|
||||
for (int i = 0; i < configured_count && logged_count < max_headers; i++) {
|
||||
for (int j = 0; j < present_count; j++) {
|
||||
if (header_name_matches(configured[i], present[j])) {
|
||||
logged_count++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
assert_int_equal(logged_count, 2);
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Header value with special JSON characters */
|
||||
static void test_header_value_json_special(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
const char *value = "test\"value\\with\tspecial";
|
||||
char *truncated = truncate_header_value(pool, value, 256);
|
||||
|
||||
/* Truncation should preserve the value */
|
||||
assert_string_equal(truncated, "test\"value\\with\tspecial");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Unicode in header value (UTF-8) */
|
||||
static void test_header_value_unicode(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
apr_pool_create(&pool, NULL);
|
||||
|
||||
const char *value = "Mozilla/5.0 (compatible; 日本語)";
|
||||
char *result = truncate_header_value(pool, value, 50);
|
||||
|
||||
/* Should be truncated but valid */
|
||||
assert_non_null(result);
|
||||
assert_true(strlen(result) <= 50);
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
static int group_setup(void **state)
|
||||
{
|
||||
(void)state;
|
||||
return apr_initialize();
|
||||
}
|
||||
|
||||
static int group_teardown(void **state)
|
||||
{
|
||||
(void)state;
|
||||
apr_terminate();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
const struct CMUnitTest tests[] = {
|
||||
cmocka_unit_test(test_header_truncation_within_limit),
|
||||
cmocka_unit_test(test_header_truncation_exact_limit),
|
||||
cmocka_unit_test(test_header_truncation_exceeds_limit),
|
||||
cmocka_unit_test(test_header_truncation_limit_one),
|
||||
cmocka_unit_test(test_header_truncation_null),
|
||||
cmocka_unit_test(test_header_truncation_empty),
|
||||
cmocka_unit_test(test_header_name_matching_case_insensitive),
|
||||
cmocka_unit_test(test_header_name_matching_different),
|
||||
cmocka_unit_test(test_header_count_limit),
|
||||
cmocka_unit_test(test_header_value_json_special),
|
||||
cmocka_unit_test(test_header_value_unicode),
|
||||
};
|
||||
|
||||
return cmocka_run_group_tests(tests, group_setup, group_teardown);
|
||||
}
|
||||
@ -1,266 +0,0 @@
|
||||
/*
|
||||
* test_json_serialization.c - Unit tests for JSON serialization
|
||||
*/
|
||||
|
||||
#include <stdarg.h>
|
||||
#include <stddef.h>
|
||||
#include <setjmp.h>
|
||||
#include <cmocka.h>
|
||||
#include <string.h>
|
||||
#include <stdio.h>
|
||||
#include <apr_pools.h>
|
||||
#include <apr_strings.h>
|
||||
#include <apr_time.h>
|
||||
#include <apr_lib.h>
|
||||
#include <apr_general.h>
|
||||
|
||||
typedef struct {
|
||||
char *data;
|
||||
size_t len;
|
||||
size_t cap;
|
||||
apr_pool_t *pool;
|
||||
} testbuf_t;
|
||||
|
||||
static void testbuf_init(testbuf_t *buf, apr_pool_t *pool, size_t initial_capacity)
|
||||
{
|
||||
buf->pool = pool;
|
||||
buf->cap = initial_capacity;
|
||||
buf->len = 0;
|
||||
buf->data = apr_palloc(pool, initial_capacity);
|
||||
buf->data[0] = '\0';
|
||||
}
|
||||
|
||||
static void testbuf_append(testbuf_t *buf, const char *str, size_t len)
|
||||
{
|
||||
if (str == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (len == (size_t)-1) {
|
||||
len = strlen(str);
|
||||
}
|
||||
|
||||
if (buf->len + len + 1 > buf->cap) {
|
||||
size_t new_cap = (buf->len + len + 1) * 2;
|
||||
char *new_data = apr_palloc(buf->pool, new_cap);
|
||||
memcpy(new_data, buf->data, buf->len + 1); /* Copy including null terminator */
|
||||
buf->data = new_data;
|
||||
buf->cap = new_cap;
|
||||
}
|
||||
|
||||
memcpy(buf->data + buf->len, str, len);
|
||||
buf->len += len;
|
||||
buf->data[buf->len] = '\0';
|
||||
}
|
||||
|
||||
static void testbuf_append_char(testbuf_t *buf, char c)
|
||||
{
|
||||
if (buf->len + 2 > buf->cap) {
|
||||
size_t new_cap = (buf->cap * 2);
|
||||
char *new_data = apr_palloc(buf->pool, new_cap);
|
||||
memcpy(new_data, buf->data, buf->len + 1);
|
||||
buf->data = new_data;
|
||||
buf->cap = new_cap;
|
||||
}
|
||||
|
||||
buf->data[buf->len++] = c;
|
||||
buf->data[buf->len] = '\0';
|
||||
}
|
||||
|
||||
/* Mock JSON string escaping function for testing */
|
||||
static void append_json_string(testbuf_t *buf, const char *str)
|
||||
{
|
||||
if (str == NULL) {
|
||||
return;
|
||||
}
|
||||
|
||||
for (const char *p = str; *p; p++) {
|
||||
char c = *p;
|
||||
switch (c) {
|
||||
case '"': testbuf_append(buf, "\\\"", 2); break;
|
||||
case '\\': testbuf_append(buf, "\\\\", 2); break;
|
||||
case '\b': testbuf_append(buf, "\\b", 2); break;
|
||||
case '\f': testbuf_append(buf, "\\f", 2); break;
|
||||
case '\n': testbuf_append(buf, "\\n", 2); break;
|
||||
case '\r': testbuf_append(buf, "\\r", 2); break;
|
||||
case '\t': testbuf_append(buf, "\\t", 2); break;
|
||||
default:
|
||||
if ((unsigned char)c < 0x20) {
|
||||
char unicode[8];
|
||||
apr_snprintf(unicode, sizeof(unicode), "\\u%04x", (unsigned char)c);
|
||||
testbuf_append(buf, unicode, (size_t)-1);
|
||||
} else {
|
||||
testbuf_append_char(buf, c);
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Test: Empty string */
|
||||
static void test_json_escape_empty_string(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 256);
|
||||
|
||||
append_json_string(&buf, "");
|
||||
|
||||
assert_string_equal(buf.data, "");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Simple string without special characters */
|
||||
static void test_json_escape_simple_string(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 256);
|
||||
|
||||
append_json_string(&buf, "hello world");
|
||||
|
||||
assert_string_equal(buf.data, "hello world");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: String with double quotes */
|
||||
static void test_json_escape_quotes(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 256);
|
||||
|
||||
append_json_string(&buf, "hello \"world\"");
|
||||
|
||||
assert_string_equal(buf.data, "hello \\\"world\\\"");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: String with backslashes */
|
||||
static void test_json_escape_backslashes(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 256);
|
||||
|
||||
append_json_string(&buf, "path\\to\\file");
|
||||
|
||||
assert_string_equal(buf.data, "path\\\\to\\\\file");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: String with newlines and tabs */
|
||||
static void test_json_escape_newlines_tabs(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 256);
|
||||
|
||||
append_json_string(&buf, "line1\nline2\ttab");
|
||||
|
||||
assert_string_equal(buf.data, "line1\\nline2\\ttab");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: String with control characters */
|
||||
static void test_json_escape_control_chars(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 256);
|
||||
|
||||
/* Test with bell character (0x07) - use octal literal */
|
||||
append_json_string(&buf, "test\007bell");
|
||||
|
||||
/* Should contain unicode escape for bell (0x07) */
|
||||
assert_true(strstr(buf.data, "\\u0007") != NULL);
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: NULL string */
|
||||
static void test_json_escape_null_string(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 256);
|
||||
|
||||
append_json_string(&buf, NULL);
|
||||
|
||||
assert_string_equal(buf.data, "");
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
/* Test: Complex user agent string */
|
||||
static void test_json_escape_user_agent(void **state)
|
||||
{
|
||||
apr_pool_t *pool;
|
||||
testbuf_t buf;
|
||||
const char *ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \"Test\"";
|
||||
(void)state;
|
||||
|
||||
apr_pool_create(&pool, NULL);
|
||||
testbuf_init(&buf, pool, 512);
|
||||
|
||||
append_json_string(&buf, ua);
|
||||
|
||||
assert_true(strstr(buf.data, "\\\"Test\\\"") != NULL);
|
||||
|
||||
apr_pool_destroy(pool);
|
||||
}
|
||||
|
||||
static int group_setup(void **state)
|
||||
{
|
||||
(void)state;
|
||||
return apr_initialize();
|
||||
}
|
||||
|
||||
static int group_teardown(void **state)
|
||||
{
|
||||
(void)state;
|
||||
apr_terminate();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
const struct CMUnitTest tests[] = {
|
||||
cmocka_unit_test(test_json_escape_empty_string),
|
||||
cmocka_unit_test(test_json_escape_simple_string),
|
||||
cmocka_unit_test(test_json_escape_quotes),
|
||||
cmocka_unit_test(test_json_escape_backslashes),
|
||||
cmocka_unit_test(test_json_escape_newlines_tabs),
|
||||
cmocka_unit_test(test_json_escape_control_chars),
|
||||
cmocka_unit_test(test_json_escape_null_string),
|
||||
cmocka_unit_test(test_json_escape_user_agent),
|
||||
};
|
||||
|
||||
return cmocka_run_group_tests(tests, group_setup, group_teardown);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@ -1,37 +0,0 @@
|
||||
# Git files
|
||||
.git
|
||||
.gitignore
|
||||
.gitattributes
|
||||
|
||||
# Qwen
|
||||
.qwen
|
||||
.qwenignore
|
||||
|
||||
# Build artifacts
|
||||
dist/
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test results
|
||||
test-results/
|
||||
coverage.out
|
||||
coverage.html
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Temporary files
|
||||
tmp/
|
||||
temp/
|
||||
*.tmp
|
||||
*.bak
|
||||
|
||||
# Docker compose override
|
||||
docker-compose.override.yml
|
||||
@ -1,7 +0,0 @@
|
||||
# sentinel configuration — DO NOT COMMIT real values
|
||||
# Copy to .env and fill in for local development
|
||||
JA4SENTINEL_INTERFACE=eth0
|
||||
JA4SENTINEL_PORTS=443,8443
|
||||
JA4SENTINEL_BPF_FILTER=
|
||||
JA4SENTINEL_FLOW_TIMEOUT=30
|
||||
JA4SENTINEL_PACKET_BUFFER_SIZE=1000
|
||||
@ -1,125 +0,0 @@
|
||||
name: Build RPM Package
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'v*'
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
paths:
|
||||
- 'go/**'
|
||||
- 'cmd/**'
|
||||
- 'internal/**'
|
||||
- 'api/**'
|
||||
- 'packaging/**'
|
||||
- 'Makefile'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- 'Dockerfile.package'
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- master
|
||||
paths:
|
||||
- 'go/**'
|
||||
- 'cmd/**'
|
||||
- 'internal/**'
|
||||
- 'api/**'
|
||||
- 'packaging/**'
|
||||
- 'Makefile'
|
||||
- 'go.mod'
|
||||
- 'go.sum'
|
||||
- 'Dockerfile.package'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
version:
|
||||
description: 'Version to build (e.g., 1.0.0)'
|
||||
required: false
|
||||
default: '1.0.0-dev'
|
||||
|
||||
env:
|
||||
GO_VERSION: '1.24'
|
||||
PACKAGE_NAME: ja4sentinel
|
||||
|
||||
jobs:
|
||||
build-rpm:
|
||||
name: Build RPM Packages (CentOS 7, Rocky 8/9/10)
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Determine version
|
||||
id: version
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
VERSION="${{ github.event.inputs.version }}"
|
||||
elif [[ "${{ github.ref }}" == refs/tags/v* ]]; then
|
||||
VERSION="${{ github.ref_name#v }}"
|
||||
else
|
||||
VERSION="0.0.0-$(git rev-parse --short HEAD)"
|
||||
fi
|
||||
echo "version=${VERSION}" >> $GITHUB_OUTPUT
|
||||
echo "Building version: ${VERSION}"
|
||||
|
||||
- name: Build RPM packages in Docker
|
||||
run: |
|
||||
docker build --no-cache \
|
||||
-t ${PACKAGE_NAME}-packager \
|
||||
--build-arg VERSION="${{ steps.version.outputs.version }}" \
|
||||
-f Dockerfile.package .
|
||||
|
||||
# Extract RPM packages from image
|
||||
mkdir -p build/rpm/el8 build/rpm/el9 build/rpm/el10
|
||||
docker run --rm -v $(pwd)/build:/output ${PACKAGE_NAME}-packager sh -c \
|
||||
'cp -r /packages/rpm/el8 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el9 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el10 /output/rpm/'
|
||||
|
||||
- name: List build artifacts
|
||||
run: |
|
||||
echo "=== Build Artifacts ==="
|
||||
echo "Rocky Linux 8 (el8):"
|
||||
ls -lah build/rpm/el8/ || echo " (no packages)"
|
||||
echo "Rocky Linux 9 (el9):"
|
||||
ls -lah build/rpm/el9/ || echo " (no packages)"
|
||||
echo "AlmaLinux/Rocky 10 (el10):"
|
||||
ls -lah build/rpm/el10/ || echo " (no packages)"
|
||||
|
||||
# Generate checksums
|
||||
find build/rpm -name "*.rpm" -exec sha256sum {} \; > build/rpm/checksums.txt
|
||||
cat build/rpm/checksums.txt
|
||||
|
||||
- name: Upload RPM artifacts
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${PACKAGE_NAME}-rpm-x86_64
|
||||
path: build/rpm/**/*.rpm
|
||||
retention-days: 30
|
||||
|
||||
- name: Upload checksum artifact
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${PACKAGE_NAME}-rpm-checksums
|
||||
path: build/rpm/checksums.txt
|
||||
retention-days: 30
|
||||
|
||||
- name: Create release and upload assets (on tag)
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
files: |
|
||||
build/rpm/el8/*.rpm
|
||||
build/rpm/el9/*.rpm
|
||||
build/rpm/el10/*.rpm
|
||||
generate_release_notes: true
|
||||
make_latest: true
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
59
old/services/sentinel/.gitignore
vendored
59
old/services/sentinel/.gitignore
vendored
@ -1,59 +0,0 @@
|
||||
# AIDER
|
||||
.aider*
|
||||
.qwen/
|
||||
.qwenignore
|
||||
|
||||
# Build artifacts
|
||||
dist/
|
||||
build/
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
*.o
|
||||
|
||||
# Go
|
||||
*.test
|
||||
*.out
|
||||
coverage.out
|
||||
coverage.html
|
||||
go.work
|
||||
go.work.sum
|
||||
|
||||
# Docker
|
||||
*.dockerfile.local
|
||||
docker-compose.override.yml
|
||||
|
||||
# IDE
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
|
||||
# Temporary files
|
||||
tmp/
|
||||
temp/
|
||||
*.tmp
|
||||
*.bak
|
||||
|
||||
# Local config (copie de config.yml.example)
|
||||
config.yml
|
||||
|
||||
# Runtime artifacts
|
||||
*.pid
|
||||
*.sock
|
||||
|
||||
# Integration test artifacts
|
||||
test-results/
|
||||
|
||||
# Test artifacts
|
||||
packaging/test/*.rpm
|
||||
|
||||
# Build artifacts
|
||||
packages/
|
||||
|
||||
# Binary (root level only)
|
||||
/ja4sentinel
|
||||
ja4sentinel-linux-amd64
|
||||
@ -1,37 +0,0 @@
|
||||
FROM golang:1.24-alpine AS builder
|
||||
|
||||
RUN apk add --no-cache git make libpcap-dev gcc musl-dev linux-headers
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy workspace and shared module first (better caching)
|
||||
COPY go.work go.work.sum* ./
|
||||
COPY shared/go/ja4common/ ./shared/go/ja4common/
|
||||
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
|
||||
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
|
||||
|
||||
WORKDIR /build/services/sentinel
|
||||
RUN go mod download || true
|
||||
|
||||
COPY services/sentinel/ /build/services/sentinel/
|
||||
|
||||
ARG VERSION=dev
|
||||
ARG BUILD_TIME=unknown
|
||||
ARG GIT_COMMIT=unknown
|
||||
|
||||
RUN mkdir -p dist && \
|
||||
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \
|
||||
CGO_LDFLAGS="-Wl,-Bstatic -lpcap -Wl,-Bdynamic" \
|
||||
go build -buildvcs=false \
|
||||
-ldflags "-X main.Version=${VERSION} -X main.BuildTime=${BUILD_TIME} -X main.GitCommit=${GIT_COMMIT}" \
|
||||
-o dist/sentinel ./cmd/ja4sentinel
|
||||
|
||||
FROM alpine:latest
|
||||
RUN apk add --no-cache ca-certificates
|
||||
RUN addgroup -S sentinel && adduser -S sentinel -G sentinel
|
||||
RUN mkdir -p /var/lib/sentinel /var/run /etc/sentinel /var/log/sentinel
|
||||
COPY --from=builder /build/services/sentinel/dist/sentinel /usr/local/bin/sentinel
|
||||
RUN chown -R sentinel:sentinel /var/lib/sentinel /var/log/sentinel
|
||||
USER sentinel
|
||||
WORKDIR /var/lib/sentinel
|
||||
ENTRYPOINT ["/usr/local/bin/sentinel"]
|
||||
@ -1,25 +0,0 @@
|
||||
# Development and test image for sentinel (was ja4sentinel)
|
||||
# Build context: monorepo root (ja4-platform/)
|
||||
# Usage: docker build -f services/sentinel/Dockerfile.dev -t sentinel-dev .
|
||||
FROM golang:1.24-alpine
|
||||
|
||||
RUN apk add --no-cache git make libpcap-dev gcc musl-dev linux-headers
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
# Copy Go workspace and shared module first for better layer caching
|
||||
COPY go.work go.work.sum* ./
|
||||
COPY shared/go/ja4common/ ./shared/go/ja4common/
|
||||
|
||||
# Copy service module descriptor then download deps
|
||||
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
|
||||
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
|
||||
|
||||
WORKDIR /build/services/sentinel
|
||||
RUN go mod download || true
|
||||
|
||||
# Copy full service source
|
||||
COPY services/sentinel/ /build/services/sentinel/
|
||||
|
||||
# Default: run tests with race detector
|
||||
CMD ["go", "test", "-race", "-v", "./..."]
|
||||
@ -1,109 +0,0 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
# =============================================================================
|
||||
# sentinel — Dockerfile de packaging RPM (Rocky Linux 8/9, AlmaLinux 10)
|
||||
# Build context: monorepo root (ja4-platform/)
|
||||
# Méthode: 1 builder Go → 1 rpm-builder (rpmbuild, 3 × dist) → 1 output alpine
|
||||
# =============================================================================
|
||||
|
||||
# =============================================================================
|
||||
# Stage 1: Builder — compilation du binaire Go sur Rocky Linux 9
|
||||
# Rocky Linux 9 comme base builder assure la compatibilité binaire sur toutes
|
||||
# les distros cibles (el8/el9/el10 sont ABI-compatibles pour les libs system).
|
||||
# =============================================================================
|
||||
FROM rockylinux:9 AS builder
|
||||
|
||||
WORKDIR /build
|
||||
|
||||
RUN dnf install -y epel-release && \
|
||||
dnf config-manager --set-enabled crb && \
|
||||
dnf install -y golang git libpcap-devel gcc make && \
|
||||
dnf clean all
|
||||
|
||||
# Copie du workspace Go et du module partagé en premier (meilleur cache)
|
||||
COPY go.work go.work.sum* ./
|
||||
COPY shared/go/ja4common/ ./shared/go/ja4common/
|
||||
COPY services/sentinel/go.mod services/sentinel/go.sum* ./services/sentinel/
|
||||
COPY services/correlator/go.mod services/correlator/go.sum* ./services/correlator/
|
||||
|
||||
WORKDIR /build/services/sentinel
|
||||
RUN go mod download || true
|
||||
|
||||
COPY services/sentinel/ /build/services/sentinel/
|
||||
|
||||
ARG VERSION=dev
|
||||
ARG BUILD_TIME=""
|
||||
ARG GIT_COMMIT=""
|
||||
|
||||
RUN mkdir -p dist && \
|
||||
CGO_ENABLED=1 GOOS=linux GOARCH=amd64 \
|
||||
go build -buildvcs=false \
|
||||
-ldflags "-X main.Version=${VERSION} -X main.BuildTime=${BUILD_TIME} -X main.GitCommit=${GIT_COMMIT}" \
|
||||
-o dist/sentinel \
|
||||
./cmd/ja4sentinel
|
||||
|
||||
# =============================================================================
|
||||
# Stage 2: rpm-builder — construction des RPMs avec rpmbuild
|
||||
# Un seul stage, trois appels rpmbuild successifs (el8, el9, el10).
|
||||
# =============================================================================
|
||||
FROM rockylinux:9 AS rpm-builder
|
||||
|
||||
WORKDIR /package
|
||||
|
||||
ARG VERSION=dev
|
||||
|
||||
RUN dnf install -y rpm-build rpmdevtools systemd-rpm-macros && dnf clean all
|
||||
|
||||
RUN mkdir -p /root/rpmbuild/{BUILD,RPMS,SOURCES,SPECS,SRPMS} && \
|
||||
mkdir -p /root/rpmbuild/SOURCES/logrotate && \
|
||||
mkdir -p /packages/rpm/{el8,el9,el10}
|
||||
|
||||
# Spec et fichiers sources
|
||||
COPY services/sentinel/packaging/rpm/ja4sentinel.spec /root/rpmbuild/SPECS/ja4sentinel.spec
|
||||
COPY --from=builder /build/services/sentinel/dist/sentinel /root/rpmbuild/SOURCES/ja4sentinel
|
||||
COPY services/sentinel/packaging/systemd/ja4sentinel.service /root/rpmbuild/SOURCES/ja4sentinel.service
|
||||
COPY services/sentinel/packaging/logrotate/ja4sentinel /root/rpmbuild/SOURCES/logrotate/ja4sentinel
|
||||
COPY services/sentinel/config.yml.example /root/rpmbuild/SOURCES/config.yml
|
||||
|
||||
RUN chmod 755 /root/rpmbuild/SOURCES/ja4sentinel && \
|
||||
chmod 644 /root/rpmbuild/SOURCES/ja4sentinel.service && \
|
||||
chmod 644 /root/rpmbuild/SOURCES/logrotate/ja4sentinel && \
|
||||
chmod 640 /root/rpmbuild/SOURCES/config.yml
|
||||
|
||||
# el8
|
||||
RUN rpmbuild --define "_topdir /root/rpmbuild" \
|
||||
--define "dist .el8" \
|
||||
--define "build_version ${VERSION}" \
|
||||
--target x86_64 \
|
||||
-bb /root/rpmbuild/SPECS/ja4sentinel.spec && \
|
||||
cp /root/rpmbuild/RPMS/x86_64/*.el8.x86_64.rpm /packages/rpm/el8/
|
||||
|
||||
# el9
|
||||
RUN rpmbuild --define "_topdir /root/rpmbuild" \
|
||||
--define "dist .el9" \
|
||||
--define "build_version ${VERSION}" \
|
||||
--target x86_64 \
|
||||
-bb /root/rpmbuild/SPECS/ja4sentinel.spec && \
|
||||
cp /root/rpmbuild/RPMS/x86_64/*.el9.x86_64.rpm /packages/rpm/el9/
|
||||
|
||||
# el10
|
||||
RUN rpmbuild --define "_topdir /root/rpmbuild" \
|
||||
--define "dist .el10" \
|
||||
--define "build_version ${VERSION}" \
|
||||
--target x86_64 \
|
||||
-bb /root/rpmbuild/SPECS/ja4sentinel.spec && \
|
||||
cp /root/rpmbuild/RPMS/x86_64/*.el10.x86_64.rpm /packages/rpm/el10/
|
||||
|
||||
# =============================================================================
|
||||
# Stage 3: output — image finale contenant uniquement les RPMs
|
||||
# =============================================================================
|
||||
FROM alpine:latest AS output
|
||||
|
||||
WORKDIR /packages
|
||||
COPY --from=rpm-builder /packages/rpm/el8/*.rpm /packages/rpm/el8/
|
||||
COPY --from=rpm-builder /packages/rpm/el9/*.rpm /packages/rpm/el9/
|
||||
COPY --from=rpm-builder /packages/rpm/el10/*.rpm /packages/rpm/el10/
|
||||
|
||||
CMD ["sh", "-c", \
|
||||
"echo '=== RPM el8 ===' && ls -la /packages/rpm/el8/ && \
|
||||
echo '' && echo '=== RPM el9 ===' && ls -la /packages/rpm/el9/ && \
|
||||
echo '' && echo '=== RPM el10 ===' && ls -la /packages/rpm/el10/"]
|
||||
@ -1,105 +0,0 @@
|
||||
# Test server for generating TLS traffic in integration tests
|
||||
FROM golang:1.23-alpine
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Create a simple TLS server for testing
|
||||
RUN cat > main.go << 'EOF'
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
port := flag.String("port", "8443", "Port to listen on")
|
||||
flag.Parse()
|
||||
|
||||
// Generate self-signed certificate
|
||||
cert, err := generateSelfSignedCert()
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to generate certificate: %v", err)
|
||||
}
|
||||
|
||||
config := &tls.Config{
|
||||
Certificates: []tls.Certificate{cert},
|
||||
MinVersion: tls.VersionTLS12,
|
||||
}
|
||||
|
||||
listener, err := tls.Listen("tcp", ":"+*port, config)
|
||||
if err != nil {
|
||||
log.Fatalf("Failed to start TLS listener: %v", err)
|
||||
}
|
||||
defer listener.Close()
|
||||
|
||||
log.Printf("TLS test server listening on port %s", *port)
|
||||
|
||||
http.Serve(listener, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(http.StatusOK)
|
||||
w.Write([]byte("Hello from TLS test server"))
|
||||
}))
|
||||
}
|
||||
|
||||
func generateSelfSignedCert() (tls.Certificate, error) {
|
||||
// Generate private key
|
||||
priv, err := rsa.GenerateKey(rand.Reader, 2048)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
|
||||
// Create certificate template
|
||||
template := x509.Certificate{
|
||||
SerialNumber: big.NewInt(1),
|
||||
Subject: pkix.Name{
|
||||
Organization: []string{"JA4Sentinel Test"},
|
||||
CommonName: "localhost",
|
||||
},
|
||||
NotBefore: time.Now(),
|
||||
NotAfter: time.Now().Add(24 * time.Hour),
|
||||
KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
|
||||
BasicConstraintsValid: true,
|
||||
IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback},
|
||||
DNSNames: []string{"localhost"},
|
||||
}
|
||||
|
||||
// Create certificate
|
||||
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
|
||||
// Encode certificate
|
||||
certPEM := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: certDER,
|
||||
})
|
||||
|
||||
// Encode private key
|
||||
keyPEM := pem.EncodeToMemory(&pem.Block{
|
||||
Type: "RSA PRIVATE KEY",
|
||||
Bytes: x509.MarshalPKCS1PrivateKey(priv),
|
||||
})
|
||||
|
||||
// Load certificate
|
||||
return tls.X509KeyPair(certPEM, keyPEM)
|
||||
}
|
||||
EOF
|
||||
|
||||
RUN go mod init test-server && go mod tidy
|
||||
|
||||
EXPOSE 8443
|
||||
|
||||
CMD ["go", "run", "main.go", "-port", "8443"]
|
||||
@ -1,158 +0,0 @@
|
||||
.PHONY: build build-docker test test-docker test-integration lint clean help docker-build-dev docker-build-runtime package package-rpm
|
||||
|
||||
# Docker parameters
|
||||
DOCKER=docker
|
||||
DOCKER_BUILD=$(DOCKER) build
|
||||
DOCKER_RUN=$(DOCKER) run
|
||||
DOCKER_COMPOSE=docker compose
|
||||
|
||||
# Image names
|
||||
DEV_IMAGE=ja4sentinel-dev:latest
|
||||
RUNTIME_IMAGE=ja4sentinel-runtime:latest
|
||||
TEST_SERVER_IMAGE=ja4sentinel-test-server:latest
|
||||
|
||||
# Binary name
|
||||
BINARY_NAME=ja4sentinel
|
||||
BINARY_PATH=./cmd/ja4sentinel
|
||||
DIST_DIR=dist
|
||||
BUILD_DIR=build
|
||||
|
||||
# RPM build directory
|
||||
RPM_DIR=$(DIST_DIR)/rpm
|
||||
|
||||
# Package version (extract default from spec file, can be overridden)
|
||||
PKG_VERSION ?= $(shell grep '^%define spec_version' packaging/rpm/ja4sentinel.spec | tail -1 | awk '{print $$3}')
|
||||
|
||||
# Build flags
|
||||
VERSION=$(PKG_VERSION)
|
||||
BUILD_TIME=$(shell date -u '+%Y-%m-%d_%H:%M:%S')
|
||||
GIT_COMMIT=$(shell git rev-parse --short HEAD 2>/dev/null || echo "unknown")
|
||||
|
||||
LDFLAGS=-ldflags "-X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME) -X main.GitCommit=$(GIT_COMMIT)"
|
||||
|
||||
# Default target
|
||||
all: docker-build-dev test-docker
|
||||
|
||||
## build: Build the ja4sentinel binary locally
|
||||
build:
|
||||
mkdir -p $(DIST_DIR)
|
||||
go build -buildvcs=false $(LDFLAGS) -o $(DIST_DIR)/$(BINARY_NAME) $(BINARY_PATH)
|
||||
|
||||
## build-linux: Build for Linux (amd64)
|
||||
build-linux:
|
||||
mkdir -p $(DIST_DIR)
|
||||
GOOS=linux GOARCH=amd64 go build -buildvcs=false $(LDFLAGS) -o $(DIST_DIR)/$(BINARY_NAME)-linux-amd64 $(BINARY_PATH)
|
||||
|
||||
## docker-build-dev: Build the development Docker image
|
||||
docker-build-dev:
|
||||
$(DOCKER_BUILD) -t $(DEV_IMAGE) -f Dockerfile.dev .
|
||||
|
||||
## docker-build-runtime: Build the runtime Docker image (multi-stage build)
|
||||
docker-build-runtime:
|
||||
$(DOCKER_BUILD) -t $(RUNTIME_IMAGE) -f Dockerfile .
|
||||
|
||||
## docker-build-test-server: Build the test server image
|
||||
docker-build-test-server:
|
||||
$(DOCKER_BUILD) -t $(TEST_SERVER_IMAGE) -f Dockerfile.test-server .
|
||||
|
||||
## test: Run unit tests locally
|
||||
test:
|
||||
go test -v ./...
|
||||
|
||||
## test-docker: Run unit tests inside Docker container
|
||||
test-docker: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) go test -v ./...
|
||||
|
||||
## test-race: Run tests with race detector in Docker
|
||||
test-race: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) go test -race -v ./...
|
||||
|
||||
## test-coverage: Run tests with coverage report in Docker
|
||||
test-coverage: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) sh -c \
|
||||
"go test -v -coverprofile=coverage.out ./... && go tool cover -html=coverage.out -o coverage.html"
|
||||
|
||||
## test-integration: Run integration tests in Docker
|
||||
test-integration: docker-build-dev docker-build-test-server
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml build --no-cache
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from ja4sentinel-test
|
||||
|
||||
## test-integration-clean: Run integration tests and clean up afterward
|
||||
test-integration-clean: docker-build-dev docker-build-test-server
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml build --no-cache
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml up --abort-on-container-exit --exit-code-from ja4sentinel-test
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml down -v
|
||||
|
||||
## lint: Run linters in Docker
|
||||
lint: docker-build-dev
|
||||
$(DOCKER_RUN) --rm -v $(PWD):/app -w /app $(DEV_IMAGE) sh -c \
|
||||
"go vet ./... && echo 'Running gofmt check...' && gofmt -l . | grep -v '^vendor/' | grep -v '^path/' || true"
|
||||
|
||||
## fmt: Format all Go files
|
||||
fmt:
|
||||
gofmt -w .
|
||||
|
||||
## package: Build RPM packages for all target distributions
|
||||
package: package-rpm
|
||||
|
||||
## package-rpm: Build RPM packages for Rocky Linux 8/9/10, AlmaLinux (requires Docker)
|
||||
package-rpm:
|
||||
mkdir -p $(RPM_DIR)/el8 $(RPM_DIR)/el9 $(RPM_DIR)/el10
|
||||
@echo "Building RPM packages for Rocky Linux 8/9, AlmaLinux 10..."
|
||||
docker build --target output -t ja4sentinel-rpm-packager:latest \
|
||||
--build-arg VERSION=$(PKG_VERSION) \
|
||||
-f Dockerfile.package .
|
||||
@echo "Extracting RPM packages from Docker image..."
|
||||
@docker run --rm -v $(PWD)/$(RPM_DIR):/output/rpm ja4sentinel-rpm-packager:latest sh -c \
|
||||
'cp -r /packages/rpm/el8 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el9 /output/rpm/ && \
|
||||
cp -r /packages/rpm/el10 /output/rpm/'
|
||||
@echo "RPM packages created:"
|
||||
@echo " Rocky Linux 8 (el8):"
|
||||
ls -la $(RPM_DIR)/el8/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " Rocky Linux 9 (el9):"
|
||||
ls -la $(RPM_DIR)/el9/ 2>/dev/null || echo " (no packages)"
|
||||
@echo " AlmaLinux/Rocky 10 (el10):"
|
||||
ls -la $(RPM_DIR)/el10/ 2>/dev/null || echo " (no packages)"
|
||||
|
||||
## test-package-rpm: Test RPM package installation in Docker
|
||||
test-package-rpm: package-rpm
|
||||
./packaging/test/test-rpm.sh
|
||||
|
||||
## test-package: Test RPM package installation
|
||||
test-package: test-package-rpm
|
||||
|
||||
## ci: Full CI pipeline (tests, build, packages, package tests)
|
||||
ci: ci-test ci-build ci-package ci-package-test
|
||||
|
||||
## ci-test: Run all tests for CI
|
||||
ci-test: test lint
|
||||
|
||||
## ci-build: Build for CI (production binary)
|
||||
ci-build: build-linux
|
||||
|
||||
## ci-package: Build all packages for CI
|
||||
ci-package: package
|
||||
|
||||
## ci-package-test: Test all packages for CI
|
||||
ci-package-test: test-package
|
||||
|
||||
## clean: Clean build artifacts and Docker images
|
||||
clean:
|
||||
rm -rf $(DIST_DIR)/
|
||||
rm -rf $(BUILD_DIR)/
|
||||
rm -f coverage.out coverage.html
|
||||
$(DOCKER) rmi $(DEV_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(RUNTIME_IMAGE) 2>/dev/null || true
|
||||
$(DOCKER) rmi $(TEST_SERVER_IMAGE) 2>/dev/null || true
|
||||
|
||||
## clean-all: Clean everything including containers and volumes
|
||||
clean-all: clean
|
||||
$(DOCKER_COMPOSE) -f docker-compose.test.yml down -v --remove-orphans
|
||||
|
||||
## help: Show this help message
|
||||
help:
|
||||
@echo "Usage: make [target]"
|
||||
@echo ""
|
||||
@echo "Targets:"
|
||||
@sed -n 's/^##//p' $(MAKEFILE_LIST) | column -t -s ':' | sed -e 's/^/ /'
|
||||
@ -1,291 +0,0 @@
|
||||
# JA4Sentinel
|
||||
|
||||
Outil Go pour capturer le trafic réseau sur un serveur Linux, extraire les handshakes TLS côté client, générer les signatures JA4, enrichir avec des métadonnées IP/TCP, et loguer les résultats vers une ou plusieurs sorties configurables.
|
||||
|
||||
## Fonctionnalités
|
||||
|
||||
- **Capture réseau** : Écoute sur une interface réseau avec filtres BPF configurables
|
||||
- **Parsing TLS** : Extraction des ClientHello TLS depuis les flux TCP
|
||||
- **Fingerprinting** : Génération des empreintes JA4 et JA3 pour chaque client
|
||||
- **Métadonnées** : Enrichissement avec IPMeta (TTL, IP ID, DF) et TCPMeta (window, MSS, options)
|
||||
- **Sorties multiples** : stdout, fichier JSON, socket UNIX (combinables via MultiWriter)
|
||||
- **Logging structuré** : Logs JSON sur stdout/stderr pour intégration avec systemd/journald
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ Capture │ ──▶ │ TLSParse │ ──▶ │ Fingerprint │ ──▶ │ Output │
|
||||
│ (pcap) │ │ (ClientHello)│ │ (JA4) │ │ (JSON logs) │
|
||||
└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘
|
||||
│ │ │ │
|
||||
▼ ▼ ▼ ▼
|
||||
api.RawPacket api.TLSClientHello api.Fingerprints api.LogRecord
|
||||
```
|
||||
|
||||
### Modules
|
||||
|
||||
| Module | Responsabilités |
|
||||
|--------|-----------------|
|
||||
| `config` | Chargement et validation de la configuration (YAML, env, CLI) |
|
||||
| `capture` | Capture des paquets réseau via libpcap |
|
||||
| `tlsparse` | Extraction des ClientHello TLS avec suivi d'état de flux |
|
||||
| `fingerprint` | Génération JA4/JA3 via `psanford/tlsfingerprint` |
|
||||
| `output` | Écriture des logs vers stdout, fichier, socket UNIX |
|
||||
| `logging` | Logs structurés JSON pour le diagnostic du service |
|
||||
|
||||
## Installation
|
||||
|
||||
### Prérequis
|
||||
|
||||
- Go 1.24+
|
||||
- libpcap-dev (pour la compilation)
|
||||
- Docker (pour les tests et le déploiement)
|
||||
|
||||
### Note sur libpcap
|
||||
|
||||
**Le binaire est compilé sur Rocky Linux 9** pour une compatibilité maximale avec toutes les distributions RHEL/Rocky/AlmaLinux.
|
||||
libpcap est requis à l'exécution et sera installé automatiquement par le gestionnaire de packages.
|
||||
|
||||
### Packages système
|
||||
|
||||
#### Rocky Linux / RHEL / AlmaLinux (.rpm)
|
||||
|
||||
```bash
|
||||
# Télécharger le package
|
||||
wget https://github.com/your-repo/ja4sentinel/releases/latest/download/ja4sentinel.rpm
|
||||
|
||||
# Installer
|
||||
sudo dnf install ./ja4sentinel.rpm
|
||||
|
||||
# Activer le service
|
||||
sudo systemctl enable ja4sentinel
|
||||
sudo systemctl start ja4sentinel
|
||||
|
||||
# Vérifier le statut
|
||||
sudo systemctl status ja4sentinel
|
||||
```
|
||||
|
||||
#### Distributions supportées
|
||||
|
||||
- Rocky Linux 8, 9, 10
|
||||
- AlmaLinux 8, 9, 10
|
||||
- RHEL 8, 9, 10
|
||||
|
||||
## Configuration
|
||||
|
||||
### Fichier de configuration (YAML)
|
||||
|
||||
```yaml
|
||||
core:
|
||||
interface: eth0
|
||||
listen_ports: [443, 8443]
|
||||
bpf_filter: ""
|
||||
flow_timeout_sec: 30
|
||||
|
||||
outputs:
|
||||
- type: stdout
|
||||
enabled: true
|
||||
- type: file
|
||||
enabled: true
|
||||
params:
|
||||
path: /var/log/ja4sentinel/ja4.log
|
||||
- type: unix_socket
|
||||
enabled: true
|
||||
params:
|
||||
socket_path: /var/run/logcorrelator/network.socket
|
||||
```
|
||||
|
||||
### Variables d'environnement
|
||||
|
||||
| Variable | Description |
|
||||
|----------|-------------|
|
||||
| `JA4SENTINEL_INTERFACE` | Interface réseau (ex: `eth0`) |
|
||||
| `JA4SENTINEL_PORTS` | Ports à surveiller (ex: `443,8443`) |
|
||||
| `JA4SENTINEL_BPF_FILTER` | Filtre BPF personnalisé |
|
||||
| `JA4SENTINEL_FLOW_TIMEOUT` | Timeout de flux en secondes (défaut: 30) |
|
||||
|
||||
### Ligne de commande
|
||||
|
||||
```bash
|
||||
ja4sentinel --config /etc/ja4sentinel/config.yml
|
||||
ja4sentinel --version
|
||||
```
|
||||
|
||||
## Format des logs
|
||||
|
||||
### Logs de service (stdout/stderr)
|
||||
|
||||
```json
|
||||
{
|
||||
"timestamp": 1708876543210000000,
|
||||
"level": "INFO",
|
||||
"component": "capture",
|
||||
"message": "Starting packet capture",
|
||||
"interface": "eth0"
|
||||
}
|
||||
```
|
||||
|
||||
### Logs métier (JA4)
|
||||
|
||||
```json
|
||||
{
|
||||
"src_ip": "192.168.1.100",
|
||||
"src_port": 54321,
|
||||
"dst_ip": "10.0.0.1",
|
||||
"dst_port": 443,
|
||||
"ip_meta_ttl": 64,
|
||||
"ip_meta_total_length": 512,
|
||||
"ip_meta_id": 12345,
|
||||
"ip_meta_df": true,
|
||||
"tcp_meta_window_size": 65535,
|
||||
"tcp_meta_mss": 1460,
|
||||
"tcp_meta_window_scale": 7,
|
||||
"tcp_meta_options": "MSS,WS,SACK,TS",
|
||||
"ja4": "t13d1516h2_8daaf6152771_02cb136f2775",
|
||||
"ja4_hash": "8daaf6152771_02cb136f2775",
|
||||
"ja3": "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
|
||||
"ja3_hash": "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d"
|
||||
}
|
||||
```
|
||||
|
||||
## Tests
|
||||
|
||||
### Tests unitaires
|
||||
|
||||
```bash
|
||||
# En local
|
||||
make test
|
||||
|
||||
# Dans Docker
|
||||
make test-docker
|
||||
|
||||
# Avec détection de race conditions
|
||||
make test-race
|
||||
|
||||
# Avec rapport de couverture
|
||||
make test-coverage
|
||||
```
|
||||
|
||||
### Tests d'intégration
|
||||
|
||||
```bash
|
||||
# Lance les tests bout-à-bout dans Docker
|
||||
make test-integration
|
||||
|
||||
# Nettoyage après tests
|
||||
make test-integration-clean
|
||||
```
|
||||
|
||||
## Déploiement systemd
|
||||
|
||||
Exemple de fichier de service `/etc/systemd/system/ja4sentinel.service` :
|
||||
|
||||
```ini
|
||||
[Unit]
|
||||
Description=JA4 client fingerprinting daemon
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=ja4sentinel
|
||||
Group=ja4sentinel
|
||||
ExecStart=/usr/local/bin/ja4sentinel --config /etc/ja4sentinel/config.yml
|
||||
Restart=on-failure
|
||||
RestartSec=5
|
||||
Environment=JA4SENTINEL_LOG_LEVEL=info
|
||||
|
||||
# Security
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=full
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
CapabilityBoundingSet=CAP_NET_RAW CAP_NET_ADMIN
|
||||
AmbientCapabilities=CAP_NET_RAW CAP_NET_ADMIN
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
```
|
||||
|
||||
## Exemples d'utilisation
|
||||
|
||||
### Surveillance du trafic HTTPS
|
||||
|
||||
```yaml
|
||||
core:
|
||||
interface: eth0
|
||||
listen_ports: [443]
|
||||
outputs:
|
||||
- type: stdout
|
||||
enabled: true
|
||||
```
|
||||
|
||||
### Export vers socket UNIX pour traitement externe
|
||||
|
||||
```yaml
|
||||
core:
|
||||
interface: eth0
|
||||
listen_ports: [443, 8443]
|
||||
outputs:
|
||||
- type: unix_socket
|
||||
enabled: true
|
||||
params:
|
||||
socket_path: /var/run/logcorrelator/network.socket
|
||||
# log_level: debug # debug, info, warn, error (défaut: error)
|
||||
```
|
||||
|
||||
### Logging fichier + stdout
|
||||
|
||||
```yaml
|
||||
core:
|
||||
interface: ens192
|
||||
listen_ports: [443]
|
||||
flow_timeout_sec: 60
|
||||
outputs:
|
||||
- type: stdout
|
||||
enabled: true
|
||||
- type: file
|
||||
enabled: true
|
||||
params:
|
||||
path: /var/log/ja4sentinel/ja4.json
|
||||
```
|
||||
|
||||
## Développement
|
||||
|
||||
### Linting
|
||||
|
||||
```bash
|
||||
make lint
|
||||
```
|
||||
|
||||
### Formatage
|
||||
|
||||
```bash
|
||||
make fmt
|
||||
```
|
||||
|
||||
### Nettoyage
|
||||
|
||||
```bash
|
||||
# Supprime les binaires et images Docker
|
||||
make clean
|
||||
|
||||
# Supprime aussi les conteneurs et volumes
|
||||
make clean-all
|
||||
```
|
||||
|
||||
## Licence
|
||||
|
||||
À définir.
|
||||
|
||||
## Contribuer
|
||||
|
||||
1. Fork le projet
|
||||
2. Créer une branche de feature (`git checkout -b feature/amélioration`)
|
||||
3. Commit les changements (`git commit -am 'Ajout fonctionnalité'`)
|
||||
4. Push (`git push origin feature/amélioration`)
|
||||
5. Ouvrir une Pull Request
|
||||
|
||||
---
|
||||
|
||||
**Voir `architecture.yml` pour la documentation complète de l'architecture.**
|
||||
@ -1,307 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ServiceLog represents internal service logging for diagnostics
|
||||
type ServiceLog struct {
|
||||
Level string `json:"level"`
|
||||
Component string `json:"component"`
|
||||
Message string `json:"message"`
|
||||
Details map[string]string `json:"details,omitempty"`
|
||||
Timestamp int64 `json:"timestamp,omitempty"` // Unix nanoseconds (auto-set by logger)
|
||||
TraceID string `json:"trace_id,omitempty"` // Optional distributed tracing ID
|
||||
ConnID string `json:"conn_id,omitempty"` // Optional TCP flow identifier
|
||||
}
|
||||
|
||||
// Config holds basic network and TLS configuration
|
||||
type Config struct {
|
||||
Interface string `yaml:"interface" json:"interface"`
|
||||
ListenPorts []uint16 `yaml:"listen_ports" json:"listen_ports"`
|
||||
BPFFilter string `yaml:"bpf_filter" json:"bpf_filter,omitempty"`
|
||||
LocalIPs []string `yaml:"local_ips" json:"local_ips,omitempty"` // Local IPs to monitor (empty = auto-detect, excludes loopback)
|
||||
ExcludeSourceIPs []string `yaml:"exclude_source_ips" json:"exclude_source_ips,omitempty"` // Source IPs or CIDR ranges to exclude (e.g., ["10.0.0.0/8", "192.168.1.1"])
|
||||
FlowTimeoutSec int `yaml:"flow_timeout_sec" json:"flow_timeout_sec,omitempty"` // Timeout for TLS handshake extraction (default: 30)
|
||||
PacketBufferSize int `yaml:"packet_buffer_size" json:"packet_buffer_size,omitempty"` // Buffer size for packet channel (default: 1000)
|
||||
LogLevel string `yaml:"log_level" json:"log_level,omitempty"` // Log level: debug, info, warn, error (default: info)
|
||||
}
|
||||
|
||||
// IPMeta contains IP metadata for stack fingerprinting
|
||||
type IPMeta struct {
|
||||
TTL uint8 `json:"ttl"`
|
||||
TotalLength uint16 `json:"total_length"`
|
||||
IPID uint16 `json:"id"`
|
||||
DF bool `json:"df"`
|
||||
}
|
||||
|
||||
// TCPMeta contains TCP metadata for stack fingerprinting
|
||||
type TCPMeta struct {
|
||||
WindowSize uint16 `json:"window_size"`
|
||||
MSS uint16 `json:"mss,omitempty"`
|
||||
WindowScale uint8 `json:"window_scale,omitempty"`
|
||||
Options []string `json:"options"`
|
||||
OptionKinds []uint8 `json:"-"` // Raw TCP option kind numbers for JA4T
|
||||
}
|
||||
|
||||
// RawPacket represents a raw packet captured from the network
|
||||
type RawPacket struct {
|
||||
Data []byte `json:"-"` // Raw packet data including link-layer header
|
||||
Timestamp int64 `json:"timestamp"` // nanoseconds since epoch
|
||||
LinkType int `json:"-"` // Link type (1=Ethernet, 101=Linux SLL, etc.)
|
||||
}
|
||||
|
||||
// TLSClientHello represents a client-side TLS ClientHello with IP/TCP metadata
|
||||
type TLSClientHello struct {
|
||||
SrcIP string `json:"src_ip"`
|
||||
SrcPort uint16 `json:"src_port"`
|
||||
DstIP string `json:"dst_ip"`
|
||||
DstPort uint16 `json:"dst_port"`
|
||||
Payload []byte `json:"-"` // Not serialized
|
||||
IPMeta IPMeta `json:"ip_meta"`
|
||||
TCPMeta TCPMeta `json:"tcp_meta"`
|
||||
ConnID string `json:"conn_id,omitempty"` // Unique flow identifier
|
||||
SNI string `json:"tls_sni,omitempty"` // Server Name Indication
|
||||
ALPN string `json:"tls_alpn,omitempty"` // Application-Layer Protocol Negotiation
|
||||
TLSVersion string `json:"tls_version,omitempty"` // Max TLS version supported
|
||||
SynToCHMs *uint32 `json:"syn_to_clienthello_ms,omitempty"` // Time from SYN to ClientHello (ms)
|
||||
}
|
||||
|
||||
// Fingerprints contains TLS fingerprints for a client flow
|
||||
// Note: JA4Hash is kept for internal use but not serialized to LogRecord
|
||||
// as the JA4 format already includes its own hash portions
|
||||
type Fingerprints struct {
|
||||
JA4 string `json:"ja4"`
|
||||
JA4Hash string `json:"ja4_hash,omitempty"` // Internal use, not serialized to LogRecord
|
||||
JA4T string `json:"ja4t,omitempty"`
|
||||
JA3 string `json:"ja3,omitempty"`
|
||||
JA3Hash string `json:"ja3_hash,omitempty"`
|
||||
}
|
||||
|
||||
// LogRecord is the final log record, serialized as a flat JSON object
|
||||
type LogRecord struct {
|
||||
SrcIP string `json:"src_ip"`
|
||||
SrcPort uint16 `json:"src_port"`
|
||||
DstIP string `json:"dst_ip"`
|
||||
DstPort uint16 `json:"dst_port"`
|
||||
|
||||
// Flattened IPMeta fields
|
||||
IPTTL uint8 `json:"ip_meta_ttl"`
|
||||
IPTotalLen uint16 `json:"ip_meta_total_length"`
|
||||
IPID uint16 `json:"ip_meta_id"`
|
||||
IPDF bool `json:"ip_meta_df"`
|
||||
|
||||
// Flattened TCPMeta fields
|
||||
TCPWindow uint16 `json:"tcp_meta_window_size"`
|
||||
TCPMSS *uint16 `json:"tcp_meta_mss,omitempty"`
|
||||
TCPWScale *uint8 `json:"tcp_meta_window_scale,omitempty"`
|
||||
TCPOptions string `json:"tcp_meta_options"` // comma-separated list
|
||||
|
||||
// Correlation & Triage
|
||||
ConnID string `json:"conn_id,omitempty"` // Unique flow identifier
|
||||
SensorID string `json:"sensor_id,omitempty"` // Sensor/captor identifier
|
||||
|
||||
// TLS elements (ClientHello)
|
||||
TLSVersion string `json:"tls_version,omitempty"` // Max TLS version announced by client
|
||||
SNI string `json:"tls_sni,omitempty"` // Server Name Indication
|
||||
ALPN string `json:"tls_alpn,omitempty"` // Application-Layer Protocol Negotiation
|
||||
|
||||
// Behavioral detection (Timing)
|
||||
SynToCHMs *uint32 `json:"syn_to_clienthello_ms,omitempty"` // Time from SYN to ClientHello (ms)
|
||||
|
||||
// Fingerprints
|
||||
// Note: ja4_hash is NOT included - the JA4 format already includes its own hash portions
|
||||
JA4 string `json:"ja4"`
|
||||
JA4T string `json:"ja4t,omitempty"`
|
||||
JA3 string `json:"ja3,omitempty"`
|
||||
JA3Hash string `json:"ja3_hash,omitempty"`
|
||||
|
||||
// Timestamp in nanoseconds since Unix epoch
|
||||
Timestamp int64 `json:"timestamp"`
|
||||
}
|
||||
|
||||
// OutputConfig defines configuration for a single log output
|
||||
type OutputConfig struct {
|
||||
Type string `yaml:"type" json:"type"` // unix_socket, stdout, file, etc.
|
||||
Enabled bool `yaml:"enabled" json:"enabled"` // whether this output is active
|
||||
AsyncBuffer int `yaml:"async_buffer" json:"async_buffer"` // queue size for async writes (e.g., 5000)
|
||||
Params map[string]string `yaml:"params" json:"params"` // specific parameters like socket_path, path, etc.
|
||||
}
|
||||
|
||||
// AppConfig is the complete ja4sentinel configuration
|
||||
type AppConfig struct {
|
||||
Core Config `yaml:"core" json:"core"`
|
||||
Outputs []OutputConfig `yaml:"outputs" json:"outputs"`
|
||||
}
|
||||
|
||||
// Loader defines the interface for loading application configuration.
|
||||
// Implementations must read configuration from a YAML file, merge with
|
||||
// environment variables (JA4SENTINEL_*), and validate the final result.
|
||||
type Loader interface {
|
||||
Load() (AppConfig, error)
|
||||
}
|
||||
|
||||
// Capture defines the interface for capturing raw network packets.
|
||||
// Implementations must listen on a configured network interface, apply
|
||||
// BPF filters for specified ports, and emit RawPacket objects to a channel.
|
||||
// The Close method must be called to release resources (e.g., pcap handle).
|
||||
type Capture interface {
|
||||
Run(cfg Config, out chan<- RawPacket) error
|
||||
Close() error
|
||||
GetStats() (received, sent, dropped uint64)
|
||||
}
|
||||
|
||||
// Parser defines the interface for extracting TLS ClientHello messages
|
||||
// from raw network packets. Implementations must track TCP connection states,
|
||||
// reassemble fragmented handshakes, and return TLSClientHello objects with
|
||||
// IP/TCP metadata. Returns nil for non-TLS or non-ClientHello packets.
|
||||
type Parser interface {
|
||||
Process(pkt RawPacket) (*TLSClientHello, error)
|
||||
Close() error
|
||||
GetMetrics() (retransmit, gapDetected, bufferExceeded, segmentExceeded uint64)
|
||||
}
|
||||
|
||||
// Engine defines the interface for generating TLS fingerprints.
|
||||
// Implementations must analyze TLS ClientHello payloads and produce
|
||||
// JA4 (required) and optionally JA3 fingerprint strings.
|
||||
type Engine interface {
|
||||
FromClientHello(ch TLSClientHello) (*Fingerprints, error)
|
||||
}
|
||||
|
||||
// Writer defines the generic interface for writing log records.
|
||||
// Implementations must serialize LogRecord objects and send them to
|
||||
// a destination (stdout, file, UNIX socket, etc.).
|
||||
type Writer interface {
|
||||
Write(rec LogRecord) error
|
||||
}
|
||||
|
||||
// UnixSocketWriter extends Writer with a Close method for UNIX socket cleanup.
|
||||
// Implementations must connect to a UNIX socket at the specified path and
|
||||
// write JSON-encoded LogRecord objects. Reconnection logic should be
|
||||
// implemented for transient socket failures.
|
||||
type UnixSocketWriter interface {
|
||||
Writer
|
||||
Close() error
|
||||
}
|
||||
|
||||
// MultiWriter extends Writer to support multiple output destinations.
|
||||
// Implementations must write each LogRecord to all registered writers
|
||||
// and provide methods to add writers and close all connections.
|
||||
type MultiWriter interface {
|
||||
Writer
|
||||
Add(writer Writer)
|
||||
CloseAll() error
|
||||
}
|
||||
|
||||
// Builder defines the interface for constructing output writers from configuration.
|
||||
// Implementations must parse AppConfig.Outputs and create appropriate Writer
|
||||
// instances (StdoutWriter, FileWriter, UnixSocketWriter), combining them
|
||||
// into a MultiWriter if multiple outputs are configured.
|
||||
type Builder interface {
|
||||
NewFromConfig(cfg AppConfig) (Writer, error)
|
||||
}
|
||||
|
||||
// Logger defines the interface for structured service logging.
|
||||
// Implementations must emit JSON-formatted log entries to stdout/stderr
|
||||
// with support for multiple log levels (DEBUG, INFO, WARN, ERROR).
|
||||
// Each log entry includes timestamp, level, component, message, and optional details.
|
||||
type Logger interface {
|
||||
Debug(component, message string, details map[string]string)
|
||||
Info(component, message string, details map[string]string)
|
||||
Warn(component, message string, details map[string]string)
|
||||
Error(component, message string, details map[string]string)
|
||||
}
|
||||
|
||||
// Reopenable defines the interface for components that support log file rotation.
|
||||
// Implementations must reopen their output files when receiving a SIGHUP signal.
|
||||
// This is used by systemctl reload to switch to new log files after logrotate.
|
||||
type Reopenable interface {
|
||||
Reopen() error
|
||||
}
|
||||
|
||||
// Helper functions for creating and converting records
|
||||
|
||||
// NewLogRecord creates a flattened LogRecord from TLSClientHello and Fingerprints.
|
||||
// Converts TCPMeta options to a comma-separated string and creates pointer values
|
||||
// for optional fields (MSS, WindowScale) to support proper JSON omitempty behavior.
|
||||
// If fingerprints is nil, the JA4/JA3 fields will be empty strings.
|
||||
// Note: JA4Hash is intentionally NOT included in LogRecord as the JA4 format
|
||||
// already includes its own hash portions (the full 38-character JA4 string is sufficient).
|
||||
func NewLogRecord(ch TLSClientHello, fp *Fingerprints) LogRecord {
|
||||
opts := ""
|
||||
if len(ch.TCPMeta.Options) > 0 {
|
||||
opts = strings.Join(ch.TCPMeta.Options, ",")
|
||||
}
|
||||
|
||||
// Helper to create pointer from value for optional fields
|
||||
var mssPtr *uint16
|
||||
if ch.TCPMeta.MSS != 0 {
|
||||
mssPtr = &ch.TCPMeta.MSS
|
||||
}
|
||||
|
||||
var wScalePtr *uint8
|
||||
if ch.TCPMeta.WindowScale != 0 {
|
||||
wScalePtr = &ch.TCPMeta.WindowScale
|
||||
}
|
||||
|
||||
rec := LogRecord{
|
||||
SrcIP: ch.SrcIP,
|
||||
SrcPort: ch.SrcPort,
|
||||
DstIP: ch.DstIP,
|
||||
DstPort: ch.DstPort,
|
||||
IPTTL: ch.IPMeta.TTL,
|
||||
IPTotalLen: ch.IPMeta.TotalLength,
|
||||
IPID: ch.IPMeta.IPID,
|
||||
IPDF: ch.IPMeta.DF,
|
||||
TCPWindow: ch.TCPMeta.WindowSize,
|
||||
TCPMSS: mssPtr,
|
||||
TCPWScale: wScalePtr,
|
||||
TCPOptions: opts,
|
||||
ConnID: ch.ConnID,
|
||||
SNI: ch.SNI,
|
||||
ALPN: ch.ALPN,
|
||||
TLSVersion: ch.TLSVersion,
|
||||
SynToCHMs: ch.SynToCHMs,
|
||||
Timestamp: time.Now().UnixNano(),
|
||||
}
|
||||
|
||||
if fp != nil {
|
||||
rec.JA4 = fp.JA4
|
||||
rec.JA4T = fp.JA4T
|
||||
rec.JA3 = fp.JA3
|
||||
rec.JA3Hash = fp.JA3Hash
|
||||
}
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// Default values and constants
|
||||
|
||||
const (
|
||||
DefaultInterface = "eth0"
|
||||
DefaultPort = 443
|
||||
DefaultBPFFilter = ""
|
||||
DefaultFlowTimeout = 30 // seconds
|
||||
DefaultPacketBuffer = 1000 // packet channel buffer size
|
||||
DefaultLogLevel = "info"
|
||||
)
|
||||
|
||||
// DefaultConfig returns an AppConfig with sensible default values.
|
||||
// Uses eth0 as the default interface, port 443 for monitoring,
|
||||
// no BPF filter, a 30-second flow timeout, and a 1000-packet
|
||||
// channel buffer. Returns an empty outputs slice (caller must
|
||||
// configure outputs explicitly).
|
||||
func DefaultConfig() AppConfig {
|
||||
return AppConfig{
|
||||
Core: Config{
|
||||
Interface: DefaultInterface,
|
||||
ListenPorts: []uint16{DefaultPort},
|
||||
BPFFilter: DefaultBPFFilter,
|
||||
FlowTimeoutSec: DefaultFlowTimeout,
|
||||
PacketBufferSize: DefaultPacketBuffer,
|
||||
LogLevel: DefaultLogLevel,
|
||||
},
|
||||
Outputs: []OutputConfig{},
|
||||
}
|
||||
}
|
||||
@ -1,340 +0,0 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNewLogRecord(t *testing.T) {
|
||||
synToCHMs := uint32(150)
|
||||
tests := []struct {
|
||||
name string
|
||||
clientHello TLSClientHello
|
||||
fingerprints *Fingerprints
|
||||
wantNil bool
|
||||
}{
|
||||
{
|
||||
name: "complete record with fingerprints",
|
||||
clientHello: TLSClientHello{
|
||||
SrcIP: "192.168.1.100",
|
||||
SrcPort: 54321,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
ConnID: "flow-abc123",
|
||||
SNI: "example.com",
|
||||
ALPN: "h2",
|
||||
TLSVersion: "1.3",
|
||||
SynToCHMs: &synToCHMs,
|
||||
IPMeta: IPMeta{
|
||||
TTL: 64,
|
||||
TotalLength: 512,
|
||||
IPID: 12345,
|
||||
DF: true,
|
||||
},
|
||||
TCPMeta: TCPMeta{
|
||||
WindowSize: 65535,
|
||||
MSS: 1460,
|
||||
WindowScale: 7,
|
||||
Options: []string{"MSS", "WS", "SACK", "TS"},
|
||||
},
|
||||
},
|
||||
fingerprints: &Fingerprints{
|
||||
JA4: "t13d1516h2_8daaf6152771_02cb136f2775",
|
||||
JA4Hash: "8daaf6152771_02cb136f2775", // Internal use only
|
||||
JA3: "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
|
||||
JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d",
|
||||
},
|
||||
wantNil: false,
|
||||
},
|
||||
{
|
||||
name: "record without fingerprints",
|
||||
clientHello: TLSClientHello{
|
||||
SrcIP: "192.168.1.100",
|
||||
SrcPort: 54321,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
ConnID: "flow-xyz789",
|
||||
SNI: "test.example.com",
|
||||
ALPN: "http/1.1",
|
||||
TLSVersion: "1.2",
|
||||
IPMeta: IPMeta{
|
||||
TTL: 64,
|
||||
TotalLength: 512,
|
||||
IPID: 12345,
|
||||
DF: true,
|
||||
},
|
||||
TCPMeta: TCPMeta{
|
||||
WindowSize: 65535,
|
||||
MSS: 1460,
|
||||
WindowScale: 7,
|
||||
Options: []string{"MSS", "WS"},
|
||||
},
|
||||
},
|
||||
fingerprints: nil,
|
||||
wantNil: false,
|
||||
},
|
||||
{
|
||||
name: "record with zero values for optional fields",
|
||||
clientHello: TLSClientHello{
|
||||
SrcIP: "192.168.1.100",
|
||||
SrcPort: 54321,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
IPMeta: IPMeta{
|
||||
TTL: 0,
|
||||
TotalLength: 0,
|
||||
IPID: 0,
|
||||
DF: false,
|
||||
},
|
||||
TCPMeta: TCPMeta{
|
||||
WindowSize: 0,
|
||||
MSS: 0,
|
||||
WindowScale: 0,
|
||||
Options: []string{},
|
||||
},
|
||||
},
|
||||
fingerprints: nil,
|
||||
wantNil: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
rec := NewLogRecord(tt.clientHello, tt.fingerprints)
|
||||
|
||||
// Verify timestamp is set
|
||||
if rec.Timestamp == 0 {
|
||||
t.Error("Timestamp should be set")
|
||||
}
|
||||
|
||||
// Verify basic fields
|
||||
if rec.SrcIP != tt.clientHello.SrcIP {
|
||||
t.Errorf("SrcIP = %v, want %v", rec.SrcIP, tt.clientHello.SrcIP)
|
||||
}
|
||||
if rec.SrcPort != tt.clientHello.SrcPort {
|
||||
t.Errorf("SrcPort = %v, want %v", rec.SrcPort, tt.clientHello.SrcPort)
|
||||
}
|
||||
if rec.DstIP != tt.clientHello.DstIP {
|
||||
t.Errorf("DstIP = %v, want %v", rec.DstIP, tt.clientHello.DstIP)
|
||||
}
|
||||
if rec.DstPort != tt.clientHello.DstPort {
|
||||
t.Errorf("DstPort = %v, want %v", rec.DstPort, tt.clientHello.DstPort)
|
||||
}
|
||||
|
||||
// Verify IPMeta fields
|
||||
if rec.IPTTL != tt.clientHello.IPMeta.TTL {
|
||||
t.Errorf("IPTTL = %v, want %v", rec.IPTTL, tt.clientHello.IPMeta.TTL)
|
||||
}
|
||||
if rec.IPTotalLen != tt.clientHello.IPMeta.TotalLength {
|
||||
t.Errorf("IPTotalLen = %v, want %v", rec.IPTotalLen, tt.clientHello.IPMeta.TotalLength)
|
||||
}
|
||||
if rec.IPID != tt.clientHello.IPMeta.IPID {
|
||||
t.Errorf("IPID = %v, want %v", rec.IPID, tt.clientHello.IPMeta.IPID)
|
||||
}
|
||||
if rec.IPDF != tt.clientHello.IPMeta.DF {
|
||||
t.Errorf("IPDF = %v, want %v", rec.IPDF, tt.clientHello.IPMeta.DF)
|
||||
}
|
||||
|
||||
// Verify TCPMeta fields
|
||||
if rec.TCPWindow != tt.clientHello.TCPMeta.WindowSize {
|
||||
t.Errorf("TCPWindow = %v, want %v", rec.TCPWindow, tt.clientHello.TCPMeta.WindowSize)
|
||||
}
|
||||
|
||||
// Verify optional fields (MSS, WindowScale)
|
||||
if tt.clientHello.TCPMeta.MSS != 0 {
|
||||
if rec.TCPMSS == nil {
|
||||
t.Error("TCPMSS should not be nil when MSS != 0")
|
||||
} else if *rec.TCPMSS != tt.clientHello.TCPMeta.MSS {
|
||||
t.Errorf("TCPMSS = %v, want %v", *rec.TCPMSS, tt.clientHello.TCPMeta.MSS)
|
||||
}
|
||||
} else {
|
||||
if rec.TCPMSS != nil {
|
||||
t.Error("TCPMSS should be nil when MSS == 0")
|
||||
}
|
||||
}
|
||||
|
||||
if tt.clientHello.TCPMeta.WindowScale != 0 {
|
||||
if rec.TCPWScale == nil {
|
||||
t.Error("TCPWScale should not be nil when WindowScale != 0")
|
||||
} else if *rec.TCPWScale != tt.clientHello.TCPMeta.WindowScale {
|
||||
t.Errorf("TCPWScale = %v, want %v", *rec.TCPWScale, tt.clientHello.TCPMeta.WindowScale)
|
||||
}
|
||||
} else {
|
||||
if rec.TCPWScale != nil {
|
||||
t.Error("TCPWScale should be nil when WindowScale == 0")
|
||||
}
|
||||
}
|
||||
|
||||
// Verify new TLS fields
|
||||
if rec.ConnID != tt.clientHello.ConnID {
|
||||
t.Errorf("ConnID = %v, want %v", rec.ConnID, tt.clientHello.ConnID)
|
||||
}
|
||||
if rec.SNI != tt.clientHello.SNI {
|
||||
t.Errorf("SNI = %v, want %v", rec.SNI, tt.clientHello.SNI)
|
||||
}
|
||||
if rec.ALPN != tt.clientHello.ALPN {
|
||||
t.Errorf("ALPN = %v, want %v", rec.ALPN, tt.clientHello.ALPN)
|
||||
}
|
||||
if rec.TLSVersion != tt.clientHello.TLSVersion {
|
||||
t.Errorf("TLSVersion = %v, want %v", rec.TLSVersion, tt.clientHello.TLSVersion)
|
||||
}
|
||||
if tt.clientHello.SynToCHMs != nil {
|
||||
if rec.SynToCHMs == nil {
|
||||
t.Error("SynToCHMs should not be nil")
|
||||
} else if *rec.SynToCHMs != *tt.clientHello.SynToCHMs {
|
||||
t.Errorf("SynToCHMs = %v, want %v", *rec.SynToCHMs, *tt.clientHello.SynToCHMs)
|
||||
}
|
||||
}
|
||||
|
||||
// Verify fingerprints (note: JA4Hash is NOT in LogRecord per architecture)
|
||||
if tt.fingerprints != nil {
|
||||
if rec.JA4 != tt.fingerprints.JA4 {
|
||||
t.Errorf("JA4 = %v, want %v", rec.JA4, tt.fingerprints.JA4)
|
||||
}
|
||||
// JA4Hash is intentionally NOT in LogRecord (architecture decision)
|
||||
// JA3Hash is still present as it's the MD5 of JA3 (needed for exploitation)
|
||||
if rec.JA3 != tt.fingerprints.JA3 {
|
||||
t.Errorf("JA3 = %v, want %v", rec.JA3, tt.fingerprints.JA3)
|
||||
}
|
||||
if rec.JA3Hash != tt.fingerprints.JA3Hash {
|
||||
t.Errorf("JA3Hash = %v, want %v", rec.JA3Hash, tt.fingerprints.JA3Hash)
|
||||
}
|
||||
} else {
|
||||
if rec.JA4 != "" {
|
||||
t.Error("JA4 should be empty when fingerprints is nil")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultConfig(t *testing.T) {
|
||||
cfg := DefaultConfig()
|
||||
|
||||
if cfg.Core.Interface != DefaultInterface {
|
||||
t.Errorf("Core.Interface = %v, want %v", cfg.Core.Interface, DefaultInterface)
|
||||
}
|
||||
if len(cfg.Core.ListenPorts) != 1 {
|
||||
t.Errorf("Core.ListenPorts length = %v, want 1", len(cfg.Core.ListenPorts))
|
||||
}
|
||||
if cfg.Core.ListenPorts[0] != DefaultPort {
|
||||
t.Errorf("Core.ListenPorts[0] = %v, want %v", cfg.Core.ListenPorts[0], DefaultPort)
|
||||
}
|
||||
if cfg.Core.BPFFilter != DefaultBPFFilter {
|
||||
t.Errorf("Core.BPFFilter = %v, want %v", cfg.Core.BPFFilter, DefaultBPFFilter)
|
||||
}
|
||||
if cfg.Core.FlowTimeoutSec != DefaultFlowTimeout {
|
||||
t.Errorf("Core.FlowTimeoutSec = %v, want %v", cfg.Core.FlowTimeoutSec, DefaultFlowTimeout)
|
||||
}
|
||||
if len(cfg.Outputs) != 0 {
|
||||
t.Errorf("Outputs length = %v, want 0", len(cfg.Outputs))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogRecordConversion(t *testing.T) {
|
||||
// Test that NewLogRecord correctly converts TCPMeta options to comma-separated string
|
||||
clientHello := TLSClientHello{
|
||||
SrcIP: "192.168.1.100",
|
||||
SrcPort: 54321,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
TCPMeta: TCPMeta{
|
||||
WindowSize: 65535,
|
||||
MSS: 1460,
|
||||
WindowScale: 7,
|
||||
Options: []string{"MSS", "WS", "SACK", "TS"},
|
||||
},
|
||||
}
|
||||
|
||||
rec := NewLogRecord(clientHello, nil)
|
||||
|
||||
// Verify options are joined with comma
|
||||
expectedOpts := "MSS,WS,SACK,TS"
|
||||
if rec.TCPOptions != expectedOpts {
|
||||
t.Errorf("TCPOptions = %v, want %v", rec.TCPOptions, expectedOpts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLogRecordNoJA4Hash(t *testing.T) {
|
||||
// Verify that JA4Hash is NOT included in LogRecord per architecture decision
|
||||
clientHello := TLSClientHello{
|
||||
SrcIP: "192.168.1.100",
|
||||
SrcPort: 54321,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
}
|
||||
fingerprints := &Fingerprints{
|
||||
JA4: "t13d1516h2_8daaf6152771_02cb136f2775",
|
||||
JA4Hash: "8daaf6152771_02cb136f2775", // Should NOT appear in LogRecord
|
||||
JA3: "771,4865-4866-4867,0-23-65281,29-23-24,0",
|
||||
JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d",
|
||||
}
|
||||
|
||||
rec := NewLogRecord(clientHello, fingerprints)
|
||||
|
||||
// JA4Hash is NOT in LogRecord (architecture decision)
|
||||
// The JA4 format already includes its own hash portions
|
||||
|
||||
// But JA4 should be present
|
||||
if rec.JA4 != fingerprints.JA4 {
|
||||
t.Errorf("JA4 = %v, want %v", rec.JA4, fingerprints.JA4)
|
||||
}
|
||||
|
||||
// JA3Hash should still be present (it's the MD5 of JA3, which is needed)
|
||||
if rec.JA3Hash != fingerprints.JA3Hash {
|
||||
t.Errorf("JA3Hash = %v, want %v", rec.JA3Hash, fingerprints.JA3Hash)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOutputConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config OutputConfig
|
||||
wantEnabled bool
|
||||
wantAsyncBuf int
|
||||
}{
|
||||
{
|
||||
name: "stdout output with async buffer",
|
||||
config: OutputConfig{
|
||||
Type: "stdout",
|
||||
Enabled: true,
|
||||
AsyncBuffer: 5000,
|
||||
Params: map[string]string{},
|
||||
},
|
||||
wantEnabled: true,
|
||||
wantAsyncBuf: 5000,
|
||||
},
|
||||
{
|
||||
name: "unix_socket output with default async buffer",
|
||||
config: OutputConfig{
|
||||
Type: "unix_socket",
|
||||
Enabled: true,
|
||||
AsyncBuffer: 0, // Default
|
||||
Params: map[string]string{"socket_path": "/var/run/test.sock"},
|
||||
},
|
||||
wantEnabled: true,
|
||||
wantAsyncBuf: 0,
|
||||
},
|
||||
{
|
||||
name: "disabled output",
|
||||
config: OutputConfig{
|
||||
Type: "file",
|
||||
Enabled: false,
|
||||
AsyncBuffer: 1000,
|
||||
Params: map[string]string{"path": "/var/log/test.log"},
|
||||
},
|
||||
wantEnabled: false,
|
||||
wantAsyncBuf: 1000,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
if tt.config.Enabled != tt.wantEnabled {
|
||||
t.Errorf("Enabled = %v, want %v", tt.config.Enabled, tt.wantEnabled)
|
||||
}
|
||||
if tt.config.AsyncBuffer != tt.wantAsyncBuf {
|
||||
t.Errorf("AsyncBuffer = %v, want %v", tt.config.AsyncBuffer, tt.wantAsyncBuf)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -1,524 +0,0 @@
|
||||
version: 1
|
||||
|
||||
project:
|
||||
name: ja4sentinel
|
||||
description: >
|
||||
Outil Go pour capturer le trafic réseau sur un serveur Linux,
|
||||
extraire les handshakes TLS côté client, générer les signatures JA4
|
||||
(via psanford/tlsfingerprint), enrichir avec des métadonnées IP/TCP,
|
||||
et loguer les résultats (IP, ports, JA4, meta) vers une ou plusieurs
|
||||
sorties configurables (socket UNIX par défaut, stdout, fichier, ...).
|
||||
Le service est géré par systemd avec support de rotation des logs via logrotate.
|
||||
La commande `systemctl reload ja4sentinel` permet de réouvrir les fichiers de log
|
||||
après rotation (signal SIGHUP).
|
||||
languages:
|
||||
- go
|
||||
goals:
|
||||
- "Développement bloc par bloc avec interfaces simples et stables."
|
||||
- "Focalisé sur JA4 client (le serveur est connu/local)."
|
||||
- "Séparation claire des responsabilités (capture, parsing, fingerprint, output)."
|
||||
- "Tests unitaires pour chaque fonction publique."
|
||||
- "Tests d’intégration dans des conteneurs Docker."
|
||||
- "Commentaires standardisés, code évolutif avec changements minimaux."
|
||||
|
||||
modules:
|
||||
- name: config
|
||||
path: "internal/config"
|
||||
description: "Chargement et validation de la configuration (fichier, env, CLI)."
|
||||
responsibilities:
|
||||
- "Lire le fichier de configuration (YAML par défaut)."
|
||||
- "Fusionner avec les overrides env/CLI."
|
||||
- "Construire une api.AppConfig cohérente."
|
||||
allowed_dependencies: []
|
||||
forbidden_dependencies:
|
||||
- "capture"
|
||||
- "tlsparse"
|
||||
- "fingerprint"
|
||||
- "output"
|
||||
|
||||
- name: capture
|
||||
path: "internal/capture"
|
||||
description: "Capture des paquets réseau (pcap/raw socket) sur Linux."
|
||||
responsibilities:
|
||||
- "Ouvrir l’interface réseau configurée."
|
||||
- "Appliquer les filtres (ports, BPF, protocole)."
|
||||
- "Observer les flux TCP côté client vers les ports d’intérêt."
|
||||
- "Extraire les en-têtes IP/TCP utiles (IPMeta, TCPMeta)."
|
||||
- "Convertir les paquets en objets RawPacket."
|
||||
allowed_dependencies:
|
||||
- "config"
|
||||
- "api"
|
||||
forbidden_dependencies:
|
||||
- "tlsparse"
|
||||
- "fingerprint"
|
||||
- "output"
|
||||
|
||||
- name: tlsparse
|
||||
path: "internal/tlsparse"
|
||||
description: "Extraction des ClientHello TLS côté client à partir des paquets capturés."
|
||||
responsibilities:
|
||||
- "Décoder les couches IP/TCP jusqu'au payload TLS."
|
||||
- "Identifier le ClientHello TLS du client sur les ports configurés."
|
||||
- "Assembler les segments si nécessaire pour obtenir un ClientHello complet."
|
||||
- "Produire des TLSClientHello enrichis avec IPMeta et TCPMeta."
|
||||
- "Filtrer les IPs source exclues via le module ipfilter (avant parsing TLS)."
|
||||
- "Compter les paquets filtrés pour statistiques (GetFilterStats)."
|
||||
allowed_dependencies:
|
||||
- "config"
|
||||
- "capture"
|
||||
- "api"
|
||||
- "ipfilter"
|
||||
forbidden_dependencies:
|
||||
- "output"
|
||||
|
||||
- name: fingerprint
|
||||
path: "internal/fingerprint"
|
||||
description: "Génération des empreintes JA4 à partir des ClientHello TLS."
|
||||
responsibilities:
|
||||
- "Utiliser psanford/tlsfingerprint pour analyser le ClientHello."
|
||||
- "Générer la chaîne JA4 (et éventuellement JA3) côté client."
|
||||
- "Encapsuler les résultats dans un type Fingerprints."
|
||||
allowed_dependencies:
|
||||
- "config"
|
||||
- "tlsparse"
|
||||
- "api"
|
||||
forbidden_dependencies:
|
||||
- "capture"
|
||||
|
||||
- name: output
|
||||
path: "internal/output"
|
||||
description: "Sortie asynchrone ultra-rapide des résultats (JA4 + meta)."
|
||||
responsibilities:
|
||||
- "Prendre en entrée les Fingerprints et les métadonnées réseau."
|
||||
- "Formater les données en enregistrements log (JSON ou autre format simple)."
|
||||
- "Gérer une file d'attente interne (buffer channel) pour rendre l'écriture non-bloquante pour la capture."
|
||||
- "Sérialiser le JSON le plus rapidement possible (ex: pool d'allocations, librairies optimisées comme goccy/go-json)."
|
||||
- "Envoyer les enregistrements vers une ou plusieurs sorties (socket UNIX DGRAM, stdout, fichier, ...)."
|
||||
- "Gérer un MultiWriter pour combiner plusieurs outputs sans modifier le reste du code."
|
||||
allowed_dependencies:
|
||||
- "config"
|
||||
- "api"
|
||||
forbidden_dependencies:
|
||||
- "capture"
|
||||
- "tlsparse"
|
||||
- "fingerprint"
|
||||
|
||||
- name: logging
|
||||
path: "internal/logging"
|
||||
description: "Logs structurés JSON pour le service (stdout/stderr)."
|
||||
responsibilities:
|
||||
- "Fournir une fabrique de loggers (LoggerFactory)."
|
||||
- "Émettre des logs au format JSON lines sur stdout."
|
||||
- "Supporter les niveaux : debug, info, warn, error."
|
||||
- "Inclure timestamp, niveau, composant, message et détails optionnels."
|
||||
allowed_dependencies:
|
||||
- "api"
|
||||
forbidden_dependencies:
|
||||
- "config"
|
||||
- "capture"
|
||||
- "tlsparse"
|
||||
- "fingerprint"
|
||||
- "output"
|
||||
|
||||
- name: ipfilter
|
||||
path: "internal/ipfilter"
|
||||
description: "Filtrage des adresses IP source par correspondance IP/CIDR."
|
||||
responsibilities:
|
||||
- "Charger une liste d'IPs ou plages CIDR à exclure."
|
||||
- "Vérifier si une IP source correspond à une entrée de la liste d'exclusion."
|
||||
- "Supporter IPv4 et IPv6."
|
||||
- "Validation des formats IP et CIDR lors du chargement de la config."
|
||||
allowed_dependencies: []
|
||||
forbidden_dependencies:
|
||||
- "config"
|
||||
- "capture"
|
||||
- "tlsparse"
|
||||
- "fingerprint"
|
||||
- "output"
|
||||
|
||||
- name: cmd_ja4sentinel
|
||||
path: "cmd/ja4sentinel"
|
||||
description: "Point d'entrée de l'application (main)."
|
||||
responsibilities:
|
||||
- "Charger la configuration via le module config."
|
||||
- "Construire les instances des modules (capture, tlsparse, fingerprint, output, logging)."
|
||||
- "Brancher les modules entre eux selon l'architecture pipeline."
|
||||
- "Gérer les signaux système (arrêt propre)."
|
||||
- "Gérer le signal SIGHUP pour la rotation des logs (systemctl reload)."
|
||||
- "Logger les statistiques du filtre IP au démarrage et à l'arrêt (debug)."
|
||||
allowed_dependencies:
|
||||
- "config"
|
||||
- "capture"
|
||||
- "tlsparse"
|
||||
- "fingerprint"
|
||||
- "output"
|
||||
- "api"
|
||||
- "logging"
|
||||
forbidden_dependencies: []
|
||||
|
||||
api:
|
||||
types:
|
||||
- name: "api.ServiceLog"
|
||||
description: "Log interne du service ja4sentinel (diagnostic)."
|
||||
fields:
|
||||
- { name: Level, type: "string", description: "niveau: debug, info, warn, error." }
|
||||
- { name: Component, type: "string", description: "module concerné (capture, tlsparse, ...)." }
|
||||
- { name: Message, type: "string", description: "texte du log." }
|
||||
- { name: Details, type: "map[string]string", description: "infos additionnelles (erreurs, IDs...)." }
|
||||
- { name: Timestamp, type: "int64", description: "Timestamp en nanosecondes (auto-rempli par le logger)." }
|
||||
- { name: TraceID, type: "string", description: "ID de tracing distribué (optionnel)." }
|
||||
- { name: ConnID, type: "string", description: "Identifiant de flux TCP (optionnel)." }
|
||||
|
||||
- name: "api.Config"
|
||||
description: "Configuration réseau et TLS de base."
|
||||
fields:
|
||||
- { name: Interface, type: "string", description: "Nom de l'interface réseau (ex: eth0)." }
|
||||
- { name: ListenPorts, type: "[]uint16", description: "Ports TCP à surveiller (ex: [443, 8443])." }
|
||||
- { name: BPFFilter, type: "string", description: "Filtre BPF optionnel pour la capture." }
|
||||
- { name: LocalIPs, type: "[]string", description: "IPs locales à surveiller (vide = auto-détection, exclut loopback)." }
|
||||
- { name: ExcludeSourceIPs,type: "[]string", description: "IPs sources ou plages CIDR à exclure (ex: [\"10.0.0.0/8\", \"192.168.1.1\"]). Validé par le module config." }
|
||||
- { name: FlowTimeoutSec, type: "int", description: "Timeout en secondes pour l'extraction du handshake TLS (défaut: 30)." }
|
||||
- { name: PacketBufferSize,type: "int", description: "Taille du buffer du canal de paquets (défaut: 1000). Pour les environnements à fort trafic." }
|
||||
- { name: LogLevel, type: "string", description: "Niveau de log : debug, info, warn, error (défaut: info). Configuration via fichier YAML uniquement (pas d'override env dans systemd)." }
|
||||
|
||||
- name: "api.IPMeta"
|
||||
description: "Métadonnées IP pour fingerprinting de stack."
|
||||
fields:
|
||||
- { name: TTL, type: "uint8", description: "TTL initial observé." }
|
||||
- { name: TotalLength, type: "uint16", description: "Taille totale du paquet IP." }
|
||||
- { name: IPID, type: "uint16", description: "Identifiant IP du paquet." }
|
||||
- { name: DF, type: "bool", description: "Flag Don't Fragment." }
|
||||
|
||||
- name: "api.TCPMeta"
|
||||
description: "Métadonnées TCP pour fingerprinting de stack."
|
||||
fields:
|
||||
- { name: WindowSize, type: "uint16", description: "Fenêtre initiale TCP." }
|
||||
- { name: MSS, type: "uint16", description: "Maximum Segment Size (option TCP)." }
|
||||
- { name: WindowScale, type: "uint8", description: "Facteur de scaling (option TCP)." }
|
||||
- { name: Options, type: "[]string", description: "Liste ordonnée des options TCP (ex: [MSS, SACK, TS])." }
|
||||
|
||||
- name: "api.RawPacket"
|
||||
description: "Paquet brut capturé sur le réseau (vue minimale)."
|
||||
fields:
|
||||
- { name: Data, type: "[]byte", description: "Contenu brut du paquet." }
|
||||
- { name: Timestamp, type: "int64", description: "Timestamp (nanos / epoch) de capture." }
|
||||
|
||||
- name: "api.TLSClientHello"
|
||||
description: "Représentation d'un ClientHello TLS client, avec meta IP/TCP."
|
||||
fields:
|
||||
- { name: SrcIP, type: "string", description: "Adresse IP source (client)." }
|
||||
- { name: SrcPort, type: "uint16", description: "Port source (client)." }
|
||||
- { name: DstIP, type: "string", description: "Adresse IP destination (serveur)." }
|
||||
- { name: DstPort, type: "uint16", description: "Port destination (serveur)." }
|
||||
- { name: Payload, type: "[]byte", description: "Bytes correspondant au ClientHello TLS." }
|
||||
- { name: IPMeta, type: "api.IPMeta", description: "Métadonnées IP observées côté client." }
|
||||
- { name: TCPMeta, type: "api.TCPMeta", description: "Métadonnées TCP observées côté client." }
|
||||
- { name: ConnID, type: "string", description: "Identifiant unique du flux TCP (extension pour corrélation)." }
|
||||
- { name: SNI, type: "string", description: "Server Name Indication extrait du ClientHello (extension)." }
|
||||
- { name: ALPN, type: "string", description: "ALPN protocols négociés (extension)." }
|
||||
- { name: TLSVersion,type: "string", description: "Version TLS maximale annoncée (extension)." }
|
||||
- { name: SynToCHMs,type: "*uint32", description: "Temps SYN->ClientHello en ms (extension pour détection comportementale)." }
|
||||
|
||||
- name: "api.Fingerprints"
|
||||
description: "Empreintes TLS pour un flux client."
|
||||
fields:
|
||||
- { name: JA4, type: "string", description: "Signature JA4 client." }
|
||||
- { name: JA4Hash, type: "string", description: "Hash JA4 client." }
|
||||
- { name: JA3, type: "string", description: "Signature JA3 (optionnel, si calculée)." }
|
||||
- { name: JA3Hash, type: "string", description: "Hash JA3 (optionnel)." }
|
||||
|
||||
- name: "api.LogRecord"
|
||||
description: "Enregistrement de log final, sérialisé en JSON objet plat."
|
||||
json_object: true
|
||||
fields:
|
||||
- { name: SrcIP, type: "string", json_key: "src_ip" }
|
||||
- { name: SrcPort, type: "uint16", json_key: "src_port" }
|
||||
- { name: DstIP, type: "string", json_key: "dst_ip" }
|
||||
- { name: DstPort, type: "uint16", json_key: "dst_port" }
|
||||
|
||||
# IPMeta flatten
|
||||
- { name: IPTTL, type: "uint8", json_key: "ip_meta_ttl" }
|
||||
- { name: IPTotalLen, type: "uint16", json_key: "ip_meta_total_length" }
|
||||
- { name: IPID, type: "uint16", json_key: "ip_meta_id" }
|
||||
- { name: IPDF, type: "bool", json_key: "ip_meta_df" }
|
||||
|
||||
# TCPMeta flatten
|
||||
- { name: TCPWindow, type: "uint16", json_key: "tcp_meta_window_size" }
|
||||
- { name: TCPMSS, type: "*uint16", json_key: "tcp_meta_mss", optional: true, description: "Pointeur (nil si non présent, 0 si absent)." }
|
||||
- { name: TCPWScale, type: "*uint8", json_key: "tcp_meta_window_scale", optional: true, description: "Pointeur (nil si non présent, 0 si absent)." }
|
||||
- { name: TCPOptions, type: "string", json_key: "tcp_meta_options" }
|
||||
|
||||
# Fingerprints
|
||||
- { name: JA4, type: "string", json_key: "ja4", description: "Le format JA4 inclut nativement ses propres hachages (parties b et c), pas besoin de ja4_hash séparé." }
|
||||
- { name: JA3, type: "string", json_key: "ja3", description: "Chaîne brute JA3 (variable)." }
|
||||
- { name: JA3Hash, type: "string", json_key: "ja3_hash", description: "Hachage MD5 indispensable pour exploiter la chaîne JA3." }
|
||||
|
||||
# --- Corrélation & Triage ---
|
||||
- { name: ConnID, type: "string", json_key: "conn_id", optional: true, description: "Identifiant unique du flux (ex: hash de src_ip:src_port-dst_ip:dst_port) pour corréler facilement plusieurs événements liés à une même session TCP." }
|
||||
- { name: SensorID, type: "string", json_key: "sensor_id", optional: true, description: "Nom ou identifiant du serveur/capteur qui a généré le log. Indispensable pour du déploiement à grande échelle." }
|
||||
|
||||
# --- Éléments TLS (ClientHello) ---
|
||||
- { name: TLSVersion, type: "string", json_key: "tls_version", optional: true, description: "Version TLS maximale supportée annoncée par le client (ex: 1.2, 1.3). Utile pour repérer les clients obsolètes." }
|
||||
- { name: SNI, type: "string", json_key: "tls_sni", optional: true, description: "Server Name Indication en clair. Crucial pour détecter le domaine visé par le client (C2, DGA, etc.)." }
|
||||
- { name: ALPN, type: "string", json_key: "tls_alpn", optional: true, description: "Application-Layer Protocol Negotiation (ex: h2, http/1.1). Aide à différencier le trafic web légitime d'un tunnel personnalisé." }
|
||||
|
||||
# --- Détection comportementale (Timing) ---
|
||||
- { name: SynToCHMs, type: "*uint32", json_key: "syn_to_clienthello_ms", optional: true, description: "Temps écoulé (en millisecondes) entre l'observation du SYN et l'envoi du ClientHello complet." }
|
||||
|
||||
# Timestamp
|
||||
- { name: Timestamp, type: "int64", json_key: "timestamp", description: "Wall-clock timestamp in nanoseconds since Unix epoch (auto-filled by NewLogRecord)." }
|
||||
|
||||
|
||||
- name: "api.OutputConfig"
|
||||
description: "Configuration d’une sortie de logs."
|
||||
fields:
|
||||
- { name: Type, type: "string", description: "Type d’output (unix_socket, stdout, file, ...)." }
|
||||
- { name: AsyncBuffer, type: "int", description: "Taille de la file d'attente avant envoi asynchrone (ex: 5000)." }
|
||||
- { name: Enabled, type: "bool", description: "Active ou non cette sortie." }
|
||||
- { name: Params, type: "map[string]string", description: "Paramètres spécifiques (socket_path, path, ...)." }
|
||||
|
||||
- name: "api.AppConfig"
|
||||
description: "Configuration complète de ja4sentinel."
|
||||
fields:
|
||||
- { name: Core, type: "api.Config", description: "Paramètres réseau + TLS." }
|
||||
- { name: Outputs, type: "[]api.OutputConfig", description: "Liste des outputs configurés." }
|
||||
|
||||
interfaces:
|
||||
- name: "config.Loader"
|
||||
description: "Charge la configuration (fichier + env + CLI)."
|
||||
module: "config"
|
||||
methods:
|
||||
- name: "Load"
|
||||
params: []
|
||||
returns:
|
||||
- { type: "api.AppConfig" }
|
||||
- { type: "error" }
|
||||
|
||||
- name: "capture.Capture"
|
||||
description: "Source de paquets réseau bruts côté client."
|
||||
module: "capture"
|
||||
methods:
|
||||
- name: "Run"
|
||||
params:
|
||||
- { name: cfg, type: "api.Config" }
|
||||
- { name: out, type: "chan<- api.RawPacket" }
|
||||
returns:
|
||||
- { type: "error" }
|
||||
notes:
|
||||
- "Doit respecter les filtres (ports, BPF) définis dans la configuration."
|
||||
- "Ne connaît pas le format TLS ni JA4."
|
||||
- name: "Close"
|
||||
params: []
|
||||
returns:
|
||||
- { type: "error" }
|
||||
notes:
|
||||
- "Libère les ressources (handle pcap, etc.). Doit être appelé après Run()."
|
||||
|
||||
- name: "tlsparse.Parser"
|
||||
description: "Transforme des RawPacket en TLSClientHello (côté client uniquement)."
|
||||
module: "tlsparse"
|
||||
methods:
|
||||
- name: "Process"
|
||||
params:
|
||||
- { name: pkt, type: "api.RawPacket" }
|
||||
returns:
|
||||
- { type: "*api.TLSClientHello" }
|
||||
- { type: "error" }
|
||||
notes:
|
||||
- "Retourne nil si le paquet ne contient pas (ou plus) de ClientHello."
|
||||
- "Pour chaque flux, s'arrête une fois le ClientHello complet obtenu."
|
||||
- name: "Close"
|
||||
params: []
|
||||
returns:
|
||||
- { type: "error" }
|
||||
notes:
|
||||
- "Arrête les goroutines en arrière-plan et nettoie les états de flux."
|
||||
|
||||
- name: "fingerprint.Engine"
|
||||
description: "Génère les empreintes JA4 (et JA3 éventuellement) à partir d’un ClientHello."
|
||||
module: "fingerprint"
|
||||
methods:
|
||||
- name: "FromClientHello"
|
||||
params:
|
||||
- { name: ch, type: "api.TLSClientHello" }
|
||||
returns:
|
||||
- { type: "*api.Fingerprints" }
|
||||
- { type: "error" }
|
||||
notes:
|
||||
- "Utilise github.com/psanford/tlsfingerprint en interne."
|
||||
- "Focalisé sur le JA4 client (le côté serveur est déjà connu)."
|
||||
|
||||
- name: "output.Writer"
|
||||
description: "Interface générique pour écrire les résultats."
|
||||
module: "output"
|
||||
methods:
|
||||
- name: "Write"
|
||||
params:
|
||||
- { name: rec, type: "api.LogRecord" }
|
||||
returns:
|
||||
- { type: "error" }
|
||||
notes:
|
||||
- "Ne connaît pas la capture ni les détails de parsing TLS."
|
||||
|
||||
- name: "output.UnixSocketWriter"
|
||||
description: "Implémentation de Writer envoyant les logs sur une socket UNIX."
|
||||
module: "output"
|
||||
implements: "output.Writer"
|
||||
config:
|
||||
- { name: socket_path, type: "string", description: "Chemin de la socket UNIX DGRAM (ex: /var/run/logcorrelator/network.sock)." }
|
||||
|
||||
- name: "output.MultiWriter"
|
||||
description: "Combinaison de plusieurs Writer configurés."
|
||||
module: "output"
|
||||
implements: "output.Writer"
|
||||
config:
|
||||
- { name: writers, type: "[]output.Writer", description: "Liste de Writers concrets à appeler." }
|
||||
|
||||
- name: "output.Builder"
|
||||
description: "Construit les Writers à partir de api.AppConfig."
|
||||
module: "output"
|
||||
methods:
|
||||
- name: "NewFromConfig"
|
||||
params:
|
||||
- { name: cfg, type: "api.AppConfig" }
|
||||
returns:
|
||||
- { type: "output.Writer" }
|
||||
- { type: "error" }
|
||||
notes:
|
||||
- "Doit supporter plusieurs outputs simultanés via un MultiWriter."
|
||||
|
||||
- name: "logging.LoggerFactory"
|
||||
description: "Fabrique de loggers structurés JSON."
|
||||
module: "logging"
|
||||
methods:
|
||||
- name: "NewLogger"
|
||||
params:
|
||||
- { name: level, type: "string" }
|
||||
returns:
|
||||
- { type: "api.Logger" }
|
||||
- name: "NewDefaultLogger"
|
||||
params: []
|
||||
returns:
|
||||
- { type: "api.Logger" }
|
||||
notes:
|
||||
- "Les logs sont émis en JSON lines sur stdout pour systemd/journald."
|
||||
|
||||
- name: "api.Logger"
|
||||
description: "Interface de logging pour tous les modules."
|
||||
module: "logging"
|
||||
methods:
|
||||
- name: "Debug"
|
||||
params:
|
||||
- { name: component, type: "string" }
|
||||
- { name: message, type: "string" }
|
||||
- { name: details, type: "map[string]string" }
|
||||
- name: "Info"
|
||||
params:
|
||||
- { name: component, type: "string" }
|
||||
- { name: message, type: "string" }
|
||||
- { name: details, type: "map[string]string" }
|
||||
- name: "Warn"
|
||||
params:
|
||||
- { name: component, type: "string" }
|
||||
- { name: message, type: "string" }
|
||||
- { name: details, type: "map[string]string" }
|
||||
- name: "Error"
|
||||
params:
|
||||
- { name: component, type: "string" }
|
||||
- { name: message, type: "string" }
|
||||
- { name: details, type: "map[string]string" }
|
||||
notes:
|
||||
- "Tous les logs passent par stdout/stderr (pas de fichiers directs)."
|
||||
|
||||
architecture:
|
||||
style: "pipeline"
|
||||
flow:
|
||||
- from: "capture.Capture"
|
||||
to: "tlsparse.Parser"
|
||||
via: "api.RawPacket"
|
||||
- from: "tlsparse.Parser"
|
||||
to: "fingerprint.Engine"
|
||||
via: "api.TLSClientHello"
|
||||
- from: "fingerprint.Engine"
|
||||
to: "output.Writer"
|
||||
via: "api.LogRecord"
|
||||
constraints:
|
||||
- id: "client_only"
|
||||
description: "On ne calcule que les empreintes JA4 côté client (pas côté serveur)."
|
||||
- id: "no_back_dependencies"
|
||||
description: "Pas de dépendances en arrière (output ne dépend pas de fingerprint, etc.)."
|
||||
- id: "simple_messages"
|
||||
description: "Les communications entre blocs utilisent uniquement les types définis dans api.*."
|
||||
- id: "no_global_state"
|
||||
description: "Pas de variables globales partagées entre blocs pour la logique principale."
|
||||
|
||||
flow_control:
|
||||
connection_states:
|
||||
description: "États simplifiés d'un flux TCP pour minimiser la capture."
|
||||
states:
|
||||
- name: "NEW"
|
||||
description: "Observation d'un SYN client sur un port surveillé, création d'un état minimal (IP/TCP meta)."
|
||||
- name: "WAIT_CLIENT_HELLO"
|
||||
description: "Accumulation des segments TCP nécessaires pour extraire un ClientHello complet."
|
||||
- name: "JA4_DONE"
|
||||
description: "JA4 calculé et logué, on arrête de suivre ce flux."
|
||||
rules:
|
||||
- "Pas de tableaux imbriqués ni d'objets deeply nested."
|
||||
- "Toutes les métadonnées IP/TCP sont flatten sous forme de champs scalaires nommés."
|
||||
- "Les noms de champs suivent la convention: ip_meta_*, tcp_meta_*, ja*."
|
||||
- "Pas de champ ja4_hash : le format JA4 intègre déjà son propre hachage tronqué, la chaîne complète de 38 caractères suffit."
|
||||
logrecord_schema:
|
||||
# Exemple de mapping pour api.LogRecord (résumé)
|
||||
- "conn_id"
|
||||
- "sensor_id"
|
||||
- "src_ip"
|
||||
- "src_port"
|
||||
- "dst_ip"
|
||||
- "dst_port"
|
||||
- "ip_meta_ttl"
|
||||
- "ip_meta_total_length"
|
||||
- "ip_meta_id"
|
||||
- "ip_meta_df"
|
||||
- "tcp_meta_window_size"
|
||||
- "tcp_meta_mss"
|
||||
- "tcp_meta_window_scale"
|
||||
- "tcp_meta_options" # string joinée, ex: 'MSS,SACK,TS,NOP,WS'
|
||||
- "tls_version"
|
||||
- "tls_sni"
|
||||
- "tls_alpn"
|
||||
- "syn_to_clienthello_ms"
|
||||
- "ja4"
|
||||
- "ja3"
|
||||
- "ja3_hash"
|
||||
|
||||
packaging:
|
||||
rpm:
|
||||
description: "Package RPM pour déploiement sur serveurs Linux."
|
||||
files:
|
||||
- path: "/etc/logrotate.d/ja4sentinel"
|
||||
description: "Script logrotate pour la rotation des fichiers de log."
|
||||
note: "Fourni par le RPM, configure la rotation quotidienne avec compression."
|
||||
- path: "/etc/systemd/system/ja4sentinel.service"
|
||||
description: "Unité systemd pour la gestion du service."
|
||||
note: "Doit inclure Type=notify et ExecReload=/bin/kill -HUP $MAINPID pour supporter systemctl reload. PAS de variable Environment=JA4SENTINEL_LOG_LEVEL pour respecter la config fichier."
|
||||
logrotate:
|
||||
description: "Configuration logrotate pour la rotation des logs."
|
||||
behavior:
|
||||
- "Rotation quotidienne ou selon taille."
|
||||
- "Compression des logs archivés."
|
||||
- "Envoi du signal SIGHUP au service après rotation pour réouvrir les fichiers."
|
||||
reload_mechanism:
|
||||
- "systemctl reload ja4sentinel déclenche le handler SIGHUP."
|
||||
- "Le service réouvre ses fichiers de log sans redémarrage complet."
|
||||
|
||||
config_loading:
|
||||
priority:
|
||||
- "1. Fichier de configuration YAML (config.yml)"
|
||||
- "2. Variables d'environnement JA4SENTINEL_* (sauf log_level depuis v1.1.11)"
|
||||
- "3. Arguments CLI (--config)"
|
||||
notes:
|
||||
- "Depuis v1.1.11, la variable JA4SENTINEL_LOG_LEVEL n'est plus définie dans le service systemd."
|
||||
- "Le log_level doit être configuré exclusivement dans le fichier YAML."
|
||||
- "exclude_source_ips est uniquement chargé depuis le fichier YAML (pas d'override env)."
|
||||
- "La fusion des configs utilise mergeConfigs() qui préserve les valeurs non-overridées."
|
||||
|
||||
@ -1,357 +0,0 @@
|
||||
// Package main provides the entry point for ja4sentinel
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-systemd/v22/daemon"
|
||||
"github.com/antitbone/ja4/sentinel/api"
|
||||
"github.com/antitbone/ja4/sentinel/internal/capture"
|
||||
"github.com/antitbone/ja4/sentinel/internal/config"
|
||||
"github.com/antitbone/ja4/sentinel/internal/fingerprint"
|
||||
"github.com/antitbone/ja4/sentinel/internal/logging"
|
||||
"github.com/antitbone/ja4/sentinel/internal/output"
|
||||
"github.com/antitbone/ja4/sentinel/internal/tlsparse"
|
||||
)
|
||||
|
||||
var (
|
||||
// Version information (set via ldflags)
|
||||
Version = "1.1.15"
|
||||
BuildTime = "unknown"
|
||||
GitCommit = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Parse command-line flags
|
||||
configPath := flag.String("config", "", "Path to configuration file (YAML)")
|
||||
version := flag.Bool("version", false, "Show version information")
|
||||
flag.Parse()
|
||||
|
||||
if *version {
|
||||
fmt.Printf("ja4sentinel version %s (built %s, commit %s)\n", Version, BuildTime, GitCommit)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Load configuration
|
||||
cfgLoader := config.NewLoader(*configPath)
|
||||
appConfig, err := cfgLoader.Load()
|
||||
if err != nil {
|
||||
// Create logger with default level for error reporting
|
||||
loggerFactory := &logging.LoggerFactory{}
|
||||
appLogger := loggerFactory.NewDefaultLogger()
|
||||
appLogger.Error("main", "Failed to load configuration", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Create logger factory with configured log level
|
||||
loggerFactory := &logging.LoggerFactory{}
|
||||
appLogger := loggerFactory.NewLogger(appConfig.Core.LogLevel)
|
||||
|
||||
appLogger.Info("main", "Starting ja4sentinel", map[string]string{
|
||||
"version": Version,
|
||||
"build_time": BuildTime,
|
||||
"git_commit": GitCommit,
|
||||
})
|
||||
|
||||
appLogger.Info("main", "Configuration loaded", map[string]string{
|
||||
"interface": appConfig.Core.Interface,
|
||||
"listen_ports": formatPorts(appConfig.Core.ListenPorts),
|
||||
"log_level": appConfig.Core.LogLevel,
|
||||
})
|
||||
|
||||
// Create context with cancellation for graceful shutdown
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Signal readiness to systemd
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyReady); err != nil {
|
||||
appLogger.Warn("main", "Failed to send READY notification to systemd", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Start watchdog goroutine if enabled
|
||||
watchdogInterval, err := daemon.SdWatchdogEnabled(false)
|
||||
if err != nil {
|
||||
appLogger.Warn("main", "Failed to check watchdog status", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
if watchdogInterval > 0 {
|
||||
appLogger.Info("main", "systemd watchdog enabled", map[string]string{
|
||||
"interval": watchdogInterval.String(),
|
||||
})
|
||||
go func() {
|
||||
ticker := time.NewTicker(watchdogInterval / 2)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyWatchdog); err != nil {
|
||||
appLogger.Warn("main", "Failed to send WATCHDOG notification", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Setup signal handling for shutdown and log rotation
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
|
||||
// Create pipeline components
|
||||
captureEngine := capture.New()
|
||||
parser := tlsparse.NewParserWithTimeoutAndFilter(
|
||||
time.Duration(appConfig.Core.FlowTimeoutSec)*time.Second,
|
||||
appConfig.Core.ExcludeSourceIPs,
|
||||
)
|
||||
fingerprintEngine := fingerprint.NewEngine()
|
||||
|
||||
// Log exclusion configuration with debug details
|
||||
if len(appConfig.Core.ExcludeSourceIPs) > 0 {
|
||||
appLogger.Info("main", "Source IP exclusion enabled", map[string]string{
|
||||
"exclude_count": fmt.Sprintf("%d", len(appConfig.Core.ExcludeSourceIPs)),
|
||||
"exclude_ips": strings.Join(appConfig.Core.ExcludeSourceIPs, ", "),
|
||||
})
|
||||
appLogger.Debug("tlsparse", "IP filter configured", map[string]string{
|
||||
"filter_entries": strings.Join(appConfig.Core.ExcludeSourceIPs, ", "),
|
||||
})
|
||||
} else {
|
||||
appLogger.Debug("tlsparse", "IP filter disabled (no exclusions configured)", nil)
|
||||
}
|
||||
|
||||
// Log filter stats at startup (debug mode)
|
||||
filteredCount, hasFilter := parser.GetFilterStats()
|
||||
if hasFilter {
|
||||
appLogger.Debug("tlsparse", "IP filter initialized", map[string]string{
|
||||
"filtered_packets": fmt.Sprintf("%d", filteredCount),
|
||||
})
|
||||
}
|
||||
|
||||
// Create output builder with error callback for socket connection errors
|
||||
outputBuilder := output.NewBuilder().WithErrorCallback(func(socketPath string, err error, attempt int) {
|
||||
appLogger.Error("output", "UNIX socket connection failed", map[string]string{
|
||||
"socket_path": socketPath,
|
||||
"error": err.Error(),
|
||||
"attempt": fmt.Sprintf("%d", attempt),
|
||||
})
|
||||
})
|
||||
|
||||
outputWriter, err := outputBuilder.NewFromConfig(appConfig)
|
||||
if err != nil {
|
||||
appLogger.Error("main", "Failed to create output writer", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Create channel for raw packets (configurable buffer size)
|
||||
bufferSize := appConfig.Core.PacketBufferSize
|
||||
if bufferSize <= 0 {
|
||||
bufferSize = 1000 // Default fallback
|
||||
}
|
||||
packetChan := make(chan api.RawPacket, bufferSize)
|
||||
|
||||
// Start capture goroutine
|
||||
captureErrChan := make(chan error, 1)
|
||||
go func() {
|
||||
appLogger.Info("capture", "Starting packet capture", map[string]string{
|
||||
"interface": appConfig.Core.Interface,
|
||||
})
|
||||
err := captureEngine.Run(appConfig.Core, packetChan)
|
||||
close(packetChan) // Close channel to signal packet processor to shut down
|
||||
captureErrChan <- err
|
||||
}()
|
||||
|
||||
// Log capture diagnostics after a short delay to allow initialization
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
ifName, localIPs, bpfFilter, linkType := captureEngine.GetDiagnostics()
|
||||
appLogger.Debug("capture", "Capture initialized", map[string]string{
|
||||
"interface": ifName,
|
||||
"link_type": fmt.Sprintf("%d", linkType),
|
||||
"local_ips": strings.Join(localIPs, ", "),
|
||||
"bpf_filter": bpfFilter,
|
||||
})
|
||||
}()
|
||||
|
||||
// Process packets
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
appLogger.Info("main", "Packet processor shutting down", nil)
|
||||
return
|
||||
case pkt, ok := <-packetChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Parse TLS ClientHello
|
||||
clientHello, err := parser.Process(pkt)
|
||||
if err != nil {
|
||||
appLogger.Warn("tlsparse", "Failed to parse TLS ClientHello", map[string]string{
|
||||
"error": err.Error(),
|
||||
"packet_len": fmt.Sprintf("%d", len(pkt.Data)),
|
||||
"link_type": fmt.Sprintf("%d", pkt.LinkType),
|
||||
"timestamp": fmt.Sprintf("%d", pkt.Timestamp),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if clientHello == nil {
|
||||
continue // Not a TLS ClientHello packet
|
||||
}
|
||||
|
||||
appLogger.Debug("tlsparse", "ClientHello extracted", map[string]string{
|
||||
"src_ip": clientHello.SrcIP,
|
||||
"src_port": fmt.Sprintf("%d", clientHello.SrcPort),
|
||||
"dst_ip": clientHello.DstIP,
|
||||
"dst_port": fmt.Sprintf("%d", clientHello.DstPort),
|
||||
})
|
||||
|
||||
// Generate fingerprints
|
||||
fingerprints, err := fingerprintEngine.FromClientHello(*clientHello)
|
||||
if err != nil {
|
||||
appLogger.Warn("fingerprint", "Failed to generate fingerprints", map[string]string{
|
||||
"error": err.Error(),
|
||||
"src_ip": clientHello.SrcIP,
|
||||
"src_port": fmt.Sprintf("%d", clientHello.SrcPort),
|
||||
"dst_ip": clientHello.DstIP,
|
||||
"dst_port": fmt.Sprintf("%d", clientHello.DstPort),
|
||||
"conn_id": clientHello.ConnID,
|
||||
"payload_len": fmt.Sprintf("%d", len(clientHello.Payload)),
|
||||
"sni": clientHello.SNI,
|
||||
"tls_version": clientHello.TLSVersion,
|
||||
"alpn": clientHello.ALPN,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
appLogger.Debug("fingerprint", "Fingerprints generated", map[string]string{
|
||||
"src_ip": clientHello.SrcIP,
|
||||
"ja4": fingerprints.JA4,
|
||||
})
|
||||
|
||||
// Create log record
|
||||
logRecord := api.NewLogRecord(*clientHello, fingerprints)
|
||||
|
||||
// Write output
|
||||
if err := outputWriter.Write(logRecord); err != nil {
|
||||
appLogger.Error("output", "Failed to write log record", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for shutdown signal or capture error
|
||||
for {
|
||||
select {
|
||||
case sig := <-sigChan:
|
||||
switch sig {
|
||||
case syscall.SIGHUP:
|
||||
// Handle log rotation - reopen output files
|
||||
appLogger.Info("main", "Received SIGHUP, reopening log files", nil)
|
||||
if mw, ok := outputWriter.(api.Reopenable); ok {
|
||||
if err := mw.Reopen(); err != nil {
|
||||
appLogger.Error("main", "Failed to reopen log files", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else {
|
||||
appLogger.Info("main", "Log files reopened successfully", nil)
|
||||
}
|
||||
} else {
|
||||
appLogger.Warn("main", "Output writer does not support log rotation", nil)
|
||||
}
|
||||
case syscall.SIGINT, syscall.SIGTERM:
|
||||
appLogger.Info("main", "Received shutdown signal", map[string]string{
|
||||
"signal": sig.String(),
|
||||
})
|
||||
goto shutdown
|
||||
}
|
||||
case err := <-captureErrChan:
|
||||
if err != nil {
|
||||
appLogger.Error("capture", "Capture engine failed", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
goto shutdown
|
||||
}
|
||||
}
|
||||
|
||||
shutdown:
|
||||
// Graceful shutdown
|
||||
appLogger.Info("main", "Shutting down...", nil)
|
||||
|
||||
// Signal stopping to systemd
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyStopping); err != nil {
|
||||
appLogger.Warn("main", "Failed to send STOPPING notification to systemd", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
cancel()
|
||||
|
||||
// Close components
|
||||
if err := captureEngine.Close(); err != nil {
|
||||
appLogger.Error("main", "Failed to close capture engine", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
if err := parser.Close(); err != nil {
|
||||
appLogger.Error("main", "Failed to close parser", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Log final filter stats
|
||||
filteredCount, hasFilter = parser.GetFilterStats()
|
||||
if hasFilter {
|
||||
appLogger.Info("tlsparse", "IP filter statistics", map[string]string{
|
||||
"total_filtered_packets": fmt.Sprintf("%d", filteredCount),
|
||||
})
|
||||
}
|
||||
|
||||
if mw, ok := outputWriter.(interface{ CloseAll() error }); ok {
|
||||
if err := mw.CloseAll(); err != nil {
|
||||
appLogger.Error("main", "Failed to close output writers", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
} else if closer, ok := outputWriter.(interface{ Close() error }); ok {
|
||||
if err := closer.Close(); err != nil {
|
||||
appLogger.Error("main", "Failed to close output writer", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
appLogger.Info("main", "ja4sentinel stopped", nil)
|
||||
}
|
||||
|
||||
// formatPorts formats a slice of ports as a comma-separated string
|
||||
func formatPorts(ports []uint16) string {
|
||||
if len(ports) == 0 {
|
||||
return ""
|
||||
}
|
||||
result := fmt.Sprintf("%d", ports[0])
|
||||
for _, port := range ports[1:] {
|
||||
result += fmt.Sprintf(",%d", port)
|
||||
}
|
||||
return result
|
||||
}
|
||||
@ -1,221 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestFormatPorts(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
ports []uint16
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty slice",
|
||||
ports: []uint16{},
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "single port",
|
||||
ports: []uint16{443},
|
||||
want: "443",
|
||||
},
|
||||
{
|
||||
name: "multiple ports",
|
||||
ports: []uint16{443, 8443, 9443},
|
||||
want: "443,8443,9443",
|
||||
},
|
||||
{
|
||||
name: "two ports",
|
||||
ports: []uint16{80, 443},
|
||||
want: "80,443",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := formatPorts(tt.ports)
|
||||
if got != tt.want {
|
||||
t.Errorf("formatPorts() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMain_VersionFlag_VerifiesOutput tests that the version flag produces correct output
|
||||
// Note: This test verifies the version variables are set correctly
|
||||
func TestMain_VersionFlag_VerifiesOutput(t *testing.T) {
|
||||
// Verify version variables are set
|
||||
if Version == "" {
|
||||
t.Error("Version should not be empty")
|
||||
}
|
||||
if BuildTime == "" {
|
||||
t.Error("BuildTime should not be empty")
|
||||
}
|
||||
if GitCommit == "" {
|
||||
t.Error("GitCommit should not be empty")
|
||||
}
|
||||
|
||||
// Verify version format
|
||||
expectedPrefix := "ja4sentinel version"
|
||||
got := getVersionString()
|
||||
if !strings.HasPrefix(got, expectedPrefix) {
|
||||
t.Errorf("getVersionString() = %v, should start with %v", got, expectedPrefix)
|
||||
}
|
||||
}
|
||||
|
||||
// getVersionString returns the version string (helper for testing)
|
||||
func getVersionString() string {
|
||||
return "ja4sentinel version " + Version + " (built " + BuildTime + ", commit " + GitCommit + ")"
|
||||
}
|
||||
|
||||
func TestFlagParsing(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
wantConfig string
|
||||
wantVersion bool
|
||||
}{
|
||||
{
|
||||
name: "config flag",
|
||||
args: []string{"ja4sentinel", "-config", "/path/to/config.yml"},
|
||||
wantConfig: "/path/to/config.yml",
|
||||
wantVersion: false,
|
||||
},
|
||||
{
|
||||
name: "version flag",
|
||||
args: []string{"ja4sentinel", "-version"},
|
||||
wantConfig: "",
|
||||
wantVersion: true,
|
||||
},
|
||||
{
|
||||
name: "no flags",
|
||||
args: []string{"ja4sentinel"},
|
||||
wantConfig: "",
|
||||
wantVersion: false,
|
||||
},
|
||||
{
|
||||
name: "config with long form",
|
||||
args: []string{"ja4sentinel", "--config", "/etc/ja4sentinel/config.yml"},
|
||||
wantConfig: "/etc/ja4sentinel/config.yml",
|
||||
wantVersion: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
fs := flag.NewFlagSet("test", flag.ContinueOnError)
|
||||
configPath := fs.String("config", "", "Path to configuration file (YAML)")
|
||||
version := fs.Bool("version", false, "Show version information")
|
||||
|
||||
err := fs.Parse(tt.args[1:])
|
||||
if err != nil {
|
||||
t.Fatalf("Flag parsing failed: %v", err)
|
||||
}
|
||||
|
||||
if *configPath != tt.wantConfig {
|
||||
t.Errorf("config = %v, want %v", *configPath, tt.wantConfig)
|
||||
}
|
||||
if *version != tt.wantVersion {
|
||||
t.Errorf("version = %v, want %v", *version, tt.wantVersion)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestMain_WithInvalidConfig tests that main exits gracefully with invalid config
|
||||
func TestMain_WithInvalidConfig(t *testing.T) {
|
||||
// This test verifies that the application handles config errors gracefully
|
||||
// We can't easily test the full main() function, but we can test the
|
||||
// config loading and error handling paths
|
||||
t.Log("Note: Full main() testing requires integration tests with mocked dependencies")
|
||||
}
|
||||
|
||||
// TestSignalHandling_VerifiesConstants tests that signal constants are defined
|
||||
func TestSignalHandling_VerifiesConstants(t *testing.T) {
|
||||
// Verify that we import the required packages for signal handling
|
||||
// This test ensures the imports are present
|
||||
t.Log("syscall and os/signal packages are imported for signal handling")
|
||||
}
|
||||
|
||||
// TestGracefulShutdown_SimulatesSignal tests graceful shutdown behavior
|
||||
func TestGracefulShutdown_SimulatesSignal(t *testing.T) {
|
||||
// This test documents the expected shutdown behavior
|
||||
// Full testing requires integration tests with actual signal sending
|
||||
|
||||
expectedBehavior := `
|
||||
Graceful shutdown sequence:
|
||||
1. Receive SIGINT or SIGTERM
|
||||
2. Stop packet capture
|
||||
3. Close output writers
|
||||
4. Flush pending logs
|
||||
5. Exit cleanly
|
||||
`
|
||||
t.Log(expectedBehavior)
|
||||
}
|
||||
|
||||
// TestLogRotation_SIGHUP tests SIGHUP handling for log rotation
|
||||
func TestLogRotation_SIGHUP(t *testing.T) {
|
||||
// This test documents the expected log rotation behavior
|
||||
// Full testing requires integration tests with actual SIGHUP signal
|
||||
|
||||
expectedBehavior := `
|
||||
Log rotation sequence (SIGHUP):
|
||||
1. Receive SIGHUP
|
||||
2. Reopen all reopenable writers (FileWriter, MultiWriter)
|
||||
3. Continue operation with new file handles
|
||||
4. No data loss during rotation
|
||||
`
|
||||
t.Log(expectedBehavior)
|
||||
}
|
||||
|
||||
// TestMain_ConfigValidation tests config validation before starting
|
||||
func TestMain_ConfigValidation(t *testing.T) {
|
||||
// Test that invalid configs are rejected before starting the pipeline
|
||||
tests := []struct {
|
||||
name string
|
||||
configErr string
|
||||
}{
|
||||
{
|
||||
name: "empty_interface",
|
||||
configErr: "interface cannot be empty",
|
||||
},
|
||||
{
|
||||
name: "no_listen_ports",
|
||||
configErr: "at least one listen port required",
|
||||
},
|
||||
{
|
||||
name: "invalid_output_type",
|
||||
configErr: "unknown output type",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
// Verify that these error conditions are documented
|
||||
t.Logf("Expected error for %s: %s", tt.name, tt.configErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPipelineConstruction verifies the pipeline is built correctly
|
||||
func TestPipelineConstruction(t *testing.T) {
|
||||
// This test documents the expected pipeline construction
|
||||
// Full testing requires integration tests
|
||||
|
||||
expectedPipeline := `
|
||||
Pipeline construction:
|
||||
1. Load configuration
|
||||
2. Create logger
|
||||
3. Create capture engine
|
||||
4. Create TLS parser
|
||||
5. Create fingerprint engine
|
||||
6. Create output writer(s)
|
||||
7. Connect pipeline: capture -> parser -> fingerprint -> output
|
||||
8. Start signal handling
|
||||
9. Run capture loop
|
||||
`
|
||||
t.Log(expectedPipeline)
|
||||
}
|
||||
@ -1,57 +0,0 @@
|
||||
# Sample configuration file for ja4sentinel
|
||||
# Copy to config.yml and adjust as needed
|
||||
|
||||
core:
|
||||
# Network interface to capture traffic from
|
||||
# "any" captures on all interfaces (default, recommended)
|
||||
# Or specify a specific interface (e.g., eth0, ens192, etc.)
|
||||
interface: any
|
||||
|
||||
# TCP ports to monitor for TLS handshakes
|
||||
listen_ports:
|
||||
- 443
|
||||
- 8443
|
||||
|
||||
# Optional BPF filter (leave empty for auto-generated filter based on listen_ports and local_ips)
|
||||
bpf_filter: ""
|
||||
|
||||
# Local IP addresses to monitor (traffic destined to these IPs will be captured)
|
||||
# Leave empty for auto-detection (recommended) - excludes loopback addresses
|
||||
# Or specify manually: ["192.168.1.10", "10.0.0.5", "2001:db8::1"]
|
||||
local_ips: []
|
||||
|
||||
# Source IP addresses or CIDR ranges to exclude from capture
|
||||
# Useful for filtering out internal traffic, health checks, or monitoring systems
|
||||
# Examples: ["10.0.0.0/8", "192.168.1.1", "172.16.0.0/12"]
|
||||
exclude_source_ips: []
|
||||
|
||||
# Timeout in seconds for TLS handshake extraction (default: 30)
|
||||
flow_timeout_sec: 30
|
||||
|
||||
# Buffer size for packet channel (default: 1000, increase for high-traffic environments)
|
||||
packet_buffer_size: 1000
|
||||
|
||||
# Log level: debug, info, warn, error (default: info)
|
||||
# Can be overridden by JA4SENTINEL_LOG_LEVEL environment variable
|
||||
log_level: info
|
||||
|
||||
outputs:
|
||||
# Output to UNIX socket (for systemd/journald or other consumers)
|
||||
# Only JSON LogRecord data is sent - no diagnostic logs
|
||||
- type: unix_socket
|
||||
enabled: true
|
||||
params:
|
||||
socket_path: /var/run/logcorrelator/network.socket
|
||||
|
||||
# Output to stdout (JSON lines)
|
||||
# Diagnostic logs (error, debug, warning) should go here
|
||||
# - type: stdout
|
||||
# enabled: false
|
||||
# params: {}
|
||||
|
||||
# Output to file
|
||||
# Only JSON LogRecord data is sent - no diagnostic logs
|
||||
# - type: file
|
||||
# enabled: false
|
||||
# params:
|
||||
# path: /var/log/ja4sentinel/ja4.log
|
||||
@ -1,49 +0,0 @@
|
||||
# Docker Compose for integration testing
|
||||
# Based on architecture.yml testing.levels.integration
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
# TLS test server for generating test traffic
|
||||
tls-server:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.test-server
|
||||
image: ja4sentinel-test-server:latest
|
||||
networks:
|
||||
- test-network
|
||||
ports:
|
||||
- "8443:8443"
|
||||
command: ["-port", "8443"]
|
||||
|
||||
# ja4sentinel integration test runner
|
||||
ja4sentinel-test:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.dev
|
||||
image: ja4sentinel-dev:latest
|
||||
networks:
|
||||
- test-network
|
||||
cap_add:
|
||||
- NET_RAW
|
||||
- NET_ADMIN
|
||||
volumes:
|
||||
- ./test-results:/app/test-results
|
||||
environment:
|
||||
- JA4SENTINEL_INTERFACE=eth0
|
||||
- JA4SENTINEL_PORTS=8443
|
||||
depends_on:
|
||||
- tls-server
|
||||
command: ["make", "test-integration"]
|
||||
|
||||
# Test client that generates TLS traffic
|
||||
tls-client:
|
||||
image: curlimages/curl:latest
|
||||
networks:
|
||||
- test-network
|
||||
depends_on:
|
||||
- tls-server
|
||||
command: ["curl", "-kv", "https://tls-server:8443/"]
|
||||
|
||||
networks:
|
||||
test-network:
|
||||
driver: bridge
|
||||
@ -1,21 +0,0 @@
|
||||
module github.com/antitbone/ja4/sentinel
|
||||
|
||||
go 1.24.6
|
||||
|
||||
toolchain go1.24.13
|
||||
|
||||
require (
|
||||
github.com/google/gopacket v1.1.19
|
||||
github.com/psanford/tlsfingerprint v0.0.0-20251111180026-c742e470de9b
|
||||
gopkg.in/yaml.v3 v3.0.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 // indirect
|
||||
golang.org/x/sys v0.1.0 // indirect
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
|
||||
)
|
||||
|
||||
require github.com/antitbone/ja4/ja4common v0.1.0
|
||||
|
||||
replace github.com/antitbone/ja4/ja4common => ../../shared/go/ja4common
|
||||
@ -1,27 +0,0 @@
|
||||
github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA=
|
||||
github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w=
|
||||
github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
|
||||
github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
|
||||
github.com/psanford/tlsfingerprint v0.0.0-20251111180026-c742e470de9b h1:fsP7F1zLHZ4ozxhesg4j8qSsaJxK7Ev9fA2cUtbThec=
|
||||
github.com/psanford/tlsfingerprint v0.0.0-20251111180026-c742e470de9b/go.mod h1:F7HlBxc/I5XX6syuwpDtffw/6J+d0Q2xcUhYSbx/0Uw=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
@ -1,397 +0,0 @@
|
||||
// Package capture provides network packet capture functionality for ja4sentinel
|
||||
package capture
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/pcap"
|
||||
|
||||
"github.com/antitbone/ja4/sentinel/api"
|
||||
)
|
||||
|
||||
// Capture configuration constants
|
||||
const (
|
||||
// DefaultSnapLen is the default snapshot length for packet capture
|
||||
// Increased from 1600 to 65535 to capture full packets including large TLS handshakes
|
||||
DefaultSnapLen = 65535
|
||||
// DefaultPromiscuous is the default promiscuous mode setting
|
||||
DefaultPromiscuous = false
|
||||
// MaxBPFFilterLength is the maximum allowed length for BPF filters
|
||||
MaxBPFFilterLength = 1024
|
||||
)
|
||||
|
||||
// validBPFPattern checks if a BPF filter contains only valid characters
|
||||
// This is a basic validation to prevent injection attacks
|
||||
var validBPFPattern = regexp.MustCompile(`^[a-zA-Z0-9\s\(\)\-\_\.\*\+\?\:\=\!\&\|\<\>\[\]\/\@,]+$`)
|
||||
|
||||
// CaptureImpl implements the capture.Capture interface for packet capture
|
||||
type CaptureImpl struct {
|
||||
handle *pcap.Handle
|
||||
mu sync.Mutex
|
||||
snapLen int
|
||||
promisc bool
|
||||
isClosed bool
|
||||
localIPs []string // Local IPs to filter (dst host)
|
||||
linkType int // Link type from pcap handle
|
||||
interfaceName string // Interface name (for diagnostics)
|
||||
bpfFilter string // Applied BPF filter (for diagnostics)
|
||||
// Metrics counters (atomic)
|
||||
packetsReceived uint64 // Total packets received from interface
|
||||
packetsSent uint64 // Total packets sent to channel
|
||||
packetsDropped uint64 // Total packets dropped (channel full)
|
||||
}
|
||||
|
||||
// New creates a new capture instance
|
||||
func New() *CaptureImpl {
|
||||
return &CaptureImpl{
|
||||
snapLen: DefaultSnapLen,
|
||||
promisc: DefaultPromiscuous,
|
||||
}
|
||||
}
|
||||
|
||||
// NewWithSnapLen creates a new capture instance with custom snapshot length
|
||||
func NewWithSnapLen(snapLen int) *CaptureImpl {
|
||||
if snapLen <= 0 || snapLen > 65535 {
|
||||
snapLen = DefaultSnapLen
|
||||
}
|
||||
return &CaptureImpl{
|
||||
snapLen: snapLen,
|
||||
promisc: DefaultPromiscuous,
|
||||
}
|
||||
}
|
||||
|
||||
// Run starts network packet capture according to the configuration
|
||||
func (c *CaptureImpl) Run(cfg api.Config, out chan<- api.RawPacket) error {
|
||||
// Validate interface name (basic check)
|
||||
if cfg.Interface == "" {
|
||||
return fmt.Errorf("interface cannot be empty")
|
||||
}
|
||||
|
||||
// Find available interfaces to validate the interface exists
|
||||
ifaces, err := pcap.FindAllDevs()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to list network interfaces: %w", err)
|
||||
}
|
||||
|
||||
// Special handling for "any" interface
|
||||
interfaceFound := cfg.Interface == "any"
|
||||
if !interfaceFound {
|
||||
for _, iface := range ifaces {
|
||||
if iface.Name == cfg.Interface {
|
||||
interfaceFound = true
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
if !interfaceFound {
|
||||
return fmt.Errorf("interface %s not found (available: %v)", cfg.Interface, getInterfaceNames(ifaces))
|
||||
}
|
||||
|
||||
handle, err := pcap.OpenLive(cfg.Interface, int32(c.snapLen), c.promisc, pcap.BlockForever)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open interface %s: %w", cfg.Interface, err)
|
||||
}
|
||||
|
||||
c.mu.Lock()
|
||||
c.handle = handle
|
||||
c.mu.Unlock()
|
||||
|
||||
defer func() {
|
||||
c.mu.Lock()
|
||||
if c.handle != nil && !c.isClosed {
|
||||
c.handle.Close()
|
||||
c.handle = nil
|
||||
}
|
||||
c.mu.Unlock()
|
||||
}()
|
||||
|
||||
// Store interface name for diagnostics
|
||||
c.interfaceName = cfg.Interface
|
||||
|
||||
// Resolve local IPs for filtering (if not manually specified)
|
||||
localIPs := cfg.LocalIPs
|
||||
if len(localIPs) == 0 {
|
||||
localIPs, err = c.detectLocalIPs(cfg.Interface)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to detect local IPs: %w", err)
|
||||
}
|
||||
if len(localIPs) == 0 {
|
||||
// NAT/VIP: destination IP may not be assigned to this interface.
|
||||
// Fall back to port-only BPF filter instead of aborting.
|
||||
log.Printf("WARN capture: no local IPs found on interface %s; using port-only BPF filter (NAT/VIP mode)", cfg.Interface)
|
||||
}
|
||||
}
|
||||
c.localIPs = localIPs
|
||||
|
||||
// Build and apply BPF filter
|
||||
bpfFilter := cfg.BPFFilter
|
||||
if bpfFilter == "" {
|
||||
bpfFilter = c.buildBPFFilter(cfg.ListenPorts, localIPs)
|
||||
}
|
||||
c.bpfFilter = bpfFilter
|
||||
|
||||
// Validate BPF filter before applying
|
||||
if err := validateBPFFilter(bpfFilter); err != nil {
|
||||
return fmt.Errorf("invalid BPF filter: %w", err)
|
||||
}
|
||||
|
||||
err = handle.SetBPFFilter(bpfFilter)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to set BPF filter '%s': %w", bpfFilter, err)
|
||||
}
|
||||
|
||||
// Store link type once, after the handle is fully configured (BPF filter applied).
|
||||
// A single write avoids the race where packetToRawPacket reads a stale value
|
||||
// that existed before the BPF filter was set.
|
||||
c.mu.Lock()
|
||||
c.linkType = int(handle.LinkType())
|
||||
c.mu.Unlock()
|
||||
|
||||
packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
|
||||
|
||||
for packet := range packetSource.Packets() {
|
||||
// Convert packet to RawPacket
|
||||
rawPkt := c.packetToRawPacket(packet)
|
||||
if rawPkt != nil {
|
||||
atomic.AddUint64(&c.packetsReceived, 1)
|
||||
select {
|
||||
case out <- *rawPkt:
|
||||
// Packet sent successfully
|
||||
atomic.AddUint64(&c.packetsSent, 1)
|
||||
default:
|
||||
// Channel full, drop packet
|
||||
atomic.AddUint64(&c.packetsDropped, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateBPFFilter performs basic validation of BPF filter strings
|
||||
func validateBPFFilter(filter string) error {
|
||||
if filter == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
if len(filter) > MaxBPFFilterLength {
|
||||
return fmt.Errorf("BPF filter too long (max %d characters)", MaxBPFFilterLength)
|
||||
}
|
||||
|
||||
// Check for potentially dangerous patterns
|
||||
if !validBPFPattern.MatchString(filter) {
|
||||
return fmt.Errorf("BPF filter contains invalid characters")
|
||||
}
|
||||
|
||||
// Check for unbalanced parentheses
|
||||
openParens := 0
|
||||
for _, ch := range filter {
|
||||
if ch == '(' {
|
||||
openParens++
|
||||
} else if ch == ')' {
|
||||
openParens--
|
||||
if openParens < 0 {
|
||||
return fmt.Errorf("BPF filter has unbalanced parentheses")
|
||||
}
|
||||
}
|
||||
}
|
||||
if openParens != 0 {
|
||||
return fmt.Errorf("BPF filter has unbalanced parentheses")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// getInterfaceNames extracts interface names from a list of devices
|
||||
func getInterfaceNames(ifaces []pcap.Interface) []string {
|
||||
names := make([]string, len(ifaces))
|
||||
for i, iface := range ifaces {
|
||||
names[i] = iface.Name
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// detectLocalIPs detects local IP addresses on the specified interface
|
||||
// Excludes loopback addresses (127.0.0.0/8, ::1) and IPv6 link-local (fe80::)
|
||||
func (c *CaptureImpl) detectLocalIPs(interfaceName string) ([]string, error) {
|
||||
var localIPs []string
|
||||
|
||||
// Special case: "any" interface - get all non-loopback IPs
|
||||
if interfaceName == "any" {
|
||||
ifaces, err := net.Interfaces()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to list interfaces: %w", err)
|
||||
}
|
||||
|
||||
for _, iface := range ifaces {
|
||||
// Skip loopback interfaces
|
||||
if iface.Flags&net.FlagLoopback != 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
addrs, err := iface.Addrs()
|
||||
if err != nil {
|
||||
continue // Skip this interface, try others
|
||||
}
|
||||
|
||||
for _, addr := range addrs {
|
||||
ip := extractIP(addr)
|
||||
if ip != nil && !ip.IsLoopback() && !ip.IsLinkLocalUnicast() {
|
||||
localIPs = append(localIPs, ip.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return localIPs, nil
|
||||
}
|
||||
|
||||
// Specific interface - get IPs from that interface only
|
||||
iface, err := net.InterfaceByName(interfaceName)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get interface %s: %w", interfaceName, err)
|
||||
}
|
||||
|
||||
addrs, err := iface.Addrs()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get addresses for %s: %w", interfaceName, err)
|
||||
}
|
||||
|
||||
for _, addr := range addrs {
|
||||
ip := extractIP(addr)
|
||||
if ip != nil && !ip.IsLoopback() && !ip.IsLinkLocalUnicast() {
|
||||
localIPs = append(localIPs, ip.String())
|
||||
}
|
||||
}
|
||||
|
||||
return localIPs, nil
|
||||
}
|
||||
|
||||
// extractIP extracts the IP address from a net.Addr
|
||||
func extractIP(addr net.Addr) net.IP {
|
||||
switch v := addr.(type) {
|
||||
case *net.IPNet:
|
||||
ip := v.IP
|
||||
// Return IPv4 as 4-byte, IPv6 as 16-byte
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
return ip
|
||||
case *net.IPAddr:
|
||||
ip := v.IP
|
||||
if ip4 := ip.To4(); ip4 != nil {
|
||||
return ip4
|
||||
}
|
||||
return ip
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// buildBPFFilter builds a BPF filter for the specified ports and local IPs
|
||||
// Filter: (tcp dst port 443 or tcp dst port 8443) and (dst host 192.168.1.10 or dst host 10.0.0.5)
|
||||
// Uses "tcp dst port" to only capture client→server traffic (not server→client responses)
|
||||
func (c *CaptureImpl) buildBPFFilter(ports []uint16, localIPs []string) string {
|
||||
if len(ports) == 0 {
|
||||
return "tcp"
|
||||
}
|
||||
|
||||
// Build port filter (dst port only to avoid capturing server responses)
|
||||
portParts := make([]string, len(ports))
|
||||
for i, port := range ports {
|
||||
portParts[i] = fmt.Sprintf("tcp dst port %d", port)
|
||||
}
|
||||
portFilter := "(" + strings.Join(portParts, ") or (") + ")"
|
||||
|
||||
// Build destination host filter
|
||||
if len(localIPs) == 0 {
|
||||
return portFilter
|
||||
}
|
||||
|
||||
hostParts := make([]string, len(localIPs))
|
||||
for i, ip := range localIPs {
|
||||
// Handle IPv6 addresses
|
||||
if strings.Contains(ip, ":") {
|
||||
hostParts[i] = fmt.Sprintf("dst host %s", ip)
|
||||
} else {
|
||||
hostParts[i] = fmt.Sprintf("dst host %s", ip)
|
||||
}
|
||||
}
|
||||
hostFilter := "(" + strings.Join(hostParts, ") or (") + ")"
|
||||
|
||||
// Combine port and host filters
|
||||
return portFilter + " and " + hostFilter
|
||||
}
|
||||
|
||||
// joinString joins strings with a separator (kept for backward compatibility)
|
||||
func joinString(parts []string, sep string) string {
|
||||
if len(parts) == 0 {
|
||||
return ""
|
||||
}
|
||||
result := parts[0]
|
||||
for _, part := range parts[1:] {
|
||||
result += sep + part
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// packetToRawPacket converts a gopacket packet to RawPacket
|
||||
// Uses the raw packet bytes from the link layer
|
||||
func (c *CaptureImpl) packetToRawPacket(packet gopacket.Packet) *api.RawPacket {
|
||||
// Try to get link layer contents + payload for full packet
|
||||
var data []byte
|
||||
|
||||
linkLayer := packet.LinkLayer()
|
||||
if linkLayer != nil {
|
||||
// Combine link layer contents with payload to get full packet
|
||||
data = append(data, linkLayer.LayerContents()...)
|
||||
data = append(data, linkLayer.LayerPayload()...)
|
||||
} else {
|
||||
// Fallback to packet.Data()
|
||||
data = packet.Data()
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
return &api.RawPacket{
|
||||
Data: data,
|
||||
Timestamp: packet.Metadata().Timestamp.UnixNano(),
|
||||
LinkType: c.linkType,
|
||||
}
|
||||
}
|
||||
|
||||
// Close properly closes the capture handle
|
||||
func (c *CaptureImpl) Close() error {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
|
||||
if c.handle != nil && !c.isClosed {
|
||||
c.handle.Close()
|
||||
c.handle = nil
|
||||
c.isClosed = true
|
||||
return nil
|
||||
}
|
||||
c.isClosed = true
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetStats returns capture statistics (for monitoring/debugging)
|
||||
func (c *CaptureImpl) GetStats() (received, sent, dropped uint64) {
|
||||
return atomic.LoadUint64(&c.packetsReceived),
|
||||
atomic.LoadUint64(&c.packetsSent),
|
||||
atomic.LoadUint64(&c.packetsDropped)
|
||||
}
|
||||
|
||||
// GetDiagnostics returns capture diagnostics information (for debugging)
|
||||
func (c *CaptureImpl) GetDiagnostics() (interfaceName string, localIPs []string, bpfFilter string, linkType int) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.interfaceName, c.localIPs, c.bpfFilter, c.linkType
|
||||
}
|
||||
@ -1,661 +0,0 @@
|
||||
package capture
|
||||
|
||||
import (
|
||||
"net"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/antitbone/ja4/sentinel/api"
|
||||
|
||||
"github.com/google/gopacket"
|
||||
"github.com/google/gopacket/layers"
|
||||
"github.com/google/gopacket/pcap"
|
||||
)
|
||||
|
||||
func TestCaptureImpl_Run_EmptyInterface(t *testing.T) {
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
|
||||
cfg := api.Config{
|
||||
Interface: "",
|
||||
ListenPorts: []uint16{443},
|
||||
}
|
||||
|
||||
out := make(chan api.RawPacket, 10)
|
||||
err := c.Run(cfg, out)
|
||||
|
||||
if err == nil {
|
||||
t.Error("Run() with empty interface should return error")
|
||||
}
|
||||
if err.Error() != "interface cannot be empty" {
|
||||
t.Errorf("Run() error = %v, want 'interface cannot be empty'", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_Run_NonExistentInterface(t *testing.T) {
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
|
||||
cfg := api.Config{
|
||||
Interface: "nonexistent_interface_xyz123",
|
||||
ListenPorts: []uint16{443},
|
||||
}
|
||||
|
||||
out := make(chan api.RawPacket, 10)
|
||||
err := c.Run(cfg, out)
|
||||
|
||||
if err == nil {
|
||||
t.Error("Run() with non-existent interface should return error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_Run_InvalidBPFFilter(t *testing.T) {
|
||||
// Get a real interface name
|
||||
ifaces, err := pcap.FindAllDevs()
|
||||
if err != nil || len(ifaces) == 0 {
|
||||
t.Skip("No network interfaces available for testing")
|
||||
}
|
||||
|
||||
c := New()
|
||||
cfg := api.Config{
|
||||
Interface: ifaces[0].Name,
|
||||
ListenPorts: []uint16{443},
|
||||
BPFFilter: "invalid; rm -rf /", // Invalid characters
|
||||
}
|
||||
|
||||
out := make(chan api.RawPacket, 10)
|
||||
err = c.Run(cfg, out)
|
||||
|
||||
if err == nil {
|
||||
t.Error("Run() with invalid BPF filter should return error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_Run_ChannelFull_DropsPackets(t *testing.T) {
|
||||
// This test verifies that when the output channel is full,
|
||||
// packets are dropped gracefully (non-blocking write)
|
||||
|
||||
// We can't easily test the full Run() loop without real interfaces,
|
||||
// but we can verify the channel behavior with a small buffer
|
||||
out := make(chan api.RawPacket, 1)
|
||||
|
||||
// Fill the channel
|
||||
out <- api.RawPacket{Data: []byte{1, 2, 3}, Timestamp: time.Now().UnixNano()}
|
||||
|
||||
// Channel should be full now, select default should trigger
|
||||
done := make(chan bool)
|
||||
go func() {
|
||||
select {
|
||||
case out <- api.RawPacket{Data: []byte{4, 5, 6}, Timestamp: time.Now().UnixNano()}:
|
||||
done <- false // Would block
|
||||
default:
|
||||
done <- true // Dropped as expected
|
||||
}
|
||||
}()
|
||||
|
||||
dropped := <-done
|
||||
if !dropped {
|
||||
t.Error("Expected packet to be dropped when channel is full")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPacketToRawPacket(t *testing.T) {
|
||||
t.Run("valid_packet", func(t *testing.T) {
|
||||
// Create a simple TCP packet
|
||||
eth := layers.Ethernet{
|
||||
SrcMAC: []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55},
|
||||
DstMAC: []byte{0x66, 0x77, 0x88, 0x99, 0xAA, 0xBB},
|
||||
EthernetType: layers.EthernetTypeIPv4,
|
||||
}
|
||||
ip := layers.IPv4{
|
||||
Version: 4,
|
||||
TTL: 64,
|
||||
Protocol: layers.IPProtocolTCP,
|
||||
SrcIP: []byte{192, 168, 1, 1},
|
||||
DstIP: []byte{10, 0, 0, 1},
|
||||
}
|
||||
tcp := layers.TCP{
|
||||
SrcPort: 12345,
|
||||
DstPort: 443,
|
||||
}
|
||||
tcp.SetNetworkLayerForChecksum(&ip)
|
||||
|
||||
buf := gopacket.NewSerializeBuffer()
|
||||
opts := gopacket.SerializeOptions{}
|
||||
gopacket.SerializeLayers(buf, opts, ð, &ip, &tcp)
|
||||
|
||||
packet := gopacket.NewPacket(buf.Bytes(), layers.LinkTypeEthernet, gopacket.Default)
|
||||
|
||||
// Create capture instance for method call
|
||||
c := New()
|
||||
rawPkt := c.packetToRawPacket(packet)
|
||||
|
||||
if rawPkt == nil {
|
||||
t.Fatal("packetToRawPacket() returned nil for valid packet")
|
||||
}
|
||||
if len(rawPkt.Data) == 0 {
|
||||
t.Error("packetToRawPacket() returned empty data")
|
||||
}
|
||||
if rawPkt.Timestamp == 0 {
|
||||
t.Error("packetToRawPacket() returned zero timestamp")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("empty_packet", func(t *testing.T) {
|
||||
// Create packet with no data
|
||||
packet := gopacket.NewPacket([]byte{}, layers.LinkTypeEthernet, gopacket.Default)
|
||||
|
||||
c := New()
|
||||
rawPkt := c.packetToRawPacket(packet)
|
||||
|
||||
if rawPkt != nil {
|
||||
t.Error("packetToRawPacket() should return nil for empty packet")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil_packet", func(t *testing.T) {
|
||||
// packetToRawPacket will panic with nil packet due to Metadata() call
|
||||
// This is expected behavior - the function is not designed to handle nil
|
||||
defer func() {
|
||||
if r := recover(); r == nil {
|
||||
t.Error("packetToRawPacket() with nil packet should panic")
|
||||
}
|
||||
}()
|
||||
c := New()
|
||||
var packet gopacket.Packet
|
||||
_ = c.packetToRawPacket(packet)
|
||||
})
|
||||
}
|
||||
|
||||
func TestGetInterfaceNames(t *testing.T) {
|
||||
t.Run("empty_list", func(t *testing.T) {
|
||||
names := getInterfaceNames([]pcap.Interface{})
|
||||
if len(names) != 0 {
|
||||
t.Errorf("getInterfaceNames() with empty list = %v, want []", names)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("single_interface", func(t *testing.T) {
|
||||
ifaces := []pcap.Interface{
|
||||
{Name: "eth0"},
|
||||
}
|
||||
names := getInterfaceNames(ifaces)
|
||||
if len(names) != 1 || names[0] != "eth0" {
|
||||
t.Errorf("getInterfaceNames() = %v, want [eth0]", names)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("multiple_interfaces", func(t *testing.T) {
|
||||
ifaces := []pcap.Interface{
|
||||
{Name: "eth0"},
|
||||
{Name: "lo"},
|
||||
{Name: "docker0"},
|
||||
}
|
||||
names := getInterfaceNames(ifaces)
|
||||
if len(names) != 3 {
|
||||
t.Errorf("getInterfaceNames() returned %d names, want 3", len(names))
|
||||
}
|
||||
expected := []string{"eth0", "lo", "docker0"}
|
||||
for i, name := range names {
|
||||
if name != expected[i] {
|
||||
t.Errorf("getInterfaceNames()[%d] = %s, want %s", i, name, expected[i])
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestValidateBPFFilter(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
filter string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "empty filter",
|
||||
filter: "",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid simple filter",
|
||||
filter: "tcp port 443",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "valid complex filter",
|
||||
filter: "(tcp port 443) or (tcp port 8443)",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "filter with special chars",
|
||||
filter: "tcp port 443 and host 192.168.1.1",
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "too long filter",
|
||||
filter: string(make([]byte, MaxBPFFilterLength+1)),
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unbalanced parentheses - extra open",
|
||||
filter: "(tcp port 443",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unbalanced parentheses - extra close",
|
||||
filter: "tcp port 443)",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid characters - semicolon",
|
||||
filter: "tcp port 443; rm -rf /",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid characters - backtick",
|
||||
filter: "tcp port `whoami`",
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "invalid characters - dollar",
|
||||
filter: "tcp port $HOME",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
err := validateBPFFilter(tt.filter)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("validateBPFFilter() error = %v, wantErr %v", err, tt.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinString(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
parts []string
|
||||
sep string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "empty slice",
|
||||
parts: []string{},
|
||||
sep: ") or (",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "single element",
|
||||
parts: []string{"tcp port 443"},
|
||||
sep: ") or (",
|
||||
want: "tcp port 443",
|
||||
},
|
||||
{
|
||||
name: "multiple elements",
|
||||
parts: []string{"tcp port 443", "tcp port 8443"},
|
||||
sep: ") or (",
|
||||
want: "tcp port 443) or (tcp port 8443",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := joinString(tt.parts, tt.sep)
|
||||
if got != tt.want {
|
||||
t.Errorf("joinString() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewCapture(t *testing.T) {
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
if c.snapLen != DefaultSnapLen {
|
||||
t.Errorf("snapLen = %d, want %d", c.snapLen, DefaultSnapLen)
|
||||
}
|
||||
if c.promisc != DefaultPromiscuous {
|
||||
t.Errorf("promisc = %v, want %v", c.promisc, DefaultPromiscuous)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewWithSnapLen(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
snapLen int
|
||||
wantSnapLen int
|
||||
}{
|
||||
{
|
||||
name: "valid snapLen",
|
||||
snapLen: 2048,
|
||||
wantSnapLen: 2048,
|
||||
},
|
||||
{
|
||||
name: "zero snapLen uses default",
|
||||
snapLen: 0,
|
||||
wantSnapLen: DefaultSnapLen,
|
||||
},
|
||||
{
|
||||
name: "negative snapLen uses default",
|
||||
snapLen: -100,
|
||||
wantSnapLen: DefaultSnapLen,
|
||||
},
|
||||
{
|
||||
name: "too large snapLen uses default",
|
||||
snapLen: 100000,
|
||||
wantSnapLen: DefaultSnapLen,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
c := NewWithSnapLen(tt.snapLen)
|
||||
if c == nil {
|
||||
t.Fatal("NewWithSnapLen() returned nil")
|
||||
}
|
||||
if c.snapLen != tt.wantSnapLen {
|
||||
t.Errorf("snapLen = %d, want %d", c.snapLen, tt.wantSnapLen)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_Close(t *testing.T) {
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
|
||||
// Close should not panic on fresh instance
|
||||
if err := c.Close(); err != nil {
|
||||
t.Errorf("Close() error = %v", err)
|
||||
}
|
||||
|
||||
// Multiple closes should be safe
|
||||
if err := c.Close(); err != nil {
|
||||
t.Errorf("Close() second call error = %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidateBPFFilter_BalancedParentheses(t *testing.T) {
|
||||
// Test various balanced parentheses scenarios
|
||||
validFilters := []string{
|
||||
"(tcp port 443)",
|
||||
"((tcp port 443))",
|
||||
"(tcp port 443) or (tcp port 8443)",
|
||||
"((tcp port 443) or (tcp port 8443))",
|
||||
"(tcp port 443 and host 1.2.3.4) or (tcp port 8443)",
|
||||
}
|
||||
|
||||
for _, filter := range validFilters {
|
||||
t.Run(filter, func(t *testing.T) {
|
||||
if err := validateBPFFilter(filter); err != nil {
|
||||
t.Errorf("validateBPFFilter(%q) unexpected error = %v", filter, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_detectLocalIPs(t *testing.T) {
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
|
||||
t.Run("any_interface", func(t *testing.T) {
|
||||
ips, err := c.detectLocalIPs("any")
|
||||
if err != nil {
|
||||
t.Errorf("detectLocalIPs(any) error = %v", err)
|
||||
}
|
||||
// Should return at least one non-loopback IP or empty if none available
|
||||
for _, ip := range ips {
|
||||
if ip == "127.0.0.1" || ip == "::1" {
|
||||
t.Errorf("detectLocalIPs(any) should exclude loopback, got %s", ip)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("loopback_excluded", func(t *testing.T) {
|
||||
ips, err := c.detectLocalIPs("any")
|
||||
if err != nil {
|
||||
t.Skipf("Skipping loopback test: %v", err)
|
||||
}
|
||||
// Verify no loopback addresses are included
|
||||
for _, ip := range ips {
|
||||
if ip == "127.0.0.1" {
|
||||
t.Error("detectLocalIPs should exclude 127.0.0.1")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestCaptureImpl_detectLocalIPs_SpecificInterface(t *testing.T) {
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
|
||||
// Test with a non-existent interface
|
||||
_, err := c.detectLocalIPs("nonexistent_interface_xyz")
|
||||
if err == nil {
|
||||
t.Error("detectLocalIPs with non-existent interface should return error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_extractIP(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
addr net.Addr
|
||||
wantIPv4 bool
|
||||
wantIPv6 bool
|
||||
}{
|
||||
{
|
||||
name: "IPv4",
|
||||
addr: &net.IPNet{
|
||||
IP: net.ParseIP("192.168.1.10"),
|
||||
Mask: net.CIDRMask(24, 32),
|
||||
},
|
||||
wantIPv4: true,
|
||||
},
|
||||
{
|
||||
name: "IPv6",
|
||||
addr: &net.IPNet{
|
||||
IP: net.ParseIP("2001:db8::1"),
|
||||
Mask: net.CIDRMask(64, 128),
|
||||
},
|
||||
wantIPv6: true,
|
||||
},
|
||||
{
|
||||
name: "nil",
|
||||
addr: nil,
|
||||
wantIPv4: false,
|
||||
wantIPv6: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := extractIP(tt.addr)
|
||||
if tt.wantIPv4 {
|
||||
if got == nil || got.To4() == nil {
|
||||
t.Error("extractIP() should return IPv4 address")
|
||||
}
|
||||
}
|
||||
if tt.wantIPv6 {
|
||||
if got == nil || got.To4() != nil {
|
||||
t.Error("extractIP() should return IPv6 address")
|
||||
}
|
||||
}
|
||||
if !tt.wantIPv4 && !tt.wantIPv6 {
|
||||
if got != nil {
|
||||
t.Error("extractIP() should return nil for nil address")
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_buildBPFFilter(t *testing.T) {
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
ports []uint16
|
||||
localIPs []string
|
||||
wantParts []string // Parts that should be in the filter
|
||||
}{
|
||||
{
|
||||
name: "no ports",
|
||||
ports: []uint16{},
|
||||
localIPs: []string{},
|
||||
wantParts: []string{"tcp"},
|
||||
},
|
||||
{
|
||||
name: "single port no IPs",
|
||||
ports: []uint16{443},
|
||||
localIPs: []string{},
|
||||
wantParts: []string{"tcp dst port 443"},
|
||||
},
|
||||
{
|
||||
name: "single port with single IP",
|
||||
ports: []uint16{443},
|
||||
localIPs: []string{"192.168.1.10"},
|
||||
wantParts: []string{"tcp dst port 443", "dst host 192.168.1.10"},
|
||||
},
|
||||
{
|
||||
name: "multiple ports with multiple IPs",
|
||||
ports: []uint16{443, 8443},
|
||||
localIPs: []string{"192.168.1.10", "10.0.0.5"},
|
||||
wantParts: []string{"tcp dst port 443", "tcp dst port 8443", "dst host 192.168.1.10", "dst host 10.0.0.5"},
|
||||
},
|
||||
{
|
||||
name: "IPv6 address",
|
||||
ports: []uint16{443},
|
||||
localIPs: []string{"2001:db8::1"},
|
||||
wantParts: []string{"tcp dst port 443", "dst host 2001:db8::1"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := c.buildBPFFilter(tt.ports, tt.localIPs)
|
||||
for _, part := range tt.wantParts {
|
||||
if !strings.Contains(got, part) {
|
||||
t.Errorf("buildBPFFilter() = %q, should contain %q", got, part)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_Run_AnyInterface(t *testing.T) {
|
||||
t.Skip("integration: pcap on 'any' interface blocks until close; run with -run=Integration in a real network env")
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
cfg := api.Config{
|
||||
Interface: "any",
|
||||
ListenPorts: []uint16{443},
|
||||
LocalIPs: []string{"192.168.1.10"},
|
||||
}
|
||||
out := make(chan api.RawPacket, 10)
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- c.Run(cfg, out) }()
|
||||
// Allow up to 300ms for the handle to open (or fail immediately)
|
||||
select {
|
||||
case err := <-errCh:
|
||||
// Immediate error: permission or "not found"
|
||||
if err != nil && strings.Contains(err.Error(), "not found") {
|
||||
t.Errorf("Run() with 'any' interface should be valid, got: %v", err)
|
||||
}
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
// Run() started successfully (blocking on packets) — close to stop it
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func TestCaptureImpl_Run_WithManualLocalIPs(t *testing.T) {
|
||||
t.Skip("integration: pcap on 'any' interface blocks until close; run with -run=Integration in a real network env")
|
||||
c := New()
|
||||
if c == nil {
|
||||
t.Fatal("New() returned nil")
|
||||
}
|
||||
cfg := api.Config{
|
||||
Interface: "any",
|
||||
ListenPorts: []uint16{443},
|
||||
LocalIPs: []string{"192.168.1.10", "10.0.0.5"},
|
||||
}
|
||||
out := make(chan api.RawPacket, 10)
|
||||
errCh := make(chan error, 1)
|
||||
go func() { errCh <- c.Run(cfg, out) }()
|
||||
select {
|
||||
case err := <-errCh:
|
||||
if err != nil && strings.Contains(err.Error(), "not found") {
|
||||
t.Errorf("Run() with manual LocalIPs should be valid, got: %v", err)
|
||||
}
|
||||
case <-time.After(300 * time.Millisecond):
|
||||
c.Close()
|
||||
}
|
||||
}
|
||||
|
||||
// TestCaptureImpl_LinkTypeInitializedOnce verifies that linkType is set exactly once,
|
||||
// after the BPF filter is applied (Bug 2 fix: removed the redundant early assignment).
|
||||
func TestCaptureImpl_LinkTypeInitializedOnce(t *testing.T) {
|
||||
c := New()
|
||||
// Fresh instance: linkType must be zero before Run() is called.
|
||||
if c.linkType != 0 {
|
||||
t.Errorf("new CaptureImpl should have linkType=0, got %d", c.linkType)
|
||||
}
|
||||
|
||||
// GetDiagnostics reflects linkType correctly.
|
||||
_, _, _, lt := c.GetDiagnostics()
|
||||
if lt != 0 {
|
||||
t.Errorf("GetDiagnostics() linkType before Run() should be 0, got %d", lt)
|
||||
}
|
||||
|
||||
// Simulate what Run() does: set linkType once under the mutex.
|
||||
c.mu.Lock()
|
||||
c.linkType = 1 // 1 = Ethernet
|
||||
c.mu.Unlock()
|
||||
|
||||
_, _, _, lt = c.GetDiagnostics()
|
||||
if lt != 1 {
|
||||
t.Errorf("GetDiagnostics() linkType after set = %d, want 1", lt)
|
||||
}
|
||||
}
|
||||
|
||||
// TestBuildBPFFilter_NoLocalIPs verifies Bug 3 fix: when no local IPs are
|
||||
// available (NAT/VIP), buildBPFFilter returns a port-only filter.
|
||||
func TestBuildBPFFilter_NoLocalIPs(t *testing.T) {
|
||||
c := New()
|
||||
filter := c.buildBPFFilter([]uint16{443}, nil)
|
||||
if strings.Contains(filter, "dst host") {
|
||||
t.Errorf("port-only filter expected when localIPs nil, got: %s", filter)
|
||||
}
|
||||
if !strings.Contains(filter, "tcp dst port 443") {
|
||||
t.Errorf("expected tcp dst port 443, got: %s", filter)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildBPFFilter_EmptyLocalIPs(t *testing.T) {
|
||||
c := New()
|
||||
filter := c.buildBPFFilter([]uint16{443, 8443}, []string{})
|
||||
if strings.Contains(filter, "dst host") {
|
||||
t.Errorf("port-only filter expected when localIPs empty, got: %s", filter)
|
||||
}
|
||||
if !strings.Contains(filter, "tcp dst port 443") || !strings.Contains(filter, "tcp dst port 8443") {
|
||||
t.Errorf("expected both ports in filter, got: %s", filter)
|
||||
}
|
||||
}
|
||||
@ -1,295 +0,0 @@
|
||||
// Package config provides configuration loading and validation for ja4sentinel
|
||||
package config
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
"github.com/antitbone/ja4/sentinel/api"
|
||||
)
|
||||
|
||||
// LoaderImpl implements the api.Loader interface for configuration loading
|
||||
type LoaderImpl struct {
|
||||
configPath string
|
||||
}
|
||||
|
||||
// NewLoader creates a new configuration loader
|
||||
func NewLoader(configPath string) *LoaderImpl {
|
||||
return &LoaderImpl{
|
||||
configPath: configPath,
|
||||
}
|
||||
}
|
||||
|
||||
// Load reads and merges configuration from file, environment variables, and CLI
|
||||
func (l *LoaderImpl) Load() (api.AppConfig, error) {
|
||||
config := api.DefaultConfig()
|
||||
|
||||
path := l.configPath
|
||||
explicit := path != ""
|
||||
if !explicit {
|
||||
path = "config.yml"
|
||||
}
|
||||
|
||||
fileConfig, err := l.loadFromFile(path)
|
||||
if err == nil {
|
||||
config = mergeConfigs(config, fileConfig)
|
||||
} else if !(!explicit && errors.Is(err, os.ErrNotExist)) {
|
||||
return config, fmt.Errorf("failed to load config file: %w", err)
|
||||
}
|
||||
|
||||
// Override with environment variables
|
||||
config = l.loadFromEnv(config)
|
||||
|
||||
// Validate the final configuration
|
||||
if err := l.validate(config); err != nil {
|
||||
return config, fmt.Errorf("invalid configuration: %w", err)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// loadFromFile reads configuration from a YAML file
|
||||
func (l *LoaderImpl) loadFromFile(path string) (api.AppConfig, error) {
|
||||
config := api.AppConfig{}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return config, fmt.Errorf("failed to read config file: %w", err)
|
||||
}
|
||||
|
||||
err = yaml.Unmarshal(data, &config)
|
||||
if err != nil {
|
||||
return config, fmt.Errorf("failed to parse config file: %w", err)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// loadFromEnv overrides configuration with environment variables
|
||||
func (l *LoaderImpl) loadFromEnv(config api.AppConfig) api.AppConfig {
|
||||
// JA4SENTINEL_INTERFACE
|
||||
if val := os.Getenv("JA4SENTINEL_INTERFACE"); val != "" {
|
||||
config.Core.Interface = val
|
||||
}
|
||||
|
||||
// JA4SENTINEL_PORTS (comma-separated list)
|
||||
if val := os.Getenv("JA4SENTINEL_PORTS"); val != "" {
|
||||
ports := parsePorts(val)
|
||||
if len(ports) > 0 {
|
||||
config.Core.ListenPorts = ports
|
||||
}
|
||||
}
|
||||
|
||||
// JA4SENTINEL_BPF_FILTER
|
||||
if val := os.Getenv("JA4SENTINEL_BPF_FILTER"); val != "" {
|
||||
config.Core.BPFFilter = val
|
||||
}
|
||||
|
||||
// JA4SENTINEL_FLOW_TIMEOUT (in seconds)
|
||||
if val := os.Getenv("JA4SENTINEL_FLOW_TIMEOUT"); val != "" {
|
||||
if timeout, err := strconv.Atoi(val); err == nil && timeout > 0 {
|
||||
config.Core.FlowTimeoutSec = timeout
|
||||
}
|
||||
}
|
||||
|
||||
// JA4SENTINEL_PACKET_BUFFER_SIZE
|
||||
if val := os.Getenv("JA4SENTINEL_PACKET_BUFFER_SIZE"); val != "" {
|
||||
if size, err := strconv.Atoi(val); err == nil && size > 0 {
|
||||
config.Core.PacketBufferSize = size
|
||||
}
|
||||
}
|
||||
|
||||
// Note: JA4SENTINEL_LOG_LEVEL is intentionally NOT loaded from env.
|
||||
// log_level must be configured exclusively via the YAML config file.
|
||||
|
||||
return config
|
||||
}
|
||||
|
||||
// parsePorts parses a comma-separated list of ports
|
||||
func parsePorts(s string) []uint16 {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
parts := strings.Split(s, ",")
|
||||
ports := make([]uint16, 0, len(parts))
|
||||
seen := make(map[uint16]struct{}, len(parts))
|
||||
|
||||
for _, part := range parts {
|
||||
part = strings.TrimSpace(part)
|
||||
if part == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
port, err := strconv.ParseUint(part, 10, 16)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
p := uint16(port)
|
||||
if p == 0 {
|
||||
continue
|
||||
}
|
||||
if _, exists := seen[p]; exists {
|
||||
continue
|
||||
}
|
||||
seen[p] = struct{}{}
|
||||
ports = append(ports, p)
|
||||
}
|
||||
|
||||
return ports
|
||||
}
|
||||
|
||||
// mergeConfigs merges two configs, with override taking precedence
|
||||
func mergeConfigs(base, override api.AppConfig) api.AppConfig {
|
||||
result := base
|
||||
|
||||
if override.Core.Interface != "" {
|
||||
result.Core.Interface = override.Core.Interface
|
||||
}
|
||||
|
||||
if len(override.Core.ListenPorts) > 0 {
|
||||
result.Core.ListenPorts = override.Core.ListenPorts
|
||||
}
|
||||
|
||||
if override.Core.BPFFilter != "" {
|
||||
result.Core.BPFFilter = override.Core.BPFFilter
|
||||
}
|
||||
|
||||
if override.Core.FlowTimeoutSec > 0 {
|
||||
result.Core.FlowTimeoutSec = override.Core.FlowTimeoutSec
|
||||
}
|
||||
|
||||
if override.Core.PacketBufferSize > 0 {
|
||||
result.Core.PacketBufferSize = override.Core.PacketBufferSize
|
||||
}
|
||||
|
||||
if override.Core.LogLevel != "" {
|
||||
result.Core.LogLevel = override.Core.LogLevel
|
||||
}
|
||||
|
||||
// Merge exclude_source_ips (override takes precedence)
|
||||
if len(override.Core.ExcludeSourceIPs) > 0 {
|
||||
result.Core.ExcludeSourceIPs = override.Core.ExcludeSourceIPs
|
||||
}
|
||||
|
||||
if len(override.Outputs) > 0 {
|
||||
result.Outputs = override.Outputs
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// validate checks if the configuration is valid
|
||||
func (l *LoaderImpl) validate(config api.AppConfig) error {
|
||||
if strings.TrimSpace(config.Core.Interface) == "" {
|
||||
return fmt.Errorf("interface cannot be empty")
|
||||
}
|
||||
|
||||
if len(config.Core.ListenPorts) == 0 {
|
||||
return fmt.Errorf("at least one listen port is required")
|
||||
}
|
||||
for _, p := range config.Core.ListenPorts {
|
||||
if p == 0 {
|
||||
return fmt.Errorf("listen port 0 is invalid")
|
||||
}
|
||||
}
|
||||
|
||||
if config.Core.FlowTimeoutSec <= 0 || config.Core.FlowTimeoutSec > 300 {
|
||||
return fmt.Errorf("flow_timeout_sec must be between 1 and 300")
|
||||
}
|
||||
|
||||
if config.Core.PacketBufferSize <= 0 || config.Core.PacketBufferSize > 1_000_000 {
|
||||
return fmt.Errorf("packet_buffer_size must be between 1 and 1000000")
|
||||
}
|
||||
|
||||
// Validate log level
|
||||
validLogLevels := map[string]struct{}{
|
||||
"debug": {},
|
||||
"info": {},
|
||||
"warn": {},
|
||||
"error": {},
|
||||
}
|
||||
if config.Core.LogLevel != "" {
|
||||
if _, ok := validLogLevels[config.Core.LogLevel]; !ok {
|
||||
return fmt.Errorf("log_level must be one of: debug, info, warn, error")
|
||||
}
|
||||
}
|
||||
|
||||
// Validate exclude_source_ips (if provided)
|
||||
if len(config.Core.ExcludeSourceIPs) > 0 {
|
||||
for i, ip := range config.Core.ExcludeSourceIPs {
|
||||
if ip == "" {
|
||||
return fmt.Errorf("exclude_source_ips[%d]: entry cannot be empty", i)
|
||||
}
|
||||
// Basic validation: check if it looks like an IP or CIDR
|
||||
if !strings.Contains(ip, "/") {
|
||||
// Single IP - basic check
|
||||
if !isValidIP(ip) {
|
||||
return fmt.Errorf("exclude_source_ips[%d]: invalid IP address %q", i, ip)
|
||||
}
|
||||
} else {
|
||||
// CIDR - basic check
|
||||
if !isValidCIDR(ip) {
|
||||
return fmt.Errorf("exclude_source_ips[%d]: invalid CIDR %q", i, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
allowedTypes := map[string]struct{}{
|
||||
"stdout": {},
|
||||
"file": {},
|
||||
"unix_socket": {},
|
||||
}
|
||||
|
||||
// Validate outputs
|
||||
for i, output := range config.Outputs {
|
||||
outputType := strings.TrimSpace(output.Type)
|
||||
if outputType == "" {
|
||||
return fmt.Errorf("output[%d]: type cannot be empty", i)
|
||||
}
|
||||
if _, ok := allowedTypes[outputType]; !ok {
|
||||
return fmt.Errorf("output[%d]: unknown type %q", i, outputType)
|
||||
}
|
||||
|
||||
switch outputType {
|
||||
case "file":
|
||||
if strings.TrimSpace(output.Params["path"]) == "" {
|
||||
return fmt.Errorf("output[%d]: file output requires non-empty path", i)
|
||||
}
|
||||
case "unix_socket":
|
||||
if strings.TrimSpace(output.Params["socket_path"]) == "" {
|
||||
return fmt.Errorf("output[%d]: unix_socket output requires non-empty socket_path", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ToJSON converts config to JSON string for debugging
|
||||
func ToJSON(config api.AppConfig) string {
|
||||
data, err := json.MarshalIndent(config, "", " ")
|
||||
if err != nil {
|
||||
return fmt.Sprintf("error marshaling config: %v", err)
|
||||
}
|
||||
return string(data)
|
||||
}
|
||||
|
||||
// isValidIP checks if a string is a valid IP address using net.ParseIP
|
||||
func isValidIP(ip string) bool {
|
||||
return net.ParseIP(ip) != nil
|
||||
}
|
||||
|
||||
// isValidCIDR checks if a string is a valid CIDR notation using net.ParseCIDR
|
||||
func isValidCIDR(cidr string) bool {
|
||||
_, _, err := net.ParseCIDR(cidr)
|
||||
return err == nil
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user