chore: suppression des services obsolètes (sentinel, correlator, mod-reqin-log)
Remplacés par l'agent ja4ebpf (eBPF CO-RE). Nettoyage complet : Supprimé : - old/ (archive de l'ancienne architecture) - services/correlator/ (logcorrelator Go) - services/sentinel/ (capture pcap Go) - services/mod-reqin-log/ (module Apache C) - shared/go/ja4common/ (lib Go partagée — plus importée par ja4ebpf) - tests/integration/platform/ (test correlator+sentinel+httpd) - tests/integration/docker-compose.yml (compose ancienne archi) - tests/integration/run-tests.sh (runner correlator/sentinel) - tests/integration/verify_mvs.py (script orphelin) Nettoyé : - go.work : retire ./shared/go/ja4common - services/ja4ebpf/go.mod : retire replace ja4common (jamais importé) - services/ja4ebpf/Dockerfile* : retire les COPY ja4common inutiles - Makefile : retire test-ja4common-python, test-integration*, targets obsolètes - tests/integration/README.md : réécrit pour l'architecture ja4ebpf Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
@ -1,97 +0,0 @@
|
||||
# =============================================================================
|
||||
# Platform container — Rocky Linux 9
|
||||
# Runs: Apache (HTTPS) + mod-reqin-log + sentinel + correlator
|
||||
#
|
||||
# Multi-stage:
|
||||
# 1. go-builder — compile correlator (static, no CGO) on golang image
|
||||
# 2. platform — Rocky Linux 9: builds sentinel (CGO+libpcap), mod-reqin-log,
|
||||
# installs Apache, runs everything
|
||||
#
|
||||
# sentinel is compiled on Rocky so it links against the same libpcap as runtime.
|
||||
# This mirrors RPM packaging where build and target are the same distro.
|
||||
# =============================================================================
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 1: Build correlator (static binary, no CGO — distro-independent)
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM golang:1.24 AS go-builder
|
||||
|
||||
WORKDIR /src
|
||||
COPY go.work go.work.sum* ./
|
||||
COPY shared/go/ja4common/ shared/go/ja4common/
|
||||
COPY services/correlator/ services/correlator/
|
||||
COPY services/sentinel/ services/sentinel/
|
||||
|
||||
RUN cd services/correlator && \
|
||||
CGO_ENABLED=0 go build -ldflags="-s -w" -o /out/correlator ./cmd/logcorrelator
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stage 2: Rocky Linux 9 — build sentinel + mod-reqin-log, then run everything
|
||||
# ---------------------------------------------------------------------------
|
||||
FROM rockylinux:9
|
||||
|
||||
# Install build deps + runtime deps
|
||||
RUN dnf install -y --allowerasing \
|
||||
httpd httpd-devel mod_ssl \
|
||||
apr-devel apr-util-devel \
|
||||
gcc make redhat-rpm-config \
|
||||
libpcap \
|
||||
golang \
|
||||
procps-ng curl \
|
||||
&& dnf install -y --enablerepo=crb libpcap-devel \
|
||||
&& dnf clean all
|
||||
|
||||
# -- Build sentinel on Rocky (CGO + libpcap from Rocky repos) ---------------
|
||||
COPY go.work go.work.sum* /tmp/sentinel-build/
|
||||
COPY shared/go/ja4common/ /tmp/sentinel-build/shared/go/ja4common/
|
||||
COPY services/sentinel/ /tmp/sentinel-build/services/sentinel/
|
||||
COPY services/correlator/ /tmp/sentinel-build/services/correlator/
|
||||
RUN cd /tmp/sentinel-build/services/sentinel && \
|
||||
CGO_ENABLED=1 go build -ldflags="-s -w" -o /usr/local/bin/sentinel ./cmd/ja4sentinel && \
|
||||
rm -rf /tmp/sentinel-build /root/go
|
||||
|
||||
# -- Build mod-reqin-log from source -----------------------------------------
|
||||
COPY services/mod-reqin-log/src/ /tmp/mod-reqin-log/src/
|
||||
COPY services/mod-reqin-log/Makefile /tmp/mod-reqin-log/Makefile
|
||||
RUN cd /tmp/mod-reqin-log && make all && \
|
||||
cp modules/mod_reqin_log.so /usr/lib64/httpd/modules/ 2>/dev/null || \
|
||||
cp build/.libs/mod_reqin_log.so /usr/lib64/httpd/modules/ && \
|
||||
rm -rf /tmp/mod-reqin-log
|
||||
|
||||
# -- Copy correlator from builder (static binary, no deps) -------------------
|
||||
COPY --from=go-builder /out/correlator /usr/local/bin/correlator
|
||||
|
||||
# -- Create runtime directories ----------------------------------------------
|
||||
RUN mkdir -p /var/run/logcorrelator \
|
||||
/var/log/logcorrelator \
|
||||
/var/log/ja4sentinel \
|
||||
/etc/logcorrelator \
|
||||
/etc/ja4sentinel
|
||||
|
||||
# -- Correlator config -------------------------------------------------------
|
||||
COPY tests/integration/platform/correlator.yml /etc/logcorrelator/correlator.yml
|
||||
|
||||
# -- Sentinel config ----------------------------------------------------------
|
||||
COPY tests/integration/platform/sentinel.yml /etc/ja4sentinel/config.yml
|
||||
|
||||
# -- Apache config (HTTPS + mod-reqin-log) ------------------------------------
|
||||
COPY tests/integration/platform/httpd-integration.conf /etc/httpd/conf.d/integration.conf
|
||||
|
||||
# -- Generate self-signed TLS certificate -------------------------------------
|
||||
RUN openssl req -x509 -nodes -days 365 \
|
||||
-subj "/CN=platform.test" \
|
||||
-newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/localhost.key \
|
||||
-out /etc/pki/tls/certs/localhost.crt
|
||||
|
||||
# -- Simple health endpoint for Apache ---------------------------------------
|
||||
RUN mkdir -p /var/www/html && \
|
||||
echo '{"status":"ok"}' > /var/www/html/health
|
||||
|
||||
# -- Entrypoint (manages all processes) --------------------------------------
|
||||
COPY tests/integration/platform/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
EXPOSE 80 443
|
||||
|
||||
CMD ["/entrypoint.sh"]
|
||||
@ -1,48 +0,0 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# clickhouse-init.sh — Pre-process shared SQL files for integration testing
|
||||
#
|
||||
# Copies SQL from /initdb-src/ to /tmp, patches credentials, then executes.
|
||||
# =============================================================================
|
||||
set -e
|
||||
|
||||
SRC_DIR="/initdb-src"
|
||||
TMP_DIR="/tmp/initdb-patched"
|
||||
USER_FILES="/var/lib/clickhouse/user_files"
|
||||
mkdir -p "$TMP_DIR"
|
||||
|
||||
# Copier les CSV de référence dans user_files (dictionnaires navigateurs)
|
||||
for csv in "$SRC_DIR"/*.csv; do
|
||||
[ -f "$csv" ] || continue
|
||||
fname=$(basename "$csv")
|
||||
if [ ! -f "$USER_FILES/$fname" ]; then
|
||||
cp "$csv" "$USER_FILES/$fname"
|
||||
echo "[init] CSV copié : $fname"
|
||||
fi
|
||||
done
|
||||
|
||||
for f in "$SRC_DIR"/*.sql; do
|
||||
[ -f "$f" ] || continue
|
||||
base=$(basename "$f")
|
||||
echo "[init] Patching $base"
|
||||
sed \
|
||||
-e "s/USER 'admin'/USER 'default'/g" \
|
||||
-e "s/PASSWORD 'CHANGE_ME'/PASSWORD ''/g" \
|
||||
-e "s/PASSWORD 'ChangeMe'/PASSWORD ''/g" \
|
||||
"$f" > "$TMP_DIR/$base"
|
||||
done
|
||||
|
||||
for f in "$TMP_DIR"/*.sql; do
|
||||
[ -f "$f" ] || continue
|
||||
base=$(basename "$f")
|
||||
echo "[init] Executing $base"
|
||||
# 10_perf_indexes.sql uses ALTER TABLE ADD INDEX which may fail if index
|
||||
# already exists — allow non-zero exit for migration/perf scripts
|
||||
if [[ "$base" == 10_* ]]; then
|
||||
clickhouse-client --multiquery < "$f" || echo "[init] WARNING: $base had errors (expected for duplicate indexes)"
|
||||
else
|
||||
clickhouse-client --multiquery < "$f"
|
||||
fi
|
||||
done
|
||||
|
||||
echo "[init] All SQL files executed successfully"
|
||||
@ -1,51 +0,0 @@
|
||||
# Correlator config for integration tests
|
||||
log:
|
||||
level: DEBUG
|
||||
|
||||
inputs:
|
||||
unix_sockets:
|
||||
- name: http
|
||||
source_type: A
|
||||
path: /var/run/logcorrelator/http.socket
|
||||
format: json
|
||||
socket_permissions: "0666"
|
||||
- name: network
|
||||
source_type: B
|
||||
path: /var/run/logcorrelator/network.socket
|
||||
format: json
|
||||
socket_permissions: "0666"
|
||||
|
||||
outputs:
|
||||
clickhouse:
|
||||
enabled: true
|
||||
dsn: clickhouse://default:@clickhouse:9000/ja4_logs
|
||||
table: http_logs_raw
|
||||
batch_size: 10
|
||||
flush_interval_ms: 500
|
||||
max_buffer_size: 5000
|
||||
drop_on_overflow: false
|
||||
async_insert: true
|
||||
timeout_ms: 5000
|
||||
|
||||
file:
|
||||
enabled: true
|
||||
path: /var/log/logcorrelator/correlated.log
|
||||
|
||||
stdout:
|
||||
enabled: true
|
||||
|
||||
correlation:
|
||||
time_window:
|
||||
value: 10
|
||||
unit: s
|
||||
orphan_policy:
|
||||
apache_always_emit: true
|
||||
apache_emit_delay_ms: 1000
|
||||
network_emit: false
|
||||
matching:
|
||||
mode: one_to_many
|
||||
buffers:
|
||||
max_http_items: 10000
|
||||
max_network_items: 20000
|
||||
ttl:
|
||||
network_ttl_s: 120
|
||||
@ -1,59 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# Platform entrypoint — starts correlator, Apache, sentinel in order
|
||||
# =============================================================================
|
||||
set -eo pipefail
|
||||
|
||||
log() { echo "[entrypoint] $(date +%H:%M:%S) $*"; }
|
||||
|
||||
CORRELATOR_PID=""
|
||||
HTTPD_PID=""
|
||||
SENTINEL_PID=""
|
||||
|
||||
cleanup() {
|
||||
log "Shutting down..."
|
||||
[ -n "$SENTINEL_PID" ] && kill "$SENTINEL_PID" 2>/dev/null || true
|
||||
[ -n "$CORRELATOR_PID" ] && kill "$CORRELATOR_PID" 2>/dev/null || true
|
||||
httpd -k stop 2>/dev/null || true
|
||||
wait 2>/dev/null || true
|
||||
log "All processes stopped."
|
||||
}
|
||||
trap cleanup EXIT SIGTERM SIGINT
|
||||
|
||||
# -- 1. Start correlator (creates Unix sockets) ------------------------------
|
||||
log "Starting correlator..."
|
||||
correlator -config /etc/logcorrelator/correlator.yml &
|
||||
CORRELATOR_PID=$!
|
||||
|
||||
# Wait for correlator to create its sockets
|
||||
for i in $(seq 1 30); do
|
||||
if [ -S /var/run/logcorrelator/http.socket ] && [ -S /var/run/logcorrelator/network.socket ]; then
|
||||
log "Correlator sockets ready."
|
||||
break
|
||||
fi
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
if [ ! -S /var/run/logcorrelator/http.socket ]; then
|
||||
log "ERROR: correlator sockets not created after 15s"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# -- 2. Start Apache (with mod-reqin-log writing to http.socket) -------------
|
||||
log "Starting Apache..."
|
||||
httpd -DFOREGROUND &
|
||||
HTTPD_PID=$!
|
||||
sleep 2
|
||||
|
||||
# -- 3. Start sentinel (captures network traffic) ----------------------------
|
||||
log "Starting sentinel..."
|
||||
sentinel -config /etc/ja4sentinel/config.yml &
|
||||
SENTINEL_PID=$!
|
||||
|
||||
log "All services started. PIDs: correlator=$CORRELATOR_PID httpd=$HTTPD_PID sentinel=$SENTINEL_PID"
|
||||
|
||||
# -- Wait for any process to exit (indicates failure) -------------------------
|
||||
wait -n "$CORRELATOR_PID" "$HTTPD_PID" "$SENTINEL_PID" 2>/dev/null || true
|
||||
EXIT_CODE=$?
|
||||
log "A process exited with code $EXIT_CODE — triggering shutdown."
|
||||
exit $EXIT_CODE
|
||||
@ -1,40 +0,0 @@
|
||||
# Integration test Apache config — HTTPS + mod-reqin-log
|
||||
|
||||
# Load mod-reqin-log
|
||||
LoadModule reqin_log_module modules/mod_reqin_log.so
|
||||
|
||||
# Enable HTTP/2 negotiation (mod_http2 loaded by default on Rocky 9)
|
||||
Protocols h2 http/1.1
|
||||
|
||||
# mod_remoteip: trust X-Forwarded-For from Docker internal subnets.
|
||||
# mod_reqin_log reads r->useragent_ip which mod_remoteip updates,
|
||||
# so the XFF IP appears as src_ip in the correlated logs.
|
||||
LoadModule remoteip_module modules/mod_remoteip.so
|
||||
RemoteIPHeader X-Forwarded-For
|
||||
RemoteIPInternalProxy 172.0.0.0/8
|
||||
RemoteIPInternalProxy 192.168.0.0/16
|
||||
RemoteIPInternalProxy 10.0.0.0/8
|
||||
|
||||
# Enable mod-reqin-log with correlator socket
|
||||
JsonSockLogEnabled On
|
||||
JsonSockLogSocket "/var/run/logcorrelator/http.socket"
|
||||
JsonSockLogHeaders X-Request-Id User-Agent Referer X-Forwarded-For \
|
||||
Sec-CH-UA Sec-CH-UA-Mobile Sec-CH-UA-Platform \
|
||||
Sec-Fetch-Dest Sec-Fetch-Mode Sec-Fetch-Site \
|
||||
Accept Accept-Language Accept-Encoding Content-Type
|
||||
JsonSockLogMaxHeaders 25
|
||||
JsonSockLogMaxHeaderValueLen 256
|
||||
JsonSockLogReconnectInterval 5
|
||||
JsonSockLogErrorReportInterval 5
|
||||
JsonSockLogLevel DEBUG
|
||||
|
||||
# HTTPS virtual host (port 443 already configured by mod_ssl)
|
||||
<VirtualHost *:80>
|
||||
ServerName platform.test
|
||||
DocumentRoot /var/www/html
|
||||
|
||||
# Simple test pages
|
||||
<Location /health>
|
||||
Require all granted
|
||||
</Location>
|
||||
</VirtualHost>
|
||||
@ -1,18 +0,0 @@
|
||||
# Sentinel config for integration tests
|
||||
core:
|
||||
interface: eth0
|
||||
listen_ports:
|
||||
- 443
|
||||
flow_timeout_sec: 30
|
||||
packet_buffer_size: 1000
|
||||
log_level: debug
|
||||
|
||||
outputs:
|
||||
- type: unix_socket
|
||||
enabled: true
|
||||
async_buffer: 5000
|
||||
params:
|
||||
socket_path: /var/run/logcorrelator/network.socket
|
||||
|
||||
- type: stdout
|
||||
enabled: true
|
||||
Reference in New Issue
Block a user