fix(ja4ebpf): split bpf2go generate into Ja4Tc + Ja4Ssl, fix RPM systemd-rpm-macros

- Use two separate //go:generate directives (Ja4Tc for tc_capture.c, Ja4Ssl for uprobe_ssl.c) to avoid duplicate LICENSE symbol and multi-file clang issue - Update loader.go to hold tcObjs/sslObjs separately with correct field names: UprobeSslSetFd, UprobeSslReadEntry, UretprobeSslReadExit, KprobeAccept4Entry, KretprobeAccept4Exit - Add systemd-rpm-macros to all three RPM build stages (el8/el9/el10) so that %{_unitdir} macro resolves correctly - RPMs now build successfully for el8, el9, el10 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-11 23:21:11 +02:00
parent a1e4c1dad5
commit 3b047b680a
155 changed files with 197011 additions and 599 deletions
--- a/tests/integration/nginx-varnish/docker-compose.yml
+++ b/tests/integration/nginx-varnish/docker-compose.yml
@ -0,0 +1,83 @@
+# =============================================================================
+# Stack nginx + varnish — tests d'intégration ja4ebpf
+#
+# Architecture :
+#   traffic-gen ─HTTPS→ nginx (TLS, port 443) ─HTTP→ varnish (port 6081)
+#                                                         ↓
+#                                               backend HTTP simple (port 8080)
+#                          ↑
+#                       ja4ebpf (uprobe nginx/libssl + hook TC)
+#
+# nginx termine le TLS et proxyfie vers Varnish en HTTP/1.1 (cleartext).
+# Varnish ajoute un en-tête Via et le header X-Varnish pour les tests.
+# ja4ebpf capture côté nginx : JA4 depuis le ClientHello, L7 depuis SSL_read.
+# =============================================================================
+
+services:
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.8
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: ja4_processing
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - ../platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh
+      - ../../../shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro
+      - ../../../shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro
+      - ../../../shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro
+      - ../../../shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro
+      - ../../../shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro
+      - ../../../shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro
+      - ../../../shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro
+      - ../../../shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro
+      - ../../../shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro
+      - ../../../shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro
+      - ../../../shared/clickhouse/10_perf_indexes.sql:/initdb-src/10_perf_indexes.sql:ro
+      - ../../../shared/clickhouse/11_views.sql:/initdb-src/11_views.sql:ro
+      - ../../../shared/clickhouse/12_thesis_features.sql:/initdb-src/12_thesis_features.sql:ro
+      - ../../../shared/data/browser_h2.csv:/initdb-src/browser_h2.csv:ro
+      - ../platform/csv-stubs:/var/lib/clickhouse/user_files
+    ports: ["9000:9000","8123:8123"]
+    healthcheck:
+      test: ["CMD","clickhouse-client","--query","SELECT 1"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks: [ja4net]
+
+  platform:
+    build:
+      context: ../../..
+      dockerfile: tests/integration/nginx-varnish/platform/Dockerfile
+      args:
+        BASE_IMAGE: ${PLATFORM_BASE_IMAGE:-rockylinux:9}
+    hostname: platform
+    cap_add: [NET_ADMIN, BPF, SYS_PTRACE]
+    privileged: true
+    environment:
+      JA4EBPF_CH_ADDR: "clickhouse:9000"
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+    ports: ["443:443","80:80"]
+    healthcheck:
+      test: ["CMD","curl","-sfk","https://localhost/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks: [ja4net]
+
+  traffic-gen:
+    build:
+      context: ../traffic-gen
+    hostname: traffic-gen
+    depends_on:
+      platform:
+        condition: service_healthy
+    networks: [ja4net]
+
+networks:
+  ja4net:
+    driver: bridge
--- a/tests/integration/nginx-varnish/platform/Dockerfile
+++ b/tests/integration/nginx-varnish/platform/Dockerfile
@ -0,0 +1,50 @@
+# =============================================================================
+# Platform nginx + varnish — Rocky Linux 9
+# nginx (TLS frontend) → Varnish (HTTP cache) → backend HTTP simple
+# =============================================================================
+
+FROM golang:1.24-bookworm AS go-builder
+
+RUN apt-get update && apt-get install -y clang llvm libbpf-dev && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+COPY go.work go.work.sum* ./
+COPY shared/go/ja4common/go.mod shared/go/ja4common/go.sum* ./shared/go/ja4common/
+COPY services/ja4ebpf/go.mod services/ja4ebpf/go.sum* ./services/ja4ebpf/
+RUN cd services/ja4ebpf && go mod download 2>/dev/null; true
+
+COPY shared/go/ja4common/ ./shared/go/ja4common/
+COPY services/ja4ebpf/ ./services/ja4ebpf/
+
+WORKDIR /build/services/ja4ebpf
+RUN GOWORK=off go generate ./internal/loader/ && \
+    GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    go build -ldflags="-s -w" -o /out/ja4ebpf ./cmd/ja4ebpf/
+
+# ── Runtime : nginx + varnish + backend + ja4ebpf ─────────────────────────────
+ARG BASE_IMAGE=rockylinux:9
+FROM ${BASE_IMAGE}
+
+RUN dnf install -y epel-release && \
+    dnf install -y nginx varnish openssl curl python3 && \
+    dnf clean all
+
+COPY --from=go-builder /out/ja4ebpf /usr/local/bin/ja4ebpf
+
+# Certificat TLS auto-signé pour nginx
+RUN openssl req -x509 -nodes -days 365 \
+        -subj "/CN=platform.test" \
+        -newkey rsa:2048 \
+        -keyout /etc/pki/tls/private/nginx.key \
+        -out /etc/pki/tls/certs/nginx.crt && \
+    mkdir -p /var/www/html /var/log/nginx /run/nginx /run/varnish && \
+    echo '{"status":"ok","stack":"nginx-varnish"}' > /var/www/html/health
+
+COPY tests/integration/nginx-varnish/platform/nginx.conf    /etc/nginx/nginx.conf
+COPY tests/integration/nginx-varnish/platform/varnish.vcl   /etc/varnish/default.vcl
+COPY tests/integration/nginx-varnish/platform/ja4ebpf.yml   /etc/ja4ebpf/config.yml
+COPY tests/integration/nginx-varnish/platform/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 80 443
+CMD ["/entrypoint.sh"]
--- a/tests/integration/nginx-varnish/platform/entrypoint.sh
+++ b/tests/integration/nginx-varnish/platform/entrypoint.sh
@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Entrypoint — stack nginx + varnish + backend HTTP + ja4ebpf
+# Ordre : backend → varnish → nginx → ja4ebpf
+# =============================================================================
+set -eo pipefail
+
+log() { echo "[entrypoint:nginx-varnish] $(date +%H:%M:%S) $*"; }
+
+BACKEND_PID="" VARNISH_PID="" NGINX_PID="" JA4EBPF_PID=""
+
+cleanup() {
+    log "Arrêt des processus…"
+    for pid in "$JA4EBPF_PID" "$NGINX_PID" "$VARNISH_PID" "$BACKEND_PID"; do
+        [ -n "$pid" ] && kill "$pid" 2>/dev/null || true
+    done
+    wait 2>/dev/null || true
+}
+trap cleanup EXIT SIGTERM SIGINT
+
+# ── 1. Backend HTTP simple (Python) ──────────────────────────────────────────
+log "Démarrage du backend HTTP (port 8080)…"
+python3 -c "
+import http.server, socketserver, json
+
+class H(http.server.BaseHTTPRequestHandler):
+    def log_message(self, *a): pass
+    def do_GET(self):
+        body = json.dumps({'status':'ok','backend':'python','path':self.path}).encode()
+        self.send_response(200)
+        self.send_header('Content-Type','application/json')
+        self.send_header('Content-Length', len(body))
+        self.end_headers()
+        self.wfile.write(body)
+    def do_POST(self):
+        n = int(self.headers.get('Content-Length',0))
+        self.rfile.read(n)
+        body = b'{\"result\":\"accepted\"}'
+        self.send_response(200)
+        self.send_header('Content-Type','application/json')
+        self.send_header('Content-Length', len(body))
+        self.end_headers()
+        self.wfile.write(body)
+
+with socketserver.TCPServer(('127.0.0.1', 8080), H) as s:
+    s.serve_forever()
+" &
+BACKEND_PID=$!
+sleep 1
+log "Backend HTTP démarré (PID $BACKEND_PID)"
+
+# ── 2. Démarrage de Varnish ───────────────────────────────────────────────────
+log "Démarrage de Varnish (port 6081)…"
+varnishd \
+    -F \
+    -f /etc/varnish/default.vcl \
+    -a "0.0.0.0:6081,HTTP" \
+    -s malloc,64m \
+    -T 127.0.0.1:6082 &
+VARNISH_PID=$!
+
+for i in $(seq 1 20); do
+    if varnishadm -T 127.0.0.1:6082 status 2>/dev/null | grep -q "Child in state running"; then
+        log "Varnish opérationnel (PID $VARNISH_PID)"; break
+    fi
+    sleep 0.5
+done
+
+# ── 3. Démarrage de nginx ─────────────────────────────────────────────────────
+log "Démarrage de nginx…"
+nginx
+NGINX_PID=$(cat /run/nginx/nginx.pid 2>/dev/null || pgrep nginx | head -1)
+
+for i in $(seq 1 20); do
+    if curl -sf http://localhost/health >/dev/null 2>&1; then
+        log "nginx opérationnel (PID $NGINX_PID)"; break
+    fi
+    sleep 0.5
+done
+
+# ── 4. Démarrage de ja4ebpf ───────────────────────────────────────────────────
+log "Démarrage de ja4ebpf (uprobes nginx/libssl + hook TC)…"
+ja4ebpf -config /etc/ja4ebpf/config.yml &
+JA4EBPF_PID=$!
+
+log "Stack complète — backend=$BACKEND_PID varnish=$VARNISH_PID nginx=$NGINX_PID ja4ebpf=$JA4EBPF_PID"
+
+# ── 5. Supervision ────────────────────────────────────────────────────────────
+while true; do
+    for pid_var in BACKEND_PID VARNISH_PID JA4EBPF_PID; do
+        pid="${!pid_var}"
+        if [ -n "$pid" ] && ! kill -0 "$pid" 2>/dev/null; then
+            log "$pid_var (PID $pid) s'est arrêté — fin"
+            exit 1
+        fi
+    done
+    # nginx master process via PID file
+    NGINX_PID=$(cat /run/nginx/nginx.pid 2>/dev/null || echo "")
+    if [ -z "$NGINX_PID" ] || ! kill -0 "$NGINX_PID" 2>/dev/null; then
+        log "nginx master s'est arrêté — fin"
+        exit 1
+    fi
+    sleep 2
+done
--- a/tests/integration/nginx-varnish/platform/ja4ebpf.yml
+++ b/tests/integration/nginx-varnish/platform/ja4ebpf.yml
@ -0,0 +1,30 @@
+# Configuration ja4ebpf — stack nginx + varnish
+# TLS terminé par nginx → uprobe sur libssl.so.3 (liée par nginx).
+# Varnish reçoit le trafic HTTP cleartext : pas de SSL_read côté varnish.
+
+interface: eth0
+
+ssl_probes:
+  # nginx lie libssl.so.3 pour la terminaison TLS.
+  # L'uprobe SSL_read capture les données HTTP/1.1 et HTTP/2
+  # déchiffrées juste avant que nginx les traite.
+  - executable: /usr/lib64/libssl.so.3
+    symbol: SSL_read
+
+clickhouse:
+  addr: "clickhouse:9000"
+  database: "ja4_logs"
+  table: "http_logs_raw"
+  username: "default"
+  password: ""
+  tls: false
+  batch_size: 100
+  flush_every: "1s"
+
+timeouts:
+  session_expiry: "500ms"
+  slowloris: "10s"
+
+log:
+  level: "info"
+  format: "json"
--- a/tests/integration/nginx-varnish/platform/nginx.conf
+++ b/tests/integration/nginx-varnish/platform/nginx.conf
@ -0,0 +1,74 @@
+# nginx.conf — Stack nginx + varnish
+# nginx : TLS frontend (port 443) + proxy vers Varnish (port 6081)
+# Port 80 : redirige vers HTTPS
+
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /run/nginx/nginx.pid;
+
+events {
+    worker_connections 1024;
+    use epoll;
+}
+
+http {
+    include      /etc/nginx/mime.types;
+    default_type application/octet-stream;
+    sendfile on;
+    keepalive_timeout 65;
+    keepalive_requests 200;
+
+    log_format main '$remote_addr [$time_local] "$request" $status '
+                    'ssl=$ssl_protocol via_varnish=$upstream_http_x_varnish';
+    access_log /var/log/nginx/access.log main;
+
+    # Backend Varnish (HTTP cleartext, pas de TLS)
+    upstream varnish_backend {
+        server 127.0.0.1:6081;
+        keepalive 32;
+    }
+
+    # ── Port 80 ─────────────────────────────────────────────────────────────
+    server {
+        listen 80;
+        server_name _;
+        location /health {
+            return 200 '{"status":"ok","stack":"nginx-varnish"}';
+            add_header Content-Type application/json;
+        }
+        location / {
+            return 301 https://$host$request_uri;
+        }
+    }
+
+    # ── Port 443 (TLS frontend → proxy Varnish) ──────────────────────────────
+    server {
+        listen 443 ssl;
+        http2 on;
+        server_name _;
+
+        ssl_certificate     /etc/pki/tls/certs/nginx.crt;
+        ssl_certificate_key /etc/pki/tls/private/nginx.key;
+        ssl_protocols       TLSv1.2 TLSv1.3;
+        ssl_ciphers         HIGH:!aNULL:!MD5;
+        ssl_session_cache   shared:SSL:10m;
+
+        # Transmission de l'IP réelle du client vers Varnish
+        proxy_set_header X-Real-IP       $remote_addr;
+        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header Host            $http_host;
+
+        location /health {
+            return 200 '{"status":"ok","stack":"nginx-varnish","tls":true}';
+            add_header Content-Type application/json;
+        }
+
+        location / {
+            proxy_pass         http://varnish_backend;
+            proxy_http_version 1.1;
+            # HTTP/2 terminé côté nginx ; nginx→Varnish est HTTP/1.1 cleartext
+            proxy_set_header   Connection "";
+        }
+    }
+}
--- a/tests/integration/nginx-varnish/platform/varnish.vcl
+++ b/tests/integration/nginx-varnish/platform/varnish.vcl
@ -0,0 +1,56 @@
+vcl 4.1;
+# =============================================================================
+# varnish.vcl — Stack nginx + varnish
+# Varnish reçoit le trafic HTTP/1.1 depuis nginx (après terminaison TLS).
+# Il proxyfie vers le backend HTTP simple sur 127.0.0.1:8080.
+# =============================================================================
+
+backend default {
+    .host = "127.0.0.1";
+    .port = "8080";
+    .probe = {
+        .url = "/health";
+        .interval = 5s;
+        .timeout  = 2s;
+        .window   = 3;
+        .threshold = 2;
+    }
+}
+
+sub vcl_recv {
+    # Normalisation de l'URL (suppression du double slash)
+    if (req.url ~ "//") {
+        set req.url = regsuball(req.url, "//+", "/");
+    }
+
+    # Transmission des en-têtes de provenance (déjà ajoutés par nginx)
+    if (req.http.X-Forwarded-For) {
+        set req.http.X-Forwarded-For = req.http.X-Forwarded-For;
+    }
+
+    # Ne pas mettre en cache les requêtes POST/PUT/DELETE
+    if (req.method != "GET" && req.method != "HEAD") {
+        return (pass);
+    }
+
+    # Cache les requêtes GET/HEAD
+    return (hash);
+}
+
+sub vcl_backend_response {
+    # TTL de cache court pour les tests (évite les faux positifs)
+    set beresp.ttl = 5s;
+    set beresp.grace = 10s;
+}
+
+sub vcl_deliver {
+    # Ajout des en-têtes de debug Varnish (vérifiés par les tests)
+    if (obj.hits > 0) {
+        set resp.http.X-Cache = "HIT";
+    } else {
+        set resp.http.X-Cache = "MISS";
+    }
+    # X-Varnish est ajouté automatiquement par Varnish
+    set resp.http.Via = "1.1 varnish (Varnish/test)";
+    return (deliver);
+}
--- a/tests/integration/nginx-varnish/run-tests.sh
+++ b/tests/integration/nginx-varnish/run-tests.sh
@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+# =============================================================================
+# run-tests.sh — Tests d'intégration stack nginx + varnish + ja4ebpf
+# =============================================================================
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+export STACK_NAME="nginx-varnish"
+export COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
+
+for arg in "$@"; do
+    case "$arg" in
+        --no-down)    export KEEP_UP=true ;;
+        --build-only) export BUILD_ONLY=true ;;
+    esac
+done
+
+source "$SCRIPT_DIR/../lib/run-stack-tests.sh"
+
+# ── Vérifications spécifiques nginx+varnish ───────────────────────────────────
+stack_verify_extra() {
+    # Vérifie la présence du header Via Varnish dans les réponses HTTPS
+    local via_hdr
+    via_hdr=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        curl -skI https://localhost/ 2>/dev/null | grep -i "^Via:" | tr -d '\r' || echo "")
+    if echo "$via_hdr" | grep -qi "varnish"; then
+        pass "Header Via: varnish présent — traffic bien routé nginx→Varnish"
+    else
+        warn "Header Via varnish absent (via='$via_hdr') — Varnish peut-être bypassé"
+    fi
+
+    # Vérifie X-Cache (HIT/MISS de Varnish)
+    local xcache1 xcache2
+    xcache1=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        curl -sk https://localhost/api/cached -o /dev/null -w "%{http_code}" 2>/dev/null || echo "")
+    # Deuxième requête sur le même chemin → doit être un HIT Varnish
+    xcache2=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        curl -skI https://localhost/api/cached 2>/dev/null | grep -i "^X-Cache:" | tr -d '\r' || echo "")
+    if echo "$xcache2" | grep -qi "HIT"; then
+        pass "Cache Varnish actif : X-Cache: HIT sur requête répétée"
+    else
+        warn "Cache Varnish non confirmé (X-Cache='$xcache2') — TTL cache peut-être expiré"
+    fi
+
+    # Vérifie que HTTP/2 est terminé côté nginx (pas propagé vers Varnish)
+    local http_ver
+    http_ver=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        curl -sk --http2 -w "%{http_version}" -o /dev/null https://localhost/ 2>/dev/null || echo "")
+    if [ "$http_ver" = "2" ]; then
+        pass "HTTP/2 terminé par nginx (client↔nginx h2, nginx↔varnish HTTP/1.1)"
+    else
+        warn "HTTP/2 non négocié côté client (version: '$http_ver')"
+    fi
+
+    # Vérifie que ja4ebpf tourne
+    local ja4_pid
+    ja4_pid=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        pgrep -x ja4ebpf 2>/dev/null | head -1 || echo "")
+    [ -n "$ja4_pid" ] && pass "ja4ebpf actif (PID $ja4_pid)" \
+                      || fail "ja4ebpf introuvable"
+
+    # Dans cette stack, les requêtes L7 passent via Varnish :
+    # on vérifie que header_order_signature est capturé malgré le proxy.
+    local sig_count
+    sig_count=$(ch_query "SELECT count() FROM ja4_logs.http_logs_raw WHERE header_order_signature != ''")
+    if [ "${sig_count:-0}" -gt 0 ] 2>/dev/null; then
+        pass "Signature ordre en-têtes capturée : $sig_count enregistrements"
+    else
+        warn "Signature ordre en-têtes absente (uprobe SSL_read peut-être inactif)"
+    fi
+}
+
+run_all_phases