fix(ja4ebpf): split bpf2go generate into Ja4Tc + Ja4Ssl, fix RPM systemd-rpm-macros

- Use two separate //go:generate directives (Ja4Tc for tc_capture.c, Ja4Ssl for uprobe_ssl.c) to avoid duplicate LICENSE symbol and multi-file clang issue - Update loader.go to hold tcObjs/sslObjs separately with correct field names: UprobeSslSetFd, UprobeSslReadEntry, UretprobeSslReadExit, KprobeAccept4Entry, KretprobeAccept4Exit - Add systemd-rpm-macros to all three RPM build stages (el8/el9/el10) so that %{_unitdir} macro resolves correctly - RPMs now build successfully for el8, el9, el10 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
2026-04-11 23:21:11 +02:00
parent a1e4c1dad5
commit 3b047b680a
155 changed files with 197011 additions and 599 deletions
--- a/tests/integration/nginx/docker-compose.yml
+++ b/tests/integration/nginx/docker-compose.yml
@ -0,0 +1,82 @@
+# =============================================================================
+# Stack nginx — tests d'intégration ja4ebpf
+#
+# Architecture :
+#   traffic-gen ─HTTPS→ nginx (TLS, HTTP/1.1+h2) ─→ ClickHouse
+#                          ↑
+#                       ja4ebpf (uprobe nginx + hook TC)
+#
+# ja4ebpf attache ses uprobes directement sur /usr/sbin/nginx (qui lie OpenSSL).
+# Le hook TC ingress capture les paquets TCP SYN + TLS ClientHello sur eth0.
+# =============================================================================
+
+services:
+
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.8
+    hostname: clickhouse
+    environment:
+      CLICKHOUSE_DB: ja4_processing
+      CLICKHOUSE_USER: default
+      CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: 1
+    volumes:
+      - ../platform/clickhouse-init.sh:/docker-entrypoint-initdb.d/00_init.sh
+      - ../../../shared/clickhouse/00_database.sql:/initdb-src/00_database.sql:ro
+      - ../../../shared/clickhouse/01_raw_tables.sql:/initdb-src/01_raw_tables.sql:ro
+      - ../../../shared/clickhouse/02_dictionaries.sql:/initdb-src/02_dictionaries.sql:ro
+      - ../../../shared/clickhouse/03_anubis_tables.sql:/initdb-src/03_anubis_tables.sql:ro
+      - ../../../shared/clickhouse/04_mv_http_logs.sql:/initdb-src/04_mv_http_logs.sql:ro
+      - ../../../shared/clickhouse/05_aggregation_tables.sql:/initdb-src/05_aggregation_tables.sql:ro
+      - ../../../shared/clickhouse/06_ml_tables.sql:/initdb-src/06_ml_tables.sql:ro
+      - ../../../shared/clickhouse/07_ai_features_view.sql:/initdb-src/07_ai_features_view.sql:ro
+      - ../../../shared/clickhouse/08_users.sql:/initdb-src/08_users.sql:ro
+      - ../../../shared/clickhouse/09_audit_table.sql:/initdb-src/09_audit_table.sql:ro
+      - ../../../shared/clickhouse/10_perf_indexes.sql:/initdb-src/10_perf_indexes.sql:ro
+      - ../../../shared/clickhouse/11_views.sql:/initdb-src/11_views.sql:ro
+      - ../../../shared/clickhouse/12_thesis_features.sql:/initdb-src/12_thesis_features.sql:ro
+      - ../../../shared/data/browser_h2.csv:/initdb-src/browser_h2.csv:ro
+      - ../platform/csv-stubs:/var/lib/clickhouse/user_files
+    ports: ["9000:9000","8123:8123"]
+    healthcheck:
+      test: ["CMD","clickhouse-client","--query","SELECT 1"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks: [ja4net]
+
+  platform:
+    build:
+      context: ../../..
+      dockerfile: tests/integration/nginx/platform/Dockerfile
+      args:
+        BASE_IMAGE: ${PLATFORM_BASE_IMAGE:-rockylinux:9}
+    hostname: platform
+    # ja4ebpf nécessite CAP_BPF + CAP_NET_ADMIN pour charger eBPF
+    # et CAP_SYS_PTRACE pour attacher les uprobes.
+    cap_add: [NET_ADMIN, BPF, SYS_PTRACE]
+    privileged: true          # requis sur certains kernels pour bpf()
+    environment:
+      JA4EBPF_CH_ADDR: "clickhouse:9000"
+    depends_on:
+      clickhouse:
+        condition: service_healthy
+    ports: ["443:443","80:80"]
+    healthcheck:
+      test: ["CMD","curl","-sfk","https://localhost/health"]
+      interval: 5s
+      timeout: 3s
+      retries: 30
+    networks: [ja4net]
+
+  traffic-gen:
+    build:
+      context: ../traffic-gen
+    hostname: traffic-gen
+    depends_on:
+      platform:
+        condition: service_healthy
+    networks: [ja4net]
+
+networks:
+  ja4net:
+    driver: bridge
--- a/tests/integration/nginx/platform/Dockerfile
+++ b/tests/integration/nginx/platform/Dockerfile
@ -0,0 +1,56 @@
+# =============================================================================
+# Platform nginx — Rocky Linux 9
+# Construit ja4ebpf (eBPF CO-RE) + nginx avec HTTP/2 et TLS.
+#
+# Multi-stage :
+#   ebpf-builder  — compile les programmes eBPF C avec clang
+#   go-builder    — compile ja4ebpf (go generate + go build)
+#   runtime       — nginx + binaire ja4ebpf sur Rocky Linux 9
+# =============================================================================
+
+# ── Stage 1 : build ja4ebpf ──────────────────────────────────────────────────
+FROM golang:1.24-bookworm AS go-builder
+
+RUN apt-get update && apt-get install -y clang llvm libbpf-dev && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /build
+COPY go.work go.work.sum* ./
+COPY shared/go/ja4common/go.mod shared/go/ja4common/go.sum* ./shared/go/ja4common/
+COPY services/ja4ebpf/go.mod services/ja4ebpf/go.sum* ./services/ja4ebpf/
+RUN cd services/ja4ebpf && go mod download 2>/dev/null; true
+
+COPY shared/go/ja4common/ ./shared/go/ja4common/
+COPY services/ja4ebpf/ ./services/ja4ebpf/
+
+WORKDIR /build/services/ja4ebpf
+RUN GOWORK=off go generate ./internal/loader/ && \
+    GOWORK=off CGO_ENABLED=0 GOOS=linux GOARCH=amd64 \
+    go build -ldflags="-s -w" -o /out/ja4ebpf ./cmd/ja4ebpf/
+
+# ── Stage 2 : runtime nginx + ja4ebpf ────────────────────────────────────────
+ARG BASE_IMAGE=rockylinux:9
+FROM ${BASE_IMAGE}
+
+RUN dnf install -y epel-release && \
+    dnf install -y nginx openssl curl && \
+    dnf clean all
+
+COPY --from=go-builder /out/ja4ebpf /usr/local/bin/ja4ebpf
+
+# Configuration nginx : HTTP/2 + TLS auto-signé
+RUN openssl req -x509 -nodes -days 365 \
+        -subj "/CN=platform.test" \
+        -newkey rsa:2048 \
+        -keyout /etc/pki/tls/private/nginx.key \
+        -out /etc/pki/tls/certs/nginx.crt && \
+    mkdir -p /var/www/html && \
+    echo '{"status":"ok","stack":"nginx"}' > /var/www/html/health && \
+    mkdir -p /var/log/nginx /run/nginx
+
+COPY tests/integration/nginx/platform/nginx.conf /etc/nginx/nginx.conf
+COPY tests/integration/nginx/platform/ja4ebpf.yml /etc/ja4ebpf/config.yml
+COPY tests/integration/nginx/platform/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+EXPOSE 80 443
+CMD ["/entrypoint.sh"]
--- a/tests/integration/nginx/platform/entrypoint.sh
+++ b/tests/integration/nginx/platform/entrypoint.sh
@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# =============================================================================
+# Entrypoint — stack nginx + ja4ebpf
+# Ordre : nginx → ja4ebpf (uprobes sur nginx/libssl)
+# =============================================================================
+set -eo pipefail
+
+log() { echo "[entrypoint:nginx] $(date +%H:%M:%S) $*"; }
+
+NGINX_PID=""
+JA4EBPF_PID=""
+
+cleanup() {
+    log "Arrêt des processus…"
+    [ -n "$JA4EBPF_PID" ] && kill "$JA4EBPF_PID" 2>/dev/null || true
+    [ -n "$NGINX_PID"   ] && kill "$NGINX_PID"   2>/dev/null || true
+    wait 2>/dev/null || true
+}
+trap cleanup EXIT SIGTERM SIGINT
+
+# ── 1. Démarrage de nginx ─────────────────────────────────────────────────
+log "Démarrage de nginx…"
+nginx
+NGINX_PID=$(cat /run/nginx/nginx.pid 2>/dev/null || pgrep nginx | head -1)
+
+# Attendre que nginx soit opérationnel
+for i in $(seq 1 20); do
+    if curl -sf http://localhost/health >/dev/null 2>&1; then
+        log "nginx opérationnel (PID $NGINX_PID)"
+        break
+    fi
+    sleep 0.5
+done
+
+# ── 2. Démarrage de ja4ebpf ───────────────────────────────────────────────
+log "Démarrage de ja4ebpf (attache uprobes sur libssl)…"
+ja4ebpf -config /etc/ja4ebpf/config.yml &
+JA4EBPF_PID=$!
+
+log "Stack démarrée — nginx PID=$NGINX_PID ja4ebpf PID=$JA4EBPF_PID"
+
+# ── 3. Supervision ────────────────────────────────────────────────────────
+# nginx fonctionne en daemon : surveiller le process master via le PID file.
+# ja4ebpf tourne en foreground.
+while true; do
+    # Vérifier que nginx est toujours en vie
+    if ! kill -0 "$NGINX_PID" 2>/dev/null; then
+        NGINX_PID=$(cat /run/nginx/nginx.pid 2>/dev/null || echo "")
+        if [ -z "$NGINX_PID" ] || ! kill -0 "$NGINX_PID" 2>/dev/null; then
+            log "nginx s'est arrêté — fin de l'entrypoint"
+            break
+        fi
+    fi
+    # Vérifier que ja4ebpf est toujours en vie
+    if ! kill -0 "$JA4EBPF_PID" 2>/dev/null; then
+        log "ja4ebpf s'est arrêté (code: $?) — fin de l'entrypoint"
+        break
+    fi
+    sleep 2
+done
--- a/tests/integration/nginx/platform/ja4ebpf.yml
+++ b/tests/integration/nginx/platform/ja4ebpf.yml
@ -0,0 +1,29 @@
+# Configuration ja4ebpf — stack nginx
+# ja4ebpf attache ses uprobes sur le processus nginx qui lie OpenSSL directement.
+# Sur Rocky Linux 9, nginx utilise libssl.so.3 via dlopen ou liaison dynamique.
+
+interface: eth0
+
+ssl_probes:
+  # nginx lie OpenSSL : les appels SSL_read sont dans la librairie partagée.
+  # Le fichier réel (pas le symlink) est requis pour l'uprobe.
+  - executable: /usr/lib64/libssl.so.3
+    symbol: SSL_read
+
+clickhouse:
+  addr: "clickhouse:9000"
+  database: "ja4_logs"
+  table: "http_logs_raw"
+  username: "default"
+  password: ""
+  tls: false
+  batch_size: 100
+  flush_every: "1s"
+
+timeouts:
+  session_expiry: "500ms"
+  slowloris: "10s"
+
+log:
+  level: "info"
+  format: "json"
--- a/tests/integration/nginx/platform/nginx.conf
+++ b/tests/integration/nginx/platform/nginx.conf
@ -0,0 +1,82 @@
+# nginx.conf — Stack de test nginx + ja4ebpf
+# HTTP/2 activé (h2) pour tester le parsing de la preface HTTP/2 par ja4ebpf.
+
+user nginx;
+worker_processes auto;
+error_log /var/log/nginx/error.log warn;
+pid /run/nginx/nginx.pid;
+
+events {
+    worker_connections 1024;
+    use epoll;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+    sendfile on;
+    keepalive_timeout 65;
+    keepalive_requests 200;
+
+    # Log format étendu (debug)
+    log_format main '$remote_addr - $remote_user [$time_local] '
+                    '"$request" $status $body_bytes_sent '
+                    '"$http_referer" "$http_user_agent" '
+                    'ssl_protocol=$ssl_protocol ssl_cipher=$ssl_cipher';
+
+    access_log /var/log/nginx/access.log main;
+
+    # ── Serveur HTTP (port 80) ─────────────────────────────────────────────
+    server {
+        listen 80;
+        server_name _;
+
+        location /health {
+            return 200 '{"status":"ok","stack":"nginx"}';
+            add_header Content-Type application/json;
+        }
+
+        # Redirection HTTPS optionnelle (trafic HTTP testé directement)
+        location / {
+            root /var/www/html;
+            index index.html;
+            try_files $uri $uri/ =404;
+        }
+    }
+
+    # ── Serveur HTTPS (port 443) avec HTTP/2 ──────────────────────────────
+    server {
+        listen 443 ssl;
+        http2 on;
+        server_name _;
+
+        ssl_certificate     /etc/pki/tls/certs/nginx.crt;
+        ssl_certificate_key /etc/pki/tls/private/nginx.key;
+
+        # Suites de chiffrement variées pour générer plusieurs JA4 distincts
+        ssl_protocols       TLSv1.2 TLSv1.3;
+        ssl_ciphers         HIGH:!aNULL:!MD5;
+        ssl_prefer_server_ciphers off;
+
+        # Activation du session resumption (teste le parsing ja4ebpf de session IDs)
+        ssl_session_cache   shared:SSL:10m;
+        ssl_session_timeout 10m;
+
+        location /health {
+            return 200 '{"status":"ok","stack":"nginx","tls":true}';
+            add_header Content-Type application/json;
+        }
+
+        location / {
+            root /var/www/html;
+            index index.html;
+            try_files $uri $uri/ =404;
+        }
+
+        # Endpoint POST pour tester la capture des requêtes avec body
+        location /api/ {
+            return 200 '{"result":"accepted"}';
+            add_header Content-Type application/json;
+        }
+    }
+}
--- a/tests/integration/nginx/run-tests.sh
+++ b/tests/integration/nginx/run-tests.sh
@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# =============================================================================
+# run-tests.sh — Tests d'intégration stack nginx + ja4ebpf
+#
+# Usage :
+#   ./run-tests.sh              # build + start + test + down
+#   ./run-tests.sh --no-down    # garder la stack après les tests
+#   ./run-tests.sh --build-only # build uniquement
+# =============================================================================
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Variables de configuration pour la bibliothèque partagée
+export STACK_NAME="nginx"
+export COMPOSE_FILE="$SCRIPT_DIR/docker-compose.yml"
+
+for arg in "$@"; do
+    case "$arg" in
+        --no-down)    export KEEP_UP=true ;;
+        --build-only) export BUILD_ONLY=true ;;
+    esac
+done
+
+# Chargement de la bibliothèque commune
+source "$SCRIPT_DIR/../lib/run-stack-tests.sh"
+
+# ── Vérifications spécifiques nginx ──────────────────────────────────────────
+stack_verify_extra() {
+    # Vérifie que nginx répond avec le bon Server header
+    local server_hdr
+    server_hdr=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        curl -skI https://localhost/ 2>/dev/null | grep -i "^Server:" | tr -d '\r' || echo "")
+    if echo "$server_hdr" | grep -qi "nginx"; then
+        pass "Header Server: nginx présent dans les réponses HTTPS"
+    else
+        warn "Header Server nginx non détecté (réponse: '$server_hdr')"
+    fi
+
+    # Vérifie HTTP/2 négocié (ALPN h2)
+    local alpn_result
+    alpn_result=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        curl -sk --http2 -w "%{http_version}" -o /dev/null https://localhost/ 2>/dev/null || echo "")
+    if [ "$alpn_result" = "2" ]; then
+        pass "HTTP/2 négocié avec succès sur nginx (ALPN h2)"
+    else
+        warn "HTTP/2 non négocié (version: '$alpn_result') — vérifier la config TLS nginx"
+    fi
+
+    # Vérifie que ja4ebpf est bien en cours d'exécution
+    local ja4_running
+    ja4_running=$(docker compose -f "$COMPOSE_FILE" exec -T platform \
+        pgrep -x ja4ebpf 2>/dev/null | head -1 || echo "")
+    if [ -n "$ja4_running" ]; then
+        pass "Processus ja4ebpf actif (PID $ja4_running)"
+    else
+        fail "Processus ja4ebpf introuvable dans le conteneur platform"
+    fi
+
+    # Vérifie SNI capturé (ja4ebpf parse le ClientHello → extrait SNI)
+    local sni_count
+    sni_count=$(ch_query "SELECT count() FROM ja4_logs.http_logs_raw WHERE sni != ''")
+    if [ "${sni_count:-0}" -gt 0 ] 2>/dev/null; then
+        local sni_sample
+        sni_sample=$(ch_query "SELECT sni FROM ja4_logs.http_logs_raw WHERE sni != '' LIMIT 1")
+        pass "SNI capturé : $sni_count enregistrements (exemple : '$sni_sample')"
+    else
+        warn "Aucun SNI capturé (trafic TLS peut-être sans extension SNI)"
+    fi
+}
+
+# Lancement de tous les phases
+run_all_phases