From d81463a5891d3b68527b0c01e2dbb041dd55f114 Mon Sep 17 00:00:00 2001 From: Jacquin Antoine Date: Mon, 13 Apr 2026 01:03:57 +0200 Subject: [PATCH] =?UTF-8?q?fix(tests):=20rewrite=20test-rpm.sh=20for=203?= =?UTF-8?q?=20distros=20=C3=97=203=20stacks=20RPM=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Complete rewrite of the RPM test harness to properly test ja4ebpf across CentOS 8, Rocky Linux 9/10 with nginx, apache, and hitch+varnish stacks. Key fixes: - Replace nested heredoc-over-SSH with upload-then-execute script pattern - Fix Apache SSLCertificateKey → SSLCertificateKeyFile directive - Fix hitch config: backend string syntax, user=nobody, combined PEM, --daemon - Fix timeout+shell-function incompatibility (timeout can't exec functions) - Fix subshell result counting by parsing output file instead of variables Co-Authored-By: Claude Opus 4.6 --- tests/vm/test-rpm.sh | 372 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 372 insertions(+) create mode 100755 tests/vm/test-rpm.sh diff --git a/tests/vm/test-rpm.sh b/tests/vm/test-rpm.sh new file mode 100755 index 0000000..00a7373 --- /dev/null +++ b/tests/vm/test-rpm.sh @@ -0,0 +1,372 @@ +#!/usr/bin/env bash +# test-rpm.sh — Test ja4ebpf via RPM sur les 3 distros × 3 stacks +# +# Usage: +# make rpm-ja4ebpf && ./tests/vm/test-rpm.sh [VM] [STACK] +# ./tests/vm/test-rpm.sh rocky9 nginx +# ./tests/vm/test-rpm.sh all all +# +# VMs: centos8 rocky9 rocky10 (or "all") +# Stacks: nginx apache hitch-varnish (or "all") +set -euo pipefail + +PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)" +VM_DIR="$PROJECT_ROOT/tests/vm" +RPM_BASE="$PROJECT_ROOT/services/ja4ebpf/dist/output" + +# Timeouts (seconds) +TEST_TIMEOUT=120 # max per VM×stack test +SSH_TIMEOUT=90 # max per SSH command +UPLOAD_TIMEOUT=60 # max for RPM upload + +VM="${1:-rocky9}" +STACK="${2:-nginx}" + +# Wrapper vagrant — always runs from the Vagrantfile directory +v() { (cd "$VM_DIR" && command vagrant "$@"); } + +# SSH with timeout — uses bash -c to avoid timeout+function issue +vssh() { + local VM="$1"; shift + timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- \"\$@\"" -- "$@" 2>/dev/null +} + +vm_to_el() { + case "$1" in + centos8) echo "el8" ;; + rocky9) echo "el9" ;; + rocky10) echo "el10" ;; + *) echo "UNKNOWN" ;; + esac +} + +ALL_VMS="centos8 rocky9 rocky10" +ALL_STACKS="nginx apache hitch-varnish" +[ "$VM" = "all" ] && VMS="$ALL_VMS" || VMS="$VM" +[ "$STACK" = "all" ] && STACKS="$ALL_STACKS" || STACKS="$STACK" + +PASS=0; FAIL=0; SKIP=0; RESULTS=() + +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ Test RPM ja4ebpf : VMS=$VMS STACKS=$STACKS" +echo "╚══════════════════════════════════════════════════════════════╝" + +# Check RPMs exist +for vm in $VMS; do + el=$(vm_to_el "$vm") + rpm=$(ls -t $RPM_BASE/$el/ja4ebpf-*.rpm 2>/dev/null | head -1) + if [ -z "$rpm" ]; then + echo "ERROR: no RPM for $vm ($el) in $RPM_BASE/$el/"; echo "Run: make rpm-ja4ebpf"; exit 1 + fi + echo " $vm → $(basename $rpm)" +done + +# ── Generate remote setup script ───────────────────────────────────── + +generate_setup_script() { + local STACK="$1" + local SETUP_SCRIPT="/tmp/ja4-setup-$$$.sh" + + cat > "$SETUP_SCRIPT" << 'SETUP_EOF' +#!/bin/bash +STACK="__STACK__" + +# ── Stack functions ────────────────────────────────────────────────── + +setup_nginx() { + mkdir -p /run/nginx /var/www/html + echo '{"ok":true}' > /var/www/html/health + openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \ + -keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true + cat > /etc/nginx/nginx.conf << 'NGINX_EOF' +worker_processes 1; +events { worker_connections 64; } +http { + access_log off; + server { + listen 80; + listen 443 ssl; + ssl_certificate /etc/pki/tls/certs/test.pem; + ssl_certificate_key /etc/pki/tls/private/test.key; + root /var/www/html; + } +} +NGINX_EOF + nginx && echo " nginx ready" +} + +setup_apache() { + mkdir -p /var/www/html + echo '{"ok":true}' > /var/www/html/health + openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \ + -keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true + [ -f /etc/httpd/conf.d/ssl.conf ] && mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.disabled 2>/dev/null || true + cat > /etc/httpd/conf.d/test.conf << 'APACHE_EOF' +Listen 8080 + + DocumentRoot /var/www/html + +Listen 8443 + + DocumentRoot /var/www/html + SSLEngine on + SSLCertificateFile /etc/pki/tls/certs/test.pem + SSLCertificateKeyFile /etc/pki/tls/private/test.key + +APACHE_EOF + httpd && echo " httpd ready" +} + +setup_hitch_varnish() { + mkdir -p /var/www/html + echo '{"ok":true}' > /var/www/html/health + openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \ + -keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true + + # Start simple HTTP backend + nohup python3 -c 'import http.server, os; os.chdir("/var/www/html"); http.server.HTTPServer(("127.0.0.1", 8081), http.server.SimpleHTTPRequestHandler).serve_forever()' /dev/null 2>&1 & + sleep 1 + + # Varnish + mkdir -p /etc/varnish + cat > /etc/varnish/default.vcl << 'VARNISH_EOF' +vcl 4.1; +backend default { .host = "127.0.0.1"; .port = "8081"; } +VARNISH_EOF + varnishd -a :80 -f /etc/varnish/default.vcl -s malloc,64m 2>/dev/null || true + sleep 1 + + # Hitch (combined PEM: key + cert) + mkdir -p /etc/hitch + cat /etc/pki/tls/private/test.key /etc/pki/tls/certs/test.pem > /etc/pki/tls/private/hitch.pem + cat > /etc/hitch/hitch.conf << 'HITCH_EOF' +frontend = { + host = "0.0.0.0" + port = "443" + tls = on + pem-file = "/etc/pki/tls/private/hitch.pem" +} +backend = "[127.0.0.1]:80" +user = "nobody" +HITCH_EOF + hitch --daemon --config=/etc/hitch/hitch.conf >/dev/null 2>&1 && echo ' hitch started' || echo ' hitch failed' + sleep 1 + echo " hitch+varnish ready" +} + +# ── Main setup ────────────────────────────────────────────────────── + +# [A] Install RPM +echo " [A] Install RPM..." +pkill ja4ebpf 2>/dev/null || true; sleep 1 +yum remove -y ja4ebpf 2>/dev/null || true +yum install -y /tmp/ja4ebpf-test.rpm 2>&1 | tail -3 +[ -x /usr/sbin/ja4ebpf ] && echo " binary OK" || { echo " FAIL: binary not found"; exit 1; } + +# [B] Configure +echo " [B] Configure..." +mkdir -p /etc/ja4ebpf +cat > /etc/ja4ebpf/config.yml << 'CONF_EOF' +interface: eth0 +ssl_lib_path: "/usr/lib64/libssl.so.3" +debug: true +clickhouse: + dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs" + batch_size: 50 + flush_secs: 1 +correlation: + timeout_ms: 500 + slowloris_ms: 10000 +log: + level: "debug" + format: "text" +CONF_EOF + +# [C] Start web stack +echo " [C] Stack: $STACK..." +nginx -s stop 2>/dev/null || true +httpd -k stop 2>/dev/null || true +pkill varnishd 2>/dev/null || true +pkill hitch 2>/dev/null || true +pkill -f 'python3 -c' 2>/dev/null || true +sleep 1 + +case "$STACK" in + nginx) setup_nginx ;; + apache) setup_apache ;; + hitch-varnish) setup_hitch_varnish ;; +esac + +# Open firewall +firewall-cmd --add-service=http --add-service=https 2>/dev/null || true +firewall-cmd --add-port=8080/tcp --add-port=8443/tcp 2>/dev/null || true + +# [D] Start ja4ebpf +echo " [D] Start ja4ebpf..." +JA4EBPF_CONFIG=/etc/ja4ebpf/config.yml /usr/sbin/ja4ebpf > /tmp/ja4-test.log 2>&1 & +JA4PID=$! +sleep 3 + +if ! kill -0 $JA4PID 2>/dev/null; then + echo " FAIL: ja4ebpf crashed" + cat /tmp/ja4-test.log + exit 1 +fi +echo " PID=$JA4PID" + +# Check TC ingress filter +if tc filter show dev eth0 ingress 2>/dev/null | grep -qi "bpf\|direct-action"; then + echo " TC: attached" +else + echo " WARN: TC filter not detected" +fi +SETUP_EOF + + # Replace stack placeholder + sed -i "s/__STACK__/$STACK/" "$SETUP_SCRIPT" + echo "$SETUP_SCRIPT" +} + +# ── Test one VM×stack combination ─────────────────────────────────── + +test_vm_stack() { + local VM="$1" + local STACK="$2" + local LABEL="$VM/$STACK" + local START_TIME=$(date +%s) + + echo "" + echo "── $LABEL ─────────────────────────────────────────────" + + # Check VM running + if ! v status "$VM" 2>/dev/null | grep -q "running"; then + echo " SKIP: VM not running" + echo " SKIP: VM not running (VM off)" + return + fi + + # Find RPM + local EL=$(vm_to_el "$VM") + local RPM=$(ls -t $RPM_BASE/$EL/ja4ebpf-*.rpm | head -1) + echo " RPM: $(basename $RPM)" + + # Upload RPM + if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $RPM /tmp/ja4ebpf-test.rpm $VM" 2>/dev/null; then + echo " FAIL: RPM upload timeout" + return + fi + + # Get VM IP + local VM_IP=$(vssh "$VM" "ip -4 addr show eth0" | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}') + if [ -z "$VM_IP" ]; then + echo " SKIP: no eth0 IP" + return + fi + echo " IP: $VM_IP" + + # Generate and upload setup script + local SETUP_SCRIPT=$(generate_setup_script "$STACK") + if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $SETUP_SCRIPT /tmp/ja4-setup.sh $VM" 2>/dev/null; then + echo " FAIL: setup script upload timeout" + rm -f "$SETUP_SCRIPT" + return + fi + rm -f "$SETUP_SCRIPT" + + # Execute setup script on VM via SSH + if ! timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo bash /tmp/ja4-setup.sh'" 2>/dev/null; then + echo " FAIL: VM setup timeout/error" + return + fi + + # [E] Generate traffic from HOST + echo " [E] Traffic host→VM..." + local HTTP_PORT HTTPS_PORT + case "$STACK" in + nginx) HTTP_PORT=80; HTTPS_PORT=443 ;; + apache) HTTP_PORT=8080; HTTPS_PORT=8443 ;; + hitch-varnish) HTTP_PORT=80; HTTPS_PORT=443 ;; + esac + + for i in $(seq 1 3); do + curl -sf --max-time 5 "http://$VM_IP:$HTTP_PORT/health" -o /dev/null 2>&1 && echo " HTTP $i: OK" || echo " HTTP $i: FAIL" + curl -skf --max-time 5 "https://$VM_IP:$HTTPS_PORT/health" -o /dev/null 2>&1 && echo " HTTPS $i: OK" || echo " HTTPS $i: FAIL" + done + + # [F] Wait for debug dump + sleep 6 + + # [G] Collect results + echo " [G] Results..." + vssh "$VM" "sudo cat /tmp/ja4-test.log" > /tmp/vm-ja4-log.txt 2>/dev/null || true + + # Cleanup VM + timeout 15 bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo pkill ja4ebpf 2>/dev/null; sudo nginx -s stop 2>/dev/null; sudo httpd -k stop 2>/dev/null; sudo pkill varnishd 2>/dev/null; sudo pkill hitch 2>/dev/null; sudo pkill -f python3 2>/dev/null; echo cleanup_done'" 2>/dev/null > /dev/null || true + + # Parse on host side + local LAST_DEBUG=$(grep "\[debug\] BPF:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1) + local LAST_GO=$(grep "\[debug\] GO:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1) + local SESSIONS=$(grep -c "session pr" /tmp/vm-ja4-log.txt 2>/dev/null || echo 0) + local SYN_SUB=$(echo "$LAST_DEBUG" | sed -n 's/.*SYN_SUB=\([0-9]*\).*/\1/p') + SYN_SUB=${SYN_SUB:-0} + + local ELAPSED=$(( $(date +%s) - START_TIME )) + echo " BPF: $LAST_DEBUG" + echo " GO: $LAST_GO" + echo " sessions=$SESSIONS SYN_SUB=$SYN_SUB (${ELAPSED}s)" + + if [ "$SYN_SUB" -gt 0 ] 2>/dev/null; then + echo " PASS: SYN_SUB=$SYN_SUB sessions=$SESSIONS" + else + echo " FAIL: SYN_SUB=0 (no TC capture)" + fi +} + +# ── Main loop ──────────────────────────────────────────────────────── + +for v in $VMS; do + for s in $STACKS; do + LABEL="$v/$s" + TMPFILE=$(mktemp /tmp/ja4-test-XXXXXX.log) + + # Run test_vm_stack in background, capture output + ( + test_vm_stack "$v" "$s" + ) > "$TMPFILE" 2>&1 & + TESTPID=$! + + # Wait with timeout + if ! timeout --signal=TERM --kill-after=10 "$TEST_TIMEOUT" bash -c "while kill -0 $TESTPID 2>/dev/null; do sleep 1; done"; then + kill -9 $TESTPID 2>/dev/null || true + wait $TESTPID 2>/dev/null || true + echo "" + echo "── $LABEL ─────────────────────────────────────────────" + echo " TIMEOUT: exceeded ${TEST_TIMEOUT}s" + else + wait $TESTPID 2>/dev/null + fi + cat "$TMPFILE" + + # Parse result from output + if grep -q "^ PASS:" "$TMPFILE" 2>/dev/null; then + PASS=$((PASS + 1)); RESULTS+=("PASS $LABEL") + elif grep -q "^ FAIL:" "$TMPFILE" 2>/dev/null; then + FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL") + elif grep -q "^ SKIP:" "$TMPFILE" 2>/dev/null; then + SKIP=$((SKIP + 1)); RESULTS+=("SKIP $LABEL") + else + FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL (no result)") + fi + rm -f "$TMPFILE" + done +done + +# ── Summary ────────────────────────────────────────────────────────── + +echo "" +echo "╔══════════════════════════════════════════════════════════════╗" +echo "║ SUMMARY: $PASS pass / $FAIL fail / $SKIP skip" +echo "╚══════════════════════════════════════════════════════════════╝" +for r in "${RESULTS[@]}"; do echo " $r"; done + +[ "$FAIL" -eq 0 ] && exit 0 || exit 1