fix(tests): rewrite test-rpm.sh for 3 distros × 3 stacks RPM validation
Complete rewrite of the RPM test harness to properly test ja4ebpf across CentOS 8, Rocky Linux 9/10 with nginx, apache, and hitch+varnish stacks. Key fixes: - Replace nested heredoc-over-SSH with upload-then-execute script pattern - Fix Apache SSLCertificateKey → SSLCertificateKeyFile directive - Fix hitch config: backend string syntax, user=nobody, combined PEM, --daemon - Fix timeout+shell-function incompatibility (timeout can't exec functions) - Fix subshell result counting by parsing output file instead of variables Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
372
tests/vm/test-rpm.sh
Executable file
372
tests/vm/test-rpm.sh
Executable file
@ -0,0 +1,372 @@
|
||||
#!/usr/bin/env bash
|
||||
# test-rpm.sh — Test ja4ebpf via RPM sur les 3 distros × 3 stacks
|
||||
#
|
||||
# Usage:
|
||||
# make rpm-ja4ebpf && ./tests/vm/test-rpm.sh [VM] [STACK]
|
||||
# ./tests/vm/test-rpm.sh rocky9 nginx
|
||||
# ./tests/vm/test-rpm.sh all all
|
||||
#
|
||||
# VMs: centos8 rocky9 rocky10 (or "all")
|
||||
# Stacks: nginx apache hitch-varnish (or "all")
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||||
VM_DIR="$PROJECT_ROOT/tests/vm"
|
||||
RPM_BASE="$PROJECT_ROOT/services/ja4ebpf/dist/output"
|
||||
|
||||
# Timeouts (seconds)
|
||||
TEST_TIMEOUT=120 # max per VM×stack test
|
||||
SSH_TIMEOUT=90 # max per SSH command
|
||||
UPLOAD_TIMEOUT=60 # max for RPM upload
|
||||
|
||||
VM="${1:-rocky9}"
|
||||
STACK="${2:-nginx}"
|
||||
|
||||
# Wrapper vagrant — always runs from the Vagrantfile directory
|
||||
v() { (cd "$VM_DIR" && command vagrant "$@"); }
|
||||
|
||||
# SSH with timeout — uses bash -c to avoid timeout+function issue
|
||||
vssh() {
|
||||
local VM="$1"; shift
|
||||
timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- \"\$@\"" -- "$@" 2>/dev/null
|
||||
}
|
||||
|
||||
vm_to_el() {
|
||||
case "$1" in
|
||||
centos8) echo "el8" ;;
|
||||
rocky9) echo "el9" ;;
|
||||
rocky10) echo "el10" ;;
|
||||
*) echo "UNKNOWN" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
ALL_VMS="centos8 rocky9 rocky10"
|
||||
ALL_STACKS="nginx apache hitch-varnish"
|
||||
[ "$VM" = "all" ] && VMS="$ALL_VMS" || VMS="$VM"
|
||||
[ "$STACK" = "all" ] && STACKS="$ALL_STACKS" || STACKS="$STACK"
|
||||
|
||||
PASS=0; FAIL=0; SKIP=0; RESULTS=()
|
||||
|
||||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||||
echo "║ Test RPM ja4ebpf : VMS=$VMS STACKS=$STACKS"
|
||||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||||
|
||||
# Check RPMs exist
|
||||
for vm in $VMS; do
|
||||
el=$(vm_to_el "$vm")
|
||||
rpm=$(ls -t $RPM_BASE/$el/ja4ebpf-*.rpm 2>/dev/null | head -1)
|
||||
if [ -z "$rpm" ]; then
|
||||
echo "ERROR: no RPM for $vm ($el) in $RPM_BASE/$el/"; echo "Run: make rpm-ja4ebpf"; exit 1
|
||||
fi
|
||||
echo " $vm → $(basename $rpm)"
|
||||
done
|
||||
|
||||
# ── Generate remote setup script ─────────────────────────────────────
|
||||
|
||||
generate_setup_script() {
|
||||
local STACK="$1"
|
||||
local SETUP_SCRIPT="/tmp/ja4-setup-$$$.sh"
|
||||
|
||||
cat > "$SETUP_SCRIPT" << 'SETUP_EOF'
|
||||
#!/bin/bash
|
||||
STACK="__STACK__"
|
||||
|
||||
# ── Stack functions ──────────────────────────────────────────────────
|
||||
|
||||
setup_nginx() {
|
||||
mkdir -p /run/nginx /var/www/html
|
||||
echo '{"ok":true}' > /var/www/html/health
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
|
||||
cat > /etc/nginx/nginx.conf << 'NGINX_EOF'
|
||||
worker_processes 1;
|
||||
events { worker_connections 64; }
|
||||
http {
|
||||
access_log off;
|
||||
server {
|
||||
listen 80;
|
||||
listen 443 ssl;
|
||||
ssl_certificate /etc/pki/tls/certs/test.pem;
|
||||
ssl_certificate_key /etc/pki/tls/private/test.key;
|
||||
root /var/www/html;
|
||||
}
|
||||
}
|
||||
NGINX_EOF
|
||||
nginx && echo " nginx ready"
|
||||
}
|
||||
|
||||
setup_apache() {
|
||||
mkdir -p /var/www/html
|
||||
echo '{"ok":true}' > /var/www/html/health
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
|
||||
[ -f /etc/httpd/conf.d/ssl.conf ] && mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.disabled 2>/dev/null || true
|
||||
cat > /etc/httpd/conf.d/test.conf << 'APACHE_EOF'
|
||||
Listen 8080
|
||||
<VirtualHost *:8080>
|
||||
DocumentRoot /var/www/html
|
||||
</VirtualHost>
|
||||
Listen 8443
|
||||
<VirtualHost *:8443>
|
||||
DocumentRoot /var/www/html
|
||||
SSLEngine on
|
||||
SSLCertificateFile /etc/pki/tls/certs/test.pem
|
||||
SSLCertificateKeyFile /etc/pki/tls/private/test.key
|
||||
</VirtualHost>
|
||||
APACHE_EOF
|
||||
httpd && echo " httpd ready"
|
||||
}
|
||||
|
||||
setup_hitch_varnish() {
|
||||
mkdir -p /var/www/html
|
||||
echo '{"ok":true}' > /var/www/html/health
|
||||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||||
-keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
|
||||
|
||||
# Start simple HTTP backend
|
||||
nohup python3 -c 'import http.server, os; os.chdir("/var/www/html"); http.server.HTTPServer(("127.0.0.1", 8081), http.server.SimpleHTTPRequestHandler).serve_forever()' </dev/null >/dev/null 2>&1 &
|
||||
sleep 1
|
||||
|
||||
# Varnish
|
||||
mkdir -p /etc/varnish
|
||||
cat > /etc/varnish/default.vcl << 'VARNISH_EOF'
|
||||
vcl 4.1;
|
||||
backend default { .host = "127.0.0.1"; .port = "8081"; }
|
||||
VARNISH_EOF
|
||||
varnishd -a :80 -f /etc/varnish/default.vcl -s malloc,64m 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
# Hitch (combined PEM: key + cert)
|
||||
mkdir -p /etc/hitch
|
||||
cat /etc/pki/tls/private/test.key /etc/pki/tls/certs/test.pem > /etc/pki/tls/private/hitch.pem
|
||||
cat > /etc/hitch/hitch.conf << 'HITCH_EOF'
|
||||
frontend = {
|
||||
host = "0.0.0.0"
|
||||
port = "443"
|
||||
tls = on
|
||||
pem-file = "/etc/pki/tls/private/hitch.pem"
|
||||
}
|
||||
backend = "[127.0.0.1]:80"
|
||||
user = "nobody"
|
||||
HITCH_EOF
|
||||
hitch --daemon --config=/etc/hitch/hitch.conf >/dev/null 2>&1 && echo ' hitch started' || echo ' hitch failed'
|
||||
sleep 1
|
||||
echo " hitch+varnish ready"
|
||||
}
|
||||
|
||||
# ── Main setup ──────────────────────────────────────────────────────
|
||||
|
||||
# [A] Install RPM
|
||||
echo " [A] Install RPM..."
|
||||
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
||||
yum remove -y ja4ebpf 2>/dev/null || true
|
||||
yum install -y /tmp/ja4ebpf-test.rpm 2>&1 | tail -3
|
||||
[ -x /usr/sbin/ja4ebpf ] && echo " binary OK" || { echo " FAIL: binary not found"; exit 1; }
|
||||
|
||||
# [B] Configure
|
||||
echo " [B] Configure..."
|
||||
mkdir -p /etc/ja4ebpf
|
||||
cat > /etc/ja4ebpf/config.yml << 'CONF_EOF'
|
||||
interface: eth0
|
||||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||||
debug: true
|
||||
clickhouse:
|
||||
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
||||
batch_size: 50
|
||||
flush_secs: 1
|
||||
correlation:
|
||||
timeout_ms: 500
|
||||
slowloris_ms: 10000
|
||||
log:
|
||||
level: "debug"
|
||||
format: "text"
|
||||
CONF_EOF
|
||||
|
||||
# [C] Start web stack
|
||||
echo " [C] Stack: $STACK..."
|
||||
nginx -s stop 2>/dev/null || true
|
||||
httpd -k stop 2>/dev/null || true
|
||||
pkill varnishd 2>/dev/null || true
|
||||
pkill hitch 2>/dev/null || true
|
||||
pkill -f 'python3 -c' 2>/dev/null || true
|
||||
sleep 1
|
||||
|
||||
case "$STACK" in
|
||||
nginx) setup_nginx ;;
|
||||
apache) setup_apache ;;
|
||||
hitch-varnish) setup_hitch_varnish ;;
|
||||
esac
|
||||
|
||||
# Open firewall
|
||||
firewall-cmd --add-service=http --add-service=https 2>/dev/null || true
|
||||
firewall-cmd --add-port=8080/tcp --add-port=8443/tcp 2>/dev/null || true
|
||||
|
||||
# [D] Start ja4ebpf
|
||||
echo " [D] Start ja4ebpf..."
|
||||
JA4EBPF_CONFIG=/etc/ja4ebpf/config.yml /usr/sbin/ja4ebpf > /tmp/ja4-test.log 2>&1 &
|
||||
JA4PID=$!
|
||||
sleep 3
|
||||
|
||||
if ! kill -0 $JA4PID 2>/dev/null; then
|
||||
echo " FAIL: ja4ebpf crashed"
|
||||
cat /tmp/ja4-test.log
|
||||
exit 1
|
||||
fi
|
||||
echo " PID=$JA4PID"
|
||||
|
||||
# Check TC ingress filter
|
||||
if tc filter show dev eth0 ingress 2>/dev/null | grep -qi "bpf\|direct-action"; then
|
||||
echo " TC: attached"
|
||||
else
|
||||
echo " WARN: TC filter not detected"
|
||||
fi
|
||||
SETUP_EOF
|
||||
|
||||
# Replace stack placeholder
|
||||
sed -i "s/__STACK__/$STACK/" "$SETUP_SCRIPT"
|
||||
echo "$SETUP_SCRIPT"
|
||||
}
|
||||
|
||||
# ── Test one VM×stack combination ───────────────────────────────────
|
||||
|
||||
test_vm_stack() {
|
||||
local VM="$1"
|
||||
local STACK="$2"
|
||||
local LABEL="$VM/$STACK"
|
||||
local START_TIME=$(date +%s)
|
||||
|
||||
echo ""
|
||||
echo "── $LABEL ─────────────────────────────────────────────"
|
||||
|
||||
# Check VM running
|
||||
if ! v status "$VM" 2>/dev/null | grep -q "running"; then
|
||||
echo " SKIP: VM not running"
|
||||
echo " SKIP: VM not running (VM off)"
|
||||
return
|
||||
fi
|
||||
|
||||
# Find RPM
|
||||
local EL=$(vm_to_el "$VM")
|
||||
local RPM=$(ls -t $RPM_BASE/$EL/ja4ebpf-*.rpm | head -1)
|
||||
echo " RPM: $(basename $RPM)"
|
||||
|
||||
# Upload RPM
|
||||
if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $RPM /tmp/ja4ebpf-test.rpm $VM" 2>/dev/null; then
|
||||
echo " FAIL: RPM upload timeout"
|
||||
return
|
||||
fi
|
||||
|
||||
# Get VM IP
|
||||
local VM_IP=$(vssh "$VM" "ip -4 addr show eth0" | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
|
||||
if [ -z "$VM_IP" ]; then
|
||||
echo " SKIP: no eth0 IP"
|
||||
return
|
||||
fi
|
||||
echo " IP: $VM_IP"
|
||||
|
||||
# Generate and upload setup script
|
||||
local SETUP_SCRIPT=$(generate_setup_script "$STACK")
|
||||
if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $SETUP_SCRIPT /tmp/ja4-setup.sh $VM" 2>/dev/null; then
|
||||
echo " FAIL: setup script upload timeout"
|
||||
rm -f "$SETUP_SCRIPT"
|
||||
return
|
||||
fi
|
||||
rm -f "$SETUP_SCRIPT"
|
||||
|
||||
# Execute setup script on VM via SSH
|
||||
if ! timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo bash /tmp/ja4-setup.sh'" 2>/dev/null; then
|
||||
echo " FAIL: VM setup timeout/error"
|
||||
return
|
||||
fi
|
||||
|
||||
# [E] Generate traffic from HOST
|
||||
echo " [E] Traffic host→VM..."
|
||||
local HTTP_PORT HTTPS_PORT
|
||||
case "$STACK" in
|
||||
nginx) HTTP_PORT=80; HTTPS_PORT=443 ;;
|
||||
apache) HTTP_PORT=8080; HTTPS_PORT=8443 ;;
|
||||
hitch-varnish) HTTP_PORT=80; HTTPS_PORT=443 ;;
|
||||
esac
|
||||
|
||||
for i in $(seq 1 3); do
|
||||
curl -sf --max-time 5 "http://$VM_IP:$HTTP_PORT/health" -o /dev/null 2>&1 && echo " HTTP $i: OK" || echo " HTTP $i: FAIL"
|
||||
curl -skf --max-time 5 "https://$VM_IP:$HTTPS_PORT/health" -o /dev/null 2>&1 && echo " HTTPS $i: OK" || echo " HTTPS $i: FAIL"
|
||||
done
|
||||
|
||||
# [F] Wait for debug dump
|
||||
sleep 6
|
||||
|
||||
# [G] Collect results
|
||||
echo " [G] Results..."
|
||||
vssh "$VM" "sudo cat /tmp/ja4-test.log" > /tmp/vm-ja4-log.txt 2>/dev/null || true
|
||||
|
||||
# Cleanup VM
|
||||
timeout 15 bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo pkill ja4ebpf 2>/dev/null; sudo nginx -s stop 2>/dev/null; sudo httpd -k stop 2>/dev/null; sudo pkill varnishd 2>/dev/null; sudo pkill hitch 2>/dev/null; sudo pkill -f python3 2>/dev/null; echo cleanup_done'" 2>/dev/null > /dev/null || true
|
||||
|
||||
# Parse on host side
|
||||
local LAST_DEBUG=$(grep "\[debug\] BPF:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1)
|
||||
local LAST_GO=$(grep "\[debug\] GO:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1)
|
||||
local SESSIONS=$(grep -c "session pr" /tmp/vm-ja4-log.txt 2>/dev/null || echo 0)
|
||||
local SYN_SUB=$(echo "$LAST_DEBUG" | sed -n 's/.*SYN_SUB=\([0-9]*\).*/\1/p')
|
||||
SYN_SUB=${SYN_SUB:-0}
|
||||
|
||||
local ELAPSED=$(( $(date +%s) - START_TIME ))
|
||||
echo " BPF: $LAST_DEBUG"
|
||||
echo " GO: $LAST_GO"
|
||||
echo " sessions=$SESSIONS SYN_SUB=$SYN_SUB (${ELAPSED}s)"
|
||||
|
||||
if [ "$SYN_SUB" -gt 0 ] 2>/dev/null; then
|
||||
echo " PASS: SYN_SUB=$SYN_SUB sessions=$SESSIONS"
|
||||
else
|
||||
echo " FAIL: SYN_SUB=0 (no TC capture)"
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Main loop ────────────────────────────────────────────────────────
|
||||
|
||||
for v in $VMS; do
|
||||
for s in $STACKS; do
|
||||
LABEL="$v/$s"
|
||||
TMPFILE=$(mktemp /tmp/ja4-test-XXXXXX.log)
|
||||
|
||||
# Run test_vm_stack in background, capture output
|
||||
(
|
||||
test_vm_stack "$v" "$s"
|
||||
) > "$TMPFILE" 2>&1 &
|
||||
TESTPID=$!
|
||||
|
||||
# Wait with timeout
|
||||
if ! timeout --signal=TERM --kill-after=10 "$TEST_TIMEOUT" bash -c "while kill -0 $TESTPID 2>/dev/null; do sleep 1; done"; then
|
||||
kill -9 $TESTPID 2>/dev/null || true
|
||||
wait $TESTPID 2>/dev/null || true
|
||||
echo ""
|
||||
echo "── $LABEL ─────────────────────────────────────────────"
|
||||
echo " TIMEOUT: exceeded ${TEST_TIMEOUT}s"
|
||||
else
|
||||
wait $TESTPID 2>/dev/null
|
||||
fi
|
||||
cat "$TMPFILE"
|
||||
|
||||
# Parse result from output
|
||||
if grep -q "^ PASS:" "$TMPFILE" 2>/dev/null; then
|
||||
PASS=$((PASS + 1)); RESULTS+=("PASS $LABEL")
|
||||
elif grep -q "^ FAIL:" "$TMPFILE" 2>/dev/null; then
|
||||
FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL")
|
||||
elif grep -q "^ SKIP:" "$TMPFILE" 2>/dev/null; then
|
||||
SKIP=$((SKIP + 1)); RESULTS+=("SKIP $LABEL")
|
||||
else
|
||||
FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL (no result)")
|
||||
fi
|
||||
rm -f "$TMPFILE"
|
||||
done
|
||||
done
|
||||
|
||||
# ── Summary ──────────────────────────────────────────────────────────
|
||||
|
||||
echo ""
|
||||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||||
echo "║ SUMMARY: $PASS pass / $FAIL fail / $SKIP skip"
|
||||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||||
for r in "${RESULTS[@]}"; do echo " $r"; done
|
||||
|
||||
[ "$FAIL" -eq 0 ] && exit 0 || exit 1
|
||||
Reference in New Issue
Block a user