d81463a589
Complete rewrite of the RPM test harness to properly test ja4ebpf across CentOS 8, Rocky Linux 9/10 with nginx, apache, and hitch+varnish stacks. Key fixes: - Replace nested heredoc-over-SSH with upload-then-execute script pattern - Fix Apache SSLCertificateKey → SSLCertificateKeyFile directive - Fix hitch config: backend string syntax, user=nobody, combined PEM, --daemon - Fix timeout+shell-function incompatibility (timeout can't exec functions) - Fix subshell result counting by parsing output file instead of variables Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
373 lines
13 KiB
Bash
Executable File
373 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# test-rpm.sh — Test ja4ebpf via RPM sur les 3 distros × 3 stacks
|
||
#
|
||
# Usage:
|
||
# make rpm-ja4ebpf && ./tests/vm/test-rpm.sh [VM] [STACK]
|
||
# ./tests/vm/test-rpm.sh rocky9 nginx
|
||
# ./tests/vm/test-rpm.sh all all
|
||
#
|
||
# VMs: centos8 rocky9 rocky10 (or "all")
|
||
# Stacks: nginx apache hitch-varnish (or "all")
|
||
set -euo pipefail
|
||
|
||
PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
||
VM_DIR="$PROJECT_ROOT/tests/vm"
|
||
RPM_BASE="$PROJECT_ROOT/services/ja4ebpf/dist/output"
|
||
|
||
# Timeouts (seconds)
|
||
TEST_TIMEOUT=120 # max per VM×stack test
|
||
SSH_TIMEOUT=90 # max per SSH command
|
||
UPLOAD_TIMEOUT=60 # max for RPM upload
|
||
|
||
VM="${1:-rocky9}"
|
||
STACK="${2:-nginx}"
|
||
|
||
# Wrapper vagrant — always runs from the Vagrantfile directory
|
||
v() { (cd "$VM_DIR" && command vagrant "$@"); }
|
||
|
||
# SSH with timeout — uses bash -c to avoid timeout+function issue
|
||
vssh() {
|
||
local VM="$1"; shift
|
||
timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- \"\$@\"" -- "$@" 2>/dev/null
|
||
}
|
||
|
||
vm_to_el() {
|
||
case "$1" in
|
||
centos8) echo "el8" ;;
|
||
rocky9) echo "el9" ;;
|
||
rocky10) echo "el10" ;;
|
||
*) echo "UNKNOWN" ;;
|
||
esac
|
||
}
|
||
|
||
ALL_VMS="centos8 rocky9 rocky10"
|
||
ALL_STACKS="nginx apache hitch-varnish"
|
||
[ "$VM" = "all" ] && VMS="$ALL_VMS" || VMS="$VM"
|
||
[ "$STACK" = "all" ] && STACKS="$ALL_STACKS" || STACKS="$STACK"
|
||
|
||
PASS=0; FAIL=0; SKIP=0; RESULTS=()
|
||
|
||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||
echo "║ Test RPM ja4ebpf : VMS=$VMS STACKS=$STACKS"
|
||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||
|
||
# Check RPMs exist
|
||
for vm in $VMS; do
|
||
el=$(vm_to_el "$vm")
|
||
rpm=$(ls -t $RPM_BASE/$el/ja4ebpf-*.rpm 2>/dev/null | head -1)
|
||
if [ -z "$rpm" ]; then
|
||
echo "ERROR: no RPM for $vm ($el) in $RPM_BASE/$el/"; echo "Run: make rpm-ja4ebpf"; exit 1
|
||
fi
|
||
echo " $vm → $(basename $rpm)"
|
||
done
|
||
|
||
# ── Generate remote setup script ─────────────────────────────────────
|
||
|
||
generate_setup_script() {
|
||
local STACK="$1"
|
||
local SETUP_SCRIPT="/tmp/ja4-setup-$$$.sh"
|
||
|
||
cat > "$SETUP_SCRIPT" << 'SETUP_EOF'
|
||
#!/bin/bash
|
||
STACK="__STACK__"
|
||
|
||
# ── Stack functions ──────────────────────────────────────────────────
|
||
|
||
setup_nginx() {
|
||
mkdir -p /run/nginx /var/www/html
|
||
echo '{"ok":true}' > /var/www/html/health
|
||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||
-keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
|
||
cat > /etc/nginx/nginx.conf << 'NGINX_EOF'
|
||
worker_processes 1;
|
||
events { worker_connections 64; }
|
||
http {
|
||
access_log off;
|
||
server {
|
||
listen 80;
|
||
listen 443 ssl;
|
||
ssl_certificate /etc/pki/tls/certs/test.pem;
|
||
ssl_certificate_key /etc/pki/tls/private/test.key;
|
||
root /var/www/html;
|
||
}
|
||
}
|
||
NGINX_EOF
|
||
nginx && echo " nginx ready"
|
||
}
|
||
|
||
setup_apache() {
|
||
mkdir -p /var/www/html
|
||
echo '{"ok":true}' > /var/www/html/health
|
||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||
-keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
|
||
[ -f /etc/httpd/conf.d/ssl.conf ] && mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.disabled 2>/dev/null || true
|
||
cat > /etc/httpd/conf.d/test.conf << 'APACHE_EOF'
|
||
Listen 8080
|
||
<VirtualHost *:8080>
|
||
DocumentRoot /var/www/html
|
||
</VirtualHost>
|
||
Listen 8443
|
||
<VirtualHost *:8443>
|
||
DocumentRoot /var/www/html
|
||
SSLEngine on
|
||
SSLCertificateFile /etc/pki/tls/certs/test.pem
|
||
SSLCertificateKeyFile /etc/pki/tls/private/test.key
|
||
</VirtualHost>
|
||
APACHE_EOF
|
||
httpd && echo " httpd ready"
|
||
}
|
||
|
||
setup_hitch_varnish() {
|
||
mkdir -p /var/www/html
|
||
echo '{"ok":true}' > /var/www/html/health
|
||
openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
|
||
-keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
|
||
|
||
# Start simple HTTP backend
|
||
nohup python3 -c 'import http.server, os; os.chdir("/var/www/html"); http.server.HTTPServer(("127.0.0.1", 8081), http.server.SimpleHTTPRequestHandler).serve_forever()' </dev/null >/dev/null 2>&1 &
|
||
sleep 1
|
||
|
||
# Varnish
|
||
mkdir -p /etc/varnish
|
||
cat > /etc/varnish/default.vcl << 'VARNISH_EOF'
|
||
vcl 4.1;
|
||
backend default { .host = "127.0.0.1"; .port = "8081"; }
|
||
VARNISH_EOF
|
||
varnishd -a :80 -f /etc/varnish/default.vcl -s malloc,64m 2>/dev/null || true
|
||
sleep 1
|
||
|
||
# Hitch (combined PEM: key + cert)
|
||
mkdir -p /etc/hitch
|
||
cat /etc/pki/tls/private/test.key /etc/pki/tls/certs/test.pem > /etc/pki/tls/private/hitch.pem
|
||
cat > /etc/hitch/hitch.conf << 'HITCH_EOF'
|
||
frontend = {
|
||
host = "0.0.0.0"
|
||
port = "443"
|
||
tls = on
|
||
pem-file = "/etc/pki/tls/private/hitch.pem"
|
||
}
|
||
backend = "[127.0.0.1]:80"
|
||
user = "nobody"
|
||
HITCH_EOF
|
||
hitch --daemon --config=/etc/hitch/hitch.conf >/dev/null 2>&1 && echo ' hitch started' || echo ' hitch failed'
|
||
sleep 1
|
||
echo " hitch+varnish ready"
|
||
}
|
||
|
||
# ── Main setup ──────────────────────────────────────────────────────
|
||
|
||
# [A] Install RPM
|
||
echo " [A] Install RPM..."
|
||
pkill ja4ebpf 2>/dev/null || true; sleep 1
|
||
yum remove -y ja4ebpf 2>/dev/null || true
|
||
yum install -y /tmp/ja4ebpf-test.rpm 2>&1 | tail -3
|
||
[ -x /usr/sbin/ja4ebpf ] && echo " binary OK" || { echo " FAIL: binary not found"; exit 1; }
|
||
|
||
# [B] Configure
|
||
echo " [B] Configure..."
|
||
mkdir -p /etc/ja4ebpf
|
||
cat > /etc/ja4ebpf/config.yml << 'CONF_EOF'
|
||
interface: eth0
|
||
ssl_lib_path: "/usr/lib64/libssl.so.3"
|
||
debug: true
|
||
clickhouse:
|
||
dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs"
|
||
batch_size: 50
|
||
flush_secs: 1
|
||
correlation:
|
||
timeout_ms: 500
|
||
slowloris_ms: 10000
|
||
log:
|
||
level: "debug"
|
||
format: "text"
|
||
CONF_EOF
|
||
|
||
# [C] Start web stack
|
||
echo " [C] Stack: $STACK..."
|
||
nginx -s stop 2>/dev/null || true
|
||
httpd -k stop 2>/dev/null || true
|
||
pkill varnishd 2>/dev/null || true
|
||
pkill hitch 2>/dev/null || true
|
||
pkill -f 'python3 -c' 2>/dev/null || true
|
||
sleep 1
|
||
|
||
case "$STACK" in
|
||
nginx) setup_nginx ;;
|
||
apache) setup_apache ;;
|
||
hitch-varnish) setup_hitch_varnish ;;
|
||
esac
|
||
|
||
# Open firewall
|
||
firewall-cmd --add-service=http --add-service=https 2>/dev/null || true
|
||
firewall-cmd --add-port=8080/tcp --add-port=8443/tcp 2>/dev/null || true
|
||
|
||
# [D] Start ja4ebpf
|
||
echo " [D] Start ja4ebpf..."
|
||
JA4EBPF_CONFIG=/etc/ja4ebpf/config.yml /usr/sbin/ja4ebpf > /tmp/ja4-test.log 2>&1 &
|
||
JA4PID=$!
|
||
sleep 3
|
||
|
||
if ! kill -0 $JA4PID 2>/dev/null; then
|
||
echo " FAIL: ja4ebpf crashed"
|
||
cat /tmp/ja4-test.log
|
||
exit 1
|
||
fi
|
||
echo " PID=$JA4PID"
|
||
|
||
# Check TC ingress filter
|
||
if tc filter show dev eth0 ingress 2>/dev/null | grep -qi "bpf\|direct-action"; then
|
||
echo " TC: attached"
|
||
else
|
||
echo " WARN: TC filter not detected"
|
||
fi
|
||
SETUP_EOF
|
||
|
||
# Replace stack placeholder
|
||
sed -i "s/__STACK__/$STACK/" "$SETUP_SCRIPT"
|
||
echo "$SETUP_SCRIPT"
|
||
}
|
||
|
||
# ── Test one VM×stack combination ───────────────────────────────────
|
||
|
||
test_vm_stack() {
|
||
local VM="$1"
|
||
local STACK="$2"
|
||
local LABEL="$VM/$STACK"
|
||
local START_TIME=$(date +%s)
|
||
|
||
echo ""
|
||
echo "── $LABEL ─────────────────────────────────────────────"
|
||
|
||
# Check VM running
|
||
if ! v status "$VM" 2>/dev/null | grep -q "running"; then
|
||
echo " SKIP: VM not running"
|
||
echo " SKIP: VM not running (VM off)"
|
||
return
|
||
fi
|
||
|
||
# Find RPM
|
||
local EL=$(vm_to_el "$VM")
|
||
local RPM=$(ls -t $RPM_BASE/$EL/ja4ebpf-*.rpm | head -1)
|
||
echo " RPM: $(basename $RPM)"
|
||
|
||
# Upload RPM
|
||
if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $RPM /tmp/ja4ebpf-test.rpm $VM" 2>/dev/null; then
|
||
echo " FAIL: RPM upload timeout"
|
||
return
|
||
fi
|
||
|
||
# Get VM IP
|
||
local VM_IP=$(vssh "$VM" "ip -4 addr show eth0" | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
|
||
if [ -z "$VM_IP" ]; then
|
||
echo " SKIP: no eth0 IP"
|
||
return
|
||
fi
|
||
echo " IP: $VM_IP"
|
||
|
||
# Generate and upload setup script
|
||
local SETUP_SCRIPT=$(generate_setup_script "$STACK")
|
||
if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $SETUP_SCRIPT /tmp/ja4-setup.sh $VM" 2>/dev/null; then
|
||
echo " FAIL: setup script upload timeout"
|
||
rm -f "$SETUP_SCRIPT"
|
||
return
|
||
fi
|
||
rm -f "$SETUP_SCRIPT"
|
||
|
||
# Execute setup script on VM via SSH
|
||
if ! timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo bash /tmp/ja4-setup.sh'" 2>/dev/null; then
|
||
echo " FAIL: VM setup timeout/error"
|
||
return
|
||
fi
|
||
|
||
# [E] Generate traffic from HOST
|
||
echo " [E] Traffic host→VM..."
|
||
local HTTP_PORT HTTPS_PORT
|
||
case "$STACK" in
|
||
nginx) HTTP_PORT=80; HTTPS_PORT=443 ;;
|
||
apache) HTTP_PORT=8080; HTTPS_PORT=8443 ;;
|
||
hitch-varnish) HTTP_PORT=80; HTTPS_PORT=443 ;;
|
||
esac
|
||
|
||
for i in $(seq 1 3); do
|
||
curl -sf --max-time 5 "http://$VM_IP:$HTTP_PORT/health" -o /dev/null 2>&1 && echo " HTTP $i: OK" || echo " HTTP $i: FAIL"
|
||
curl -skf --max-time 5 "https://$VM_IP:$HTTPS_PORT/health" -o /dev/null 2>&1 && echo " HTTPS $i: OK" || echo " HTTPS $i: FAIL"
|
||
done
|
||
|
||
# [F] Wait for debug dump
|
||
sleep 6
|
||
|
||
# [G] Collect results
|
||
echo " [G] Results..."
|
||
vssh "$VM" "sudo cat /tmp/ja4-test.log" > /tmp/vm-ja4-log.txt 2>/dev/null || true
|
||
|
||
# Cleanup VM
|
||
timeout 15 bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo pkill ja4ebpf 2>/dev/null; sudo nginx -s stop 2>/dev/null; sudo httpd -k stop 2>/dev/null; sudo pkill varnishd 2>/dev/null; sudo pkill hitch 2>/dev/null; sudo pkill -f python3 2>/dev/null; echo cleanup_done'" 2>/dev/null > /dev/null || true
|
||
|
||
# Parse on host side
|
||
local LAST_DEBUG=$(grep "\[debug\] BPF:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1)
|
||
local LAST_GO=$(grep "\[debug\] GO:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1)
|
||
local SESSIONS=$(grep -c "session pr" /tmp/vm-ja4-log.txt 2>/dev/null || echo 0)
|
||
local SYN_SUB=$(echo "$LAST_DEBUG" | sed -n 's/.*SYN_SUB=\([0-9]*\).*/\1/p')
|
||
SYN_SUB=${SYN_SUB:-0}
|
||
|
||
local ELAPSED=$(( $(date +%s) - START_TIME ))
|
||
echo " BPF: $LAST_DEBUG"
|
||
echo " GO: $LAST_GO"
|
||
echo " sessions=$SESSIONS SYN_SUB=$SYN_SUB (${ELAPSED}s)"
|
||
|
||
if [ "$SYN_SUB" -gt 0 ] 2>/dev/null; then
|
||
echo " PASS: SYN_SUB=$SYN_SUB sessions=$SESSIONS"
|
||
else
|
||
echo " FAIL: SYN_SUB=0 (no TC capture)"
|
||
fi
|
||
}
|
||
|
||
# ── Main loop ────────────────────────────────────────────────────────
|
||
|
||
for v in $VMS; do
|
||
for s in $STACKS; do
|
||
LABEL="$v/$s"
|
||
TMPFILE=$(mktemp /tmp/ja4-test-XXXXXX.log)
|
||
|
||
# Run test_vm_stack in background, capture output
|
||
(
|
||
test_vm_stack "$v" "$s"
|
||
) > "$TMPFILE" 2>&1 &
|
||
TESTPID=$!
|
||
|
||
# Wait with timeout
|
||
if ! timeout --signal=TERM --kill-after=10 "$TEST_TIMEOUT" bash -c "while kill -0 $TESTPID 2>/dev/null; do sleep 1; done"; then
|
||
kill -9 $TESTPID 2>/dev/null || true
|
||
wait $TESTPID 2>/dev/null || true
|
||
echo ""
|
||
echo "── $LABEL ─────────────────────────────────────────────"
|
||
echo " TIMEOUT: exceeded ${TEST_TIMEOUT}s"
|
||
else
|
||
wait $TESTPID 2>/dev/null
|
||
fi
|
||
cat "$TMPFILE"
|
||
|
||
# Parse result from output
|
||
if grep -q "^ PASS:" "$TMPFILE" 2>/dev/null; then
|
||
PASS=$((PASS + 1)); RESULTS+=("PASS $LABEL")
|
||
elif grep -q "^ FAIL:" "$TMPFILE" 2>/dev/null; then
|
||
FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL")
|
||
elif grep -q "^ SKIP:" "$TMPFILE" 2>/dev/null; then
|
||
SKIP=$((SKIP + 1)); RESULTS+=("SKIP $LABEL")
|
||
else
|
||
FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL (no result)")
|
||
fi
|
||
rm -f "$TMPFILE"
|
||
done
|
||
done
|
||
|
||
# ── Summary ──────────────────────────────────────────────────────────
|
||
|
||
echo ""
|
||
echo "╔══════════════════════════════════════════════════════════════╗"
|
||
echo "║ SUMMARY: $PASS pass / $FAIL fail / $SKIP skip"
|
||
echo "╚══════════════════════════════════════════════════════════════╝"
|
||
for r in "${RESULTS[@]}"; do echo " $r"; done
|
||
|
||
[ "$FAIL" -eq 0 ] && exit 0 || exit 1
|