ja4-platform/tests/vm/test-rpm.sh

#!/usr/bin/env bash
# test-rpm.sh — Test ja4ebpf via RPM sur les 3 distros × 3 stacks
#
# Usage:
#   make rpm-ja4ebpf && ./tests/vm/test-rpm.sh [VM] [STACK]
#   ./tests/vm/test-rpm.sh rocky9 nginx
#   ./tests/vm/test-rpm.sh all all
#
# VMs:    centos8 rocky9 rocky10 (or "all")
# Stacks: nginx apache hitch-varnish (or "all")
set -euo pipefail

PROJECT_ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
VM_DIR="$PROJECT_ROOT/tests/vm"
RPM_BASE="$PROJECT_ROOT/services/ja4ebpf/dist/output"

# Timeouts (seconds)
TEST_TIMEOUT=120      # max per VM×stack test
SSH_TIMEOUT=90        # max per SSH command
UPLOAD_TIMEOUT=60     # max for RPM upload

VM="${1:-rocky9}"
STACK="${2:-nginx}"

# Wrapper vagrant — always runs from the Vagrantfile directory
v() { (cd "$VM_DIR" && command vagrant "$@"); }

# SSH with timeout — uses bash -c to avoid timeout+function issue
vssh() {
    local VM="$1"; shift
    timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- \"\$@\"" -- "$@" 2>/dev/null
}

vm_to_el() {
    case "$1" in
        centos8) echo "el8" ;;
        rocky9)  echo "el9" ;;
        rocky10) echo "el10" ;;
        *) echo "UNKNOWN" ;;
    esac
}

ALL_VMS="centos8 rocky9 rocky10"
ALL_STACKS="nginx apache hitch-varnish"
[ "$VM" = "all" ] && VMS="$ALL_VMS" || VMS="$VM"
[ "$STACK" = "all" ] && STACKS="$ALL_STACKS" || STACKS="$STACK"

PASS=0; FAIL=0; SKIP=0; RESULTS=()

echo "╔══════════════════════════════════════════════════════════════╗"
echo "║  Test RPM ja4ebpf : VMS=$VMS  STACKS=$STACKS"
echo "╚══════════════════════════════════════════════════════════════╝"

# Check RPMs exist
for vm in $VMS; do
    el=$(vm_to_el "$vm")
    rpm=$(ls -t $RPM_BASE/$el/ja4ebpf-*.rpm 2>/dev/null | head -1)
    if [ -z "$rpm" ]; then
        echo "ERROR: no RPM for $vm ($el) in $RPM_BASE/$el/"; echo "Run: make rpm-ja4ebpf"; exit 1
    fi
    echo "  $vm → $(basename $rpm)"
done

# ── Generate remote setup script ─────────────────────────────────────

generate_setup_script() {
    local STACK="$1"
    local SETUP_SCRIPT="/tmp/ja4-setup-$$$.sh"

    cat > "$SETUP_SCRIPT" << 'SETUP_EOF'
#!/bin/bash
STACK="__STACK__"

# ── Stack functions ──────────────────────────────────────────────────

setup_nginx() {
    mkdir -p /run/nginx /var/www/html
    echo '{"ok":true}' > /var/www/html/health
    openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
        -keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
    cat > /etc/nginx/nginx.conf << 'NGINX_EOF'
worker_processes 1;
events { worker_connections 64; }
http {
    access_log off;
    server {
        listen 80;
        listen 443 ssl;
        ssl_certificate /etc/pki/tls/certs/test.pem;
        ssl_certificate_key /etc/pki/tls/private/test.key;
        root /var/www/html;
    }
}
NGINX_EOF
    nginx && echo "  nginx ready"
}

setup_apache() {
    mkdir -p /var/www/html
    echo '{"ok":true}' > /var/www/html/health
    openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
        -keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true
    [ -f /etc/httpd/conf.d/ssl.conf ] && mv /etc/httpd/conf.d/ssl.conf /etc/httpd/conf.d/ssl.conf.disabled 2>/dev/null || true
    cat > /etc/httpd/conf.d/test.conf << 'APACHE_EOF'
Listen 8080
<VirtualHost *:8080>
    DocumentRoot /var/www/html
</VirtualHost>
Listen 8443
<VirtualHost *:8443>
    DocumentRoot /var/www/html
    SSLEngine on
    SSLCertificateFile /etc/pki/tls/certs/test.pem
    SSLCertificateKeyFile /etc/pki/tls/private/test.key
</VirtualHost>
APACHE_EOF
    httpd && echo "  httpd ready"
}

setup_hitch_varnish() {
    mkdir -p /var/www/html
    echo '{"ok":true}' > /var/www/html/health
    openssl req -x509 -nodes -days 365 -subj /CN=test -newkey rsa:2048 \
        -keyout /etc/pki/tls/private/test.key -out /etc/pki/tls/certs/test.pem 2>/dev/null || true

    # Start simple HTTP backend
    nohup python3 -c 'import http.server, os; os.chdir("/var/www/html"); http.server.HTTPServer(("127.0.0.1", 8081), http.server.SimpleHTTPRequestHandler).serve_forever()' </dev/null >/dev/null 2>&1 &
    sleep 1

    # Varnish
    mkdir -p /etc/varnish
    cat > /etc/varnish/default.vcl << 'VARNISH_EOF'
vcl 4.1;
backend default { .host = "127.0.0.1"; .port = "8081"; }
VARNISH_EOF
    varnishd -a :80 -f /etc/varnish/default.vcl -s malloc,64m 2>/dev/null || true
    sleep 1

    # Hitch (combined PEM: key + cert)
    mkdir -p /etc/hitch
    cat /etc/pki/tls/private/test.key /etc/pki/tls/certs/test.pem > /etc/pki/tls/private/hitch.pem
    cat > /etc/hitch/hitch.conf << 'HITCH_EOF'
frontend = {
    host = "0.0.0.0"
    port = "443"
    tls = on
    pem-file = "/etc/pki/tls/private/hitch.pem"
}
backend = "[127.0.0.1]:80"
user = "nobody"
HITCH_EOF
    hitch --daemon --config=/etc/hitch/hitch.conf >/dev/null 2>&1 && echo '  hitch started' || echo '  hitch failed'
    sleep 1
    echo "  hitch+varnish ready"
}

# ── Main setup ──────────────────────────────────────────────────────

# [A] Install RPM
echo "  [A] Install RPM..."
pkill ja4ebpf 2>/dev/null || true; sleep 1
yum remove -y ja4ebpf 2>/dev/null || true
yum install -y /tmp/ja4ebpf-test.rpm 2>&1 | tail -3
[ -x /usr/sbin/ja4ebpf ] && echo "  binary OK" || { echo "  FAIL: binary not found"; exit 1; }

# [B] Configure
echo "  [B] Configure..."
mkdir -p /etc/ja4ebpf
cat > /etc/ja4ebpf/config.yml << 'CONF_EOF'
interfaces:
  - any
ssl_lib_path: "/usr/lib64/libssl.so.3"
listen_ports:
  - 80
  - 443
debug: true
clickhouse:
  dsn: "clickhouse://default:@127.0.0.1:9000/ja4_logs?async_insert=0"
  batch_size: 50
  flush_secs: 1
correlation:
  timeout_ms: 500
  slowloris_ms: 10000
log:
  level: "debug"
  format: "text"
CONF_EOF

# [C] Start web stack
echo "  [C] Stack: $STACK..."
nginx -s stop 2>/dev/null || true
httpd -k stop 2>/dev/null || true
pkill varnishd 2>/dev/null || true
pkill hitch 2>/dev/null || true
pkill -f 'python3 -c' 2>/dev/null || true
sleep 1

case "$STACK" in
    nginx)         setup_nginx ;;
    apache)        setup_apache ;;
    hitch-varnish) setup_hitch_varnish ;;
esac

# Open firewall
firewall-cmd --add-service=http --add-service=https 2>/dev/null || true
firewall-cmd --add-port=8080/tcp --add-port=8443/tcp 2>/dev/null || true

# [D] Start ja4ebpf
echo "  [D] Start ja4ebpf..."
JA4EBPF_CONFIG=/etc/ja4ebpf/config.yml /usr/sbin/ja4ebpf > /tmp/ja4-test.log 2>&1 &
JA4PID=$!
sleep 3

if ! kill -0 $JA4PID 2>/dev/null; then
    echo "  FAIL: ja4ebpf crashed"
    cat /tmp/ja4-test.log
    exit 1
fi
echo "  PID=$JA4PID"

# Check TC ingress filter (multi-interface)
ATTACHED=0
for IFACE in $(ls /sys/class/net/ 2>/dev/null | grep -v lo); do
    if tc filter show dev "$IFACE" ingress 2>/dev/null | grep -qi "bpf\|direct-action"; then
        ATTACHED=$((ATTACHED + 1))
    fi
done
if [ "$ATTACHED" -gt 0 ]; then
    echo "  TC: attached on $ATTACHED interface(s)"
else
    echo "  WARN: TC filter not detected on any interface"
fi
SETUP_EOF

    # Replace stack placeholder
    sed -i "s/__STACK__/$STACK/" "$SETUP_SCRIPT"
    echo "$SETUP_SCRIPT"
}

# ── Test one VM×stack combination ───────────────────────────────────

test_vm_stack() {
    local VM="$1"
    local STACK="$2"
    local LABEL="$VM/$STACK"
    local START_TIME=$(date +%s)

    echo ""
    echo "── $LABEL ─────────────────────────────────────────────"

    # Check VM running
    if ! v status "$VM" 2>/dev/null | grep -q "running"; then
        echo "  SKIP: VM not running"
        echo "  SKIP: VM not running (VM off)"
        return
    fi

    # Find RPM
    local EL=$(vm_to_el "$VM")
    local RPM=$(ls -t $RPM_BASE/$EL/ja4ebpf-*.rpm | head -1)
    echo "  RPM: $(basename $RPM)"

    # Upload RPM
    if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $RPM /tmp/ja4ebpf-test.rpm $VM" 2>/dev/null; then
        echo "  FAIL: RPM upload timeout"
        return
    fi

    # Get VM IP
    local VM_IP=$(vssh "$VM" "ip -4 addr show eth0" | awk '/inet /{sub(/\/.*/,"",$2); print $2; exit}')
    if [ -z "$VM_IP" ]; then
        echo "  SKIP: no eth0 IP"
        return
    fi
    echo "  IP: $VM_IP"

    # Generate and upload setup script
    local SETUP_SCRIPT=$(generate_setup_script "$STACK")
    if ! timeout "$UPLOAD_TIMEOUT" sh -c "cd $VM_DIR && vagrant upload $SETUP_SCRIPT /tmp/ja4-setup.sh $VM" 2>/dev/null; then
        echo "  FAIL: setup script upload timeout"
        rm -f "$SETUP_SCRIPT"
        return
    fi
    rm -f "$SETUP_SCRIPT"

    # Execute setup script on VM via SSH
    if ! timeout "$SSH_TIMEOUT" bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo bash /tmp/ja4-setup.sh'" 2>/dev/null; then
        echo "  FAIL: VM setup timeout/error"
        return
    fi

    # [E] Generate traffic from HOST
    echo "  [E] Traffic host→VM..."
    local HTTP_PORT HTTPS_PORT
    case "$STACK" in
        nginx)         HTTP_PORT=80;  HTTPS_PORT=443 ;;
        apache)        HTTP_PORT=8080; HTTPS_PORT=8443 ;;
        hitch-varnish) HTTP_PORT=80;  HTTPS_PORT=443 ;;
    esac

    for i in $(seq 1 3); do
        curl -sf --max-time 5 "http://$VM_IP:$HTTP_PORT/health" -o /dev/null 2>&1 && echo "  HTTP $i: OK" || echo "  HTTP $i: FAIL"
        curl -skf --max-time 5 "https://$VM_IP:$HTTPS_PORT/health" -o /dev/null 2>&1 && echo "  HTTPS $i: OK" || echo "  HTTPS $i: FAIL"
    done

    # [F] Wait for debug dump
    sleep 6

    # [G] Collect results
    echo "  [G] Results..."
    vssh "$VM" "sudo cat /tmp/ja4-test.log" > /tmp/vm-ja4-log.txt 2>/dev/null || true

    # Cleanup VM
    timeout 15 bash -c "cd '$VM_DIR' && vagrant ssh '$VM' -- 'sudo pkill ja4ebpf 2>/dev/null; sudo nginx -s stop 2>/dev/null; sudo httpd -k stop 2>/dev/null; sudo pkill varnishd 2>/dev/null; sudo pkill hitch 2>/dev/null; sudo pkill -f python3 2>/dev/null; echo cleanup_done'" 2>/dev/null > /dev/null || true

    # Parse on host side
    local LAST_DEBUG=$(grep "\[debug\] BPF:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1)
    local LAST_GO=$(grep "\[debug\] GO:" /tmp/vm-ja4-log.txt 2>/dev/null | tail -1)
    local SESSIONS=$(grep -c "session pr" /tmp/vm-ja4-log.txt 2>/dev/null || echo 0)
    local SYN_SUB=$(echo "$LAST_DEBUG" | sed -n 's/.*SYN_SUB=\([0-9]*\).*/\1/p')
    SYN_SUB=${SYN_SUB:-0}

    local ELAPSED=$(( $(date +%s) - START_TIME ))
    echo "    BPF: $LAST_DEBUG"
    echo "    GO:  $LAST_GO"
    echo "    sessions=$SESSIONS  SYN_SUB=$SYN_SUB  (${ELAPSED}s)"

    if [ "$SYN_SUB" -gt 0 ] 2>/dev/null; then
        echo "  PASS: SYN_SUB=$SYN_SUB sessions=$SESSIONS"
    else
        echo "  FAIL: SYN_SUB=0 (no TC capture)"
    fi
}

# ── Main loop ────────────────────────────────────────────────────────

for v in $VMS; do
    for s in $STACKS; do
        LABEL="$v/$s"
        TMPFILE=$(mktemp /tmp/ja4-test-XXXXXX.log)

        # Run test_vm_stack in background, capture output
        (
            test_vm_stack "$v" "$s"
        ) > "$TMPFILE" 2>&1 &
        TESTPID=$!

        # Wait with timeout
        if ! timeout --signal=TERM --kill-after=10 "$TEST_TIMEOUT" bash -c "while kill -0 $TESTPID 2>/dev/null; do sleep 1; done"; then
            kill -9 $TESTPID 2>/dev/null || true
            wait $TESTPID 2>/dev/null || true
            echo ""
            echo "── $LABEL ─────────────────────────────────────────────"
            echo "  TIMEOUT: exceeded ${TEST_TIMEOUT}s"
        else
            wait $TESTPID 2>/dev/null
        fi
        cat "$TMPFILE"

        # Parse result from output
        if grep -q "^  PASS:" "$TMPFILE" 2>/dev/null; then
            PASS=$((PASS + 1)); RESULTS+=("PASS $LABEL")
        elif grep -q "^  FAIL:" "$TMPFILE" 2>/dev/null; then
            FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL")
        elif grep -q "^  SKIP:" "$TMPFILE" 2>/dev/null; then
            SKIP=$((SKIP + 1)); RESULTS+=("SKIP $LABEL")
        else
            FAIL=$((FAIL + 1)); RESULTS+=("FAIL $LABEL (no result)")
        fi
        rm -f "$TMPFILE"
    done
done

# ── Summary ──────────────────────────────────────────────────────────

echo ""
echo "╔══════════════════════════════════════════════════════════════╗"
echo "║  SUMMARY: $PASS pass / $FAIL fail / $SKIP skip"
echo "╚══════════════════════════════════════════════════════════════╝"
for r in "${RESULTS[@]}"; do echo "  $r"; done

[ "$FAIL" -eq 0 ] && exit 0 || exit 1