code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: TLS capture end-to-end in integration tests

Browse Source 
- Add traffic-gen container (curlimages/curl) to send HTTPS traffic
  across Docker network so sentinel (pcap on eth0) captures ClientHello
- Seed anubis_ua_rules with catch-all rule (REGEXP_TREE needs ≥1 entry)
  so MV mv_http_logs processes raw logs without errors
- Add JA4/JA3 fingerprint verification in Phase 5 tests
- Dashboard healthcheck via python urllib (no curl in image)

Results: 59 raw logs, 59 parsed, 53 with JA4+JA3 fingerprints (TLS 1.3)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 21:04:14 +02:00
parent a985661369
commit da8357f43d
3 changed files with 56 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

38
tests/integration/run-tests.sh
View File

@ -157,13 +157,15 @@ log "============================================"
log "Phase 4: Generating test traffic"
log "============================================"
PLATFORM_IP=$(docker compose exec -T platform hostname -I | tr -d ' ')
PLATFORM_IP=$(docker compose exec -T platform hostname -I | tr -d ' \n\r')
log "Platform IP: $PLATFORM_IP"
# Send HTTPS requests to Apache (triggers mod-reqin-log + sentinel)
log "Sending 50 HTTPS requests..."
# Traffic MUST come from OUTSIDE the platform container so sentinel sees it
# on eth0. curl from localhost goes through loopback → invisible to pcap.
# We use the traffic-gen container (curlimages/curl) as the traffic source.
log "Sending 50 HTTPS requests (from traffic-gen → platform via Docker network)..."
for i in $(seq 1 50); do
docker compose exec -T platform curl -sk \
docker compose exec -T traffic-gen curl -sk \
-H "User-Agent: IntegrationTest/1.0 (test-run-$i)" \
-H "Accept: text/html,application/json" \
-H "Accept-Language: fr-FR,en-US" \
@ -171,19 +173,19 @@ for i in $(seq 1 50); do
-H "Sec-Fetch-Dest: document" \
-H "Sec-Fetch-Mode: navigate" \
-H "Sec-Fetch-Site: none" \
"https://localhost/health?test=$i" > /dev/null 2>&1 || true &
"https://platform/health?test=$i" > /dev/null 2>&1 || true &
done
wait || true
pass "50 HTTPS requests sent"
# Send varied HTTP methods
log "Sending varied HTTP methods..."
docker compose exec -T platform curl -sk -X POST -d '{"test":true}' \
docker compose exec -T traffic-gen curl -sk -X POST -d '{"test":true}' \
-H "Content-Type: application/json" \
-H "User-Agent: BotTest/2.0" \
"https://localhost/health" > /dev/null 2>&1 || true
docker compose exec -T platform curl -sk -X HEAD "https://localhost/health" > /dev/null 2>&1 || true
docker compose exec -T platform curl -sk "https://localhost/" > /dev/null 2>&1 || true
"https://platform/health" > /dev/null 2>&1 || true
docker compose exec -T traffic-gen curl -sk -X HEAD "https://platform/health" > /dev/null 2>&1 || true
docker compose exec -T traffic-gen curl -sk "https://platform/" > /dev/null 2>&1 || true
pass "Varied HTTP methods sent (POST, HEAD, GET)"
# Wait for correlator to flush batches to ClickHouse
@ -226,7 +228,23 @@ if [ "$PARSED_COUNT" -gt 0 ] 2>/dev/null; then
fi
fi
# 5d. Check correlator log file
# 5d. TLS fingerprints captured (sentinel → correlator → ClickHouse)
if [ "$PARSED_COUNT" -gt 0 ] 2>/dev/null; then
JA4_COUNT=$(ch_query "SELECT count() FROM ja4_logs.http_logs WHERE ja4 != ''")
JA3_COUNT=$(ch_query "SELECT count() FROM ja4_logs.http_logs WHERE ja3 != ''")
TLS_SAMPLE=$(ch_query "SELECT ja4, ja3_hash, tls_version FROM ja4_logs.http_logs WHERE ja4 != '' LIMIT 1 FORMAT TabSeparated")
if [ "$JA4_COUNT" -gt 0 ] 2>/dev/null; then
pass "TLS capture: $JA4_COUNT rows with JA4 fingerprints"
log " Sample: $TLS_SAMPLE"
else
warn "No JA4 fingerprints in parsed logs (sentinel may not capture loopback traffic)"
fi
if [ "$JA3_COUNT" -gt 0 ] 2>/dev/null; then
pass "TLS capture: $JA3_COUNT rows with JA3 fingerprints"
fi
fi
# 5e. Check correlator log file
CORR_LINES=$(docker compose exec -T platform wc -l < /var/log/logcorrelator/correlated.log 2>/dev/null || echo 0)
if [ "$CORR_LINES" -gt 0 ] 2>/dev/null; then
pass "Correlator file output: $CORR_LINES lines in correlated.log"