23f3012fb1
Some checks failed
Build RPM Package / Build RPM Packages (CentOS 7, Rocky 8/9/10) (push) Has been cancelled
Features: - Add ErrorCallback type for UNIX socket connection error reporting - Add WithErrorCallback option for UnixSocketWriter configuration - Add BuilderImpl.WithErrorCallback() for propagating callbacks - Add consecutive failure tracking in processQueue Testing (50+ new tests): - Add integration tests for full pipeline (capture → tlsparse → fingerprint → output) - Add tests for FileWriter.rotate() and Reopen() log rotation - Add tests for cleanupExpiredFlows() and cleanupLoop() in TLS parser - Add tests for extractSNIFromPayload() and extractJA4Hash() helpers - Add tests for config load error paths (invalid YAML, permission denied) - Add tests for capture.Run() error conditions - Add tests for signal handling documentation Documentation: - Update architecture.yml with new fields (LogLevel, TLSClientHello extensions) - Update architecture.yml with Close() methods for Capture and Parser interfaces - Update RPM spec changelog Cleanup: - Remove empty internal/api/ directory Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
403 lines
10 KiB
Go
403 lines
10 KiB
Go
// Package integration provides integration tests for the full ja4sentinel pipeline
|
|
package integration
|
|
|
|
import (
|
|
"encoding/json"
|
|
"os"
|
|
"testing"
|
|
"time"
|
|
|
|
"ja4sentinel/api"
|
|
"ja4sentinel/internal/fingerprint"
|
|
"ja4sentinel/internal/output"
|
|
"ja4sentinel/internal/tlsparse"
|
|
)
|
|
|
|
// TestFullPipeline_TLSClientHelloToFingerprint tests the pipeline from TLS ClientHello to fingerprint
|
|
func TestFullPipeline_TLSClientHelloToFingerprint(t *testing.T) {
|
|
// Create a minimal TLS 1.2 ClientHello for testing
|
|
clientHello := buildMinimalTLSClientHello()
|
|
|
|
// Step 1: Parse the ClientHello
|
|
parser := tlsparse.NewParser()
|
|
if parser == nil {
|
|
t.Fatal("NewParser() returned nil")
|
|
}
|
|
defer parser.Close()
|
|
|
|
// Create a raw packet with the ClientHello
|
|
rawPacket := api.RawPacket{
|
|
Data: buildEthernetIPPacket(clientHello),
|
|
Timestamp: time.Now().UnixNano(),
|
|
}
|
|
|
|
// Process the packet
|
|
ch, err := parser.Process(rawPacket)
|
|
if err != nil {
|
|
t.Fatalf("Process() error = %v", err)
|
|
}
|
|
if ch == nil {
|
|
t.Fatal("Process() returned nil ClientHello")
|
|
}
|
|
|
|
// Step 2: Generate fingerprints
|
|
engine := fingerprint.NewEngine()
|
|
if engine == nil {
|
|
t.Fatal("NewEngine() returned nil")
|
|
}
|
|
|
|
fp, err := engine.FromClientHello(*ch)
|
|
if err != nil {
|
|
t.Fatalf("FromClientHello() error = %v", err)
|
|
}
|
|
if fp == nil {
|
|
t.Fatal("FromClientHello() returned nil")
|
|
}
|
|
|
|
// Verify fingerprints are populated
|
|
if fp.JA4 == "" {
|
|
t.Error("JA4 should be populated")
|
|
}
|
|
if fp.JA3 == "" {
|
|
t.Error("JA3 should be populated")
|
|
}
|
|
if fp.JA3Hash == "" {
|
|
t.Error("JA3Hash should be populated")
|
|
}
|
|
}
|
|
|
|
// TestFullPipeline_FingerprintToOutput tests the pipeline from fingerprint to output
|
|
func TestFullPipeline_FingerprintToOutput(t *testing.T) {
|
|
// Create test data
|
|
clientHello := api.TLSClientHello{
|
|
SrcIP: "192.168.1.100",
|
|
SrcPort: 54321,
|
|
DstIP: "10.0.0.1",
|
|
DstPort: 443,
|
|
IPMeta: api.IPMeta{
|
|
TTL: 64,
|
|
TotalLength: 512,
|
|
IPID: 12345,
|
|
DF: true,
|
|
},
|
|
TCPMeta: api.TCPMeta{
|
|
WindowSize: 65535,
|
|
MSS: 1460,
|
|
WindowScale: 7,
|
|
Options: []string{"MSS", "SACK", "TS", "WS"},
|
|
},
|
|
ConnID: "test-flow-123",
|
|
SNI: "example.com",
|
|
ALPN: "h2",
|
|
TLSVersion: "1.3",
|
|
SynToCHMs: uint32Ptr(50),
|
|
}
|
|
|
|
// Create fingerprints
|
|
fingerprints := &api.Fingerprints{
|
|
JA4: "t13d1516h2_8daaf6152771_02cb136f2775",
|
|
JA4Hash: "8daaf6152771_02cb136f2775",
|
|
JA3: "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
|
|
JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d",
|
|
}
|
|
|
|
// Step 1: Create LogRecord
|
|
logRecord := api.NewLogRecord(clientHello, fingerprints)
|
|
logRecord.SensorID = "test-sensor"
|
|
|
|
// Step 2: Write to output (stdout writer for testing)
|
|
writer := output.NewStdoutWriter()
|
|
if writer == nil {
|
|
t.Fatal("NewStdoutWriter() returned nil")
|
|
}
|
|
|
|
// Capture stdout by using a buffer (we can't easily test stdout, so we verify the record)
|
|
// Instead, verify the LogRecord is valid JSON
|
|
data, err := json.Marshal(logRecord)
|
|
if err != nil {
|
|
t.Fatalf("json.Marshal() error = %v", err)
|
|
}
|
|
|
|
// Verify JSON is valid and contains expected fields
|
|
var result map[string]interface{}
|
|
if err := json.Unmarshal(data, &result); err != nil {
|
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
|
}
|
|
|
|
// Verify key fields
|
|
if result["src_ip"] != "192.168.1.100" {
|
|
t.Errorf("src_ip = %v, want 192.168.1.100", result["src_ip"])
|
|
}
|
|
if result["src_port"] != float64(54321) {
|
|
t.Errorf("src_port = %v, want 54321", result["src_port"])
|
|
}
|
|
if result["ja4"] != "t13d1516h2_8daaf6152771_02cb136f2775" {
|
|
t.Errorf("ja4 = %v, want t13d1516h2_8daaf6152771_02cb136f2775", result["ja4"])
|
|
}
|
|
if result["tls_sni"] != "example.com" {
|
|
t.Errorf("tls_sni = %v, want example.com", result["tls_sni"])
|
|
}
|
|
if result["sensor_id"] != "test-sensor" {
|
|
t.Errorf("sensor_id = %v, want test-sensor", result["sensor_id"])
|
|
}
|
|
}
|
|
|
|
// TestFullPipeline_EndToEnd tests the complete pipeline with file output
|
|
func TestFullPipeline_EndToEnd(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
outputPath := tmpDir + "/output.log"
|
|
|
|
// Create test ClientHello
|
|
clientHello := buildMinimalTLSClientHello()
|
|
|
|
// Step 1: Parse
|
|
parser := tlsparse.NewParser()
|
|
defer parser.Close()
|
|
|
|
rawPacket := api.RawPacket{
|
|
Data: buildEthernetIPPacket(clientHello),
|
|
Timestamp: time.Now().UnixNano(),
|
|
}
|
|
|
|
ch, err := parser.Process(rawPacket)
|
|
if err != nil {
|
|
t.Fatalf("Process() error = %v", err)
|
|
}
|
|
|
|
// Step 2: Fingerprint
|
|
engine := fingerprint.NewEngine()
|
|
fp, err := engine.FromClientHello(*ch)
|
|
if err != nil {
|
|
t.Fatalf("FromClientHello() error = %v", err)
|
|
}
|
|
|
|
// Step 3: Create LogRecord
|
|
logRecord := api.NewLogRecord(*ch, fp)
|
|
logRecord.SensorID = "test-sensor-e2e"
|
|
|
|
// Step 4: Write to file
|
|
fileWriter, err := output.NewFileWriter(outputPath)
|
|
if err != nil {
|
|
t.Fatalf("NewFileWriter() error = %v", err)
|
|
}
|
|
defer fileWriter.Close()
|
|
|
|
err = fileWriter.Write(logRecord)
|
|
if err != nil {
|
|
t.Errorf("Write() error = %v", err)
|
|
}
|
|
|
|
// Verify output file
|
|
data, err := os.ReadFile(outputPath)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile() error = %v", err)
|
|
}
|
|
|
|
if len(data) == 0 {
|
|
t.Fatal("Output file is empty")
|
|
}
|
|
|
|
// Parse and verify
|
|
var result api.LogRecord
|
|
if err := json.Unmarshal(data, &result); err != nil {
|
|
t.Fatalf("json.Unmarshal() error = %v", err)
|
|
}
|
|
|
|
if result.SensorID != "test-sensor-e2e" {
|
|
t.Errorf("SensorID = %v, want test-sensor-e2e", result.SensorID)
|
|
}
|
|
if result.JA4 == "" {
|
|
t.Error("JA4 should be populated")
|
|
}
|
|
}
|
|
|
|
// TestFullPipeline_MultiOutput tests writing to multiple outputs simultaneously
|
|
func TestFullPipeline_MultiOutput(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
filePath := tmpDir + "/multi.log"
|
|
|
|
// Create multi-writer
|
|
multiWriter := output.NewMultiWriter()
|
|
multiWriter.Add(output.NewStdoutWriter())
|
|
|
|
fileWriter, err := output.NewFileWriter(filePath)
|
|
if err != nil {
|
|
t.Fatalf("NewFileWriter() error = %v", err)
|
|
}
|
|
multiWriter.Add(fileWriter)
|
|
|
|
// Create test record
|
|
logRecord := api.LogRecord{
|
|
SrcIP: "192.168.1.1",
|
|
SrcPort: 12345,
|
|
JA4: "test-multi-output",
|
|
}
|
|
|
|
// Write to all outputs
|
|
err = multiWriter.Write(logRecord)
|
|
if err != nil {
|
|
t.Errorf("Write() error = %v", err)
|
|
}
|
|
|
|
// Verify file output
|
|
data, err := os.ReadFile(filePath)
|
|
if err != nil {
|
|
t.Fatalf("ReadFile() error = %v", err)
|
|
}
|
|
|
|
if len(data) == 0 {
|
|
t.Fatal("File output is empty")
|
|
}
|
|
}
|
|
|
|
// TestFullPipeline_ConfigToOutput tests building output from config
|
|
func TestFullPipeline_ConfigToOutput(t *testing.T) {
|
|
tmpDir := t.TempDir()
|
|
|
|
// Create config with multiple outputs
|
|
config := api.AppConfig{
|
|
Core: api.Config{
|
|
Interface: "eth0",
|
|
ListenPorts: []uint16{443},
|
|
},
|
|
Outputs: []api.OutputConfig{
|
|
{
|
|
Type: "stdout",
|
|
Enabled: true,
|
|
AsyncBuffer: 1000,
|
|
},
|
|
{
|
|
Type: "file",
|
|
Enabled: true,
|
|
AsyncBuffer: 1000,
|
|
Params: map[string]string{"path": tmpDir + "/config-output.log"},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Build writer from config
|
|
builder := output.NewBuilder()
|
|
writer, err := builder.NewFromConfig(config)
|
|
if err != nil {
|
|
t.Fatalf("NewFromConfig() error = %v", err)
|
|
}
|
|
|
|
// Verify writer is MultiWriter
|
|
_, ok := writer.(*output.MultiWriter)
|
|
if !ok {
|
|
t.Fatal("Expected MultiWriter")
|
|
}
|
|
|
|
// Test writing
|
|
logRecord := api.LogRecord{
|
|
SrcIP: "192.168.1.1",
|
|
JA4: "test-config-output",
|
|
}
|
|
|
|
err = writer.Write(logRecord)
|
|
if err != nil {
|
|
t.Errorf("Write() error = %v", err)
|
|
}
|
|
}
|
|
|
|
// Helper functions
|
|
|
|
// buildMinimalTLSClientHello creates a minimal TLS 1.2 ClientHello for testing
|
|
func buildMinimalTLSClientHello() []byte {
|
|
// Cipher suites
|
|
cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f}
|
|
compressionMethods := []byte{0x01, 0x00}
|
|
extensions := []byte{}
|
|
extLen := len(extensions)
|
|
|
|
handshakeBody := []byte{
|
|
0x03, 0x03, // Version: TLS 1.2
|
|
}
|
|
// Random (32 bytes)
|
|
for i := 0; i < 32; i++ {
|
|
handshakeBody = append(handshakeBody, 0x00)
|
|
}
|
|
handshakeBody = append(handshakeBody, 0x00) // Session ID length
|
|
|
|
// Cipher suites
|
|
cipherSuiteLen := len(cipherSuites)
|
|
handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen))
|
|
handshakeBody = append(handshakeBody, cipherSuites...)
|
|
|
|
// Compression methods
|
|
handshakeBody = append(handshakeBody, compressionMethods...)
|
|
|
|
// Extensions
|
|
handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen))
|
|
handshakeBody = append(handshakeBody, extensions...)
|
|
|
|
// Build handshake
|
|
handshakeLen := len(handshakeBody)
|
|
handshake := append([]byte{
|
|
0x01, // Handshake type: ClientHello
|
|
byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen),
|
|
}, handshakeBody...)
|
|
|
|
// Build TLS record
|
|
recordLen := len(handshake)
|
|
record := make([]byte, 5+recordLen)
|
|
record[0] = 0x16 // Handshake
|
|
record[1] = 0x03 // Version: TLS 1.2
|
|
record[2] = 0x03
|
|
record[3] = byte(recordLen >> 8)
|
|
record[4] = byte(recordLen)
|
|
copy(record[5:], handshake)
|
|
|
|
return record
|
|
}
|
|
|
|
// buildEthernetIPPacket wraps a TLS payload in Ethernet/IP/TCP headers
|
|
func buildEthernetIPPacket(tlsPayload []byte) []byte {
|
|
// This is a simplified packet structure for testing
|
|
// Real packets would have proper Ethernet, IP, and TCP headers
|
|
|
|
// Ethernet header (14 bytes)
|
|
eth := make([]byte, 14)
|
|
eth[12] = 0x08 // EtherType: IPv4
|
|
eth[13] = 0x00
|
|
|
|
// IP header (20 bytes)
|
|
ip := make([]byte, 20)
|
|
ip[0] = 0x45 // Version 4, IHL 5
|
|
ip[1] = 0x00 // DSCP/ECN
|
|
ip[2] = byte((20 + 20 + len(tlsPayload)) >> 8) // Total length
|
|
ip[3] = byte((20 + 20 + len(tlsPayload)) & 0xFF)
|
|
ip[8] = 64 // TTL
|
|
ip[9] = 6 // Protocol: TCP
|
|
ip[12] = 192
|
|
ip[13] = 168
|
|
ip[14] = 1
|
|
ip[15] = 100 // Src IP: 192.168.1.100
|
|
ip[16] = 10
|
|
ip[17] = 0
|
|
ip[18] = 0
|
|
ip[19] = 1 // Dst IP: 10.0.0.1
|
|
|
|
// TCP header (20 bytes)
|
|
tcp := make([]byte, 20)
|
|
tcp[0] = byte(54321 >> 8) // Src port high
|
|
tcp[1] = byte(54321 & 0xFF) // Src port low
|
|
tcp[2] = byte(443 >> 8) // Dst port high
|
|
tcp[3] = byte(443 & 0xFF) // Dst port low
|
|
tcp[12] = 0x50 // Data offset (5 * 4 = 20 bytes)
|
|
tcp[13] = 0x18 // Flags: ACK, PSH
|
|
|
|
// Combine all headers with payload
|
|
packet := make([]byte, len(eth)+len(ip)+len(tcp)+len(tlsPayload))
|
|
copy(packet, eth)
|
|
copy(packet[len(eth):], ip)
|
|
copy(packet[len(eth)+len(ip):], tcp)
|
|
copy(packet[len(eth)+len(ip)+len(tcp):], tlsPayload)
|
|
|
|
return packet
|
|
}
|
|
|
|
func uint32Ptr(v uint32) *uint32 {
|
|
return &v
|
|
}
|