release: version 1.1.2 - Add error callback mechanism and comprehensive test suite

Features: - Add ErrorCallback type for UNIX socket connection error reporting - Add WithErrorCallback option for UnixSocketWriter configuration - Add BuilderImpl.WithErrorCallback() for propagating callbacks - Add consecutive failure tracking in processQueue Testing (50+ new tests): - Add integration tests for full pipeline (capture → tlsparse → fingerprint → output) - Add tests for FileWriter.rotate() and Reopen() log rotation - Add tests for cleanupExpiredFlows() and cleanupLoop() in TLS parser - Add tests for extractSNIFromPayload() and extractJA4Hash() helpers - Add tests for config load error paths (invalid YAML, permission denied) - Add tests for capture.Run() error conditions - Add tests for signal handling documentation Documentation: - Update architecture.yml with new fields (LogLevel, TLSClientHello extensions) - Update architecture.yml with Close() methods for Capture and Parser interfaces - Update RPM spec changelog Cleanup: - Remove empty internal/api/ directory Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
2026-03-02 23:24:56 +01:00
parent 6e5addd6d4
commit 23f3012fb1
10 changed files with 2058 additions and 10 deletions
--- a/internal/integration/pipeline_test.go
+++ b/internal/integration/pipeline_test.go
@ -0,0 +1,402 @@
+// Package integration provides integration tests for the full ja4sentinel pipeline
+package integration
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"ja4sentinel/api"
+	"ja4sentinel/internal/fingerprint"
+	"ja4sentinel/internal/output"
+	"ja4sentinel/internal/tlsparse"
+)
+
+// TestFullPipeline_TLSClientHelloToFingerprint tests the pipeline from TLS ClientHello to fingerprint
+func TestFullPipeline_TLSClientHelloToFingerprint(t *testing.T) {
+	// Create a minimal TLS 1.2 ClientHello for testing
+	clientHello := buildMinimalTLSClientHello()
+
+	// Step 1: Parse the ClientHello
+	parser := tlsparse.NewParser()
+	if parser == nil {
+		t.Fatal("NewParser() returned nil")
+	}
+	defer parser.Close()
+
+	// Create a raw packet with the ClientHello
+	rawPacket := api.RawPacket{
+		Data:      buildEthernetIPPacket(clientHello),
+		Timestamp: time.Now().UnixNano(),
+	}
+
+	// Process the packet
+	ch, err := parser.Process(rawPacket)
+	if err != nil {
+		t.Fatalf("Process() error = %v", err)
+	}
+	if ch == nil {
+		t.Fatal("Process() returned nil ClientHello")
+	}
+
+	// Step 2: Generate fingerprints
+	engine := fingerprint.NewEngine()
+	if engine == nil {
+		t.Fatal("NewEngine() returned nil")
+	}
+
+	fp, err := engine.FromClientHello(*ch)
+	if err != nil {
+		t.Fatalf("FromClientHello() error = %v", err)
+	}
+	if fp == nil {
+		t.Fatal("FromClientHello() returned nil")
+	}
+
+	// Verify fingerprints are populated
+	if fp.JA4 == "" {
+		t.Error("JA4 should be populated")
+	}
+	if fp.JA3 == "" {
+		t.Error("JA3 should be populated")
+	}
+	if fp.JA3Hash == "" {
+		t.Error("JA3Hash should be populated")
+	}
+}
+
+// TestFullPipeline_FingerprintToOutput tests the pipeline from fingerprint to output
+func TestFullPipeline_FingerprintToOutput(t *testing.T) {
+	// Create test data
+	clientHello := api.TLSClientHello{
+		SrcIP:   "192.168.1.100",
+		SrcPort: 54321,
+		DstIP:   "10.0.0.1",
+		DstPort: 443,
+		IPMeta: api.IPMeta{
+			TTL:         64,
+			TotalLength: 512,
+			IPID:        12345,
+			DF:          true,
+		},
+		TCPMeta: api.TCPMeta{
+			WindowSize:  65535,
+			MSS:         1460,
+			WindowScale: 7,
+			Options:     []string{"MSS", "SACK", "TS", "WS"},
+		},
+		ConnID:     "test-flow-123",
+		SNI:        "example.com",
+		ALPN:       "h2",
+		TLSVersion: "1.3",
+		SynToCHMs:  uint32Ptr(50),
+	}
+
+	// Create fingerprints
+	fingerprints := &api.Fingerprints{
+		JA4:     "t13d1516h2_8daaf6152771_02cb136f2775",
+		JA4Hash: "8daaf6152771_02cb136f2775",
+		JA3:     "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
+		JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d",
+	}
+
+	// Step 1: Create LogRecord
+	logRecord := api.NewLogRecord(clientHello, fingerprints)
+	logRecord.SensorID = "test-sensor"
+
+	// Step 2: Write to output (stdout writer for testing)
+	writer := output.NewStdoutWriter()
+	if writer == nil {
+		t.Fatal("NewStdoutWriter() returned nil")
+	}
+
+	// Capture stdout by using a buffer (we can't easily test stdout, so we verify the record)
+	// Instead, verify the LogRecord is valid JSON
+	data, err := json.Marshal(logRecord)
+	if err != nil {
+		t.Fatalf("json.Marshal() error = %v", err)
+	}
+
+	// Verify JSON is valid and contains expected fields
+	var result map[string]interface{}
+	if err := json.Unmarshal(data, &result); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	// Verify key fields
+	if result["src_ip"] != "192.168.1.100" {
+		t.Errorf("src_ip = %v, want 192.168.1.100", result["src_ip"])
+	}
+	if result["src_port"] != float64(54321) {
+		t.Errorf("src_port = %v, want 54321", result["src_port"])
+	}
+	if result["ja4"] != "t13d1516h2_8daaf6152771_02cb136f2775" {
+		t.Errorf("ja4 = %v, want t13d1516h2_8daaf6152771_02cb136f2775", result["ja4"])
+	}
+	if result["tls_sni"] != "example.com" {
+		t.Errorf("tls_sni = %v, want example.com", result["tls_sni"])
+	}
+	if result["sensor_id"] != "test-sensor" {
+		t.Errorf("sensor_id = %v, want test-sensor", result["sensor_id"])
+	}
+}
+
+// TestFullPipeline_EndToEnd tests the complete pipeline with file output
+func TestFullPipeline_EndToEnd(t *testing.T) {
+	tmpDir := t.TempDir()
+	outputPath := tmpDir + "/output.log"
+
+	// Create test ClientHello
+	clientHello := buildMinimalTLSClientHello()
+
+	// Step 1: Parse
+	parser := tlsparse.NewParser()
+	defer parser.Close()
+
+	rawPacket := api.RawPacket{
+		Data:      buildEthernetIPPacket(clientHello),
+		Timestamp: time.Now().UnixNano(),
+	}
+
+	ch, err := parser.Process(rawPacket)
+	if err != nil {
+		t.Fatalf("Process() error = %v", err)
+	}
+
+	// Step 2: Fingerprint
+	engine := fingerprint.NewEngine()
+	fp, err := engine.FromClientHello(*ch)
+	if err != nil {
+		t.Fatalf("FromClientHello() error = %v", err)
+	}
+
+	// Step 3: Create LogRecord
+	logRecord := api.NewLogRecord(*ch, fp)
+	logRecord.SensorID = "test-sensor-e2e"
+
+	// Step 4: Write to file
+	fileWriter, err := output.NewFileWriter(outputPath)
+	if err != nil {
+		t.Fatalf("NewFileWriter() error = %v", err)
+	}
+	defer fileWriter.Close()
+
+	err = fileWriter.Write(logRecord)
+	if err != nil {
+		t.Errorf("Write() error = %v", err)
+	}
+
+	// Verify output file
+	data, err := os.ReadFile(outputPath)
+	if err != nil {
+		t.Fatalf("ReadFile() error = %v", err)
+	}
+
+	if len(data) == 0 {
+		t.Fatal("Output file is empty")
+	}
+
+	// Parse and verify
+	var result api.LogRecord
+	if err := json.Unmarshal(data, &result); err != nil {
+		t.Fatalf("json.Unmarshal() error = %v", err)
+	}
+
+	if result.SensorID != "test-sensor-e2e" {
+		t.Errorf("SensorID = %v, want test-sensor-e2e", result.SensorID)
+	}
+	if result.JA4 == "" {
+		t.Error("JA4 should be populated")
+	}
+}
+
+// TestFullPipeline_MultiOutput tests writing to multiple outputs simultaneously
+func TestFullPipeline_MultiOutput(t *testing.T) {
+	tmpDir := t.TempDir()
+	filePath := tmpDir + "/multi.log"
+
+	// Create multi-writer
+	multiWriter := output.NewMultiWriter()
+	multiWriter.Add(output.NewStdoutWriter())
+
+	fileWriter, err := output.NewFileWriter(filePath)
+	if err != nil {
+		t.Fatalf("NewFileWriter() error = %v", err)
+	}
+	multiWriter.Add(fileWriter)
+
+	// Create test record
+	logRecord := api.LogRecord{
+		SrcIP:   "192.168.1.1",
+		SrcPort: 12345,
+		JA4:     "test-multi-output",
+	}
+
+	// Write to all outputs
+	err = multiWriter.Write(logRecord)
+	if err != nil {
+		t.Errorf("Write() error = %v", err)
+	}
+
+	// Verify file output
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		t.Fatalf("ReadFile() error = %v", err)
+	}
+
+	if len(data) == 0 {
+		t.Fatal("File output is empty")
+	}
+}
+
+// TestFullPipeline_ConfigToOutput tests building output from config
+func TestFullPipeline_ConfigToOutput(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	// Create config with multiple outputs
+	config := api.AppConfig{
+		Core: api.Config{
+			Interface:   "eth0",
+			ListenPorts: []uint16{443},
+		},
+		Outputs: []api.OutputConfig{
+			{
+				Type:        "stdout",
+				Enabled:     true,
+				AsyncBuffer: 1000,
+			},
+			{
+				Type:        "file",
+				Enabled:     true,
+				AsyncBuffer: 1000,
+				Params:      map[string]string{"path": tmpDir + "/config-output.log"},
+			},
+		},
+	}
+
+	// Build writer from config
+	builder := output.NewBuilder()
+	writer, err := builder.NewFromConfig(config)
+	if err != nil {
+		t.Fatalf("NewFromConfig() error = %v", err)
+	}
+
+	// Verify writer is MultiWriter
+	_, ok := writer.(*output.MultiWriter)
+	if !ok {
+		t.Fatal("Expected MultiWriter")
+	}
+
+	// Test writing
+	logRecord := api.LogRecord{
+		SrcIP: "192.168.1.1",
+		JA4:   "test-config-output",
+	}
+
+	err = writer.Write(logRecord)
+	if err != nil {
+		t.Errorf("Write() error = %v", err)
+	}
+}
+
+// Helper functions
+
+// buildMinimalTLSClientHello creates a minimal TLS 1.2 ClientHello for testing
+func buildMinimalTLSClientHello() []byte {
+	// Cipher suites
+	cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f}
+	compressionMethods := []byte{0x01, 0x00}
+	extensions := []byte{}
+	extLen := len(extensions)
+
+	handshakeBody := []byte{
+		0x03, 0x03, // Version: TLS 1.2
+	}
+	// Random (32 bytes)
+	for i := 0; i < 32; i++ {
+		handshakeBody = append(handshakeBody, 0x00)
+	}
+	handshakeBody = append(handshakeBody, 0x00) // Session ID length
+
+	// Cipher suites
+	cipherSuiteLen := len(cipherSuites)
+	handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen))
+	handshakeBody = append(handshakeBody, cipherSuites...)
+
+	// Compression methods
+	handshakeBody = append(handshakeBody, compressionMethods...)
+
+	// Extensions
+	handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen))
+	handshakeBody = append(handshakeBody, extensions...)
+
+	// Build handshake
+	handshakeLen := len(handshakeBody)
+	handshake := append([]byte{
+		0x01, // Handshake type: ClientHello
+		byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen),
+	}, handshakeBody...)
+
+	// Build TLS record
+	recordLen := len(handshake)
+	record := make([]byte, 5+recordLen)
+	record[0] = 0x16 // Handshake
+	record[1] = 0x03 // Version: TLS 1.2
+	record[2] = 0x03
+	record[3] = byte(recordLen >> 8)
+	record[4] = byte(recordLen)
+	copy(record[5:], handshake)
+
+	return record
+}
+
+// buildEthernetIPPacket wraps a TLS payload in Ethernet/IP/TCP headers
+func buildEthernetIPPacket(tlsPayload []byte) []byte {
+	// This is a simplified packet structure for testing
+	// Real packets would have proper Ethernet, IP, and TCP headers
+
+	// Ethernet header (14 bytes)
+	eth := make([]byte, 14)
+	eth[12] = 0x08 // EtherType: IPv4
+	eth[13] = 0x00
+
+	// IP header (20 bytes)
+	ip := make([]byte, 20)
+	ip[0] = 0x45 // Version 4, IHL 5
+	ip[1] = 0x00 // DSCP/ECN
+	ip[2] = byte((20 + 20 + len(tlsPayload)) >> 8) // Total length
+	ip[3] = byte((20 + 20 + len(tlsPayload)) & 0xFF)
+	ip[8] = 64 // TTL
+	ip[9] = 6  // Protocol: TCP
+	ip[12] = 192
+	ip[13] = 168
+	ip[14] = 1
+	ip[15] = 100 // Src IP: 192.168.1.100
+	ip[16] = 10
+	ip[17] = 0
+	ip[18] = 0
+	ip[19] = 1 // Dst IP: 10.0.0.1
+
+	// TCP header (20 bytes)
+	tcp := make([]byte, 20)
+	tcp[0] = byte(54321 >> 8) // Src port high
+	tcp[1] = byte(54321 & 0xFF) // Src port low
+	tcp[2] = byte(443 >> 8) // Dst port high
+	tcp[3] = byte(443 & 0xFF) // Dst port low
+	tcp[12] = 0x50 // Data offset (5 * 4 = 20 bytes)
+	tcp[13] = 0x18 // Flags: ACK, PSH
+
+	// Combine all headers with payload
+	packet := make([]byte, len(eth)+len(ip)+len(tcp)+len(tlsPayload))
+	copy(packet, eth)
+	copy(packet[len(eth):], ip)
+	copy(packet[len(eth)+len(ip):], tcp)
+	copy(packet[len(eth)+len(ip)+len(tcp):], tlsPayload)
+
+	return packet
+}
+
+func uint32Ptr(v uint32) *uint32 {
+	return &v
+}