// Package integration provides integration tests for the full ja4sentinel pipeline package integration import ( "encoding/json" "os" "testing" "time" "ja4sentinel/api" "ja4sentinel/internal/fingerprint" "ja4sentinel/internal/output" "ja4sentinel/internal/tlsparse" ) // TestFullPipeline_TLSClientHelloToFingerprint tests the pipeline from TLS ClientHello to fingerprint func TestFullPipeline_TLSClientHelloToFingerprint(t *testing.T) { // Create a minimal TLS 1.2 ClientHello for testing clientHello := buildMinimalTLSClientHello() // Step 1: Parse the ClientHello parser := tlsparse.NewParser() if parser == nil { t.Fatal("NewParser() returned nil") } defer parser.Close() // Create a raw packet with the ClientHello rawPacket := api.RawPacket{ Data: buildEthernetIPPacket(clientHello), Timestamp: time.Now().UnixNano(), } // Process the packet ch, err := parser.Process(rawPacket) if err != nil { t.Fatalf("Process() error = %v", err) } if ch == nil { t.Fatal("Process() returned nil ClientHello") } // Step 2: Generate fingerprints engine := fingerprint.NewEngine() if engine == nil { t.Fatal("NewEngine() returned nil") } fp, err := engine.FromClientHello(*ch) if err != nil { t.Fatalf("FromClientHello() error = %v", err) } if fp == nil { t.Fatal("FromClientHello() returned nil") } // Verify fingerprints are populated if fp.JA4 == "" { t.Error("JA4 should be populated") } if fp.JA3 == "" { t.Error("JA3 should be populated") } if fp.JA3Hash == "" { t.Error("JA3Hash should be populated") } } // TestFullPipeline_FingerprintToOutput tests the pipeline from fingerprint to output func TestFullPipeline_FingerprintToOutput(t *testing.T) { // Create test data clientHello := api.TLSClientHello{ SrcIP: "192.168.1.100", SrcPort: 54321, DstIP: "10.0.0.1", DstPort: 443, IPMeta: api.IPMeta{ TTL: 64, TotalLength: 512, IPID: 12345, DF: true, }, TCPMeta: api.TCPMeta{ WindowSize: 65535, MSS: 1460, WindowScale: 7, Options: []string{"MSS", "SACK", "TS", "WS"}, }, ConnID: "test-flow-123", SNI: "example.com", ALPN: "h2", TLSVersion: "1.3", SynToCHMs: uint32Ptr(50), } // Create fingerprints fingerprints := &api.Fingerprints{ JA4: "t13d1516h2_8daaf6152771_02cb136f2775", JA4Hash: "8daaf6152771_02cb136f2775", JA3: "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0", JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d", } // Step 1: Create LogRecord logRecord := api.NewLogRecord(clientHello, fingerprints) logRecord.SensorID = "test-sensor" // Step 2: Write to output (stdout writer for testing) writer := output.NewStdoutWriter() if writer == nil { t.Fatal("NewStdoutWriter() returned nil") } // Capture stdout by using a buffer (we can't easily test stdout, so we verify the record) // Instead, verify the LogRecord is valid JSON data, err := json.Marshal(logRecord) if err != nil { t.Fatalf("json.Marshal() error = %v", err) } // Verify JSON is valid and contains expected fields var result map[string]interface{} if err := json.Unmarshal(data, &result); err != nil { t.Fatalf("json.Unmarshal() error = %v", err) } // Verify key fields if result["src_ip"] != "192.168.1.100" { t.Errorf("src_ip = %v, want 192.168.1.100", result["src_ip"]) } if result["src_port"] != float64(54321) { t.Errorf("src_port = %v, want 54321", result["src_port"]) } if result["ja4"] != "t13d1516h2_8daaf6152771_02cb136f2775" { t.Errorf("ja4 = %v, want t13d1516h2_8daaf6152771_02cb136f2775", result["ja4"]) } if result["tls_sni"] != "example.com" { t.Errorf("tls_sni = %v, want example.com", result["tls_sni"]) } if result["sensor_id"] != "test-sensor" { t.Errorf("sensor_id = %v, want test-sensor", result["sensor_id"]) } } // TestFullPipeline_EndToEnd tests the complete pipeline with file output func TestFullPipeline_EndToEnd(t *testing.T) { tmpDir := t.TempDir() outputPath := tmpDir + "/output.log" // Create test ClientHello clientHello := buildMinimalTLSClientHello() // Step 1: Parse parser := tlsparse.NewParser() defer parser.Close() rawPacket := api.RawPacket{ Data: buildEthernetIPPacket(clientHello), Timestamp: time.Now().UnixNano(), } ch, err := parser.Process(rawPacket) if err != nil { t.Fatalf("Process() error = %v", err) } // Step 2: Fingerprint engine := fingerprint.NewEngine() fp, err := engine.FromClientHello(*ch) if err != nil { t.Fatalf("FromClientHello() error = %v", err) } // Step 3: Create LogRecord logRecord := api.NewLogRecord(*ch, fp) logRecord.SensorID = "test-sensor-e2e" // Step 4: Write to file fileWriter, err := output.NewFileWriter(outputPath) if err != nil { t.Fatalf("NewFileWriter() error = %v", err) } defer fileWriter.Close() err = fileWriter.Write(logRecord) if err != nil { t.Errorf("Write() error = %v", err) } // Verify output file data, err := os.ReadFile(outputPath) if err != nil { t.Fatalf("ReadFile() error = %v", err) } if len(data) == 0 { t.Fatal("Output file is empty") } // Parse and verify var result api.LogRecord if err := json.Unmarshal(data, &result); err != nil { t.Fatalf("json.Unmarshal() error = %v", err) } if result.SensorID != "test-sensor-e2e" { t.Errorf("SensorID = %v, want test-sensor-e2e", result.SensorID) } if result.JA4 == "" { t.Error("JA4 should be populated") } } // TestFullPipeline_MultiOutput tests writing to multiple outputs simultaneously func TestFullPipeline_MultiOutput(t *testing.T) { tmpDir := t.TempDir() filePath := tmpDir + "/multi.log" // Create multi-writer multiWriter := output.NewMultiWriter() multiWriter.Add(output.NewStdoutWriter()) fileWriter, err := output.NewFileWriter(filePath) if err != nil { t.Fatalf("NewFileWriter() error = %v", err) } multiWriter.Add(fileWriter) // Create test record logRecord := api.LogRecord{ SrcIP: "192.168.1.1", SrcPort: 12345, JA4: "test-multi-output", } // Write to all outputs err = multiWriter.Write(logRecord) if err != nil { t.Errorf("Write() error = %v", err) } // Verify file output data, err := os.ReadFile(filePath) if err != nil { t.Fatalf("ReadFile() error = %v", err) } if len(data) == 0 { t.Fatal("File output is empty") } } // TestFullPipeline_ConfigToOutput tests building output from config func TestFullPipeline_ConfigToOutput(t *testing.T) { tmpDir := t.TempDir() // Create config with multiple outputs config := api.AppConfig{ Core: api.Config{ Interface: "eth0", ListenPorts: []uint16{443}, }, Outputs: []api.OutputConfig{ { Type: "stdout", Enabled: true, AsyncBuffer: 1000, }, { Type: "file", Enabled: true, AsyncBuffer: 1000, Params: map[string]string{"path": tmpDir + "/config-output.log"}, }, }, } // Build writer from config builder := output.NewBuilder() writer, err := builder.NewFromConfig(config) if err != nil { t.Fatalf("NewFromConfig() error = %v", err) } // Verify writer is MultiWriter _, ok := writer.(*output.MultiWriter) if !ok { t.Fatal("Expected MultiWriter") } // Test writing logRecord := api.LogRecord{ SrcIP: "192.168.1.1", JA4: "test-config-output", } err = writer.Write(logRecord) if err != nil { t.Errorf("Write() error = %v", err) } } // Helper functions // buildMinimalTLSClientHello creates a minimal TLS 1.2 ClientHello for testing func buildMinimalTLSClientHello() []byte { // Cipher suites cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f} compressionMethods := []byte{0x01, 0x00} extensions := []byte{} extLen := len(extensions) handshakeBody := []byte{ 0x03, 0x03, // Version: TLS 1.2 } // Random (32 bytes) for i := 0; i < 32; i++ { handshakeBody = append(handshakeBody, 0x00) } handshakeBody = append(handshakeBody, 0x00) // Session ID length // Cipher suites cipherSuiteLen := len(cipherSuites) handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen)) handshakeBody = append(handshakeBody, cipherSuites...) // Compression methods handshakeBody = append(handshakeBody, compressionMethods...) // Extensions handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen)) handshakeBody = append(handshakeBody, extensions...) // Build handshake handshakeLen := len(handshakeBody) handshake := append([]byte{ 0x01, // Handshake type: ClientHello byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen), }, handshakeBody...) // Build TLS record recordLen := len(handshake) record := make([]byte, 5+recordLen) record[0] = 0x16 // Handshake record[1] = 0x03 // Version: TLS 1.2 record[2] = 0x03 record[3] = byte(recordLen >> 8) record[4] = byte(recordLen) copy(record[5:], handshake) return record } // buildEthernetIPPacket wraps a TLS payload in Ethernet/IP/TCP headers func buildEthernetIPPacket(tlsPayload []byte) []byte { // This is a simplified packet structure for testing // Real packets would have proper Ethernet, IP, and TCP headers // Ethernet header (14 bytes) eth := make([]byte, 14) eth[12] = 0x08 // EtherType: IPv4 eth[13] = 0x00 // IP header (20 bytes) ip := make([]byte, 20) ip[0] = 0x45 // Version 4, IHL 5 ip[1] = 0x00 // DSCP/ECN ip[2] = byte((20 + 20 + len(tlsPayload)) >> 8) // Total length ip[3] = byte((20 + 20 + len(tlsPayload)) & 0xFF) ip[8] = 64 // TTL ip[9] = 6 // Protocol: TCP ip[12] = 192 ip[13] = 168 ip[14] = 1 ip[15] = 100 // Src IP: 192.168.1.100 ip[16] = 10 ip[17] = 0 ip[18] = 0 ip[19] = 1 // Dst IP: 10.0.0.1 // TCP header (20 bytes) tcp := make([]byte, 20) tcp[0] = byte(54321 >> 8) // Src port high tcp[1] = byte(54321 & 0xFF) // Src port low tcp[2] = byte(443 >> 8) // Dst port high tcp[3] = byte(443 & 0xFF) // Dst port low tcp[12] = 0x50 // Data offset (5 * 4 = 20 bytes) tcp[13] = 0x18 // Flags: ACK, PSH // Combine all headers with payload packet := make([]byte, len(eth)+len(ip)+len(tcp)+len(tlsPayload)) copy(packet, eth) copy(packet[len(eth):], ip) copy(packet[len(eth)+len(ip):], tcp) copy(packet[len(eth)+len(ip)+len(tcp):], tlsPayload) return packet } func uint32Ptr(v uint32) *uint32 { return &v }