code_public/ja4sentinel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release: version 1.0.9 - Add SNI, ALPN, TLS version extraction and architecture.yml compliance
Some checks failed
Build RPM Package / Build RPM Packages (CentOS 7, Rocky 8/9/10) (push) Has been cancelled
Details

Browse Source 
New features:
- Extract SNI (Server Name Indication) from TLS ClientHello
- Extract ALPN (Application-Layer Protocol Negotiation) protocols
- Detect TLS version from ClientHello using tlsfingerprint library
- Add ConnID field for TCP flow correlation
- Add SensorID field for multi-sensor deployments
- Add SynToCHMs timing field for behavioral detection
- Add AsyncBuffer configuration for output queue sizing

Architecture changes:
- Remove JA4Hash from LogRecord (JA4 format includes its own hash portions)
- Update api.TLSClientHello with new TLS metadata fields
- Update api.LogRecord with correlation, TLS, and timing fields
- Ensure 100% compliance with architecture.yml specification

Tests:
- Add unit tests for TLS extension extraction (SNI, ALPN, Version)
- Update tests for new LogRecord schema without JA4Hash
- Add tests for AsyncBuffer configuration

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
2026-03-02 19:32:16 +01:00
parent fd162982d9
commit 965720a183
12 changed files with 854 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

270
internal/tlsparse/parser_test.go
View File

@ -7,6 +7,7 @@ import (
"ja4sentinel/api"
tlsfingerprint "github.com/psanford/tlsfingerprint"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
)
@ -444,3 +445,272 @@ func buildRawPacket(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16,
Timestamp: time.Now().UnixNano(),
}
}
func TestTLSVersionToString(t *testing.T) {
tests := []struct {
version uint16
want string
}{
{0x0301, "1.0"},
{0x0302, "1.1"},
{0x0303, "1.2"},
{0x0304, "1.3"},
{0x0300, ""},
{0x0305, ""},
}
for _, tt := range tests {
t.Run(tt.want, func(t *testing.T) {
got := tlsVersionToString(tt.version)
if got != tt.want {
t.Errorf("tlsVersionToString(%#x) = %v, want %v", tt.version, got, tt.want)
}
})
}
}
func TestExtractTLSExtensions(t *testing.T) {
tests := []struct {
name string
payload []byte
wantSNI string
wantALPN []string
wantVersion string
wantNil bool
}{
{
name: "empty payload",
payload: []byte{},
wantNil: true,
},
{
name: "too short",
payload: []byte{0x16, 0x03, 0x03},
wantNil: true,
},
{
name: "TLS 1.2 ClientHello without extensions",
payload: createTLSClientHello(0x0303),
wantVersion: "1.2",
wantNil: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := extractTLSExtensions(tt.payload)
if err != nil {
t.Errorf("extractTLSExtensions() unexpected error = %v", err)
return
}
if (got == nil) != tt.wantNil {
t.Errorf("extractTLSExtensions() = %v, wantNil %v", got == nil, tt.wantNil)
return
}
if got != nil {
if got.TLSVersion != tt.wantVersion {
t.Errorf("TLSVersion = %v, want %v", got.TLSVersion, tt.wantVersion)
}
}
})
}
}
func TestParser_ExtractsTLSFields(t *testing.T) {
parser := NewParser()
defer parser.Close()
// Create a minimal valid TLS 1.2 ClientHello with SNI and ALPN extensions
// This is a real-world-like ClientHello structure
clientHelloWithExt := createMinimalTLSClientHelloWithSNIAndALPN("example.com", []string{"h2", "http/1.1"})
// Debug: Check what extractTLSExtensions returns
extInfo, err := extractTLSExtensions(clientHelloWithExt)
if err != nil {
t.Logf("extractTLSExtensions error: %v", err)
}
if extInfo != nil {
t.Logf("extInfo: SNI=%q, ALPN=%v, Version=%q", extInfo.SNI, extInfo.ALPN, extInfo.TLSVersion)
} else {
t.Log("extInfo is nil")
}
// Also test with tlsfingerprint directly
fp, err := tlsfingerprint.ParseClientHello(clientHelloWithExt)
if err != nil {
t.Logf("tlsfingerprint error: %v", err)
} else {
t.Logf("tlsfingerprint: ALPN=%v, Version=%#x, HasSNI=%v", fp.ALPNProtocols, fp.Version, fp.HasSNI)
}
// Debug: print first bytes of ClientHello
t.Logf("ClientHello hex: % x", clientHelloWithExt[:min(50, len(clientHelloWithExt))])
srcIP := "192.168.1.100"
dstIP := "10.0.0.1"
srcPort := uint16(54321)
dstPort := uint16(443)
pkt := buildRawPacket(t, srcIP, dstIP, srcPort, dstPort, clientHelloWithExt)
result, err := parser.Process(pkt)
if err != nil {
t.Fatalf("Process() error = %v", err)
}
if result == nil {
t.Fatal("Process() should return TLSClientHello")
}
// Verify new fields are populated
if result.SNI != "example.com" {
t.Errorf("SNI = %v, want example.com", result.SNI)
}
if result.ALPN == "" {
t.Error("ALPN should not be empty")
}
if result.TLSVersion != "1.2" {
t.Errorf("TLSVersion = %v, want 1.2", result.TLSVersion)
}
if result.ConnID == "" {
t.Error("ConnID should not be empty")
}
if result.SynToCHMs == nil {
t.Error("SynToCHMs should not be nil")
}
}
// createMinimalTLSClientHelloWithSNIAndALPN creates a minimal but valid TLS 1.2 ClientHello
// with SNI and ALPN extensions
func createMinimalTLSClientHelloWithSNIAndALPN(sni string, alpnProtocols []string) []byte {
// Build SNI extension
sniExt := buildSNIExtension(sni)
// Build ALPN extension
alpnExt := buildALPNExtension(alpnProtocols)
// Build supported_versions extension (TLS 1.2)
supportedVersionsExt := []byte{0x00, 0x2b, 0x00, 0x03, 0x02, 0x03, 0x03} // type=43, len=3, TLS 1.2
// Combine extensions
extensions := append(sniExt, alpnExt...)
extensions = append(extensions, supportedVersionsExt...)
extLen := len(extensions)
// Cipher suites (minimal set)
cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f}
// 4 cipher suites: TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
// Compression methods (null only)
compressionMethods := []byte{0x01, 0x00}
// Build ClientHello handshake (without length header first)
handshakeBody := []byte{
0x03, 0x03, // Version: TLS 1.2
// Random (32 bytes)
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, // Session ID length: 0
}
// Add cipher suites (with length prefix)
cipherSuiteLen := len(cipherSuites)
handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen))
handshakeBody = append(handshakeBody, cipherSuites...)
// Add compression methods (with length prefix)
handshakeBody = append(handshakeBody, compressionMethods...)
// Add extensions (with length prefix)
handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen))
handshakeBody = append(handshakeBody, extensions...)
// Now build full handshake with type and length
handshakeLen := len(handshakeBody)
handshake := append([]byte{
0x01, // Handshake type: ClientHello
byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen), // Handshake length
}, handshakeBody...)
// Build TLS record
recordLen := len(handshake)
record := make([]byte, 5+recordLen)
record[0] = 0x16 // Handshake
record[1] = 0x03 // Version: TLS 1.2
record[2] = 0x03
record[3] = byte(recordLen >> 8)
record[4] = byte(recordLen)
copy(record[5:], handshake)
return record
}
// buildSNIExtension builds a Server Name Indication extension
func buildSNIExtension(sni string) []byte {
nameLen := len(sni)
// SNI extension structure:
// - name_list_len (2 bytes): total length of all names
// - name_type (1 byte): always 0x00 for host_name
// - name_len (2 bytes): length of this name
// - name (variable): the actual hostname
nameListLen := 1 + 2 + nameLen // name_type + name_len + name
// Extension data = name_list_len (2) + name_type (1) + name_len (2) + name
extDataLen := 2 + nameListLen
// Full extension = type (2) + length (2) + data (variable)
ext := make([]byte, 4+extDataLen)
ext[0] = 0x00 // Extension type: SNI (0)
ext[1] = 0x00
ext[2] = byte(extDataLen >> 8)
ext[3] = byte(extDataLen)
ext[4] = byte(nameListLen >> 8) // name_list_len (high byte)
ext[5] = byte(nameListLen) // name_list_len (low byte)
ext[6] = 0x00 // name_type: host_name (0)
ext[7] = byte(nameLen >> 8) // name_len (high byte)
ext[8] = byte(nameLen) // name_len (low byte)
copy(ext[9:], sni)
return ext
}
// buildALPNExtension builds an Application-Layer Protocol Negotiation extension
func buildALPNExtension(protocols []string) []byte {
// Calculate ALPN data length
// ALPN data = alpn_list_len (2) + for each protocol: length (1) + data (variable)
alpnDataLen := 0
for _, proto := range protocols {
alpnDataLen += 1 + len(proto) // length byte + protocol string
}
// Extension data = alpn_list_len (2) + protocols
extDataLen := 2 + alpnDataLen
// Full extension = type (2) + length (2) + data (variable)
ext := make([]byte, 4+extDataLen)
ext[0] = 0x00 // Extension type: ALPN (16 = 0x10)
ext[1] = 0x10
ext[2] = byte(extDataLen >> 8)
ext[3] = byte(extDataLen)
// ALPN protocol list length (2 bytes)
ext[4] = byte(alpnDataLen >> 8)
ext[5] = byte(alpnDataLen)
offset := 6
for _, proto := range protocols {
ext[offset] = byte(len(proto))
offset++
copy(ext[offset:], proto)
offset += len(proto)
}
return ext
}
// min returns the minimum of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}