code_public/ja4sentinel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release: version 1.0.9 - Add SNI, ALPN, TLS version extraction and architecture.yml compliance
Some checks failed
Build RPM Package / Build RPM Packages (CentOS 7, Rocky 8/9/10) (push) Has been cancelled
Details

Browse Source 
New features:
- Extract SNI (Server Name Indication) from TLS ClientHello
- Extract ALPN (Application-Layer Protocol Negotiation) protocols
- Detect TLS version from ClientHello using tlsfingerprint library
- Add ConnID field for TCP flow correlation
- Add SensorID field for multi-sensor deployments
- Add SynToCHMs timing field for behavioral detection
- Add AsyncBuffer configuration for output queue sizing

Architecture changes:
- Remove JA4Hash from LogRecord (JA4 format includes its own hash portions)
- Update api.TLSClientHello with new TLS metadata fields
- Update api.LogRecord with correlation, TLS, and timing fields
- Ensure 100% compliance with architecture.yml specification

Tests:
- Add unit tests for TLS extension extraction (SNI, ALPN, Version)
- Update tests for new LogRecord schema without JA4Hash
- Add tests for AsyncBuffer configuration

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
2026-03-02 19:32:16 +01:00
parent fd162982d9
commit 965720a183
12 changed files with 854 additions and 392 deletions
Download Patch File Download Diff File Expand all files Collapse all files

241
internal/tlsparse/parser.go
View File

@ -12,6 +12,7 @@ import (
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
tlsfingerprint "github.com/psanford/tlsfingerprint"
)
// ConnectionState represents the state of a TCP connection for TLS parsing
@ -220,15 +221,31 @@ func (p *ParserImpl) Process(pkt api.RawPacket) (*api.TLSClientHello, error) {
flow.State = JA4_DONE
flow.HelloBuffer = clientHello
return &api.TLSClientHello{
SrcIP: srcIP,
SrcPort: srcPort,
DstIP: dstIP,
DstPort: dstPort,
Payload: clientHello,
IPMeta: ipMeta,
TCPMeta: tcpMeta,
}, nil
// Extract TLS extensions (SNI, ALPN, TLS version)
extInfo, _ := extractTLSExtensions(clientHello)
// Generate ConnID from flow key
connID := key
ch := &api.TLSClientHello{
SrcIP: srcIP,
SrcPort: srcPort,
DstIP: dstIP,
DstPort: dstPort,
Payload: clientHello,
IPMeta: ipMeta,
TCPMeta: tcpMeta,
ConnID: connID,
SNI: extInfo.SNI,
ALPN: joinStringSlice(extInfo.ALPN, ","),
TLSVersion: extInfo.TLSVersion,
}
// Calculate SynToCHMs if we have timing info
synToCH := uint32(time.Since(flow.CreatedAt).Milliseconds())
ch.SynToCHMs = &synToCH
return ch, nil
}
// Check for fragmented ClientHello (accumulate segments)
@ -257,15 +274,31 @@ func (p *ParserImpl) Process(pkt api.RawPacket) (*api.TLSClientHello, error) {
// Complete ClientHello found
flow.State = JA4_DONE
return &api.TLSClientHello{
SrcIP: srcIP,
SrcPort: srcPort,
DstIP: dstIP,
DstPort: dstPort,
Payload: clientHello,
IPMeta: ipMeta,
TCPMeta: tcpMeta,
}, nil
// Extract TLS extensions (SNI, ALPN, TLS version)
extInfo, _ := extractTLSExtensions(clientHello)
// Generate ConnID from flow key
connID := key
ch := &api.TLSClientHello{
SrcIP: srcIP,
SrcPort: srcPort,
DstIP: dstIP,
DstPort: dstPort,
Payload: clientHello,
IPMeta: ipMeta,
TCPMeta: tcpMeta,
ConnID: connID,
SNI: extInfo.SNI,
ALPN: joinStringSlice(extInfo.ALPN, ","),
TLSVersion: extInfo.TLSVersion,
}
// Calculate SynToCHMs
synToCH := uint32(time.Since(flow.CreatedAt).Milliseconds())
ch.SynToCHMs = &synToCH
return ch, nil
}
}
@ -377,6 +410,13 @@ func extractTCPMeta(tcp *layers.TCP) api.TCPMeta {
return meta
}
// TLSExtensionInfo contains parsed TLS extension information
type TLSExtensionInfo struct {
SNI string
ALPN []string
TLSVersion string
}
// parseClientHello checks if the payload contains a TLS ClientHello and returns it
func parseClientHello(payload []byte) ([]byte, error) {
if len(payload) < 5 {
@ -419,6 +459,171 @@ func parseClientHello(payload []byte) ([]byte, error) {
return payload[:5+recordLength], nil
}
// extractTLSExtensions extracts SNI, ALPN, and TLS version from a ClientHello payload
// Uses tlsfingerprint library for ALPN and TLS version, manual parsing for SNI value
func extractTLSExtensions(payload []byte) (*TLSExtensionInfo, error) {
if len(payload) < 5 {
return nil, nil
}
// TLS record layer
contentType := payload[0]
if contentType != 22 {
return nil, nil // Not a handshake
}
version := binary.BigEndian.Uint16(payload[1:3])
recordLength := int(binary.BigEndian.Uint16(payload[3:5]))
if len(payload) < 5+recordLength {
return nil, nil // Incomplete record
}
handshakePayload := payload[5 : 5+recordLength]
if len(handshakePayload) < 1 {
return nil, nil
}
handshakeType := handshakePayload[0]
if handshakeType != 1 {
return nil, nil // Not a ClientHello
}
info := &TLSExtensionInfo{}
// Use tlsfingerprint to parse ALPN and TLS version
fp, err := tlsfingerprint.ParseClientHello(payload)
if err == nil && fp != nil {
// Extract ALPN protocols
if len(fp.ALPNProtocols) > 0 {
info.ALPN = fp.ALPNProtocols
}
// Extract TLS version
info.TLSVersion = tlsVersionToString(fp.Version)
}
// If tlsfingerprint didn't provide version, fall back to record version
if info.TLSVersion == "" {
info.TLSVersion = tlsVersionToString(version)
}
// Parse SNI manually (tlsfingerprint only provides HasSNI, not the value)
sniValue := extractSNIFromPayload(handshakePayload)
if sniValue != "" {
info.SNI = sniValue
}
return info, nil
}
// extractSNIFromPayload extracts the SNI value from a ClientHello handshake payload
// handshakePayload starts at the handshake type byte (0x01 for ClientHello)
func extractSNIFromPayload(handshakePayload []byte) string {
// handshakePayload structure:
// [0]: Handshake type (0x01 for ClientHello)
// [1:4]: Handshake length (3 bytes, big-endian)
// [4:6]: Version (2 bytes)
// [6:38]: Random (32 bytes)
// [38]: Session ID length
// ...
if len(handshakePayload) < 40 { // type(1) + len(3) + version(2) + random(32) + sessionIDLen(1)
return ""
}
// Start after type (1) + length (3) + version (2) + random (32) = 38
offset := 38
// Session ID length (1 byte)
sessionIDLen := int(handshakePayload[offset])
offset++
// Skip session ID
offset += sessionIDLen
// Cipher suites length (2 bytes)
if offset+2 > len(handshakePayload) {
return ""
}
cipherSuiteLen := int(binary.BigEndian.Uint16(handshakePayload[offset : offset+2]))
offset += 2 + cipherSuiteLen
// Compression methods length (1 byte)
if offset >= len(handshakePayload) {
return ""
}
compressionLen := int(handshakePayload[offset])
offset++
// Skip compression methods
offset += compressionLen
// Extensions length (2 bytes) - optional
if offset+2 > len(handshakePayload) {
return ""
}
extensionsLen := int(binary.BigEndian.Uint16(handshakePayload[offset : offset+2]))
offset += 2
if extensionsLen == 0 || offset+extensionsLen > len(handshakePayload) {
return ""
}
extensionsEnd := offset + extensionsLen
// Debug: log extension types found
_ = extensionsEnd // suppress unused warning in case we remove debug code
// Parse extensions to find SNI (type 0)
for offset < extensionsEnd {
if offset+4 > len(handshakePayload) {
break
}
extType := binary.BigEndian.Uint16(handshakePayload[offset : offset+2])
extLen := int(binary.BigEndian.Uint16(handshakePayload[offset+2 : offset+4]))
offset += 4
if offset+extLen > len(handshakePayload) {
break
}
extData := handshakePayload[offset : offset+extLen]
offset += extLen
if extType == 0 && len(extData) >= 5 { // SNI extension
// SNI extension structure:
// - name_list_len (2 bytes)
// - name_type (1 byte)
// - name_len (2 bytes)
// - name (variable)
// Skip name_list_len (2), read name_type (1) + name_len (2)
nameLen := int(binary.BigEndian.Uint16(extData[3:5]))
if len(extData) >= 5+nameLen {
return string(extData[5 : 5+nameLen])
}
}
}
return ""
}
// tlsVersionToString converts a TLS version number to a string
func tlsVersionToString(version uint16) string {
switch version {
case 0x0301:
return "1.0"
case 0x0302:
return "1.1"
case 0x0303:
return "1.2"
case 0x0304:
return "1.3"
default:
return ""
}
}
// IsClientHello checks if a payload contains a TLS ClientHello
func IsClientHello(payload []byte) bool {
if len(payload) < 6 {

270
internal/tlsparse/parser_test.go
View File

@ -7,6 +7,7 @@ import (
"ja4sentinel/api"
tlsfingerprint "github.com/psanford/tlsfingerprint"
"github.com/google/gopacket"
"github.com/google/gopacket/layers"
)
@ -444,3 +445,272 @@ func buildRawPacket(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16,
Timestamp: time.Now().UnixNano(),
}
}
func TestTLSVersionToString(t *testing.T) {
tests := []struct {
version uint16
want string
}{
{0x0301, "1.0"},
{0x0302, "1.1"},
{0x0303, "1.2"},
{0x0304, "1.3"},
{0x0300, ""},
{0x0305, ""},
}
for _, tt := range tests {
t.Run(tt.want, func(t *testing.T) {
got := tlsVersionToString(tt.version)
if got != tt.want {
t.Errorf("tlsVersionToString(%#x) = %v, want %v", tt.version, got, tt.want)
}
})
}
}
func TestExtractTLSExtensions(t *testing.T) {
tests := []struct {
name string
payload []byte
wantSNI string
wantALPN []string
wantVersion string
wantNil bool
}{
{
name: "empty payload",
payload: []byte{},
wantNil: true,
},
{
name: "too short",
payload: []byte{0x16, 0x03, 0x03},
wantNil: true,
},
{
name: "TLS 1.2 ClientHello without extensions",
payload: createTLSClientHello(0x0303),
wantVersion: "1.2",
wantNil: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := extractTLSExtensions(tt.payload)
if err != nil {
t.Errorf("extractTLSExtensions() unexpected error = %v", err)
return
}
if (got == nil) != tt.wantNil {
t.Errorf("extractTLSExtensions() = %v, wantNil %v", got == nil, tt.wantNil)
return
}
if got != nil {
if got.TLSVersion != tt.wantVersion {
t.Errorf("TLSVersion = %v, want %v", got.TLSVersion, tt.wantVersion)
}
}
})
}
}
func TestParser_ExtractsTLSFields(t *testing.T) {
parser := NewParser()
defer parser.Close()
// Create a minimal valid TLS 1.2 ClientHello with SNI and ALPN extensions
// This is a real-world-like ClientHello structure
clientHelloWithExt := createMinimalTLSClientHelloWithSNIAndALPN("example.com", []string{"h2", "http/1.1"})
// Debug: Check what extractTLSExtensions returns
extInfo, err := extractTLSExtensions(clientHelloWithExt)
if err != nil {
t.Logf("extractTLSExtensions error: %v", err)
}
if extInfo != nil {
t.Logf("extInfo: SNI=%q, ALPN=%v, Version=%q", extInfo.SNI, extInfo.ALPN, extInfo.TLSVersion)
} else {
t.Log("extInfo is nil")
}
// Also test with tlsfingerprint directly
fp, err := tlsfingerprint.ParseClientHello(clientHelloWithExt)
if err != nil {
t.Logf("tlsfingerprint error: %v", err)
} else {
t.Logf("tlsfingerprint: ALPN=%v, Version=%#x, HasSNI=%v", fp.ALPNProtocols, fp.Version, fp.HasSNI)
}
// Debug: print first bytes of ClientHello
t.Logf("ClientHello hex: % x", clientHelloWithExt[:min(50, len(clientHelloWithExt))])
srcIP := "192.168.1.100"
dstIP := "10.0.0.1"
srcPort := uint16(54321)
dstPort := uint16(443)
pkt := buildRawPacket(t, srcIP, dstIP, srcPort, dstPort, clientHelloWithExt)
result, err := parser.Process(pkt)
if err != nil {
t.Fatalf("Process() error = %v", err)
}
if result == nil {
t.Fatal("Process() should return TLSClientHello")
}
// Verify new fields are populated
if result.SNI != "example.com" {
t.Errorf("SNI = %v, want example.com", result.SNI)
}
if result.ALPN == "" {
t.Error("ALPN should not be empty")
}
if result.TLSVersion != "1.2" {
t.Errorf("TLSVersion = %v, want 1.2", result.TLSVersion)
}
if result.ConnID == "" {
t.Error("ConnID should not be empty")
}
if result.SynToCHMs == nil {
t.Error("SynToCHMs should not be nil")
}
}
// createMinimalTLSClientHelloWithSNIAndALPN creates a minimal but valid TLS 1.2 ClientHello
// with SNI and ALPN extensions
func createMinimalTLSClientHelloWithSNIAndALPN(sni string, alpnProtocols []string) []byte {
// Build SNI extension
sniExt := buildSNIExtension(sni)
// Build ALPN extension
alpnExt := buildALPNExtension(alpnProtocols)
// Build supported_versions extension (TLS 1.2)
supportedVersionsExt := []byte{0x00, 0x2b, 0x00, 0x03, 0x02, 0x03, 0x03} // type=43, len=3, TLS 1.2
// Combine extensions
extensions := append(sniExt, alpnExt...)
extensions = append(extensions, supportedVersionsExt...)
extLen := len(extensions)
// Cipher suites (minimal set)
cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f}
// 4 cipher suites: TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
// Compression methods (null only)
compressionMethods := []byte{0x01, 0x00}
// Build ClientHello handshake (without length header first)
handshakeBody := []byte{
0x03, 0x03, // Version: TLS 1.2
// Random (32 bytes)
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
0x00, // Session ID length: 0
}
// Add cipher suites (with length prefix)
cipherSuiteLen := len(cipherSuites)
handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen))
handshakeBody = append(handshakeBody, cipherSuites...)
// Add compression methods (with length prefix)
handshakeBody = append(handshakeBody, compressionMethods...)
// Add extensions (with length prefix)
handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen))
handshakeBody = append(handshakeBody, extensions...)
// Now build full handshake with type and length
handshakeLen := len(handshakeBody)
handshake := append([]byte{
0x01, // Handshake type: ClientHello
byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen), // Handshake length
}, handshakeBody...)
// Build TLS record
recordLen := len(handshake)
record := make([]byte, 5+recordLen)
record[0] = 0x16 // Handshake
record[1] = 0x03 // Version: TLS 1.2
record[2] = 0x03
record[3] = byte(recordLen >> 8)
record[4] = byte(recordLen)
copy(record[5:], handshake)
return record
}
// buildSNIExtension builds a Server Name Indication extension
func buildSNIExtension(sni string) []byte {
nameLen := len(sni)
// SNI extension structure:
// - name_list_len (2 bytes): total length of all names
// - name_type (1 byte): always 0x00 for host_name
// - name_len (2 bytes): length of this name
// - name (variable): the actual hostname
nameListLen := 1 + 2 + nameLen // name_type + name_len + name
// Extension data = name_list_len (2) + name_type (1) + name_len (2) + name
extDataLen := 2 + nameListLen
// Full extension = type (2) + length (2) + data (variable)
ext := make([]byte, 4+extDataLen)
ext[0] = 0x00 // Extension type: SNI (0)
ext[1] = 0x00
ext[2] = byte(extDataLen >> 8)
ext[3] = byte(extDataLen)
ext[4] = byte(nameListLen >> 8) // name_list_len (high byte)
ext[5] = byte(nameListLen) // name_list_len (low byte)
ext[6] = 0x00 // name_type: host_name (0)
ext[7] = byte(nameLen >> 8) // name_len (high byte)
ext[8] = byte(nameLen) // name_len (low byte)
copy(ext[9:], sni)
return ext
}
// buildALPNExtension builds an Application-Layer Protocol Negotiation extension
func buildALPNExtension(protocols []string) []byte {
// Calculate ALPN data length
// ALPN data = alpn_list_len (2) + for each protocol: length (1) + data (variable)
alpnDataLen := 0
for _, proto := range protocols {
alpnDataLen += 1 + len(proto) // length byte + protocol string
}
// Extension data = alpn_list_len (2) + protocols
extDataLen := 2 + alpnDataLen
// Full extension = type (2) + length (2) + data (variable)
ext := make([]byte, 4+extDataLen)
ext[0] = 0x00 // Extension type: ALPN (16 = 0x10)
ext[1] = 0x10
ext[2] = byte(extDataLen >> 8)
ext[3] = byte(extDataLen)
// ALPN protocol list length (2 bytes)
ext[4] = byte(alpnDataLen >> 8)
ext[5] = byte(alpnDataLen)
offset := 6
for _, proto := range protocols {
ext[offset] = byte(len(proto))
offset++
copy(ext[offset:], proto)
offset += len(proto)
}
return ext
}
// min returns the minimum of two integers
func min(a, b int) int {
if a < b {
return a
}
return b
}