code_public/ja4sentinel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

release: version 1.1.15 - Fix ALPN detection for malformed TLS extensions
Some checks failed
Build RPM Package / Build RPM Packages (CentOS 7, Rocky 8/9/10) (push) Has been cancelled
Details

Browse Source 
- FIX: ALPN (tls_alpn) not appearing in logs for packets with truncated extensions
- Add sanitizeTLSRecord fallback in extractTLSExtensions (tlsparse/parser.go)
- Mirrors sanitization already present in fingerprint/engine.go
- ALPN now correctly extracted even when ParseClientHello fails on raw payload
- Bump version to 1.1.15 in main.go and packaging/rpm/ja4sentinel.spec

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-03-05 14:42:15 +01:00
parent 63c91175a2
commit d22b0634da
8 changed files with 881 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

489
internal/tlsparse/parser_test.go
View File

@ -336,7 +336,7 @@ func TestProcess_DropsWhenHelloBufferExceedsLimit(t *testing.T) {
// TLS-like payload, but intentionally incomplete to trigger accumulation.
payloadChunk := []byte{0x16, 0x03, 0x03, 0x00, 0x20, 0x01} // len = 6
pkt1 := buildRawPacket(t, srcIP, dstIP, srcPort, dstPort, payloadChunk)
pkt1 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, payloadChunk, 1)
ch, err := parser.Process(pkt1)
if err != nil {
t.Fatalf("first Process() error = %v", err)
@ -354,7 +354,7 @@ func TestProcess_DropsWhenHelloBufferExceedsLimit(t *testing.T) {
t.Fatal("flow should exist after first chunk")
}
pkt2 := buildRawPacket(t, srcIP, dstIP, srcPort, dstPort, payloadChunk)
pkt2 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, payloadChunk, 1+uint32(len(payloadChunk)))
ch, err = parser.Process(pkt2)
if err != nil {
t.Fatalf("second Process() error = %v", err)
@ -403,6 +403,10 @@ func TestProcess_NonTLSNewFlowNotTracked(t *testing.T) {
}
func buildRawPacket(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16, payload []byte) api.RawPacket {
return buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, payload, 1)
}
func buildRawPacketWithSeq(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16, payload []byte, seq uint32) api.RawPacket {
t.Helper()
ip := &layers.IPv4{
@ -416,7 +420,7 @@ func buildRawPacket(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16,
tcp := &layers.TCP{
SrcPort: layers.TCPPort(srcPort),
DstPort: layers.TCPPort(dstPort),
Seq: 1,
Seq: seq,
ACK: true,
Window: 65535,
}
@ -1307,3 +1311,482 @@ func TestParser_SLLPacketType(t *testing.T) {
t.Fatal("Process() should return TLSClientHello for PACKET_HOST")
}
}
// buildSYNPacket creates a raw SYN packet (no payload) with TCP options
func buildSYNPacket(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16, mss uint16, windowScale uint8) api.RawPacket {
t.Helper()
ip := &layers.IPv4{
Version: 4,
TTL: 64,
Id: 0x1234,
Flags: layers.IPv4DontFragment,
SrcIP: net.ParseIP(srcIP).To4(),
DstIP: net.ParseIP(dstIP).To4(),
Protocol: layers.IPProtocolTCP,
}
tcp := &layers.TCP{
SrcPort: layers.TCPPort(srcPort),
DstPort: layers.TCPPort(dstPort),
Seq: 1000,
SYN: true,
Window: 65535,
Options: []layers.TCPOption{
{
OptionType: layers.TCPOptionKindMSS,
OptionLength: 4,
OptionData: []byte{byte(mss >> 8), byte(mss)},
},
{
OptionType: layers.TCPOptionKindWindowScale,
OptionLength: 3,
OptionData: []byte{windowScale},
},
{
OptionType: layers.TCPOptionKindSACKPermitted,
OptionLength: 2,
},
},
}
if err := tcp.SetNetworkLayerForChecksum(ip); err != nil {
t.Fatalf("SetNetworkLayerForChecksum() error = %v", err)
}
eth := &layers.Ethernet{
SrcMAC: net.HardwareAddr{0x00, 0x11, 0x22, 0x33, 0x44, 0x55},
DstMAC: net.HardwareAddr{0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff},
EthernetType: layers.EthernetTypeIPv4,
}
buf := gopacket.NewSerializeBuffer()
opts := gopacket.SerializeOptions{
FixLengths: true,
ComputeChecksums: true,
}
if err := gopacket.SerializeLayers(buf, opts, eth, ip, tcp); err != nil {
t.Fatalf("SerializeLayers() error = %v", err)
}
return api.RawPacket{
Data: buf.Bytes(),
Timestamp: time.Now().UnixNano(),
LinkType: 1,
}
}
func TestProcess_SYNCreatesFlowWithTCPMeta(t *testing.T) {
parser := NewParser()
defer parser.Close()
srcIP := "192.168.1.50"
dstIP := "10.0.0.1"
srcPort := uint16(44444)
dstPort := uint16(443)
expectedMSS := uint16(1460)
expectedWS := uint8(7)
// Step 1: Send SYN packet (should create flow, return nil)
synPkt := buildSYNPacket(t, srcIP, dstIP, srcPort, dstPort, expectedMSS, expectedWS)
ch, err := parser.Process(synPkt)
if err != nil {
t.Fatalf("Process(SYN) error = %v", err)
}
if ch != nil {
t.Fatal("Process(SYN) should return nil (no ClientHello yet)")
}
// Verify flow was created with correct metadata
key := flowKey(srcIP, srcPort, dstIP, dstPort)
parser.mu.RLock()
flow, exists := parser.flows[key]
parser.mu.RUnlock()
if !exists {
t.Fatal("SYN should create a flow")
}
flow.mu.Lock()
if flow.State != NEW {
t.Errorf("flow state = %v, want NEW", flow.State)
}
if flow.TCPMeta.MSS != expectedMSS {
t.Errorf("flow TCPMeta.MSS = %d, want %d", flow.TCPMeta.MSS, expectedMSS)
}
if flow.TCPMeta.WindowScale != expectedWS {
t.Errorf("flow TCPMeta.WindowScale = %d, want %d", flow.TCPMeta.WindowScale, expectedWS)
}
if flow.TCPMeta.WindowSize != 65535 {
t.Errorf("flow TCPMeta.WindowSize = %d, want 65535", flow.TCPMeta.WindowSize)
}
// Check SACK is in options
hasSACK := false
for _, opt := range flow.TCPMeta.Options {
if opt == "SACK" {
hasSACK = true
}
}
if !hasSACK {
t.Errorf("flow TCPMeta.Options = %v, want SACK", flow.TCPMeta.Options)
}
if flow.IPMeta.TTL != 64 {
t.Errorf("flow IPMeta.TTL = %d, want 64", flow.IPMeta.TTL)
}
if !flow.IPMeta.DF {
t.Error("flow IPMeta.DF should be true")
}
flow.mu.Unlock()
// Step 2: Send ClientHello data packet (SYN had Seq=1000, so data starts at 1001)
clientHello := createTLSClientHello(0x0303)
dataPkt := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, clientHello, 1001)
result, err := parser.Process(dataPkt)
if err != nil {
t.Fatalf("Process(ClientHello) error = %v", err)
}
if result == nil {
t.Fatal("Process(ClientHello) should return TLSClientHello")
}
// Verify result uses TCP metadata from SYN, not from data packet
if result.TCPMeta.MSS != expectedMSS {
t.Errorf("result TCPMeta.MSS = %d, want %d (from SYN)", result.TCPMeta.MSS, expectedMSS)
}
if result.TCPMeta.WindowScale != expectedWS {
t.Errorf("result TCPMeta.WindowScale = %d, want %d (from SYN)", result.TCPMeta.WindowScale, expectedWS)
}
if result.IPMeta.TTL != 64 {
t.Errorf("result IPMeta.TTL = %d, want 64 (from SYN)", result.IPMeta.TTL)
}
if !result.IPMeta.DF {
t.Error("result IPMeta.DF should be true (from SYN)")
}
}
func TestProcess_SynToCHMs_Timing(t *testing.T) {
parser := NewParser()
defer parser.Close()
srcIP := "192.168.1.60"
dstIP := "10.0.0.1"
srcPort := uint16(55555)
dstPort := uint16(443)
// Step 1: Send SYN
synPkt := buildSYNPacket(t, srcIP, dstIP, srcPort, dstPort, 1460, 7)
_, err := parser.Process(synPkt)
if err != nil {
t.Fatalf("Process(SYN) error = %v", err)
}
// Wait a measurable amount of time
time.Sleep(50 * time.Millisecond)
// Step 2: Send ClientHello (SYN had Seq=1000, data at 1001)
clientHello := createTLSClientHello(0x0303)
dataPkt := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, clientHello, 1001)
result, err := parser.Process(dataPkt)
if err != nil {
t.Fatalf("Process(ClientHello) error = %v", err)
}
if result == nil {
t.Fatal("Process(ClientHello) should return TLSClientHello")
}
if result.SynToCHMs == nil {
t.Fatal("SynToCHMs should not be nil")
}
// SynToCHMs should be at least 50ms (we slept 50ms)
if *result.SynToCHMs < 40 {
t.Errorf("SynToCHMs = %d ms, want >= 40ms (slept 50ms)", *result.SynToCHMs)
}
}
func TestProcess_NoSYN_StillWorks(t *testing.T) {
// Ensure backward compatibility: if no SYN is seen (e.g. capture started
// mid-connection), a ClientHello data packet still creates a flow and works.
parser := NewParser()
defer parser.Close()
srcIP := "192.168.1.70"
dstIP := "10.0.0.1"
srcPort := uint16(55666)
dstPort := uint16(443)
clientHello := createTLSClientHello(0x0303)
dataPkt := buildRawPacket(t, srcIP, dstIP, srcPort, dstPort, clientHello)
result, err := parser.Process(dataPkt)
if err != nil {
t.Fatalf("Process() error = %v", err)
}
if result == nil {
t.Fatal("Process() should return TLSClientHello even without SYN")
}
if result.SrcIP != srcIP {
t.Errorf("SrcIP = %v, want %v", result.SrcIP, srcIP)
}
}
func TestProcess_FragmentedClientHello_UsesFlowMeta(t *testing.T) {
parser := NewParser()
defer parser.Close()
srcIP := "192.168.1.80"
dstIP := "10.0.0.1"
srcPort := uint16(33333)
dstPort := uint16(443)
expectedMSS := uint16(1460)
expectedWS := uint8(7)
// Step 1: Send SYN with TCP options
synPkt := buildSYNPacket(t, srcIP, dstIP, srcPort, dstPort, expectedMSS, expectedWS)
_, err := parser.Process(synPkt)
if err != nil {
t.Fatalf("Process(SYN) error = %v", err)
}
// Step 2: Send incomplete TLS record (fragment 1, Seq=1001)
clientHello := createTLSClientHello(0x0303)
half := len(clientHello) / 2
fragment1 := clientHello[:half]
pkt1 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, fragment1, 1001)
ch, err := parser.Process(pkt1)
if err != nil {
t.Fatalf("Process(fragment1) error = %v", err)
}
if ch != nil {
t.Fatal("Process(fragment1) should return nil (incomplete)")
}
// Step 3: Send rest (fragment 2, Seq=1001+len(fragment1))
fragment2 := clientHello[half:]
pkt2 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, fragment2, 1001+uint32(half))
result, err := parser.Process(pkt2)
if err != nil {
t.Fatalf("Process(fragment2) error = %v", err)
}
if result == nil {
t.Fatal("Process(fragment2) should return complete TLSClientHello")
}
// Verify metadata comes from the SYN (flow), not from the last data fragment
if result.TCPMeta.MSS != expectedMSS {
t.Errorf("result TCPMeta.MSS = %d, want %d (from SYN)", result.TCPMeta.MSS, expectedMSS)
}
if result.TCPMeta.WindowScale != expectedWS {
t.Errorf("result TCPMeta.WindowScale = %d, want %d (from SYN)", result.TCPMeta.WindowScale, expectedWS)
}
if result.IPMeta.TTL != 64 {
t.Errorf("result IPMeta.TTL = %d, want 64 (from SYN)", result.IPMeta.TTL)
}
}
func TestProcess_TCPRetransmission_Ignored(t *testing.T) {
parser := NewParser()
defer parser.Close()
srcIP := "192.168.1.90"
dstIP := "10.0.0.1"
srcPort := uint16(44321)
dstPort := uint16(443)
// Step 1: Send SYN (Seq=1000)
synPkt := buildSYNPacket(t, srcIP, dstIP, srcPort, dstPort, 1460, 7)
_, _ = parser.Process(synPkt)
// Step 2: Send first fragment (Seq=1001)
clientHello := createTLSClientHello(0x0303)
half := len(clientHello) / 2
fragment1 := clientHello[:half]
pkt1 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, fragment1, 1001)
_, _ = parser.Process(pkt1)
// Step 3: Retransmit fragment 1 (same Seq=1001) — should be ignored
pkt1dup := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, fragment1, 1001)
ch, err := parser.Process(pkt1dup)
if err != nil {
t.Fatalf("Process(retransmit) error = %v", err)
}
if ch != nil {
t.Fatal("Process(retransmit) should return nil")
}
// Step 4: Send second fragment (correct Seq)
fragment2 := clientHello[half:]
pkt2 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, fragment2, 1001+uint32(half))
result, err := parser.Process(pkt2)
if err != nil {
t.Fatalf("Process(fragment2) error = %v", err)
}
if result == nil {
t.Fatal("Process(fragment2) should return complete TLSClientHello after retransmission")
}
}
func TestProcess_TCPGap_DropsFlow(t *testing.T) {
parser := NewParser()
defer parser.Close()
srcIP := "192.168.1.91"
dstIP := "10.0.0.1"
srcPort := uint16(44322)
dstPort := uint16(443)
// Step 1: Send SYN (Seq=1000)
synPkt := buildSYNPacket(t, srcIP, dstIP, srcPort, dstPort, 1460, 7)
_, _ = parser.Process(synPkt)
// Step 2: Send first fragment (Seq=1001)
clientHello := createTLSClientHello(0x0303)
half := len(clientHello) / 2
fragment1 := clientHello[:half]
pkt1 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, fragment1, 1001)
_, _ = parser.Process(pkt1)
// Step 3: Send fragment with gap (Seq far ahead) — should drop flow
fragment2 := clientHello[half:]
gapSeq := uint32(1001 + half + 100) // 100 bytes gap
pkt2 := buildRawPacketWithSeq(t, srcIP, dstIP, srcPort, dstPort, fragment2, gapSeq)
ch, err := parser.Process(pkt2)
if err != nil {
t.Fatalf("Process(gap) error = %v", err)
}
if ch != nil {
t.Fatal("Process(gap) should return nil")
}
// Verify flow was removed
key := flowKey(srcIP, srcPort, dstIP, dstPort)
parser.mu.RLock()
_, exists := parser.flows[key]
parser.mu.RUnlock()
if exists {
t.Fatal("flow should be removed after sequence gap")
}
}
// createTLS13ClientHelloWithSNI creates a TLS 1.3 ClientHello (record version 0x0303,
// supported_versions extension includes 0x0304)
func createTLS13ClientHelloWithSNI(sni string) []byte {
// Build SNI extension
sniExt := buildSNIExtension(sni)
// Build ALPN extension
alpnExt := buildALPNExtension([]string{"h2", "http/1.1"})
// Build supported_versions extension with TLS 1.3 (0x0304) and TLS 1.2 (0x0303)
// Extension type: 43 (0x002b), data: list_len(1) + 2 versions (4 bytes)
supportedVersionsExt := []byte{
0x00, 0x2b, // Extension type: supported_versions (43)
0x00, 0x05, // Extension data length: 5
0x04, // Supported versions list length: 4 bytes (2 versions)
0x03, 0x04, // TLS 1.3
0x03, 0x03, // TLS 1.2
}
// Combine extensions
extensions := append(sniExt, alpnExt...)
extensions = append(extensions, supportedVersionsExt...)
extLen := len(extensions)
// Cipher suites (TLS 1.3 suites)
cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f}
// Compression methods (null only)
compressionMethods := []byte{0x01, 0x00}
// Build ClientHello handshake body
handshakeBody := []byte{
0x03, 0x03, // Version: TLS 1.2 (mandatory for TLS 1.3 ClientHello)
}
// Random (32 bytes)
for i := 0; i < 32; i++ {
handshakeBody = append(handshakeBody, 0x01)
}
handshakeBody = append(handshakeBody, 0x00) // Session ID length: 0
// Add cipher suites
cipherSuiteLen := len(cipherSuites)
handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen))
handshakeBody = append(handshakeBody, cipherSuites...)
// Add compression methods
handshakeBody = append(handshakeBody, compressionMethods...)
// Add extensions
handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen))
handshakeBody = append(handshakeBody, extensions...)
// Build handshake with type and length
handshakeLen := len(handshakeBody)
handshake := append([]byte{
0x01, // Handshake type: ClientHello
byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen),
}, handshakeBody...)
// Build TLS record (version always 0x0303 for TLS 1.3)
recordLen := len(handshake)
record := make([]byte, 5+recordLen)
record[0] = 0x16 // Handshake
record[1] = 0x03 // TLS 1.2 in record layer (per TLS 1.3 spec)
record[2] = 0x03
record[3] = byte(recordLen >> 8)
record[4] = byte(recordLen)
copy(record[5:], handshake)
return record
}
func TestExtractTLSExtensions_TLS13(t *testing.T) {
payload := createTLS13ClientHelloWithSNI("example.com")
info, err := extractTLSExtensions(payload)
if err != nil {
t.Fatalf("extractTLSExtensions() error = %v", err)
}
if info == nil {
t.Fatal("extractTLSExtensions() returned nil")
}
// TLS 1.3 should be detected via supported_versions extension
if info.TLSVersion != "1.3" {
t.Errorf("TLSVersion = %q, want \"1.3\"", info.TLSVersion)
}
if info.SNI != "example.com" {
t.Errorf("SNI = %q, want \"example.com\"", info.SNI)
}
}
func TestProcess_TLS13ClientHello_CorrectVersion(t *testing.T) {
parser := NewParser()
defer parser.Close()
srcIP := "192.168.1.200"
dstIP := "10.0.0.1"
srcPort := uint16(44555)
dstPort := uint16(443)
clientHello := createTLS13ClientHelloWithSNI("tls13.example.com")
pkt := buildRawPacket(t, srcIP, dstIP, srcPort, dstPort, clientHello)
result, err := parser.Process(pkt)
if err != nil {
t.Fatalf("Process() error = %v", err)
}
if result == nil {
t.Fatal("Process() should return TLSClientHello")
}
if result.TLSVersion != "1.3" {
t.Errorf("TLSVersion = %q, want \"1.3\"", result.TLSVersion)
}
if result.SNI != "tls13.example.com" {
t.Errorf("SNI = %q, want \"tls13.example.com\"", result.SNI)
}
}