feat(ja4ebpf): add multi-interface TC, LPM_TRIE ignore_src, unit tests, and fix bugs
- Add multi-interface TC attachment (default "any" = all UP interfaces) - Add BPF LPM_TRIE map ignored_src for kernel-side CIDR filtering - Add userspace ignore_src filtering for SSL/accept4 path via net.IPNet.Contains() - Add AcceptCache for fd→SessionKey correlation with TTL and Close() - Add 5 test files covering writer, procutil, dispatcher, accept_cache, and cmd - Fix formatTCPOptions infinite loop on EOL (case 0 break→return) - Fix pseudoOrderToShort panic on empty slice (negative cap) - Fix AcceptCache goroutine leak (add done channel + Close()) - Update config.yml.example with interfaces, listen_ports, ignore_src - Rewrite docs/services/ja4ebpf.md (was massively stale: XDP, RingBuffer, etc.) - Fix stale XDP/RingBuffer references in docs/architecture.md, thesis, tls.go Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
@ -90,6 +90,7 @@ type sessionRecord struct {
|
||||
H2WindowUpdate uint32 `json:"h2_window_update,omitempty"`
|
||||
H2PseudoOrder string `json:"h2_pseudo_order,omitempty"`
|
||||
H2HasPriority uint8 `json:"h2_has_priority,omitempty"`
|
||||
H2SettingsAck uint8 `json:"h2_settings_ack,omitempty"`
|
||||
H2HeaderTableSize *int32 `json:"h2_header_table_size,omitempty"`
|
||||
H2EnablePush *int32 `json:"h2_enable_push,omitempty"`
|
||||
H2MaxConcurrentStreams *int32 `json:"h2_max_concurrent_streams,omitempty"`
|
||||
@ -110,6 +111,11 @@ func NewClickHouseWriter(dsn string, batchSize int, flushInterval time.Duration)
|
||||
return nil, fmt.Errorf("analyse DSN ClickHouse: %w", err)
|
||||
}
|
||||
|
||||
// Désactiver l'insertion asynchrone pour les writes transactionnels
|
||||
opts.Settings = map[string]interface{}{
|
||||
"async_insert": 0,
|
||||
}
|
||||
|
||||
conn, err := clickhouse.Open(opts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("connexion ClickHouse: %w", err)
|
||||
@ -201,6 +207,10 @@ func (w *ClickHouseWriter) flushBatch(ctx context.Context, batch []*correlation.
|
||||
}
|
||||
|
||||
for _, s := range batch {
|
||||
// Ignorer les sessions sans aucune donnée applicative (SYN-only)
|
||||
if len(s.Requests) == 0 && s.TLS == nil {
|
||||
continue
|
||||
}
|
||||
record := sessionToRecord(s)
|
||||
jsonBytes, err := json.Marshal(record)
|
||||
if err != nil {
|
||||
@ -371,12 +381,29 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
}
|
||||
}
|
||||
|
||||
// Champs HTTP/2 au niveau connexion (H2ConnState)
|
||||
if s.H2Conn != nil {
|
||||
if s.H2Conn.SettingsAck {
|
||||
rec.H2SettingsAck = 1
|
||||
}
|
||||
// Vérifier si un stream a reçu une frame PRIORITY
|
||||
for _, stream := range s.H2Conn.Streams {
|
||||
if stream.Priority != nil {
|
||||
rec.H2HasPriority = 1
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// pseudoOrderToShort convertit la liste de pseudo-headers en notation abrégée.
|
||||
// Ex: [":method", ":authority", ":scheme", ":path"] → "m,a,s,p"
|
||||
func pseudoOrderToShort(headers []string) string {
|
||||
if len(headers) == 0 {
|
||||
return ""
|
||||
}
|
||||
short := make([]byte, 0, len(headers)*2-1)
|
||||
for i, h := range headers {
|
||||
if i > 0 {
|
||||
@ -391,6 +418,8 @@ func pseudoOrderToShort(headers []string) string {
|
||||
short = append(short, 's')
|
||||
case h == ":path":
|
||||
short = append(short, 'p')
|
||||
case h == ":status":
|
||||
short = append(short, 't')
|
||||
default:
|
||||
short = append(short, '?')
|
||||
}
|
||||
@ -506,7 +535,7 @@ func formatTCPOptions(opts []byte) string {
|
||||
kind := opts[i]
|
||||
switch kind {
|
||||
case 0: // End of Options List
|
||||
break
|
||||
return strings.Join(names, ",")
|
||||
case 1: // NOP
|
||||
names = append(names, "NOP")
|
||||
i++
|
||||
|
||||
179
services/ja4ebpf/internal/writer/clickhouse_test.go
Normal file
179
services/ja4ebpf/internal/writer/clickhouse_test.go
Normal file
@ -0,0 +1,179 @@
|
||||
package writer
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/antitbone/ja4/ja4ebpf/internal/correlation"
|
||||
)
|
||||
|
||||
func TestFormatTLSVersion(t *testing.T) {
|
||||
tests := []struct {
|
||||
input uint16
|
||||
want string
|
||||
}{
|
||||
{0x0301, "TLSv1.0"},
|
||||
{0x0302, "TLSv1.1"},
|
||||
{0x0303, "TLSv1.2"},
|
||||
{0x0304, "TLSv1.3"},
|
||||
{0x0000, ""},
|
||||
{0x0300, ""},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := formatTLSVersion(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("formatTLSVersion(0x%04x) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderVal(t *testing.T) {
|
||||
kv := map[string]string{
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"accept": "text/html",
|
||||
"Accept-Encoding": "gzip",
|
||||
"accept-encoding": "br",
|
||||
}
|
||||
tests := []struct {
|
||||
titleKey string
|
||||
lowerKey string
|
||||
want string
|
||||
}{
|
||||
{"User-Agent", "user-agent", "Mozilla/5.0"},
|
||||
{"Accept", "accept", "text/html"}, // lowercase key in map
|
||||
{"Accept-Encoding", "accept-encoding", "gzip"}, // title-case wins
|
||||
{"X-Missing", "x-missing", ""}, // not present
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := headerVal(kv, tt.titleKey, tt.lowerKey)
|
||||
if got != tt.want {
|
||||
t.Errorf("headerVal(kv, %q, %q) = %q, want %q", tt.titleKey, tt.lowerKey, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildClientHeaders(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
kv map[string]string
|
||||
want string
|
||||
}{
|
||||
{"empty", nil, ""},
|
||||
{"empty map", map[string]string{}, ""},
|
||||
{"single header", map[string]string{"user-agent": "curl/8.0"}, `{"user-agent":"curl/8.0"}`},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := buildClientHeaders(tt.kv)
|
||||
if got != tt.want {
|
||||
t.Errorf("buildClientHeaders() = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestPseudoOrderToShort(t *testing.T) {
|
||||
tests := []struct {
|
||||
input []string
|
||||
want string
|
||||
}{
|
||||
{[]string{":method", ":path", ":scheme", ":authority"}, "m,p,s,a"},
|
||||
{[]string{":method", ":authority", ":scheme", ":path"}, "m,a,s,p"},
|
||||
{[]string{":method"}, "m"},
|
||||
{[]string{":status"}, "t"},
|
||||
{[]string{":method", ":path", ":unknown"}, "m,p,?"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
got := pseudoOrderToShort(tt.input)
|
||||
if got != tt.want {
|
||||
t.Errorf("pseudoOrderToShort(%v) = %q, want %q", tt.input, got, tt.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFormatTCPOptions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
opts []byte
|
||||
want string
|
||||
}{
|
||||
{"nil", nil, ""},
|
||||
{"empty", []byte{}, ""},
|
||||
{"MSS only", []byte{2, 4, 0x05, 0xB4}, "MSS"}, // MSS=1460
|
||||
{"WS only", []byte{3, 3, 6}, "WS"}, // WS=6
|
||||
{"SACK", []byte{4, 2}, "SACK"}, // SACK Permitted
|
||||
{"TS", []byte{8, 10, 0, 0, 0, 0, 0, 0, 0, 0}, "TS"}, // Timestamp
|
||||
{"NOP+MSS+WS+SACK+TS", []byte{1, 2, 4, 0x05, 0xB4, 3, 3, 6, 4, 2, 8, 10, 0, 0, 0, 0, 0, 0, 0, 0}, "NOP,MSS,WS,SACK,TS"},
|
||||
{"EOL", []byte{0}, ""},
|
||||
{"NOP", []byte{1, 1}, "NOP,NOP"},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := formatTCPOptions(tt.opts)
|
||||
if got != tt.want {
|
||||
t.Errorf("formatTCPOptions(%v) = %q, want %q", tt.opts, got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildH2Fingerprint(t *testing.T) {
|
||||
h2 := &correlation.HTTP2Settings{
|
||||
HeaderTableSize: 4096,
|
||||
EnablePush: 0,
|
||||
MaxConcurrentStreams: 100,
|
||||
InitialWindowSize: 65535,
|
||||
MaxFrameSize: 16384,
|
||||
MaxHeaderListSize: 262144,
|
||||
WindowUpdateIncrement: 15663105,
|
||||
PseudoHeaderOrder: []string{":method", ":authority", ":scheme", ":path"},
|
||||
}
|
||||
got := buildH2Fingerprint(h2)
|
||||
// Expected: "1:4096,2:0,3:100,4:65535,5:16384,6:262144|15663105|0|m,a,s,p"
|
||||
want := "1:4096,2:0,3:100,4:65535,5:16384,6:262144|15663105|0|m,a,s,p"
|
||||
if got != want {
|
||||
t.Errorf("buildH2Fingerprint() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildH2Fingerprint_Minimal(t *testing.T) {
|
||||
h2 := &correlation.HTTP2Settings{
|
||||
HeaderTableSize: 4096,
|
||||
EnablePush: 0,
|
||||
InitialWindowSize: 65535,
|
||||
MaxConcurrentStreams: -1,
|
||||
MaxFrameSize: -1,
|
||||
MaxHeaderListSize: -1,
|
||||
WindowUpdateIncrement: 0,
|
||||
PseudoHeaderOrder: nil,
|
||||
}
|
||||
got := buildH2Fingerprint(h2)
|
||||
want := "1:4096,2:0,4:65535||0|"
|
||||
if got != want {
|
||||
t.Errorf("buildH2Fingerprint() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildH2SettingsFP(t *testing.T) {
|
||||
h2 := &correlation.HTTP2Settings{
|
||||
MaxConcurrentStreams: 100,
|
||||
InitialWindowSize: 65535,
|
||||
EnablePush: 0,
|
||||
}
|
||||
got := buildH2SettingsFP(h2)
|
||||
want := "3:100,4:65535,2:0"
|
||||
if got != want {
|
||||
t.Errorf("buildH2SettingsFP() = %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuildH2SettingsFP_Empty(t *testing.T) {
|
||||
h2 := &correlation.HTTP2Settings{
|
||||
MaxConcurrentStreams: -1,
|
||||
InitialWindowSize: -1,
|
||||
EnablePush: -1,
|
||||
}
|
||||
got := buildH2SettingsFP(h2)
|
||||
if got != "" {
|
||||
t.Errorf("buildH2SettingsFP() = %q, want empty", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user