feat: multi-distro VM tests, ja4ebpf eBPF improvements, bot-detector scoring
ja4ebpf: - Refactor BPF TC capture with improved SYN offset handling and TCP option parsing - Enhance TLS uprobe SSL hooking for better key extraction - Add ClickHouse writer improvements for HTTP log materialized views - Update RPM spec for Rocky Linux 8/9/10, fix systemd service - Simplify loader with cleaner bpf2go integration bot-detector: - Add H2 SETTINGS per-parameter comparison in browser_matcher - Enhance browser signatures and scoring pipeline - Improve preprocessing and cycle detection infra: - Multi-distro Vagrantfile (centos8, rocky9, rocky10) with per-distro provisioning - New Makefile targets: vm-up-all, test-vm-matrix, test-vm-centos8/rocky10 - Add debug helpers and run-test-from-host.sh for host-driven VM testing - Update run-tests-vm.sh for cross-distro compatibility - Remove accidental binary blob (\004) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
@ -52,14 +52,28 @@ type sessionRecord struct {
|
||||
TLSVersion string `json:"tls_version,omitempty"`
|
||||
|
||||
// HTTP
|
||||
Method string `json:"method,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
QueryString string `json:"query_string,omitempty"`
|
||||
StatusCode *int `json:"status_code,omitempty"`
|
||||
ResponseSize *int64 `json:"response_size,omitempty"`
|
||||
DurationMS *float64 `json:"duration_ms,omitempty"`
|
||||
KeepAlives int `json:"keepalives,omitempty"`
|
||||
HeaderOrderSig string `json:"header_order_signature,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
QueryString string `json:"query_string,omitempty"`
|
||||
StatusCode *int `json:"status_code,omitempty"`
|
||||
ResponseSize *int64 `json:"response_size,omitempty"`
|
||||
DurationMS *float64 `json:"duration_ms,omitempty"`
|
||||
KeepAlives int `json:"keepalives,omitempty"`
|
||||
HeaderOrderSig string `json:"header_order_signature,omitempty"`
|
||||
|
||||
// HTTP/2 fingerprinting passif
|
||||
H2Fingerprint string `json:"h2_fingerprint,omitempty"`
|
||||
H2SettingsFP string `json:"h2_settings_fp,omitempty"`
|
||||
H2WindowUpdate uint32 `json:"h2_window_update,omitempty"`
|
||||
H2PseudoOrder string `json:"h2_pseudo_order,omitempty"`
|
||||
H2HasPriority uint8 `json:"h2_has_priority,omitempty"`
|
||||
H2HeaderTableSize int32 `json:"h2_header_table_size"`
|
||||
H2EnablePush int32 `json:"h2_enable_push"`
|
||||
H2MaxConcurrentStreams int32 `json:"h2_max_concurrent_streams"`
|
||||
H2InitialWindowSize int64 `json:"h2_initial_window_size"`
|
||||
H2MaxFrameSize int32 `json:"h2_max_frame_size"`
|
||||
H2MaxHeaderListSize int32 `json:"h2_max_header_list_size"`
|
||||
H2EnableConnectProtocol int32 `json:"h2_enable_connect_protocol"`
|
||||
}
|
||||
|
||||
// NewClickHouseWriter crée un writer et établit la connexion ClickHouse.
|
||||
@ -192,37 +206,142 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
|
||||
// Champs métadonnées IP/TCP
|
||||
if s.L3L4 != nil {
|
||||
rec.IPMetaDF = &s.L3L4.DFBit
|
||||
rec.IPMetaID = &s.L3L4.IPID
|
||||
rec.IPMetaTTL = &s.L3L4.TTL
|
||||
rec.TCPMetaWindowSize = &s.L3L4.WindowSize
|
||||
rec.IPMetaDF = &s.L3L4.DFBit
|
||||
rec.IPMetaID = &s.L3L4.IPID
|
||||
rec.IPMetaTTL = &s.L3L4.TTL
|
||||
rec.TCPMetaWindowSize = &s.L3L4.WindowSize
|
||||
rec.TCPMetaWindowScale = &s.L3L4.WindowScale
|
||||
rec.TCPMetaMSS = &s.L3L4.MSS
|
||||
rec.TCPMetaMSS = &s.L3L4.MSS
|
||||
}
|
||||
|
||||
// Champs TLS
|
||||
if s.TLS != nil {
|
||||
rec.JA4Hash = s.TLS.JA4Hash
|
||||
rec.TLSSNI = s.TLS.SNI
|
||||
rec.TLSALPN = strings.Join(s.TLS.ALPN, ",")
|
||||
rec.JA4Hash = s.TLS.JA4Hash
|
||||
rec.TLSSNI = s.TLS.SNI
|
||||
rec.TLSALPN = strings.Join(s.TLS.ALPN, ",")
|
||||
rec.TLSVersion = formatTLSVersion(s.TLS.TLSVersion)
|
||||
}
|
||||
|
||||
// Champs HTTP (dernière requête)
|
||||
if len(s.Requests) > 0 {
|
||||
last := &s.Requests[len(s.Requests)-1]
|
||||
rec.Method = last.Method
|
||||
rec.Path = last.Path
|
||||
rec.QueryString = last.QueryString
|
||||
rec.StatusCode = &last.StatusCode
|
||||
rec.ResponseSize = &last.ResponseSize
|
||||
rec.DurationMS = &last.DurationMS
|
||||
rec.HeaderOrderSig = last.HeaderOrderSig
|
||||
rec.Method = last.Method
|
||||
rec.Path = last.Path
|
||||
rec.QueryString = last.QueryString
|
||||
rec.StatusCode = &last.StatusCode
|
||||
rec.ResponseSize = &last.ResponseSize
|
||||
rec.DurationMS = &last.DurationMS
|
||||
rec.HeaderOrderSig = last.HeaderOrderSig
|
||||
|
||||
// Champs HTTP/2 passifs
|
||||
if last.HTTP2Settings != nil {
|
||||
h2 := last.HTTP2Settings
|
||||
rec.H2WindowUpdate = h2.WindowUpdateIncrement
|
||||
|
||||
// Ordre des pseudo-headers → notation abrégée "m,a,s,p"
|
||||
if len(h2.PseudoHeaderOrder) > 0 {
|
||||
rec.H2PseudoOrder = pseudoOrderToShort(h2.PseudoHeaderOrder)
|
||||
}
|
||||
|
||||
// Paramètres SETTINGS individuels (-1 = absent)
|
||||
rec.H2HeaderTableSize = h2.HeaderTableSize
|
||||
rec.H2EnablePush = h2.EnablePush
|
||||
rec.H2MaxConcurrentStreams = h2.MaxConcurrentStreams
|
||||
rec.H2InitialWindowSize = int64(h2.InitialWindowSize)
|
||||
rec.H2MaxFrameSize = h2.MaxFrameSize
|
||||
rec.H2MaxHeaderListSize = h2.MaxHeaderListSize
|
||||
|
||||
// Fingerprints composites Akamai
|
||||
rec.H2Fingerprint = buildH2Fingerprint(h2)
|
||||
rec.H2SettingsFP = buildH2SettingsFP(h2)
|
||||
}
|
||||
}
|
||||
|
||||
return rec
|
||||
}
|
||||
|
||||
// pseudoOrderToShort convertit la liste de pseudo-headers en notation abrégée.
|
||||
// Ex: [":method", ":authority", ":scheme", ":path"] → "m,a,s,p"
|
||||
func pseudoOrderToShort(headers []string) string {
|
||||
short := make([]byte, 0, len(headers)*2-1)
|
||||
for i, h := range headers {
|
||||
if i > 0 {
|
||||
short = append(short, ',')
|
||||
}
|
||||
switch {
|
||||
case h == ":method":
|
||||
short = append(short, 'm')
|
||||
case h == ":authority":
|
||||
short = append(short, 'a')
|
||||
case h == ":scheme":
|
||||
short = append(short, 's')
|
||||
case h == ":path":
|
||||
short = append(short, 'p')
|
||||
default:
|
||||
short = append(short, '?')
|
||||
}
|
||||
}
|
||||
return string(short)
|
||||
}
|
||||
|
||||
// buildH2Fingerprint construit le fingerprint composite au format Akamai.
|
||||
// Format : SETTINGS[pairs]|WINDOW_UPDATE[value]|PRIORITY[0/1]|PSEUDO_ORDER[order]
|
||||
func buildH2Fingerprint(h2 *correlation.HTTP2Settings) string {
|
||||
var b strings.Builder
|
||||
|
||||
// SETTINGS
|
||||
b.WriteString("1:")
|
||||
b.WriteString(fmt.Sprintf("%d", h2.HeaderTableSize))
|
||||
b.WriteString(",2:")
|
||||
b.WriteString(fmt.Sprintf("%d", h2.EnablePush))
|
||||
if h2.MaxConcurrentStreams >= 0 {
|
||||
b.WriteString(",3:")
|
||||
b.WriteString(fmt.Sprintf("%d", h2.MaxConcurrentStreams))
|
||||
}
|
||||
b.WriteString(",4:")
|
||||
b.WriteString(fmt.Sprintf("%d", h2.InitialWindowSize))
|
||||
if h2.MaxFrameSize >= 0 {
|
||||
b.WriteString(",5:")
|
||||
b.WriteString(fmt.Sprintf("%d", h2.MaxFrameSize))
|
||||
}
|
||||
if h2.MaxHeaderListSize >= 0 {
|
||||
b.WriteString(",6:")
|
||||
b.WriteString(fmt.Sprintf("%d", h2.MaxHeaderListSize))
|
||||
}
|
||||
|
||||
// WINDOW_UPDATE
|
||||
b.WriteByte('|')
|
||||
if h2.WindowUpdateIncrement > 0 {
|
||||
b.WriteString(fmt.Sprintf("%d", h2.WindowUpdateIncrement))
|
||||
}
|
||||
|
||||
// PRIORITY (non capturé actuellement)
|
||||
b.WriteString("|0")
|
||||
|
||||
// PSEUDO_ORDER
|
||||
b.WriteByte('|')
|
||||
if len(h2.PseudoHeaderOrder) > 0 {
|
||||
b.WriteString(pseudoOrderToShort(h2.PseudoHeaderOrder))
|
||||
}
|
||||
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// buildH2SettingsFP construit la chaîne brute des entrées SETTINGS.
|
||||
func buildH2SettingsFP(h2 *correlation.HTTP2Settings) string {
|
||||
var parts []string
|
||||
if h2.MaxConcurrentStreams >= 0 {
|
||||
parts = append(parts, fmt.Sprintf("3:%d", h2.MaxConcurrentStreams))
|
||||
}
|
||||
if h2.InitialWindowSize >= 0 {
|
||||
parts = append(parts, fmt.Sprintf("4:%d", h2.InitialWindowSize))
|
||||
}
|
||||
if h2.EnablePush >= 0 {
|
||||
parts = append(parts, fmt.Sprintf("2:%d", h2.EnablePush))
|
||||
}
|
||||
return strings.Join(parts, ",")
|
||||
}
|
||||
|
||||
// formatTLSVersion convertit la valeur numérique TLS en chaîne lisible.
|
||||
func formatTLSVersion(v uint16) string {
|
||||
switch v {
|
||||
|
||||
Reference in New Issue
Block a user