code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(e2e): add distributed E2E test framework with parametric traffic generation

Browse Source 
Add run-e2e-test.sh with CLI parameters (--hits, --http-ratio, --dns, --tls,
--src-ips, --keep-analysis, --up) for configurable traffic generation. Traffic
runs from VM endpoints with multiple source IPs (alias IPs on eth0) to produce
distinct sessions for the ML pipeline. Fix curl TLS flags (--tlsv1.2 instead
of --tls-v1-2), skip redundant local verification in distributed mode, and
fix dashboard is_available() cache that never retried after ClickHouse recovery.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
2026-04-15 00:09:32 +02:00
parent 7894d39f1c
commit f88b739992
40 changed files with 2154 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

112
services/ja4ebpf/internal/writer/clickhouse.go
View File

@ -52,14 +52,32 @@ type sessionRecord struct {
TLSVersion string `json:"tls_version,omitempty"`
// HTTP
Method string `json:"method,omitempty"`
Path string `json:"path,omitempty"`
QueryString string `json:"query_string,omitempty"`
StatusCode *int `json:"status_code,omitempty"`
ResponseSize *int64 `json:"response_size,omitempty"`
DurationMS *float64 `json:"duration_ms,omitempty"`
KeepAlives int `json:"keepalives,omitempty"`
HeaderOrderSig string `json:"header_order_signature,omitempty"`
Method string `json:"method,omitempty"`
Path string `json:"path,omitempty"`
Host string `json:"host,omitempty"`
QueryString string `json:"query_string,omitempty"`
Scheme string `json:"scheme,omitempty"`
HTTPVersion string `json:"http_version,omitempty"`
StatusCode *int `json:"status_code,omitempty"`
ResponseSize *int64 `json:"response_size,omitempty"`
DurationMS *float64 `json:"duration_ms,omitempty"`
KeepAlives int `json:"keepalives,omitempty"`
HeaderOrderSig string `json:"header_order_signature,omitempty"`
HeadersRaw string `json:"headers_raw,omitempty"`
HeaderUserAgent string `json:"header_User-Agent,omitempty"`
HeaderAccept string `json:"header_Accept,omitempty"`
HeaderAcceptEnc string `json:"header_Accept-Encoding,omitempty"`
HeaderAcceptLang string `json:"header_Accept-Language,omitempty"`
HeaderContentType string `json:"header_Content-Type,omitempty"`
HeaderXReqID string `json:"header_X-Request-Id,omitempty"`
HeaderXTraceID string `json:"header_X-Trace-Id,omitempty"`
HeaderXForwarded string `json:"header_X-Forwarded-For,omitempty"`
HeaderSecCHUA string `json:"header_Sec-CH-UA,omitempty"`
HeaderSecCHUAMobile string `json:"header_Sec-CH-UA-Mobile,omitempty"`
HeaderSecCHUAPlat string `json:"header_Sec-CH-UA-Platform,omitempty"`
HeaderSecFetchDest string `json:"header_Sec-Fetch-Dest,omitempty"`
HeaderSecFetchMode string `json:"header_Sec-Fetch-Mode,omitempty"`
HeaderSecFetchSite string `json:"header_Sec-Fetch-Site,omitempty"`
// HTTP/2 fingerprinting passif
H2Fingerprint string `json:"h2_fingerprint,omitempty"`
@ -67,13 +85,13 @@ type sessionRecord struct {
H2WindowUpdate uint32 `json:"h2_window_update,omitempty"`
H2PseudoOrder string `json:"h2_pseudo_order,omitempty"`
H2HasPriority uint8 `json:"h2_has_priority,omitempty"`
H2HeaderTableSize int32 `json:"h2_header_table_size"`
H2EnablePush int32 `json:"h2_enable_push"`
H2MaxConcurrentStreams int32 `json:"h2_max_concurrent_streams"`
H2InitialWindowSize int64 `json:"h2_initial_window_size"`
H2MaxFrameSize int32 `json:"h2_max_frame_size"`
H2MaxHeaderListSize int32 `json:"h2_max_header_list_size"`
H2EnableConnectProtocol int32 `json:"h2_enable_connect_protocol"`
H2HeaderTableSize *int32 `json:"h2_header_table_size,omitempty"`
H2EnablePush *int32 `json:"h2_enable_push,omitempty"`
H2MaxConcurrentStreams *int32 `json:"h2_max_concurrent_streams,omitempty"`
H2InitialWindowSize *int64 `json:"h2_initial_window_size,omitempty"`
H2MaxFrameSize *int32 `json:"h2_max_frame_size,omitempty"`
H2MaxHeaderListSize *int32 `json:"h2_max_header_list_size,omitempty"`
H2EnableConnectProtocol *int32 `json:"h2_enable_connect_protocol,omitempty"`
}
// NewClickHouseWriter crée un writer et établit la connexion ClickHouse.
@ -199,19 +217,26 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
Time: s.FirstSeen,
SrcIP: srcIP,
SrcPort: int(s.Key.SrcPort),
DstIP: "0.0.0.0", // destination non capturée par les sondes eBPF actuelles
DstPort: 0,
KeepAlives: len(s.Requests),
}
// Champs métadonnées IP/TCP
if s.L3L4 != nil {
rec.DstIP = fmt.Sprintf("%d.%d.%d.%d",
s.L3L4.DstIP[0], s.L3L4.DstIP[1], s.L3L4.DstIP[2], s.L3L4.DstIP[3])
rec.DstPort = int(s.L3L4.DstPort)
rec.IPMetaDF = &s.L3L4.DFBit
rec.IPMetaID = &s.L3L4.IPID
rec.IPMetaTTL = &s.L3L4.TTL
rec.TCPMetaWindowSize = &s.L3L4.WindowSize
rec.TCPMetaWindowScale = &s.L3L4.WindowScale
rec.TCPMetaMSS = &s.L3L4.MSS
// WindowScale 0xFF = absent (convention C), ne pas inclure
if s.L3L4.WindowScale != 0xFF {
rec.TCPMetaWindowScale = &s.L3L4.WindowScale
}
// MSS 0 = absent, ne pas inclure
if s.L3L4.MSS > 0 {
rec.TCPMetaMSS = &s.L3L4.MSS
}
}
// Champs TLS
@ -220,6 +245,14 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
rec.TLSSNI = s.TLS.SNI
rec.TLSALPN = strings.Join(s.TLS.ALPN, ",")
rec.TLSVersion = formatTLSVersion(s.TLS.TLSVersion)
// Fallback : si pas de Host HTTP, utiliser TLS SNI
if rec.Host == "" && s.TLS.SNI != "" {
rec.Host = s.TLS.SNI
}
// Scheme déduit de la présence TLS
if s.TLS.SNI != "" {
rec.Scheme = "https"
}
}
// Champs HTTP (dernière requête)
@ -228,11 +261,37 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
rec.Method = last.Method
rec.Path = last.Path
rec.QueryString = last.QueryString
rec.Host = last.Host
rec.Scheme = "" // sera rempli par le dispatcher si TLS
rec.HTTPVersion = last.HTTPVersion
rec.StatusCode = &last.StatusCode
rec.ResponseSize = &last.ResponseSize
rec.DurationMS = &last.DurationMS
rec.HeaderOrderSig = last.HeaderOrderSig
// En-têtes HTTP individuels
if last.HeaderKV != nil {
rec.HeaderUserAgent = last.HeaderKV["User-Agent"]
rec.HeaderAccept = last.HeaderKV["Accept"]
rec.HeaderAcceptEnc = last.HeaderKV["Accept-Encoding"]
rec.HeaderAcceptLang = last.HeaderKV["Accept-Language"]
rec.HeaderContentType = last.HeaderKV["Content-Type"]
rec.HeaderXReqID = last.HeaderKV["X-Request-Id"]
rec.HeaderXTraceID = last.HeaderKV["X-Trace-Id"]
rec.HeaderXForwarded = last.HeaderKV["X-Forwarded-For"]
rec.HeaderSecCHUA = last.HeaderKV["Sec-CH-UA"]
rec.HeaderSecCHUAMobile = last.HeaderKV["Sec-CH-UA-Mobile"]
rec.HeaderSecCHUAPlat = last.HeaderKV["Sec-CH-UA-Platform"]
rec.HeaderSecFetchDest = last.HeaderKV["Sec-Fetch-Dest"]
rec.HeaderSecFetchMode = last.HeaderKV["Sec-Fetch-Mode"]
rec.HeaderSecFetchSite = last.HeaderKV["Sec-Fetch-Site"]
}
// Construire headers_raw : ordre des noms joints par ";"
if len(last.HeaderOrder) > 0 {
rec.HeadersRaw = strings.Join(last.HeaderOrder, ";")
}
// Champs HTTP/2 passifs
if last.HTTP2Settings != nil {
h2 := last.HTTP2Settings
@ -243,13 +302,14 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
rec.H2PseudoOrder = pseudoOrderToShort(h2.PseudoHeaderOrder)
}
// Paramètres SETTINGS individuels (-1 = absent)
rec.H2HeaderTableSize = h2.HeaderTableSize
rec.H2EnablePush = h2.EnablePush
rec.H2MaxConcurrentStreams = h2.MaxConcurrentStreams
rec.H2InitialWindowSize = int64(h2.InitialWindowSize)
rec.H2MaxFrameSize = h2.MaxFrameSize
rec.H2MaxHeaderListSize = h2.MaxHeaderListSize
// Paramètres SETTINGS individuels (pointeurs : nil = absent du preface)
rec.H2HeaderTableSize = &h2.HeaderTableSize
rec.H2EnablePush = &h2.EnablePush
rec.H2MaxConcurrentStreams = &h2.MaxConcurrentStreams
h2InitWin := int64(h2.InitialWindowSize)
rec.H2InitialWindowSize = &h2InitWin
rec.H2MaxFrameSize = &h2.MaxFrameSize
rec.H2MaxHeaderListSize = &h2.MaxHeaderListSize
// Fingerprints composites Akamai
rec.H2Fingerprint = buildH2Fingerprint(h2)