feat: maximize data completeness across L3/L4/TLS/HTTP layers and add E2E test infra
Add SSL_write uprobe for HTTP response capture, HPACK decoder for HTTP/2 header extraction, and AcceptCache for reliable SSL/TC session correlation. Populate all ClickHouse fields including tcp_meta_options, ip_meta_total_length, syn_to_clienthello_ms, client_headers, TLS cipher suites/extensions, and h2_enable_connect_protocol. Increase BPF capture buffers (HTTP 512B, TLS 1024B). Add distributed E2E testing infrastructure with multi-VM Vagrant setup. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
@ -15,16 +15,17 @@ type SessionKey struct {
|
||||
|
||||
// L3L4 contient les caractéristiques réseau et transport de la connexion.
|
||||
type L3L4 struct {
|
||||
DstIP [4]byte // adresse IP destination
|
||||
DstPort uint16 // port destination
|
||||
TTL uint8 // TTL IP observé dans le SYN
|
||||
DFBit bool // bit Don't Fragment actif
|
||||
IPID uint16 // champ identification IP
|
||||
WindowSize uint16 // taille de fenêtre TCP initiale
|
||||
WindowScale uint8 // facteur d'échelle de fenêtre (0xFF = absent)
|
||||
MSS uint16 // Maximum Segment Size (0 = absent)
|
||||
TCPOptionsRaw []byte // options TCP brutes (max 40 octets)
|
||||
SYNTimestamp time.Time // horodatage du paquet SYN
|
||||
DstIP [4]byte // adresse IP destination
|
||||
DstPort uint16 // port destination
|
||||
TTL uint8 // TTL IP observé dans le SYN
|
||||
DFBit bool // bit Don't Fragment actif
|
||||
IPID uint16 // champ identification IP
|
||||
IPTotalLength uint16 // longueur totale IP (octets)
|
||||
WindowSize uint16 // taille de fenêtre TCP initiale
|
||||
WindowScale uint8 // facteur d'échelle de fenêtre (0xFF = absent)
|
||||
MSS uint16 // Maximum Segment Size (0 = absent)
|
||||
TCPOptionsRaw []byte // options TCP brutes (max 40 octets)
|
||||
SYNTimestamp time.Time // horodatage du paquet SYN
|
||||
}
|
||||
|
||||
// TLSInfo contient les données extraites du ClientHello TLS.
|
||||
@ -43,15 +44,16 @@ type TLSInfo struct {
|
||||
|
||||
// HTTP2Settings contient les paramètres SETTINGS et WINDOW_UPDATE du client HTTP/2.
|
||||
type HTTP2Settings struct {
|
||||
HeaderTableSize int32 // SETTINGS_HEADER_TABLE_SIZE (-1 si absent)
|
||||
EnablePush int32 // SETTINGS_ENABLE_PUSH
|
||||
MaxConcurrentStreams int32 // SETTINGS_MAX_CONCURRENT_STREAMS
|
||||
InitialWindowSize int32 // SETTINGS_INITIAL_WINDOW_SIZE
|
||||
MaxFrameSize int32 // SETTINGS_MAX_FRAME_SIZE
|
||||
MaxHeaderListSize int32 // SETTINGS_MAX_HEADER_LIST_SIZE
|
||||
UnknownSettings int32 // paramètre 0x7 (JA4H2)
|
||||
WindowUpdateIncrement uint32 // valeur WINDOW_UPDATE sur stream 0
|
||||
PseudoHeaderOrder []string // ordre des pseudo-headers [:method, :authority, ...]
|
||||
HeaderTableSize int32 // SETTINGS_HEADER_TABLE_SIZE (-1 si absent)
|
||||
EnablePush int32 // SETTINGS_ENABLE_PUSH
|
||||
MaxConcurrentStreams int32 // SETTINGS_MAX_CONCURRENT_STREAMS
|
||||
InitialWindowSize int32 // SETTINGS_INITIAL_WINDOW_SIZE
|
||||
MaxFrameSize int32 // SETTINGS_MAX_FRAME_SIZE
|
||||
MaxHeaderListSize int32 // SETTINGS_MAX_HEADER_LIST_SIZE
|
||||
UnknownSettings int32 // paramètre 0x7 (JA4H2)
|
||||
EnableConnectProtocol int32 // SETTINGS_ENABLE_CONNECT_PROTOCOL (0x8, RFC 8441)
|
||||
WindowUpdateIncrement uint32 // valeur WINDOW_UPDATE sur stream 0
|
||||
PseudoHeaderOrder []string // ordre des pseudo-headers [:method, :authority, ...]
|
||||
}
|
||||
|
||||
// HTTPRequest représente une requête HTTP observée dans la session.
|
||||
|
||||
@ -46,12 +46,15 @@ type sessionRecord struct {
|
||||
TCPMetaOptions string `json:"tcp_meta_options,omitempty"`
|
||||
|
||||
// TLS (noms attendus par le MV)
|
||||
JA4Hash string `json:"ja4,omitempty"`
|
||||
JA3Raw string `json:"ja3,omitempty"`
|
||||
JA3Hash string `json:"ja3_hash,omitempty"`
|
||||
TLSSNI string `json:"tls_sni,omitempty"`
|
||||
TLSALPN string `json:"tls_alpn,omitempty"`
|
||||
TLSVersion string `json:"tls_version,omitempty"`
|
||||
JA4Hash string `json:"ja4,omitempty"`
|
||||
JA3Raw string `json:"ja3,omitempty"`
|
||||
JA3Hash string `json:"ja3_hash,omitempty"`
|
||||
TLSSNI string `json:"tls_sni,omitempty"`
|
||||
TLSALPN string `json:"tls_alpn,omitempty"`
|
||||
TLSVersion string `json:"tls_version,omitempty"`
|
||||
TLSCipherSuites string `json:"tls_cipher_suites,omitempty"`
|
||||
TLSExtensions string `json:"tls_extensions,omitempty"`
|
||||
TLSSupportedGroups string `json:"tls_supported_groups,omitempty"`
|
||||
|
||||
// HTTP
|
||||
Method string `json:"method,omitempty"`
|
||||
@ -82,18 +85,22 @@ type sessionRecord struct {
|
||||
HeaderSecFetchSite string `json:"header_Sec-Fetch-Site,omitempty"`
|
||||
|
||||
// HTTP/2 fingerprinting passif
|
||||
H2Fingerprint string `json:"h2_fingerprint,omitempty"`
|
||||
H2SettingsFP string `json:"h2_settings_fp,omitempty"`
|
||||
H2WindowUpdate uint32 `json:"h2_window_update,omitempty"`
|
||||
H2PseudoOrder string `json:"h2_pseudo_order,omitempty"`
|
||||
H2HasPriority uint8 `json:"h2_has_priority,omitempty"`
|
||||
H2HeaderTableSize *int32 `json:"h2_header_table_size,omitempty"`
|
||||
H2EnablePush *int32 `json:"h2_enable_push,omitempty"`
|
||||
H2MaxConcurrentStreams *int32 `json:"h2_max_concurrent_streams,omitempty"`
|
||||
H2InitialWindowSize *int64 `json:"h2_initial_window_size,omitempty"`
|
||||
H2MaxFrameSize *int32 `json:"h2_max_frame_size,omitempty"`
|
||||
H2MaxHeaderListSize *int32 `json:"h2_max_header_list_size,omitempty"`
|
||||
H2EnableConnectProtocol *int32 `json:"h2_enable_connect_protocol,omitempty"`
|
||||
H2Fingerprint string `json:"h2_fingerprint,omitempty"`
|
||||
H2SettingsFP string `json:"h2_settings_fp,omitempty"`
|
||||
H2WindowUpdate uint32 `json:"h2_window_update,omitempty"`
|
||||
H2PseudoOrder string `json:"h2_pseudo_order,omitempty"`
|
||||
H2HasPriority uint8 `json:"h2_has_priority,omitempty"`
|
||||
H2HeaderTableSize *int32 `json:"h2_header_table_size,omitempty"`
|
||||
H2EnablePush *int32 `json:"h2_enable_push,omitempty"`
|
||||
H2MaxConcurrentStreams *int32 `json:"h2_max_concurrent_streams,omitempty"`
|
||||
H2InitialWindowSize *int64 `json:"h2_initial_window_size,omitempty"`
|
||||
H2MaxFrameSize *int32 `json:"h2_max_frame_size,omitempty"`
|
||||
H2MaxHeaderListSize *int32 `json:"h2_max_header_list_size,omitempty"`
|
||||
H2EnableConnectProtocol *int32 `json:"h2_enable_connect_protocol,omitempty"`
|
||||
|
||||
// Champs de corrélation
|
||||
ClientHeaders string `json:"client_headers,omitempty"`
|
||||
SynToClientHelloMs *int32 `json:"syn_to_clienthello_ms,omitempty"`
|
||||
}
|
||||
|
||||
// NewClickHouseWriter crée un writer et établit la connexion ClickHouse.
|
||||
@ -222,6 +229,11 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
KeepAlives: len(s.Requests),
|
||||
}
|
||||
|
||||
// Fallback dst_port pour sessions TLS sans L3L4 : 443 pour HTTPS
|
||||
if s.L3L4 == nil && s.TLS != nil {
|
||||
rec.DstPort = 443
|
||||
}
|
||||
|
||||
// Champs métadonnées IP/TCP
|
||||
if s.L3L4 != nil {
|
||||
rec.DstIP = fmt.Sprintf("%d.%d.%d.%d",
|
||||
@ -230,6 +242,9 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
rec.IPMetaDF = &s.L3L4.DFBit
|
||||
rec.IPMetaID = &s.L3L4.IPID
|
||||
rec.IPMetaTTL = &s.L3L4.TTL
|
||||
if s.L3L4.IPTotalLength > 0 {
|
||||
rec.IPMetaTotalLength = &s.L3L4.IPTotalLength
|
||||
}
|
||||
rec.TCPMetaWindowSize = &s.L3L4.WindowSize
|
||||
// WindowScale 0xFF = absent (convention C), ne pas inclure
|
||||
if s.L3L4.WindowScale != 0xFF {
|
||||
@ -239,6 +254,10 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
if s.L3L4.MSS > 0 {
|
||||
rec.TCPMetaMSS = &s.L3L4.MSS
|
||||
}
|
||||
// Options TCP : noms abrégés (MSS, WS, SACK, TS, etc.)
|
||||
if len(s.L3L4.TCPOptionsRaw) > 0 {
|
||||
rec.TCPMetaOptions = formatTCPOptions(s.L3L4.TCPOptionsRaw)
|
||||
}
|
||||
}
|
||||
|
||||
// Champs TLS
|
||||
@ -249,6 +268,25 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
rec.TLSSNI = s.TLS.SNI
|
||||
rec.TLSALPN = strings.Join(s.TLS.ALPN, ",")
|
||||
rec.TLSVersion = formatTLSVersion(s.TLS.TLSVersion)
|
||||
// Cipher suites : liste hex séparée par des tirets
|
||||
if len(s.TLS.CipherSuites) > 0 {
|
||||
parts := make([]string, len(s.TLS.CipherSuites))
|
||||
for i, cs := range s.TLS.CipherSuites {
|
||||
parts[i] = fmt.Sprintf("%04x", cs)
|
||||
}
|
||||
rec.TLSCipherSuites = strings.Join(parts, "-")
|
||||
}
|
||||
// Extensions : liste d'IDs hex séparés par des tirets
|
||||
if len(s.TLS.Extensions) > 0 {
|
||||
parts := make([]string, len(s.TLS.Extensions))
|
||||
for i, ext := range s.TLS.Extensions {
|
||||
parts[i] = fmt.Sprintf("%04x", ext)
|
||||
}
|
||||
rec.TLSExtensions = strings.Join(parts, "-")
|
||||
}
|
||||
// Supported groups : liste hex séparée par des tirets
|
||||
// (disponible via TLSInfo.Extensions — extraction depuis ClientHelloRaw si nécessaire)
|
||||
_ = s.TLS.ClientHelloRaw // réservé pour extraction future
|
||||
// Fallback : si pas de Host HTTP, utiliser TLS SNI
|
||||
if rec.Host == "" && s.TLS.SNI != "" {
|
||||
rec.Host = s.TLS.SNI
|
||||
@ -259,6 +297,12 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
}
|
||||
}
|
||||
|
||||
// syn_to_clienthello_ms : délai entre SYN et ClientHello
|
||||
if s.L3L4 != nil && s.TLS != nil && !s.L3L4.SYNTimestamp.IsZero() && !s.TLS.Timestamp.IsZero() {
|
||||
delta := int32(s.TLS.Timestamp.Sub(s.L3L4.SYNTimestamp).Milliseconds())
|
||||
rec.SynToClientHelloMs = &delta
|
||||
}
|
||||
|
||||
// Champs HTTP (dernière requête)
|
||||
if len(s.Requests) > 0 {
|
||||
last := &s.Requests[len(s.Requests)-1]
|
||||
@ -275,22 +319,25 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
rec.DurationMS = &last.DurationMS
|
||||
rec.HeaderOrderSig = last.HeaderOrderSig
|
||||
|
||||
// En-têtes HTTP individuels
|
||||
// En-têtes HTTP individuels (HTTP/1.1: clés Title-Case, HTTP/2: clés lowercase)
|
||||
if last.HeaderKV != nil {
|
||||
rec.HeaderUserAgent = last.HeaderKV["User-Agent"]
|
||||
rec.HeaderAccept = last.HeaderKV["Accept"]
|
||||
rec.HeaderAcceptEnc = last.HeaderKV["Accept-Encoding"]
|
||||
rec.HeaderAcceptLang = last.HeaderKV["Accept-Language"]
|
||||
rec.HeaderContentType = last.HeaderKV["Content-Type"]
|
||||
rec.HeaderXReqID = last.HeaderKV["X-Request-Id"]
|
||||
rec.HeaderXTraceID = last.HeaderKV["X-Trace-Id"]
|
||||
rec.HeaderXForwarded = last.HeaderKV["X-Forwarded-For"]
|
||||
rec.HeaderSecCHUA = last.HeaderKV["Sec-CH-UA"]
|
||||
rec.HeaderSecCHUAMobile = last.HeaderKV["Sec-CH-UA-Mobile"]
|
||||
rec.HeaderSecCHUAPlat = last.HeaderKV["Sec-CH-UA-Platform"]
|
||||
rec.HeaderSecFetchDest = last.HeaderKV["Sec-Fetch-Dest"]
|
||||
rec.HeaderSecFetchMode = last.HeaderKV["Sec-Fetch-Mode"]
|
||||
rec.HeaderSecFetchSite = last.HeaderKV["Sec-Fetch-Site"]
|
||||
rec.HeaderUserAgent = headerVal(last.HeaderKV, "User-Agent", "user-agent")
|
||||
rec.HeaderAccept = headerVal(last.HeaderKV, "Accept", "accept")
|
||||
rec.HeaderAcceptEnc = headerVal(last.HeaderKV, "Accept-Encoding", "accept-encoding")
|
||||
rec.HeaderAcceptLang = headerVal(last.HeaderKV, "Accept-Language", "accept-language")
|
||||
rec.HeaderContentType = headerVal(last.HeaderKV, "Content-Type", "content-type")
|
||||
rec.HeaderXReqID = headerVal(last.HeaderKV, "X-Request-Id", "x-request-id")
|
||||
rec.HeaderXTraceID = headerVal(last.HeaderKV, "X-Trace-Id", "x-trace-id")
|
||||
rec.HeaderXForwarded = headerVal(last.HeaderKV, "X-Forwarded-For", "x-forwarded-for")
|
||||
rec.HeaderSecCHUA = headerVal(last.HeaderKV, "Sec-CH-UA", "sec-ch-ua")
|
||||
rec.HeaderSecCHUAMobile = headerVal(last.HeaderKV, "Sec-CH-UA-Mobile", "sec-ch-ua-mobile")
|
||||
rec.HeaderSecCHUAPlat = headerVal(last.HeaderKV, "Sec-CH-UA-Platform", "sec-ch-ua-platform")
|
||||
rec.HeaderSecFetchDest = headerVal(last.HeaderKV, "Sec-Fetch-Dest", "sec-fetch-dest")
|
||||
rec.HeaderSecFetchMode = headerVal(last.HeaderKV, "Sec-Fetch-Mode", "sec-fetch-mode")
|
||||
rec.HeaderSecFetchSite = headerVal(last.HeaderKV, "Sec-Fetch-Site", "sec-fetch-site")
|
||||
|
||||
// client_headers : JSON des en-têtes capturés
|
||||
rec.ClientHeaders = buildClientHeaders(last.HeaderKV)
|
||||
}
|
||||
|
||||
// Construire headers_raw : ordre des noms joints par ";"
|
||||
@ -316,6 +363,7 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
|
||||
rec.H2InitialWindowSize = &h2InitWin
|
||||
rec.H2MaxFrameSize = &h2.MaxFrameSize
|
||||
rec.H2MaxHeaderListSize = &h2.MaxHeaderListSize
|
||||
rec.H2EnableConnectProtocol = &h2.EnableConnectProtocol
|
||||
|
||||
// Fingerprints composites Akamai
|
||||
rec.H2Fingerprint = buildH2Fingerprint(h2)
|
||||
@ -423,3 +471,83 @@ func formatTLSVersion(v uint16) string {
|
||||
return ""
|
||||
}
|
||||
}
|
||||
|
||||
// headerVal cherche un en-tête dans le map avec deux clés possibles :
|
||||
// HTTP/1.1 utilise Title-Case (ex: "User-Agent"), HTTP/2 utilise lowercase (ex: "user-agent").
|
||||
func headerVal(kv map[string]string, titleKey, lowerKey string) string {
|
||||
if v := kv[titleKey]; v != "" {
|
||||
return v
|
||||
}
|
||||
return kv[lowerKey]
|
||||
}
|
||||
|
||||
// buildClientHeaders sérialise les en-têtes capturés en JSON pour la colonne client_headers.
|
||||
func buildClientHeaders(kv map[string]string) string {
|
||||
if len(kv) == 0 {
|
||||
return ""
|
||||
}
|
||||
// Sérialiser en JSON trié par clé pour un résultat déterministe
|
||||
b, err := json.Marshal(kv)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// formatTCPOptions convertit les options TCP brutes en chaîne lisible.
|
||||
// Noms abrégés : MSS=2, WS=3, SACK=4, TS=8, etc.
|
||||
func formatTCPOptions(opts []byte) string {
|
||||
if len(opts) == 0 {
|
||||
return ""
|
||||
}
|
||||
var names []string
|
||||
i := 0
|
||||
for i < len(opts) {
|
||||
kind := opts[i]
|
||||
switch kind {
|
||||
case 0: // End of Options List
|
||||
break
|
||||
case 1: // NOP
|
||||
names = append(names, "NOP")
|
||||
i++
|
||||
case 2: // MSS
|
||||
names = append(names, "MSS")
|
||||
if i+3 < len(opts) {
|
||||
i += 4
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
case 3: // Window Scale
|
||||
names = append(names, "WS")
|
||||
if i+2 < len(opts) {
|
||||
i += 3
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
case 4: // SACK Permitted
|
||||
names = append(names, "SACK")
|
||||
i += 2
|
||||
case 5: // SACK
|
||||
names = append(names, "SACKb")
|
||||
if i+1 < len(opts) {
|
||||
i += int(opts[i+1])
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
case 8: // Timestamp
|
||||
names = append(names, "TS")
|
||||
if i+9 < len(opts) {
|
||||
i += 10
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
default:
|
||||
if i+1 < len(opts) && int(opts[i+1]) > 0 {
|
||||
i += int(opts[i+1])
|
||||
} else {
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
return strings.Join(names, ",")
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user