code_public/ja4-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(e2e): add distributed E2E test framework with parametric traffic generation

Browse Source 
Add run-e2e-test.sh with CLI parameters (--hits, --http-ratio, --dns, --tls,
--src-ips, --keep-analysis, --up) for configurable traffic generation. Traffic
runs from VM endpoints with multiple source IPs (alias IPs on eth0) to produce
distinct sessions for the ML pipeline. Fix curl TLS flags (--tlsv1.2 instead
of --tls-v1-2), skip redundant local verification in distributed mode, and
fix dashboard is_available() cache that never retried after ClickHouse recovery.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Jacquin Antoine
2026-04-15 00:09:32 +02:00
parent 7894d39f1c
commit f88b739992
40 changed files with 2154 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
services/ja4ebpf/bpf/tc_capture.c
View File

@ -215,13 +215,23 @@ int capture_tc(struct __sk_buff *ctx)
tls_evt->src_ip = bpf_ntohl(src_ip);
tls_evt->src_port = src_port;
tls_evt->timestamp_ns = bpf_ktime_get_ns();
tls_evt->payload_len = (__u16)avail;
/* Copie via bpf_skb_load_bytes avec taille constante 256.
/* Copie via bpf_skb_load_bytes avec tailles constantes en cascade.
* Kernel 4.18 ne supporte pas les tailles variables vers map values.
* 256 octets capture le ClientHello dans la majorité des cas. */
if (bpf_skb_load_bytes(ctx, payload_off, tls_evt, 256))
* On essaie 512 puis 256 puis 128 pour capturer SNI et extensions.
* La taille réellement copiée est stockée dans payload_len. */
if (payload_off + 512 <= pkt_len) {
bpf_skb_load_bytes(ctx, payload_off, tls_evt, 512);
tls_evt->payload_len = 512;
} else if (payload_off + 256 <= pkt_len) {
bpf_skb_load_bytes(ctx, payload_off, tls_evt, 256);
tls_evt->payload_len = 256;
} else if (payload_off + 128 <= pkt_len) {
bpf_skb_load_bytes(ctx, payload_off, tls_evt, 128);
tls_evt->payload_len = 128;
} else {
return TC_ACT_OK;
}
bpf_perf_event_output(ctx, &pb_tls_hello, BPF_F_CURRENT_CPU,
tls_evt, sizeof(*tls_evt));
@ -269,11 +279,21 @@ int capture_tc(struct __sk_buff *ctx)
h_evt->src_port = src_port;
h_evt->dst_port = dst_port;
h_evt->timestamp_ns = bpf_ktime_get_ns();
h_evt->payload_len = (__u16)avail;
/* Taille constante 256 pour compatibilité vérificateur kernel 4.18 */
if (bpf_skb_load_bytes(ctx, payload_off, h_evt, 256))
/* Copie via bpf_skb_load_bytes avec tailles constantes en cascade.
* Les requêtes HTTP sont souvent < 256 octets, on descend à 128 puis 64. */
if (payload_off + 256 <= pkt_len) {
bpf_skb_load_bytes(ctx, payload_off, h_evt, 256);
h_evt->payload_len = 256;
} else if (payload_off + 128 <= pkt_len) {
bpf_skb_load_bytes(ctx, payload_off, h_evt, 128);
h_evt->payload_len = 128;
} else if (payload_off + 64 <= pkt_len) {
bpf_skb_load_bytes(ctx, payload_off, h_evt, 64);
h_evt->payload_len = 64;
} else {
return TC_ACT_OK;
}
bpf_perf_event_output(ctx, &pb_http_plain, BPF_F_CURRENT_CPU,
h_evt, sizeof(*h_evt));

29
services/ja4ebpf/cmd/ja4ebpf/main.go
View File

@ -315,7 +315,9 @@ func consumeSynEvents(ctx context.Context, rd *perf.Reader, mgr *correlation.Man
// src_ip et src_port stockés en host byte order (bpf_ntohl/bpf_ntohs dans BPF C).
srcIPRaw := binary.LittleEndian.Uint32(data[0:4])
dstIPRaw := binary.LittleEndian.Uint32(data[4:8])
srcPort := binary.LittleEndian.Uint16(data[8:10])
dstPort := binary.LittleEndian.Uint16(data[10:12])
var key correlation.SessionKey
key.SrcIP[0] = byte(srcIPRaw >> 24)
@ -324,6 +326,12 @@ func consumeSynEvents(ctx context.Context, rd *perf.Reader, mgr *correlation.Man
key.SrcIP[3] = byte(srcIPRaw)
key.SrcPort = srcPort
var dstIP [4]byte
dstIP[0] = byte(dstIPRaw >> 24)
dstIP[1] = byte(dstIPRaw >> 16)
dstIP[2] = byte(dstIPRaw >> 8)
dstIP[3] = byte(dstIPRaw)
// Champs IP/TCP aux offsets corrects (dst_ip occupe les octets 4-7)
ttl := data[12]
dfBit := data[13] != 0
@ -342,6 +350,8 @@ func consumeSynEvents(ctx context.Context, rd *perf.Reader, mgr *correlation.Man
mgr.Update(key, func(s *correlation.SessionState) {
s.L3L4 = &correlation.L3L4{
DstIP: dstIP,
DstPort: dstPort,
TTL: ttl,
DFBit: dfBit,
IPID: ipID,
@ -420,6 +430,17 @@ func consumeTLSEvents(ctx context.Context, rd *perf.Reader, mgr *correlation.Man
ciphers = ch.CipherSuites
alpn = ch.ALPN
// Déterminer la version TLS la plus haute (comme ComputeJA4)
var tlsVer uint16
for _, v := range ch.SupportedVersions {
if !parser.IsGREASE(v) && v > tlsVer {
tlsVer = v
}
}
if tlsVer == 0 {
tlsVer = ch.HandshakeVersion
}
mgr.Update(key, func(s *correlation.SessionState) {
s.TLS = &correlation.TLSInfo{
ClientHelloRaw: payload,
@ -428,6 +449,7 @@ func consumeTLSEvents(ctx context.Context, rd *perf.Reader, mgr *correlation.Man
ALPN: alpn,
CipherSuites: ciphers,
Extensions: extensions,
TLSVersion: tlsVer,
Timestamp: time.Now(),
}
// Corréler si L3/L4 est déjà présent
@ -540,6 +562,7 @@ func consumeSSLEvents(ctx context.Context, rd *perf.Reader, mgr *correlation.Man
}
}
if len(s.Requests) == 0 {
req.HTTPVersion = "HTTP/2"
s.Requests = append(s.Requests, req)
}
_ = s.TLS // corrélation implicite
@ -559,8 +582,11 @@ func consumeSSLEvents(ctx context.Context, rd *perf.Reader, mgr *correlation.Man
Method: req.Method,
Path: req.Path,
QueryString: req.Query,
Host: req.HeaderKV["Host"],
HeaderOrder: req.Headers,
HeaderOrderSig: req.HeaderSig,
HeaderKV: req.HeaderKV,
HTTPVersion: req.Protocol,
})
_ = s.TLS // corrélation implicite
})
@ -699,8 +725,11 @@ func consumeHTTPPlainEvents(ctx context.Context, rd *perf.Reader, mgr *correlati
Method: req.Method,
Path: req.Path,
QueryString: req.Query,
Host: req.HeaderKV["Host"],
HeaderOrder: req.Headers,
HeaderOrderSig: req.HeaderSig,
HeaderKV: req.HeaderKV,
HTTPVersion: req.Protocol,
})
// Corréler si L3/L4 est déjà présent (TCP SYN capturé)
_ = s.L3L4 // corrélation implicite

25
services/ja4ebpf/internal/correlation/session.go
View File

@ -15,6 +15,8 @@ type SessionKey struct {
// L3L4 contient les caractéristiques réseau et transport de la connexion.
type L3L4 struct {
DstIP [4]byte // adresse IP destination
DstPort uint16 // port destination
TTL uint8 // TTL IP observé dans le SYN
DFBit bool // bit Don't Fragment actif
IPID uint16 // champ identification IP
@ -52,16 +54,19 @@ type HTTP2Settings struct {
// HTTPRequest représente une requête HTTP observée dans la session.
type HTTPRequest struct {
Method string // méthode HTTP (GET, POST, etc.)
Path string // chemin de la requête
QueryString string // paramètres de requête
StatusCode int // code de statut de la réponse
ResponseSize int64 // taille de la réponse en octets
DurationMS float64 // durée de traitement en millisecondes
HeaderOrder []string // ordre exact des en-têtes HTTP bruts
HeaderOrderSig string // signature de l'ordre des en-têtes (hash)
HTTP2Settings *HTTP2Settings // non nil uniquement pour HTTP/2
Timestamp time.Time // horodatage de la requête
Method string // méthode HTTP (GET, POST, etc.)
Path string // chemin de la requête
QueryString string // paramètres de requête
Host string // en-tête Host (ou :authority pour H2)
StatusCode int // code de statut de la réponse
ResponseSize int64 // taille de la réponse en octets
DurationMS float64 // durée de traitement en millisecondes
HeaderOrder []string // ordre exact des en-têtes HTTP bruts
HeaderOrderSig string // signature de l'ordre des en-têtes (hash)
HeaderKV map[string]string // valeurs des en-têtes capturés (User-Agent, etc.)
HTTPVersion string // "HTTP/1.1", "HTTP/2", etc.
HTTP2Settings *HTTP2Settings // non nil uniquement pour HTTP/2
Timestamp time.Time // horodatage de la requête
}
// SessionState représente l'état complet d'une connexion TCP corrélée.

68
services/ja4ebpf/internal/loader/ja4ssl_x86_bpfel.go
View File

@ -25,18 +25,47 @@ type Ja4SslAcceptKey struct {
Fd uint32
}
type Ja4SslHttpPlainEvent struct {
Payload [4096]uint8
SrcIp uint32
DstIp uint32
SrcPort uint16
DstPort uint16
PayloadLen uint16
TimestampNs uint64
}
type Ja4SslSslConnInfo struct {
Fd uint32
SrcIp uint32
SrcPort uint16
}
type Ja4SslSslDataEvent struct {
PidTgid uint64
Fd uint32
SrcIp uint32
SrcPort uint16
Data [4096]uint8
DataLen uint32
TimestampNs uint64
Direction uint8
}
type Ja4SslSslReadArgs struct {
SslPtr uint64
BufPtr uint64
Num uint32
}
type Ja4SslTlsHelloEvent struct {
Payload [2048]uint8
SrcIp uint32
SrcPort uint16
PayloadLen uint16
TimestampNs uint64
}
// LoadJa4Ssl returns the embedded CollectionSpec for Ja4Ssl.
func LoadJa4Ssl() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_Ja4SslBytes)
@ -89,14 +118,17 @@ type Ja4SslProgramSpecs struct {
//
// It can be passed ebpf.CollectionSpec.Assign.
type Ja4SslMapSpecs struct {
HttpBuf *ebpf.MapSpec `ebpf:"__http_buf"`
SslBuf *ebpf.MapSpec `ebpf:"__ssl_buf"`
TlsBuf *ebpf.MapSpec `ebpf:"__tls_buf"`
AcceptArgsMap *ebpf.MapSpec `ebpf:"accept_args_map"`
AcceptMap *ebpf.MapSpec `ebpf:"accept_map"`
FdConnMap *ebpf.MapSpec `ebpf:"fd_conn_map"`
RbAccept *ebpf.MapSpec `ebpf:"rb_accept"`
RbHttpPlain *ebpf.MapSpec `ebpf:"rb_http_plain"`
RbSslData *ebpf.MapSpec `ebpf:"rb_ssl_data"`
RbTcpSyn *ebpf.MapSpec `ebpf:"rb_tcp_syn"`
RbTlsHello *ebpf.MapSpec `ebpf:"rb_tls_hello"`
PbAccept *ebpf.MapSpec `ebpf:"pb_accept"`
PbHttpPlain *ebpf.MapSpec `ebpf:"pb_http_plain"`
PbSslData *ebpf.MapSpec `ebpf:"pb_ssl_data"`
PbTcpSyn *ebpf.MapSpec `ebpf:"pb_tcp_syn"`
PbTlsHello *ebpf.MapSpec `ebpf:"pb_tls_hello"`
SslArgsMap *ebpf.MapSpec `ebpf:"ssl_args_map"`
SslConnMap *ebpf.MapSpec `ebpf:"ssl_conn_map"`
}
@ -120,28 +152,34 @@ func (o *Ja4SslObjects) Close() error {
//
// It can be passed to LoadJa4SslObjects or ebpf.CollectionSpec.LoadAndAssign.
type Ja4SslMaps struct {
HttpBuf *ebpf.Map `ebpf:"__http_buf"`
SslBuf *ebpf.Map `ebpf:"__ssl_buf"`
TlsBuf *ebpf.Map `ebpf:"__tls_buf"`
AcceptArgsMap *ebpf.Map `ebpf:"accept_args_map"`
AcceptMap *ebpf.Map `ebpf:"accept_map"`
FdConnMap *ebpf.Map `ebpf:"fd_conn_map"`
RbAccept *ebpf.Map `ebpf:"rb_accept"`
RbHttpPlain *ebpf.Map `ebpf:"rb_http_plain"`
RbSslData *ebpf.Map `ebpf:"rb_ssl_data"`
RbTcpSyn *ebpf.Map `ebpf:"rb_tcp_syn"`
RbTlsHello *ebpf.Map `ebpf:"rb_tls_hello"`
PbAccept *ebpf.Map `ebpf:"pb_accept"`
PbHttpPlain *ebpf.Map `ebpf:"pb_http_plain"`
PbSslData *ebpf.Map `ebpf:"pb_ssl_data"`
PbTcpSyn *ebpf.Map `ebpf:"pb_tcp_syn"`
PbTlsHello *ebpf.Map `ebpf:"pb_tls_hello"`
SslArgsMap *ebpf.Map `ebpf:"ssl_args_map"`
SslConnMap *ebpf.Map `ebpf:"ssl_conn_map"`
}
func (m *Ja4SslMaps) Close() error {
return _Ja4SslClose(
m.HttpBuf,
m.SslBuf,
m.TlsBuf,
m.AcceptArgsMap,
m.AcceptMap,
m.FdConnMap,
m.RbAccept,
m.RbHttpPlain,
m.RbSslData,
m.RbTcpSyn,
m.RbTlsHello,
m.PbAccept,
m.PbHttpPlain,
m.PbSslData,
m.PbTcpSyn,
m.PbTlsHello,
m.SslArgsMap,
m.SslConnMap,
)

77
services/ja4ebpf/internal/loader/ja4tc_x86_bpfel.go
View File

@ -25,18 +25,47 @@ type Ja4TcAcceptKey struct {
Fd uint32
}
type Ja4TcHttpPlainEvent struct {
Payload [4096]uint8
SrcIp uint32
DstIp uint32
SrcPort uint16
DstPort uint16
PayloadLen uint16
TimestampNs uint64
}
type Ja4TcSslConnInfo struct {
Fd uint32
SrcIp uint32
SrcPort uint16
}
type Ja4TcSslDataEvent struct {
PidTgid uint64
Fd uint32
SrcIp uint32
SrcPort uint16
Data [4096]uint8
DataLen uint32
TimestampNs uint64
Direction uint8
}
type Ja4TcSslReadArgs struct {
SslPtr uint64
BufPtr uint64
Num uint32
}
type Ja4TcTlsHelloEvent struct {
Payload [2048]uint8
SrcIp uint32
SrcPort uint16
PayloadLen uint16
TimestampNs uint64
}
// LoadJa4Tc returns the embedded CollectionSpec for Ja4Tc.
func LoadJa4Tc() (*ebpf.CollectionSpec, error) {
reader := bytes.NewReader(_Ja4TcBytes)
@ -78,22 +107,26 @@ type Ja4TcSpecs struct {
//
// It can be passed ebpf.CollectionSpec.Assign.
type Ja4TcProgramSpecs struct {
CaptureXdp *ebpf.ProgramSpec `ebpf:"capture_xdp"`
CaptureTc *ebpf.ProgramSpec `ebpf:"capture_tc"`
}
// Ja4TcMapSpecs contains maps before they are loaded into the kernel.
//
// It can be passed ebpf.CollectionSpec.Assign.
type Ja4TcMapSpecs struct {
HttpBuf *ebpf.MapSpec `ebpf:"__http_buf"`
SslBuf *ebpf.MapSpec `ebpf:"__ssl_buf"`
TlsBuf *ebpf.MapSpec `ebpf:"__tls_buf"`
AcceptMap *ebpf.MapSpec `ebpf:"accept_map"`
FdConnMap *ebpf.MapSpec `ebpf:"fd_conn_map"`
RbAccept *ebpf.MapSpec `ebpf:"rb_accept"`
RbHttpPlain *ebpf.MapSpec `ebpf:"rb_http_plain"`
RbSslData *ebpf.MapSpec `ebpf:"rb_ssl_data"`
RbTcpSyn *ebpf.MapSpec `ebpf:"rb_tcp_syn"`
RbTlsHello *ebpf.MapSpec `ebpf:"rb_tls_hello"`
PbAccept *ebpf.MapSpec `ebpf:"pb_accept"`
PbHttpPlain *ebpf.MapSpec `ebpf:"pb_http_plain"`
PbSslData *ebpf.MapSpec `ebpf:"pb_ssl_data"`
PbTcpSyn *ebpf.MapSpec `ebpf:"pb_tcp_syn"`
PbTlsHello *ebpf.MapSpec `ebpf:"pb_tls_hello"`
SslArgsMap *ebpf.MapSpec `ebpf:"ssl_args_map"`
SslConnMap *ebpf.MapSpec `ebpf:"ssl_conn_map"`
TcStats *ebpf.MapSpec `ebpf:"tc_stats"`
}
// Ja4TcObjects contains all objects after they have been loaded into the kernel.
@ -115,28 +148,36 @@ func (o *Ja4TcObjects) Close() error {
//
// It can be passed to LoadJa4TcObjects or ebpf.CollectionSpec.LoadAndAssign.
type Ja4TcMaps struct {
HttpBuf *ebpf.Map `ebpf:"__http_buf"`
SslBuf *ebpf.Map `ebpf:"__ssl_buf"`
TlsBuf *ebpf.Map `ebpf:"__tls_buf"`
AcceptMap *ebpf.Map `ebpf:"accept_map"`
FdConnMap *ebpf.Map `ebpf:"fd_conn_map"`
RbAccept *ebpf.Map `ebpf:"rb_accept"`
RbHttpPlain *ebpf.Map `ebpf:"rb_http_plain"`
RbSslData *ebpf.Map `ebpf:"rb_ssl_data"`
RbTcpSyn *ebpf.Map `ebpf:"rb_tcp_syn"`
RbTlsHello *ebpf.Map `ebpf:"rb_tls_hello"`
PbAccept *ebpf.Map `ebpf:"pb_accept"`
PbHttpPlain *ebpf.Map `ebpf:"pb_http_plain"`
PbSslData *ebpf.Map `ebpf:"pb_ssl_data"`
PbTcpSyn *ebpf.Map `ebpf:"pb_tcp_syn"`
PbTlsHello *ebpf.Map `ebpf:"pb_tls_hello"`
SslArgsMap *ebpf.Map `ebpf:"ssl_args_map"`
SslConnMap *ebpf.Map `ebpf:"ssl_conn_map"`
TcStats *ebpf.Map `ebpf:"tc_stats"`
}
func (m *Ja4TcMaps) Close() error {
return _Ja4TcClose(
m.HttpBuf,
m.SslBuf,
m.TlsBuf,
m.AcceptMap,
m.FdConnMap,
m.RbAccept,
m.RbHttpPlain,
m.RbSslData,
m.RbTcpSyn,
m.RbTlsHello,
m.PbAccept,
m.PbHttpPlain,
m.PbSslData,
m.PbTcpSyn,
m.PbTlsHello,
m.SslArgsMap,
m.SslConnMap,
m.TcStats,
)
}
@ -144,12 +185,12 @@ func (m *Ja4TcMaps) Close() error {
//
// It can be passed to LoadJa4TcObjects or ebpf.CollectionSpec.LoadAndAssign.
type Ja4TcPrograms struct {
CaptureXdp *ebpf.Program `ebpf:"capture_xdp"`
CaptureTc *ebpf.Program `ebpf:"capture_tc"`
}
func (p *Ja4TcPrograms) Close() error {
return _Ja4TcClose(
p.CaptureXdp,
p.CaptureTc,
)
}

34
services/ja4ebpf/internal/parser/http1.go
View File

@ -8,12 +8,13 @@ import (
// HTTP1Request représente une requête HTTP/1.x parsée depuis le flux déchiffré.
type HTTP1Request struct {
Method string // méthode HTTP (GET, POST, …)
Path string // chemin (sans query string)
Query string // query string (sans le '?')
Protocol string // "HTTP/1.0" ou "HTTP/1.1"
Headers []string // noms des en-têtes dans l'ordre exact d'arrivée
HeaderSig string // signature : noms joints par ";"
Method string // méthode HTTP (GET, POST, …)
Path string // chemin (sans query string)
Query string // query string (sans le '?')
Protocol string // "HTTP/1.0" ou "HTTP/1.1"
Headers []string // noms des en-têtes dans l'ordre exact d'arrivée
HeaderSig string // signature : noms joints par ";"
HeaderKV map[string]string // valeurs des en-têtes clés (Host, User-Agent, etc.)
}
// HTTP1Response représente le début d'une réponse HTTP/1.x (status line).
@ -27,6 +28,14 @@ var knownMethods = []string{
"OPTIONS", "PATCH", "CONNECT", "TRACE",
}
// capturedHeaders est la liste des en-têtes dont on capture la valeur.
var capturedHeaders = []string{
"Host", "User-Agent", "Accept", "Accept-Encoding", "Accept-Language",
"Content-Type", "X-Request-Id", "X-Trace-Id", "X-Forwarded-For",
"Sec-CH-UA", "Sec-CH-UA-Mobile", "Sec-CH-UA-Platform",
"Sec-Fetch-Dest", "Sec-Fetch-Mode", "Sec-Fetch-Site",
}
// IsHTTP1Request retourne true si les premiers octets ressemblent à une
// requête HTTP/1.x (commence par une méthode reconnue suivi d'un espace).
func IsHTTP1Request(data []byte) bool {
@ -91,8 +100,9 @@ func ParseHTTP1Request(data []byte) *HTTP1Request {
query = rawPath[idx+1:]
}
// Extraire les noms d'en-têtes dans l'ordre
// Extraire les noms d'en-têtes dans l'ordre + capturer les valeurs clés
headers := make([]string, 0, len(lines)-1)
headerKV := make(map[string]string, len(capturedHeaders))
for _, line := range lines[1:] {
if line == "" {
break
@ -101,6 +111,13 @@ func ParseHTTP1Request(data []byte) *HTTP1Request {
name := strings.TrimSpace(line[:colon])
if name != "" {
headers = append(headers, name)
// Capturer la valeur si c'est un header d'intérêt
for _, key := range capturedHeaders {
if strings.EqualFold(name, key) {
headerKV[key] = strings.TrimSpace(line[colon+1:])
break
}
}
}
}
}
@ -114,6 +131,7 @@ func ParseHTTP1Request(data []byte) *HTTP1Request {
Protocol: protocol,
Headers: headers,
HeaderSig: sig,
HeaderKV: headerKV,
}
}
@ -143,4 +161,4 @@ func ParseHTTP1Response(data []byte) *HTTP1Response {
return nil
}
return &HTTP1Response{StatusCode: code}
}
}

10
services/ja4ebpf/internal/parser/tls.go
View File

@ -247,9 +247,9 @@ func parseSupportedVersions(data []byte) []uint16 {
return versions
}
// isGREASE vérifie si une valeur est une valeur GREASE (RFC 8701).
// IsGREASE vérifie si une valeur est une valeur GREASE (RFC 8701).
// Les valeurs GREASE suivent le motif 0x?A?A (ex: 0x0A0A, 0x1A1A, ...).
func isGREASE(v uint16) bool {
func IsGREASE(v uint16) bool {
return v&0x0F0F == 0x0A0A && v>>8 == v&0xFF
}
@ -279,7 +279,7 @@ func ComputeJA4(ch *ClientHello) string {
// --- Version TLS : version la plus haute annoncée ---
var tlsVer uint16
for _, v := range ch.SupportedVersions {
if !isGREASE(v) && v > tlsVer {
if !IsGREASE(v) && v > tlsVer {
tlsVer = v
}
}
@ -298,7 +298,7 @@ func ComputeJA4(ch *ClientHello) string {
// --- Comptage des cipher suites (sans GREASE) ---
var ciphers []uint16
for _, cs := range ch.CipherSuites {
if !isGREASE(cs) {
if !IsGREASE(cs) {
ciphers = append(ciphers, cs)
}
}
@ -307,7 +307,7 @@ func ComputeJA4(ch *ClientHello) string {
// --- Comptage des extensions (sans GREASE, sans SNI 0x0000) ---
var extensions []uint16
for _, ext := range ch.Extensions {
if isGREASE(ext.Type) {
if IsGREASE(ext.Type) {
continue
}
if ext.Type == 0x0000 { // SNI exclue du comptage

112
services/ja4ebpf/internal/writer/clickhouse.go
View File

@ -52,14 +52,32 @@ type sessionRecord struct {
TLSVersion string `json:"tls_version,omitempty"`
// HTTP
Method string `json:"method,omitempty"`
Path string `json:"path,omitempty"`
QueryString string `json:"query_string,omitempty"`
StatusCode *int `json:"status_code,omitempty"`
ResponseSize *int64 `json:"response_size,omitempty"`
DurationMS *float64 `json:"duration_ms,omitempty"`
KeepAlives int `json:"keepalives,omitempty"`
HeaderOrderSig string `json:"header_order_signature,omitempty"`
Method string `json:"method,omitempty"`
Path string `json:"path,omitempty"`
Host string `json:"host,omitempty"`
QueryString string `json:"query_string,omitempty"`
Scheme string `json:"scheme,omitempty"`
HTTPVersion string `json:"http_version,omitempty"`
StatusCode *int `json:"status_code,omitempty"`
ResponseSize *int64 `json:"response_size,omitempty"`
DurationMS *float64 `json:"duration_ms,omitempty"`
KeepAlives int `json:"keepalives,omitempty"`
HeaderOrderSig string `json:"header_order_signature,omitempty"`
HeadersRaw string `json:"headers_raw,omitempty"`
HeaderUserAgent string `json:"header_User-Agent,omitempty"`
HeaderAccept string `json:"header_Accept,omitempty"`
HeaderAcceptEnc string `json:"header_Accept-Encoding,omitempty"`
HeaderAcceptLang string `json:"header_Accept-Language,omitempty"`
HeaderContentType string `json:"header_Content-Type,omitempty"`
HeaderXReqID string `json:"header_X-Request-Id,omitempty"`
HeaderXTraceID string `json:"header_X-Trace-Id,omitempty"`
HeaderXForwarded string `json:"header_X-Forwarded-For,omitempty"`
HeaderSecCHUA string `json:"header_Sec-CH-UA,omitempty"`
HeaderSecCHUAMobile string `json:"header_Sec-CH-UA-Mobile,omitempty"`
HeaderSecCHUAPlat string `json:"header_Sec-CH-UA-Platform,omitempty"`
HeaderSecFetchDest string `json:"header_Sec-Fetch-Dest,omitempty"`
HeaderSecFetchMode string `json:"header_Sec-Fetch-Mode,omitempty"`
HeaderSecFetchSite string `json:"header_Sec-Fetch-Site,omitempty"`
// HTTP/2 fingerprinting passif
H2Fingerprint string `json:"h2_fingerprint,omitempty"`
@ -67,13 +85,13 @@ type sessionRecord struct {
H2WindowUpdate uint32 `json:"h2_window_update,omitempty"`
H2PseudoOrder string `json:"h2_pseudo_order,omitempty"`
H2HasPriority uint8 `json:"h2_has_priority,omitempty"`
H2HeaderTableSize int32 `json:"h2_header_table_size"`
H2EnablePush int32 `json:"h2_enable_push"`
H2MaxConcurrentStreams int32 `json:"h2_max_concurrent_streams"`
H2InitialWindowSize int64 `json:"h2_initial_window_size"`
H2MaxFrameSize int32 `json:"h2_max_frame_size"`
H2MaxHeaderListSize int32 `json:"h2_max_header_list_size"`
H2EnableConnectProtocol int32 `json:"h2_enable_connect_protocol"`
H2HeaderTableSize *int32 `json:"h2_header_table_size,omitempty"`
H2EnablePush *int32 `json:"h2_enable_push,omitempty"`
H2MaxConcurrentStreams *int32 `json:"h2_max_concurrent_streams,omitempty"`
H2InitialWindowSize *int64 `json:"h2_initial_window_size,omitempty"`
H2MaxFrameSize *int32 `json:"h2_max_frame_size,omitempty"`
H2MaxHeaderListSize *int32 `json:"h2_max_header_list_size,omitempty"`
H2EnableConnectProtocol *int32 `json:"h2_enable_connect_protocol,omitempty"`
}
// NewClickHouseWriter crée un writer et établit la connexion ClickHouse.
@ -199,19 +217,26 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
Time: s.FirstSeen,
SrcIP: srcIP,
SrcPort: int(s.Key.SrcPort),
DstIP: "0.0.0.0", // destination non capturée par les sondes eBPF actuelles
DstPort: 0,
KeepAlives: len(s.Requests),
}
// Champs métadonnées IP/TCP
if s.L3L4 != nil {
rec.DstIP = fmt.Sprintf("%d.%d.%d.%d",
s.L3L4.DstIP[0], s.L3L4.DstIP[1], s.L3L4.DstIP[2], s.L3L4.DstIP[3])
rec.DstPort = int(s.L3L4.DstPort)
rec.IPMetaDF = &s.L3L4.DFBit
rec.IPMetaID = &s.L3L4.IPID
rec.IPMetaTTL = &s.L3L4.TTL
rec.TCPMetaWindowSize = &s.L3L4.WindowSize
rec.TCPMetaWindowScale = &s.L3L4.WindowScale
rec.TCPMetaMSS = &s.L3L4.MSS
// WindowScale 0xFF = absent (convention C), ne pas inclure
if s.L3L4.WindowScale != 0xFF {
rec.TCPMetaWindowScale = &s.L3L4.WindowScale
}
// MSS 0 = absent, ne pas inclure
if s.L3L4.MSS > 0 {
rec.TCPMetaMSS = &s.L3L4.MSS
}
}
// Champs TLS
@ -220,6 +245,14 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
rec.TLSSNI = s.TLS.SNI
rec.TLSALPN = strings.Join(s.TLS.ALPN, ",")
rec.TLSVersion = formatTLSVersion(s.TLS.TLSVersion)
// Fallback : si pas de Host HTTP, utiliser TLS SNI
if rec.Host == "" && s.TLS.SNI != "" {
rec.Host = s.TLS.SNI
}
// Scheme déduit de la présence TLS
if s.TLS.SNI != "" {
rec.Scheme = "https"
}
}
// Champs HTTP (dernière requête)
@ -228,11 +261,37 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
rec.Method = last.Method
rec.Path = last.Path
rec.QueryString = last.QueryString
rec.Host = last.Host
rec.Scheme = "" // sera rempli par le dispatcher si TLS
rec.HTTPVersion = last.HTTPVersion
rec.StatusCode = &last.StatusCode
rec.ResponseSize = &last.ResponseSize
rec.DurationMS = &last.DurationMS
rec.HeaderOrderSig = last.HeaderOrderSig
// En-têtes HTTP individuels
if last.HeaderKV != nil {
rec.HeaderUserAgent = last.HeaderKV["User-Agent"]
rec.HeaderAccept = last.HeaderKV["Accept"]
rec.HeaderAcceptEnc = last.HeaderKV["Accept-Encoding"]
rec.HeaderAcceptLang = last.HeaderKV["Accept-Language"]
rec.HeaderContentType = last.HeaderKV["Content-Type"]
rec.HeaderXReqID = last.HeaderKV["X-Request-Id"]
rec.HeaderXTraceID = last.HeaderKV["X-Trace-Id"]
rec.HeaderXForwarded = last.HeaderKV["X-Forwarded-For"]
rec.HeaderSecCHUA = last.HeaderKV["Sec-CH-UA"]
rec.HeaderSecCHUAMobile = last.HeaderKV["Sec-CH-UA-Mobile"]
rec.HeaderSecCHUAPlat = last.HeaderKV["Sec-CH-UA-Platform"]
rec.HeaderSecFetchDest = last.HeaderKV["Sec-Fetch-Dest"]
rec.HeaderSecFetchMode = last.HeaderKV["Sec-Fetch-Mode"]
rec.HeaderSecFetchSite = last.HeaderKV["Sec-Fetch-Site"]
}
// Construire headers_raw : ordre des noms joints par ";"
if len(last.HeaderOrder) > 0 {
rec.HeadersRaw = strings.Join(last.HeaderOrder, ";")
}
// Champs HTTP/2 passifs
if last.HTTP2Settings != nil {
h2 := last.HTTP2Settings
@ -243,13 +302,14 @@ func sessionToRecord(s *correlation.SessionState) sessionRecord {
rec.H2PseudoOrder = pseudoOrderToShort(h2.PseudoHeaderOrder)
}
// Paramètres SETTINGS individuels (-1 = absent)
rec.H2HeaderTableSize = h2.HeaderTableSize
rec.H2EnablePush = h2.EnablePush
rec.H2MaxConcurrentStreams = h2.MaxConcurrentStreams
rec.H2InitialWindowSize = int64(h2.InitialWindowSize)
rec.H2MaxFrameSize = h2.MaxFrameSize
rec.H2MaxHeaderListSize = h2.MaxHeaderListSize
// Paramètres SETTINGS individuels (pointeurs : nil = absent du preface)
rec.H2HeaderTableSize = &h2.HeaderTableSize
rec.H2EnablePush = &h2.EnablePush
rec.H2MaxConcurrentStreams = &h2.MaxConcurrentStreams
h2InitWin := int64(h2.InitialWindowSize)
rec.H2InitialWindowSize = &h2InitWin
rec.H2MaxFrameSize = &h2.MaxFrameSize
rec.H2MaxHeaderListSize = &h2.MaxHeaderListSize
// Fingerprints composites Akamai
rec.H2Fingerprint = buildH2Fingerprint(h2)