fix(ja4ebpf): fix TLS capture, SYN offsets, TCP option parsing
- Increase MAX_TLS_PAYLOAD from 512 to 2048 bytes to capture full TLS ClientHellos (modern browsers/curl send 1000-1543 byte ClientHellos) - Fix ParseClientHello to tolerate XDP-truncated payloads: clamp recordLength and chLen to available data instead of returning error - Fix cipher suites, compression, extensions truncation to use clamping - Fix consumeSynEvents struct field offsets: dst_ip (4 bytes at offset 4) was not accounted for, causing all L3/L4 metadata to be read from wrong positions (TTL was actually dst_ip[0], windowSize was dst_port, etc.) - Add parseTCPOptions() to extract MSS and Window Scale from raw TCP options (C code sets defaults of mss=0, window_scale=0xFF, expects Go to parse) - Fix consumeAcceptEvents: skip zero-IP events to avoid phantom sessions - Fix consumeSSLEvents: filter zero-IP/port events when proc fallback fails - Add missing consumeHTTPPlainEvents goroutine (was defined but never called) - Fix race condition: SYN consumer sets Correlated=true if TLS already present - Update tls_hello_event struct offsets in Go consumer (payload_len now at offset 2054, was 518, due to payload array growing from 512 to 2048 bytes) - Remove debug logging from consumers and GC E2E verified: HTTP plain (port 80) and HTTPS (port 443) both produce fully correlated sessions in ClickHouse with correct: - ip_meta_ttl=64, ip_meta_df=true, ip_meta_id - tcp_meta_window_size=64240, tcp_meta_window_scale=10, tcp_meta_mss=1460 - ja4=t13i3010_1d37bd780c83_95d2a80e6515 - tls_alpn=http/1.1 - method=GET, path=/, header_order_signature=Host;User-Agent;Accept - correlated=1 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
183
services/ja4ebpf/internal/loader/ja4ssl_x86_bpfel.go
Normal file
183
services/ja4ebpf/internal/loader/ja4ssl_x86_bpfel.go
Normal file
@ -0,0 +1,183 @@
|
||||
// Code generated by bpf2go; DO NOT EDIT.
|
||||
//go:build 386 || amd64
|
||||
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type Ja4SslAcceptEvent struct {
|
||||
PidTgid uint64
|
||||
Fd uint32
|
||||
SrcIp uint32
|
||||
SrcPort uint16
|
||||
TimestampNs uint64
|
||||
}
|
||||
|
||||
type Ja4SslAcceptKey struct {
|
||||
PidTgid uint64
|
||||
Fd uint32
|
||||
}
|
||||
|
||||
type Ja4SslSslConnInfo struct {
|
||||
Fd uint32
|
||||
SrcIp uint32
|
||||
SrcPort uint16
|
||||
}
|
||||
|
||||
type Ja4SslSslReadArgs struct {
|
||||
SslPtr uint64
|
||||
BufPtr uint64
|
||||
Num uint32
|
||||
}
|
||||
|
||||
// LoadJa4Ssl returns the embedded CollectionSpec for Ja4Ssl.
|
||||
func LoadJa4Ssl() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_Ja4SslBytes)
|
||||
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't load Ja4Ssl: %w", err)
|
||||
}
|
||||
|
||||
return spec, err
|
||||
}
|
||||
|
||||
// LoadJa4SslObjects loads Ja4Ssl and converts it into a struct.
|
||||
//
|
||||
// The following types are suitable as obj argument:
|
||||
//
|
||||
// *Ja4SslObjects
|
||||
// *Ja4SslPrograms
|
||||
// *Ja4SslMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func LoadJa4SslObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := LoadJa4Ssl()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return spec.LoadAndAssign(obj, opts)
|
||||
}
|
||||
|
||||
// Ja4SslSpecs contains maps and programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type Ja4SslSpecs struct {
|
||||
Ja4SslProgramSpecs
|
||||
Ja4SslMapSpecs
|
||||
}
|
||||
|
||||
// Ja4SslSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type Ja4SslProgramSpecs struct {
|
||||
KprobeAccept4Entry *ebpf.ProgramSpec `ebpf:"kprobe_accept4_entry"`
|
||||
KretprobeAccept4Exit *ebpf.ProgramSpec `ebpf:"kretprobe_accept4_exit"`
|
||||
UprobeSslReadEntry *ebpf.ProgramSpec `ebpf:"uprobe_ssl_read_entry"`
|
||||
UprobeSslSetFd *ebpf.ProgramSpec `ebpf:"uprobe_ssl_set_fd"`
|
||||
UretprobeSslReadExit *ebpf.ProgramSpec `ebpf:"uretprobe_ssl_read_exit"`
|
||||
}
|
||||
|
||||
// Ja4SslMapSpecs contains maps before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type Ja4SslMapSpecs struct {
|
||||
AcceptArgsMap *ebpf.MapSpec `ebpf:"accept_args_map"`
|
||||
AcceptMap *ebpf.MapSpec `ebpf:"accept_map"`
|
||||
FdConnMap *ebpf.MapSpec `ebpf:"fd_conn_map"`
|
||||
RbAccept *ebpf.MapSpec `ebpf:"rb_accept"`
|
||||
RbHttpPlain *ebpf.MapSpec `ebpf:"rb_http_plain"`
|
||||
RbSslData *ebpf.MapSpec `ebpf:"rb_ssl_data"`
|
||||
RbTcpSyn *ebpf.MapSpec `ebpf:"rb_tcp_syn"`
|
||||
RbTlsHello *ebpf.MapSpec `ebpf:"rb_tls_hello"`
|
||||
SslArgsMap *ebpf.MapSpec `ebpf:"ssl_args_map"`
|
||||
SslConnMap *ebpf.MapSpec `ebpf:"ssl_conn_map"`
|
||||
}
|
||||
|
||||
// Ja4SslObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to LoadJa4SslObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type Ja4SslObjects struct {
|
||||
Ja4SslPrograms
|
||||
Ja4SslMaps
|
||||
}
|
||||
|
||||
func (o *Ja4SslObjects) Close() error {
|
||||
return _Ja4SslClose(
|
||||
&o.Ja4SslPrograms,
|
||||
&o.Ja4SslMaps,
|
||||
)
|
||||
}
|
||||
|
||||
// Ja4SslMaps contains all maps after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to LoadJa4SslObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type Ja4SslMaps struct {
|
||||
AcceptArgsMap *ebpf.Map `ebpf:"accept_args_map"`
|
||||
AcceptMap *ebpf.Map `ebpf:"accept_map"`
|
||||
FdConnMap *ebpf.Map `ebpf:"fd_conn_map"`
|
||||
RbAccept *ebpf.Map `ebpf:"rb_accept"`
|
||||
RbHttpPlain *ebpf.Map `ebpf:"rb_http_plain"`
|
||||
RbSslData *ebpf.Map `ebpf:"rb_ssl_data"`
|
||||
RbTcpSyn *ebpf.Map `ebpf:"rb_tcp_syn"`
|
||||
RbTlsHello *ebpf.Map `ebpf:"rb_tls_hello"`
|
||||
SslArgsMap *ebpf.Map `ebpf:"ssl_args_map"`
|
||||
SslConnMap *ebpf.Map `ebpf:"ssl_conn_map"`
|
||||
}
|
||||
|
||||
func (m *Ja4SslMaps) Close() error {
|
||||
return _Ja4SslClose(
|
||||
m.AcceptArgsMap,
|
||||
m.AcceptMap,
|
||||
m.FdConnMap,
|
||||
m.RbAccept,
|
||||
m.RbHttpPlain,
|
||||
m.RbSslData,
|
||||
m.RbTcpSyn,
|
||||
m.RbTlsHello,
|
||||
m.SslArgsMap,
|
||||
m.SslConnMap,
|
||||
)
|
||||
}
|
||||
|
||||
// Ja4SslPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to LoadJa4SslObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type Ja4SslPrograms struct {
|
||||
KprobeAccept4Entry *ebpf.Program `ebpf:"kprobe_accept4_entry"`
|
||||
KretprobeAccept4Exit *ebpf.Program `ebpf:"kretprobe_accept4_exit"`
|
||||
UprobeSslReadEntry *ebpf.Program `ebpf:"uprobe_ssl_read_entry"`
|
||||
UprobeSslSetFd *ebpf.Program `ebpf:"uprobe_ssl_set_fd"`
|
||||
UretprobeSslReadExit *ebpf.Program `ebpf:"uretprobe_ssl_read_exit"`
|
||||
}
|
||||
|
||||
func (p *Ja4SslPrograms) Close() error {
|
||||
return _Ja4SslClose(
|
||||
p.KprobeAccept4Entry,
|
||||
p.KretprobeAccept4Exit,
|
||||
p.UprobeSslReadEntry,
|
||||
p.UprobeSslSetFd,
|
||||
p.UretprobeSslReadExit,
|
||||
)
|
||||
}
|
||||
|
||||
func _Ja4SslClose(closers ...io.Closer) error {
|
||||
for _, closer := range closers {
|
||||
if err := closer.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Do not access this directly.
|
||||
//
|
||||
//go:embed ja4ssl_x86_bpfel.o
|
||||
var _Ja4SslBytes []byte
|
||||
BIN
services/ja4ebpf/internal/loader/ja4ssl_x86_bpfel.o
Normal file
BIN
services/ja4ebpf/internal/loader/ja4ssl_x86_bpfel.o
Normal file
Binary file not shown.
168
services/ja4ebpf/internal/loader/ja4tc_x86_bpfel.go
Normal file
168
services/ja4ebpf/internal/loader/ja4tc_x86_bpfel.go
Normal file
@ -0,0 +1,168 @@
|
||||
// Code generated by bpf2go; DO NOT EDIT.
|
||||
//go:build 386 || amd64
|
||||
|
||||
package loader
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
_ "embed"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
)
|
||||
|
||||
type Ja4TcAcceptEvent struct {
|
||||
PidTgid uint64
|
||||
Fd uint32
|
||||
SrcIp uint32
|
||||
SrcPort uint16
|
||||
TimestampNs uint64
|
||||
}
|
||||
|
||||
type Ja4TcAcceptKey struct {
|
||||
PidTgid uint64
|
||||
Fd uint32
|
||||
}
|
||||
|
||||
type Ja4TcSslConnInfo struct {
|
||||
Fd uint32
|
||||
SrcIp uint32
|
||||
SrcPort uint16
|
||||
}
|
||||
|
||||
type Ja4TcSslReadArgs struct {
|
||||
SslPtr uint64
|
||||
BufPtr uint64
|
||||
Num uint32
|
||||
}
|
||||
|
||||
// LoadJa4Tc returns the embedded CollectionSpec for Ja4Tc.
|
||||
func LoadJa4Tc() (*ebpf.CollectionSpec, error) {
|
||||
reader := bytes.NewReader(_Ja4TcBytes)
|
||||
spec, err := ebpf.LoadCollectionSpecFromReader(reader)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("can't load Ja4Tc: %w", err)
|
||||
}
|
||||
|
||||
return spec, err
|
||||
}
|
||||
|
||||
// LoadJa4TcObjects loads Ja4Tc and converts it into a struct.
|
||||
//
|
||||
// The following types are suitable as obj argument:
|
||||
//
|
||||
// *Ja4TcObjects
|
||||
// *Ja4TcPrograms
|
||||
// *Ja4TcMaps
|
||||
//
|
||||
// See ebpf.CollectionSpec.LoadAndAssign documentation for details.
|
||||
func LoadJa4TcObjects(obj interface{}, opts *ebpf.CollectionOptions) error {
|
||||
spec, err := LoadJa4Tc()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return spec.LoadAndAssign(obj, opts)
|
||||
}
|
||||
|
||||
// Ja4TcSpecs contains maps and programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type Ja4TcSpecs struct {
|
||||
Ja4TcProgramSpecs
|
||||
Ja4TcMapSpecs
|
||||
}
|
||||
|
||||
// Ja4TcSpecs contains programs before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type Ja4TcProgramSpecs struct {
|
||||
CaptureXdp *ebpf.ProgramSpec `ebpf:"capture_xdp"`
|
||||
}
|
||||
|
||||
// Ja4TcMapSpecs contains maps before they are loaded into the kernel.
|
||||
//
|
||||
// It can be passed ebpf.CollectionSpec.Assign.
|
||||
type Ja4TcMapSpecs struct {
|
||||
AcceptMap *ebpf.MapSpec `ebpf:"accept_map"`
|
||||
FdConnMap *ebpf.MapSpec `ebpf:"fd_conn_map"`
|
||||
RbAccept *ebpf.MapSpec `ebpf:"rb_accept"`
|
||||
RbHttpPlain *ebpf.MapSpec `ebpf:"rb_http_plain"`
|
||||
RbSslData *ebpf.MapSpec `ebpf:"rb_ssl_data"`
|
||||
RbTcpSyn *ebpf.MapSpec `ebpf:"rb_tcp_syn"`
|
||||
RbTlsHello *ebpf.MapSpec `ebpf:"rb_tls_hello"`
|
||||
SslArgsMap *ebpf.MapSpec `ebpf:"ssl_args_map"`
|
||||
SslConnMap *ebpf.MapSpec `ebpf:"ssl_conn_map"`
|
||||
}
|
||||
|
||||
// Ja4TcObjects contains all objects after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to LoadJa4TcObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type Ja4TcObjects struct {
|
||||
Ja4TcPrograms
|
||||
Ja4TcMaps
|
||||
}
|
||||
|
||||
func (o *Ja4TcObjects) Close() error {
|
||||
return _Ja4TcClose(
|
||||
&o.Ja4TcPrograms,
|
||||
&o.Ja4TcMaps,
|
||||
)
|
||||
}
|
||||
|
||||
// Ja4TcMaps contains all maps after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to LoadJa4TcObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type Ja4TcMaps struct {
|
||||
AcceptMap *ebpf.Map `ebpf:"accept_map"`
|
||||
FdConnMap *ebpf.Map `ebpf:"fd_conn_map"`
|
||||
RbAccept *ebpf.Map `ebpf:"rb_accept"`
|
||||
RbHttpPlain *ebpf.Map `ebpf:"rb_http_plain"`
|
||||
RbSslData *ebpf.Map `ebpf:"rb_ssl_data"`
|
||||
RbTcpSyn *ebpf.Map `ebpf:"rb_tcp_syn"`
|
||||
RbTlsHello *ebpf.Map `ebpf:"rb_tls_hello"`
|
||||
SslArgsMap *ebpf.Map `ebpf:"ssl_args_map"`
|
||||
SslConnMap *ebpf.Map `ebpf:"ssl_conn_map"`
|
||||
}
|
||||
|
||||
func (m *Ja4TcMaps) Close() error {
|
||||
return _Ja4TcClose(
|
||||
m.AcceptMap,
|
||||
m.FdConnMap,
|
||||
m.RbAccept,
|
||||
m.RbHttpPlain,
|
||||
m.RbSslData,
|
||||
m.RbTcpSyn,
|
||||
m.RbTlsHello,
|
||||
m.SslArgsMap,
|
||||
m.SslConnMap,
|
||||
)
|
||||
}
|
||||
|
||||
// Ja4TcPrograms contains all programs after they have been loaded into the kernel.
|
||||
//
|
||||
// It can be passed to LoadJa4TcObjects or ebpf.CollectionSpec.LoadAndAssign.
|
||||
type Ja4TcPrograms struct {
|
||||
CaptureXdp *ebpf.Program `ebpf:"capture_xdp"`
|
||||
}
|
||||
|
||||
func (p *Ja4TcPrograms) Close() error {
|
||||
return _Ja4TcClose(
|
||||
p.CaptureXdp,
|
||||
)
|
||||
}
|
||||
|
||||
func _Ja4TcClose(closers ...io.Closer) error {
|
||||
for _, closer := range closers {
|
||||
if err := closer.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Do not access this directly.
|
||||
//
|
||||
//go:embed ja4tc_x86_bpfel.o
|
||||
var _Ja4TcBytes []byte
|
||||
BIN
services/ja4ebpf/internal/loader/ja4tc_x86_bpfel.o
Normal file
BIN
services/ja4ebpf/internal/loader/ja4tc_x86_bpfel.o
Normal file
Binary file not shown.
@ -10,7 +10,6 @@ import (
|
||||
"net"
|
||||
"os"
|
||||
|
||||
"github.com/cilium/ebpf"
|
||||
"github.com/cilium/ebpf/link"
|
||||
"github.com/cilium/ebpf/ringbuf"
|
||||
"github.com/cilium/ebpf/rlimit"
|
||||
@ -122,23 +121,31 @@ func New() (*Loader, error) {
|
||||
}, nil
|
||||
}
|
||||
|
||||
// AttachTC attache le programme TC ingress sur l'interface réseau spécifiée.
|
||||
// Utilise TCX (TC eXpress) disponible depuis le noyau 6.6+.
|
||||
// AttachTC attache le programme XDP sur l'interface réseau spécifiée.
|
||||
// Essaie le mode natif XDP (driver support) puis se replie sur le mode générique
|
||||
// (SKB_MODE, compatible kernel ≥ 4.8, fonctionne dans les VMs).
|
||||
func (l *Loader) AttachTC(iface string) error {
|
||||
// Résoudre l'interface réseau par son nom
|
||||
netIface, err := net.InterfaceByName(iface)
|
||||
if err != nil {
|
||||
return fmt.Errorf("interface réseau %q introuvable: %w", iface, err)
|
||||
}
|
||||
|
||||
// Attacher le programme TC en ingress via TCX
|
||||
lnk, err := link.AttachTCX(link.TCXOptions{
|
||||
// Mode natif (meilleure performance sur serveurs avec NIC compatible XDP)
|
||||
lnk, err := link.AttachXDP(link.XDPOptions{
|
||||
Interface: netIface.Index,
|
||||
Program: l.tcObjs.CaptureTcIngress,
|
||||
Attach: ebpf.AttachTCXIngress,
|
||||
Program: l.tcObjs.CaptureXdp,
|
||||
Flags: link.XDPDriverMode,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("attachement TC ingress sur %q: %w", iface, err)
|
||||
// Repli sur le mode générique (VMs, NICs sans driver XDP natif)
|
||||
lnk, err = link.AttachXDP(link.XDPOptions{
|
||||
Interface: netIface.Index,
|
||||
Program: l.tcObjs.CaptureXdp,
|
||||
Flags: link.XDPGenericMode,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("attachement XDP sur %q (natif et générique): %w", iface, err)
|
||||
}
|
||||
}
|
||||
|
||||
l.tcLink = lnk
|
||||
|
||||
Reference in New Issue
Block a user