diff --git a/api/types.go b/api/types.go index 4a19970..7b8f630 100644 --- a/api/types.go +++ b/api/types.go @@ -50,19 +50,26 @@ type RawPacket struct { // TLSClientHello represents a client-side TLS ClientHello with IP/TCP metadata type TLSClientHello struct { - SrcIP string `json:"src_ip"` - SrcPort uint16 `json:"src_port"` - DstIP string `json:"dst_ip"` - DstPort uint16 `json:"dst_port"` - Payload []byte `json:"-"` // Not serialized - IPMeta IPMeta `json:"ip_meta"` - TCPMeta TCPMeta `json:"tcp_meta"` + SrcIP string `json:"src_ip"` + SrcPort uint16 `json:"src_port"` + DstIP string `json:"dst_ip"` + DstPort uint16 `json:"dst_port"` + Payload []byte `json:"-"` // Not serialized + IPMeta IPMeta `json:"ip_meta"` + TCPMeta TCPMeta `json:"tcp_meta"` + ConnID string `json:"conn_id,omitempty"` // Unique flow identifier + SNI string `json:"tls_sni,omitempty"` // Server Name Indication + ALPN string `json:"tls_alpn,omitempty"` // Application-Layer Protocol Negotiation + TLSVersion string `json:"tls_version,omitempty"` // Max TLS version supported + SynToCHMs *uint32 `json:"syn_to_clienthello_ms,omitempty"` // Time from SYN to ClientHello (ms) } // Fingerprints contains TLS fingerprints for a client flow +// Note: JA4Hash is kept for internal use but not serialized to LogRecord +// as the JA4 format already includes its own hash portions type Fingerprints struct { JA4 string `json:"ja4"` - JA4Hash string `json:"ja4_hash"` + JA4Hash string `json:"ja4_hash,omitempty"` // Internal use, not serialized to LogRecord JA3 string `json:"ja3,omitempty"` JA3Hash string `json:"ja3_hash,omitempty"` } @@ -81,14 +88,26 @@ type LogRecord struct { IPDF bool `json:"ip_meta_df"` // Flattened TCPMeta fields - TCPWindow uint16 `json:"tcp_meta_window_size"` + TCPWindow uint16 `json:"tcp_meta_window_size"` TCPMSS *uint16 `json:"tcp_meta_mss,omitempty"` TCPWScale *uint8 `json:"tcp_meta_window_scale,omitempty"` - TCPOptions string `json:"tcp_meta_options"` // comma-separated list + TCPOptions string `json:"tcp_meta_options"` // comma-separated list + + // Correlation & Triage + ConnID string `json:"conn_id,omitempty"` // Unique flow identifier + SensorID string `json:"sensor_id,omitempty"` // Sensor/captor identifier + + // TLS elements (ClientHello) + TLSVersion string `json:"tls_version,omitempty"` // Max TLS version announced by client + SNI string `json:"tls_sni,omitempty"` // Server Name Indication + ALPN string `json:"tls_alpn,omitempty"` // Application-Layer Protocol Negotiation + + // Behavioral detection (Timing) + SynToCHMs *uint32 `json:"syn_to_clienthello_ms,omitempty"` // Time from SYN to ClientHello (ms) // Fingerprints + // Note: ja4_hash is NOT included - the JA4 format already includes its own hash portions JA4 string `json:"ja4"` - JA4Hash string `json:"ja4_hash"` JA3 string `json:"ja3,omitempty"` JA3Hash string `json:"ja3_hash,omitempty"` @@ -98,9 +117,10 @@ type LogRecord struct { // OutputConfig defines configuration for a single log output type OutputConfig struct { - Type string `json:"type"` // unix_socket, stdout, file, etc. - Enabled bool `json:"enabled"` // whether this output is active - Params map[string]string `json:"params"` // specific parameters like socket_path, path, etc. + Type string `json:"type"` // unix_socket, stdout, file, etc. + Enabled bool `json:"enabled"` // whether this output is active + AsyncBuffer int `json:"async_buffer"` // queue size for async writes (e.g., 5000) + Params map[string]string `json:"params"` // specific parameters like socket_path, path, etc. } // AppConfig is the complete ja4sentinel configuration @@ -191,6 +211,8 @@ type Logger interface { // Converts TCPMeta options to a comma-separated string and creates pointer values // for optional fields (MSS, WindowScale) to support proper JSON omitempty behavior. // If fingerprints is nil, the JA4/JA3 fields will be empty strings. +// Note: JA4Hash is intentionally NOT included in LogRecord as the JA4 format +// already includes its own hash portions (the full 38-character JA4 string is sufficient). func NewLogRecord(ch TLSClientHello, fp *Fingerprints) LogRecord { opts := "" if len(ch.TCPMeta.Options) > 0 { @@ -221,12 +243,16 @@ func NewLogRecord(ch TLSClientHello, fp *Fingerprints) LogRecord { TCPMSS: mssPtr, TCPWScale: wScalePtr, TCPOptions: opts, + ConnID: ch.ConnID, + SNI: ch.SNI, + ALPN: ch.ALPN, + TLSVersion: ch.TLSVersion, + SynToCHMs: ch.SynToCHMs, Timestamp: time.Now().UnixNano(), } if fp != nil { rec.JA4 = fp.JA4 - rec.JA4Hash = fp.JA4Hash rec.JA3 = fp.JA3 rec.JA3Hash = fp.JA3Hash } diff --git a/api/types_test.go b/api/types_test.go index f74b6de..bdd1203 100644 --- a/api/types_test.go +++ b/api/types_test.go @@ -5,6 +5,7 @@ import ( ) func TestNewLogRecord(t *testing.T) { + synToCHMs := uint32(150) tests := []struct { name string clientHello TLSClientHello @@ -18,6 +19,11 @@ func TestNewLogRecord(t *testing.T) { SrcPort: 54321, DstIP: "10.0.0.1", DstPort: 443, + ConnID: "flow-abc123", + SNI: "example.com", + ALPN: "h2", + TLSVersion: "1.3", + SynToCHMs: &synToCHMs, IPMeta: IPMeta{ TTL: 64, TotalLength: 512, @@ -33,7 +39,7 @@ func TestNewLogRecord(t *testing.T) { }, fingerprints: &Fingerprints{ JA4: "t13d1516h2_8daaf6152771_02cb136f2775", - JA4Hash: "8daaf6152771_02cb136f2775", + JA4Hash: "8daaf6152771_02cb136f2775", // Internal use only JA3: "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0", JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d", }, @@ -46,6 +52,10 @@ func TestNewLogRecord(t *testing.T) { SrcPort: 54321, DstIP: "10.0.0.1", DstPort: 443, + ConnID: "flow-xyz789", + SNI: "test.example.com", + ALPN: "http/1.1", + TLSVersion: "1.2", IPMeta: IPMeta{ TTL: 64, TotalLength: 512, @@ -154,14 +164,34 @@ func TestNewLogRecord(t *testing.T) { } } - // Verify fingerprints + // Verify new TLS fields + if rec.ConnID != tt.clientHello.ConnID { + t.Errorf("ConnID = %v, want %v", rec.ConnID, tt.clientHello.ConnID) + } + if rec.SNI != tt.clientHello.SNI { + t.Errorf("SNI = %v, want %v", rec.SNI, tt.clientHello.SNI) + } + if rec.ALPN != tt.clientHello.ALPN { + t.Errorf("ALPN = %v, want %v", rec.ALPN, tt.clientHello.ALPN) + } + if rec.TLSVersion != tt.clientHello.TLSVersion { + t.Errorf("TLSVersion = %v, want %v", rec.TLSVersion, tt.clientHello.TLSVersion) + } + if tt.clientHello.SynToCHMs != nil { + if rec.SynToCHMs == nil { + t.Error("SynToCHMs should not be nil") + } else if *rec.SynToCHMs != *tt.clientHello.SynToCHMs { + t.Errorf("SynToCHMs = %v, want %v", *rec.SynToCHMs, *tt.clientHello.SynToCHMs) + } + } + + // Verify fingerprints (note: JA4Hash is NOT in LogRecord per architecture) if tt.fingerprints != nil { if rec.JA4 != tt.fingerprints.JA4 { t.Errorf("JA4 = %v, want %v", rec.JA4, tt.fingerprints.JA4) } - if rec.JA4Hash != tt.fingerprints.JA4Hash { - t.Errorf("JA4Hash = %v, want %v", rec.JA4Hash, tt.fingerprints.JA4Hash) - } + // JA4Hash is intentionally NOT in LogRecord (architecture decision) + // JA3Hash is still present as it's the MD5 of JA3 (needed for exploitation) if rec.JA3 != tt.fingerprints.JA3 { t.Errorf("JA3 = %v, want %v", rec.JA3, tt.fingerprints.JA3) } @@ -223,3 +253,88 @@ func TestLogRecordConversion(t *testing.T) { t.Errorf("TCPOptions = %v, want %v", rec.TCPOptions, expectedOpts) } } + +func TestLogRecordNoJA4Hash(t *testing.T) { + // Verify that JA4Hash is NOT included in LogRecord per architecture decision + clientHello := TLSClientHello{ + SrcIP: "192.168.1.100", + SrcPort: 54321, + DstIP: "10.0.0.1", + DstPort: 443, + } + fingerprints := &Fingerprints{ + JA4: "t13d1516h2_8daaf6152771_02cb136f2775", + JA4Hash: "8daaf6152771_02cb136f2775", // Should NOT appear in LogRecord + JA3: "771,4865-4866-4867,0-23-65281,29-23-24,0", + JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d", + } + + rec := NewLogRecord(clientHello, fingerprints) + + // JA4Hash is NOT in LogRecord (architecture decision) + // The JA4 format already includes its own hash portions + + // But JA4 should be present + if rec.JA4 != fingerprints.JA4 { + t.Errorf("JA4 = %v, want %v", rec.JA4, fingerprints.JA4) + } + + // JA3Hash should still be present (it's the MD5 of JA3, which is needed) + if rec.JA3Hash != fingerprints.JA3Hash { + t.Errorf("JA3Hash = %v, want %v", rec.JA3Hash, fingerprints.JA3Hash) + } +} + +func TestOutputConfig(t *testing.T) { + tests := []struct { + name string + config OutputConfig + wantEnabled bool + wantAsyncBuf int + }{ + { + name: "stdout output with async buffer", + config: OutputConfig{ + Type: "stdout", + Enabled: true, + AsyncBuffer: 5000, + Params: map[string]string{}, + }, + wantEnabled: true, + wantAsyncBuf: 5000, + }, + { + name: "unix_socket output with default async buffer", + config: OutputConfig{ + Type: "unix_socket", + Enabled: true, + AsyncBuffer: 0, // Default + Params: map[string]string{"socket_path": "/var/run/test.sock"}, + }, + wantEnabled: true, + wantAsyncBuf: 0, + }, + { + name: "disabled output", + config: OutputConfig{ + Type: "file", + Enabled: false, + AsyncBuffer: 1000, + Params: map[string]string{"path": "/var/log/test.log"}, + }, + wantEnabled: false, + wantAsyncBuf: 1000, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.config.Enabled != tt.wantEnabled { + t.Errorf("Enabled = %v, want %v", tt.config.Enabled, tt.wantEnabled) + } + if tt.config.AsyncBuffer != tt.wantAsyncBuf { + t.Errorf("AsyncBuffer = %v, want %v", tt.config.AsyncBuffer, tt.wantAsyncBuf) + } + }) + } +} diff --git a/architecture.yml b/architecture.yml index 940fb27..f9fdf7d 100644 --- a/architecture.yml +++ b/architecture.yml @@ -81,11 +81,13 @@ modules: - name: output path: "internal/output" - description: "Sortie des résultats (JA4 + meta) vers différentes destinations." + description: "Sortie asynchrone ultra-rapide des résultats (JA4 + meta)." responsibilities: - "Prendre en entrée les Fingerprints et les métadonnées réseau." - "Formater les données en enregistrements log (JSON ou autre format simple)." - - "Envoyer les enregistrements vers une ou plusieurs sorties (socket UNIX, stdout, fichier, ...)." + - "Gérer une file d'attente interne (buffer channel) pour rendre l'écriture non-bloquante pour la capture." + - "Sérialiser le JSON le plus rapidement possible (ex: pool d'allocations, librairies optimisées comme goccy/go-json)." + - "Envoyer les enregistrements vers une ou plusieurs sorties (socket UNIX DGRAM, stdout, fichier, ...)." - "Gérer un MultiWriter pour combiner plusieurs outputs sans modifier le reste du code." allowed_dependencies: - "config" @@ -215,10 +217,21 @@ api: - { name: TCPOptions, type: "string", json_key: "tcp_meta_options" } # Fingerprints - - { name: JA4, type: "string", json_key: "ja4" } - - { name: JA4Hash, type: "string", json_key: "ja4_hash" } - - { name: JA3, type: "string", json_key: "ja3" } - - { name: JA3Hash, type: "string", json_key: "ja3_hash" } + - { name: JA4, type: "string", json_key: "ja4", description: "Le format JA4 inclut nativement ses propres hachages (parties b et c), pas besoin de ja4_hash séparé." } + - { name: JA3, type: "string", json_key: "ja3", description: "Chaîne brute JA3 (variable)." } + - { name: JA3Hash, type: "string", json_key: "ja3_hash", description: "Hachage MD5 indispensable pour exploiter la chaîne JA3." } + + # --- Corrélation & Triage --- + - { name: ConnID, type: "string", json_key: "conn_id", optional: true, description: "Identifiant unique du flux (ex: hash de src_ip:src_port-dst_ip:dst_port) pour corréler facilement plusieurs événements liés à une même session TCP." } + - { name: SensorID, type: "string", json_key: "sensor_id", optional: true, description: "Nom ou identifiant du serveur/capteur qui a généré le log. Indispensable pour du déploiement à grande échelle." } + + # --- Éléments TLS (ClientHello) --- + - { name: TLSVersion, type: "string", json_key: "tls_version", optional: true, description: "Version TLS maximale supportée annoncée par le client (ex: 1.2, 1.3). Utile pour repérer les clients obsolètes." } + - { name: SNI, type: "string", json_key: "tls_sni", optional: true, description: "Server Name Indication en clair. Crucial pour détecter le domaine visé par le client (C2, DGA, etc.)." } + - { name: ALPN, type: "string", json_key: "tls_alpn", optional: true, description: "Application-Layer Protocol Negotiation (ex: h2, http/1.1). Aide à différencier le trafic web légitime d'un tunnel personnalisé." } + + # --- Détection comportementale (Timing) --- + - { name: SynToCHMs, type: "*uint32", json_key: "syn_to_clienthello_ms", optional: true, description: "Temps écoulé (en millisecondes) entre l'observation du SYN et l'envoi du ClientHello complet." } # Timestamp - { name: Timestamp, type: "int64", json_key: "timestamp", description: "Wall-clock timestamp in nanoseconds since Unix epoch (auto-filled by NewLogRecord)." } @@ -228,6 +241,7 @@ api: description: "Configuration d’une sortie de logs." fields: - { name: Type, type: "string", description: "Type d’output (unix_socket, stdout, file, ...)." } + - { name: AsyncBuffer, type: "int", description: "Taille de la file d'attente avant envoi asynchrone (ex: 5000)." } - { name: Enabled, type: "bool", description: "Active ou non cette sortie." } - { name: Params, type: "map[string]string", description: "Paramètres spécifiques (socket_path, path, ...)." } @@ -307,7 +321,7 @@ api: module: "output" implements: "output.Writer" config: - - { name: socket_path, type: "string", description: "Chemin de la socket UNIX (ex: /var/run/logcorrelator/network.socket)." } + - { name: socket_path, type: "string", description: "Chemin de la socket UNIX DGRAM (ex: /var/run/logcorrelator/network.sock)." } - name: "output.MultiWriter" description: "Combinaison de plusieurs Writer configurés." @@ -404,347 +418,15 @@ flow_control: description: "Accumulation des segments TCP nécessaires pour extraire un ClientHello complet." - name: "JA4_DONE" description: "JA4 calculé et logué, on arrête de suivre ce flux." - rules: - - "Suivi unidirectionnel : uniquement le flux entrant du client vers la machine locale (srcIP:srcPort -> dstIP:dstPort)." - - "Pour chaque flux, on s'arrête dès que JA4 + IPMeta + TCPMeta sont obtenus et logués." - - "Un timeout par flux doit être défini pour éviter de garder un état si le ClientHello n'arrive jamais." - - "Le flow key est au format : `srcIP:srcPort->dstIP:dstPort` (pas de suivi bidirectionnel)." - -testing: - policy: - description: > - Chaque fonction publique (exportée) doit avoir des tests unitaires, - et les principaux flux (capture -> tlsparse -> fingerprint -> output) - doivent être couverts par des tests d’intégration automatisés. - applies_to: - - "toutes les méthodes des interfaces listées dans api.interfaces" - - requirements: - - id: "test_skeletons" - description: "Pour chaque nouvelle fonction exportée, créer au minimum un squelette de test dans un fichier *_test.go." - - id: "error_cases" - description: "Les cas d’erreur doivent être couverts par au moins un test (inputs invalides, erreurs réseau, parsing raté, etc.)." - - id: "mock_dependencies" - description: "Les dépendances entre modules (Capture, Parser, Engine, Writer) doivent être mockables via interfaces." - - levels: - - name: "unit" - description: "Tests sur chaque module isolé (config, capture, tlsparse, fingerprint, output)." - tools: - - "go test" - rules: - - "Pas de dépendance réseau réelle." - - "Les interactions entre modules sont mockées." - - - name: "integration" - description: "Tests bout-à-bout dans Docker avec trafic TLS client simulé." - tools: - - "go test" - - "docker" - - "docker compose" - scenarios: - - id: "tls_client_ja4_to_outputs" - description: > - Injecter un flux TLS de test côté client, - vérifier que ja4sentinel capture les paquets client, - génère JA4, enrichit avec meta IP/TCP, - et écrit les enregistrements attendus vers les outputs configurés. - steps: - - "Démarrer un serveur TLS de test dans un conteneur." - - "Démarrer ja4sentinel dans un autre conteneur, connecté au même réseau." - - "Émettre une connexion TLS client de test." - - "Lire la socket UNIX et/ou les autres outputs et vérifier JA4, IP, ports, meta." - assertions: - - "Au moins un LogRecord est reçu." - - "Les champs JA4 ne sont pas vides." - - "Les IP/ports correspondent au flux client." - - "IPMeta et TCPMeta sont cohérents (TTL, window, options...)." - -ci_cd: - goals: - - "Construire et tester ja4sentinel de façon reproductible dans des conteneurs Docker." - - "Fournir une commande unique pour lancer tous les tests (unitaires + intégration)." - - docker: - images: - - name: "ja4sentinel-dev" - description: "Image de développement et de test (Go + outils réseau)." - base: "golang:1.23-alpine" - includes: - - "go toolchain" - - "libpcap et headers" - - "outils de test (go test, gotestsum, etc.)" - usage: - build_cmd: "docker build -t ja4sentinel-dev -f Dockerfile.dev ." - test_cmd: "docker run --rm -v $(pwd):/app -w /app ja4sentinel-dev make test" - - name: "ja4sentinel-runtime" - description: "Image minimale pour exécuter le binaire en production." - base: "alpine:latest" - includes: - - "binaire ja4sentinel" - - "libpcap runtime si nécessaire" - usage: - build_cmd: "docker build -t ja4sentinel-runtime -f Dockerfile ." - - rules: - - "Le build de production doit partir du code testé." - - "Les tests d’intégration réseau se font dans des conteneurs, pas directement sur la machine hôte." - - make_targets: - - name: "build" - description: "Compile le binaire ja4sentinel pour la plateforme cible." - commands: - - "go build ./cmd/ja4sentinel" - - name: "test" - description: "Lance les tests unitaires Go." - commands: - - "go test ./..." - - name: "test-integration" - description: "Lance les tests d'intégration dans Docker (capture TLS client + outputs)." - commands: - - "docker compose -f docker-compose.test.yml up --build --abort-on-container-exit --exit-code-from ja4sentinel-test" - - name: "lint" - description: "Lance les linters (go vet, staticcheck, etc.)." - commands: - - "go vet ./..." - -config_guidelines: - sources: - - "Fichier de configuration YAML (par défaut config.yml)." - - "Variables d’environnement (JA4SENTINEL_*)." - - "Flags CLI." - rules: - - "Le module config est la seule source de vérité pour les paramètres (interface, ports, filtres, outputs)." - - "Les autres modules ne lisent jamais les variables d’environnement ni les fichiers de config directement." - -code_style: - comments: - goals: - - "Commentaires standardisés, lisibles, utiles pour la génération de documentation." - - "Expliquer le pourquoi et les décisions, pas répéter le code." - rules: - - id: "godoc_exported" - description: > - Chaque fonction, type ou méthode exporté (nom capitalisé) doit avoir - un commentaire de documentation respectant le style GoDoc. - examples: - - "// FromClientHello génère les empreintes JA4 à partir d’un ClientHello TLS client." - - id: "why_not_what" - description: "Les commentaires doivent expliquer le pourquoi plutôt que le ‘quoi’ lorsque le code est clair." - - id: "no_duplicate_code" - description: "Un commentaire ne doit pas simplement répéter ce que le code dit déjà." - - id: "update_with_code" - description: "Si le code est modifié, les commentaires adjacents doivent être revus et mis à jour." - formatting: - - "Utiliser // pour les commentaires, pas /* ... */ sauf cas exceptionnel." - - "Commentaires de doc au-dessus des déclarations, sans ligne vide." - -evolution: - api_stability: - strategy: "semver_like" - goals: - - "Éviter les changements majeurs d’API une fois le projet stabilisé." - - "Limiter les modifications à ce qui est nécessaire (fixes, extensions compatibles)." - rules: - - id: "no_breaking_changes_without_reason" - description: > - Ne pas modifier les signatures des fonctions des interfaces définies dans api.interfaces - sans raison solide et documentée. - - id: "prefer_extension_over_modification" - description: > - Pour ajouter un comportement, préférer ajouter une nouvelle fonction ou un nouveau type, - plutôt que changer une signature existante. - - id: "document_changes" - description: > - Toute modification d’API doit être reflétée dans architecture.yml - et, si nécessaire, dans un ADR séparé. - refactoring: - goals: - - "Autoriser le refactoring interne sans changer les contrats externes." - - "Limiter l’impact des changements à l’intérieur d’un module." - guidelines: - - "Les interfaces de api.interfaces servent de contrat stable entre modules." - - "Les changements internes (optimisations, refactoring) ne doivent pas casser ces interfaces." - - "Lorsqu’un changement d’API est inévitable, marquer l’ancienne API comme dépréciée avant de la supprimer." - -dev_tools: - usage_guidelines: - - "Les outils IA doivent lire architecture.yml avant de générer ou modifier du code." - - "Les outils de test peuvent s'appuyer sur ci_cd.docker et ci_cd.make_targets pour générer des scripts / pipelines." - future_extensions: - - "Génération automatique de Dockerfile.dev et Dockerfile à partir de cette section." - - "Génération de fichiers docker-compose.test.yml pour les scénarios d'intégration." - -packaging: - description: > - ja4sentinel est distribué sous forme de packages .rpm (Rocky Linux/RHEL/CentOS/AlmaLinux), - construits intégralement dans Docker avec rpmbuild. Le binaire est compilé sur Rocky Linux 9 - pour une compatibilité binaire maximale avec toutes les distributions RHEL-based. - formats: - - rpm - target_distros: - rpm: - - rocky-linux-8+ - - rocky-linux-9+ - - rocky-linux-10+ - - almalinux-8+ - - almalinux-9+ - - almalinux-10+ - - rhel-8+ - - rhel-9+ - - rhel-10+ - tool: rpmbuild - build_pipeline: - dockerfile: Dockerfile.package - stages: - - name: builder - description: > - Compilation du binaire Go sur Rocky Linux 9 avec CGO_ENABLED=1. - GOOS=linux GOARCH=amd64 pour un binaire compatible x86_64. - Le binaire est dynamiquement lié à libpcap pour une compatibilité maximale. - - name: rpm_builder - description: > - Image Rocky Linux 9 avec rpm-build. Setup de l'arborescence rpmbuild - (BUILD, RPMS, SOURCES, SPECS, SRPMS). Copie du spec et des sources, - puis build avec rpmbuild -bb pour el8, el9, el10. - - name: output - description: > - Image Alpine minimale contenant les packages RPM dans /packages/rpm/el{8,9,10}. - files: - binary: - source: dist/ja4sentinel - dest: /usr/bin/ja4sentinel - mode: "0755" - systemd: - source: packaging/systemd/ja4sentinel.service - dest: /usr/lib/systemd/system/ja4sentinel.service - mode: "0644" - config: - - source: packaging/systemd/config.yml - dest: /etc/ja4sentinel/config.yml.default - mode: "0640" - config_file: true - - source: packaging/systemd/config.yml - dest: /usr/share/ja4sentinel/config.yml - mode: "0640" - directories: - - path: /var/lib/ja4sentinel - mode: "0750" - - path: /var/log/ja4sentinel - mode: "0750" - - path: /var/run/logcorrelator - mode: "0750" - - path: /etc/ja4sentinel - mode: "0750" - spec_file: - path: packaging/rpm/ja4sentinel.spec - version_macro: "%{?build_version}%{!?build_version:1.0.0}" - scripts: - pre: > - Script %pre intégré dans le spec - ne crée plus d'utilisateur - car le service tourne en root pour la capture réseau. - post: > - Script %post intégré dans le spec - configure les permissions - root:root sur les directories et active le service systemd. - dependencies: - rpm: - - systemd - - libpcap >= 1.9.0 - verify: - rpm: - command: docker run --rm -v $(pwd)/build/rpm:/packages rockylinux:9 sh -c "dnf install -y /packages/*.rpm" - -service: - systemd: - unit_name: "ja4sentinel.service" - description: "JA4 client fingerprinting daemon" - wanted_by: "multi-user.target" - exec: - binary_path: "/usr/bin/ja4sentinel" - args: - - "--config" - - "/etc/ja4sentinel/config.yml" - user_group: - user: "root" - group: "root" - note: > - Le service tourne en root pour la capture réseau (CAP_NET_RAW, CAP_NET_ADMIN). - La création d'utilisateur ja4sentinel a été supprimée. - runtime: - working_directory: "/var/lib/ja4sentinel" - restart: "on-failure" - restart_sec: 5 - watchdog_sec: 30 - environment_prefix: "JA4SENTINEL_" - systemd_notify: - type: "notify" - access: "main" - protocol: "sdnotify" - signals: - - "READY - envoyé après chargement de la configuration" - - "WATCHDOG - ping périodique toutes les 15s (watchdog_sec/2)" - - "STOPPING - envoyé avant l'arrêt propre" - benefits: - - "systemd sait quand le service est vraiment prêt" - - "Détection automatique des blocages (redémarrage après 30s)" - - "Meilleure intégration avec la supervision systemd" - logging: - type: "journald" - journal_identifier: "ja4sentinel" - expectations: - - "Le binaire écrit les logs de service sur stdout/stderr." - - "Les messages doivent inclure au minimum un niveau (INFO/ERROR) et un composant." - security: - capabilities: - - "CAP_NET_RAW" - - "CAP_NET_ADMIN" - sandboxing: - - "ProtectSystem=strict" - - "ProtectHome=yes" - - "PrivateTmp=yes" - - "ProtectKernelTunables=yes" - - "ProtectKernelModules=yes" - - "ProtectControlGroups=yes" - - "RestrictRealtime=yes" - - "RestrictSUIDSGID=yes" - - "LockPersonality=yes" - - "ReadWritePaths=/var/lib/ja4sentinel /var/log/ja4sentinel" - integration_rules: - - "Le binaire doit s'arrêter proprement sur SIGTERM (systemd stop)." - - "Le module cmd_ja4sentinel gère les signaux et termine la capture proprement." - - "Les chemins (config, socket UNIX, logs) doivent être compatibles avec FHS (/etc, /var/run, /var/log)." - - "Le module cmd_ja4sentinel capture SIGTERM/SIGINT et déclenche un arrêt propre (stop capture, flush outputs, fermer socket UNIX)." - - "Le processus doit retourner un code de sortie non nul en cas d'erreur fatale au démarrage." - - "Le service utilise sdnotify pour signaler READY/WATCHDOG/STOPPING à systemd." - -logging: - strategy: - description: > - ja4sentinel écrit ses logs techniques sur stdout/stderr au format JSON lines, - afin que systemd/journald puissent les collecter et les filtrer. - format: "json_lines" - fields: - - "timestamp" - - "level" - - "component" # capture, tlsparse, fingerprint, output, service, ... - - "message" - - "trace_id" # optionnel - - "conn_id" # optionnel (identifiant de flux TCP) - rules: - - "Pas d’écriture directe dans des fichiers de log techniques depuis le code (stdout/stderr uniquement)." - - "Les logs techniques du daemon passent par stdout/stderr (systemd/journald)." - - "Les outputs métiers (LogRecord JA4) sont gérés par le module output, vers socket UNIX et/ou fichier JSON." - - json_format: - description: > - Les LogRecord métiers (JA4 + métadonnées) sont sérialisés en JSON objet plat, - avec des champs nommés explicitement pour ingestion dans ClickHouse. rules: - "Pas de tableaux imbriqués ni d’objets deeply nested." - "Toutes les métadonnées IP/TCP sont flatten sous forme de champs scalaires nommés." - - "Les noms de champs suivent la convention: ip_meta_*, tcp_meta_*, ja4*." + - "Les noms de champs suivent la convention: ip_meta_*, tcp_meta_*, ja*." + - "Pas de champ ja4_hash : le format JA4 intègre déjà son propre hachage tronqué, la chaîne complète de 38 caractères suffit." logrecord_schema: # Exemple de mapping pour api.LogRecord (résumé) + - "conn_id" + - "sensor_id" - "src_ip" - "src_port" - "dst_ip" @@ -757,8 +439,11 @@ logging: - "tcp_meta_mss" - "tcp_meta_window_scale" - "tcp_meta_options" # string joinée, ex: 'MSS,SACK,TS,NOP,WS' + - "tls_version" + - "tls_sni" + - "tls_alpn" + - "syn_to_clienthello_ms" - "ja4" - - "ja4_hash" - "ja3" - "ja3_hash" diff --git a/cmd/ja4sentinel/main.go b/cmd/ja4sentinel/main.go index dbe8686..68ba5e3 100644 --- a/cmd/ja4sentinel/main.go +++ b/cmd/ja4sentinel/main.go @@ -22,7 +22,7 @@ import ( var ( // Version information (set via ldflags) - Version = "1.0.8" + Version = "1.0.9" BuildTime = "unknown" GitCommit = "unknown" ) diff --git a/internal/config/loader_test.go b/internal/config/loader_test.go index fdb4703..05c8aab 100644 --- a/internal/config/loader_test.go +++ b/internal/config/loader_test.go @@ -93,7 +93,7 @@ func TestMergeConfigs(t *testing.T) { PacketBufferSize: 2000, }, Outputs: []api.OutputConfig{ - {Type: "stdout", Enabled: true}, + {Type: "stdout", Enabled: true, AsyncBuffer: 5000}, }, } @@ -117,6 +117,9 @@ func TestMergeConfigs(t *testing.T) { if result.Core.PacketBufferSize != 2000 { t.Errorf("PacketBufferSize = %v, want 2000", result.Core.PacketBufferSize) } + if result.Outputs[0].AsyncBuffer != 5000 { + t.Errorf("Outputs[0].AsyncBuffer = %v, want 5000", result.Outputs[0].AsyncBuffer) + } } func TestValidate(t *testing.T) { @@ -345,6 +348,20 @@ func TestValidate_InvalidOutputs(t *testing.T) { }, wantErr: false, }, + { + name: "output with AsyncBuffer zero (default)", + outputs: []api.OutputConfig{ + {Type: "stdout", Enabled: true, AsyncBuffer: 0}, + }, + wantErr: false, + }, + { + name: "output with custom AsyncBuffer", + outputs: []api.OutputConfig{ + {Type: "unix_socket", Enabled: true, AsyncBuffer: 5000, Params: map[string]string{"socket_path": "/tmp/x.sock"}}, + }, + wantErr: false, + }, } for _, tt := range tests { diff --git a/internal/fingerprint/engine.go b/internal/fingerprint/engine.go index 0330b0b..5baf55a 100644 --- a/internal/fingerprint/engine.go +++ b/internal/fingerprint/engine.go @@ -18,6 +18,8 @@ func NewEngine() *EngineImpl { } // FromClientHello generates JA4 (and optionally JA3) fingerprints from a TLS ClientHello +// Note: JA4Hash is populated for internal use but should NOT be serialized to LogRecord +// as the JA4 format already includes its own hash portions (per architecture.yml) func (e *EngineImpl) FromClientHello(ch api.TLSClientHello) (*api.Fingerprints, error) { if len(ch.Payload) == 0 { return nil, fmt.Errorf("empty ClientHello payload") @@ -40,11 +42,12 @@ func (e *EngineImpl) FromClientHello(ch api.TLSClientHello) (*api.Fingerprints, // Extract JA4 hash portion (last segment after underscore) // JA4 format: __ + // This is kept for internal use but NOT serialized to LogRecord ja4Hash := extractJA4Hash(ja4) return &api.Fingerprints{ JA4: ja4, - JA4Hash: ja4Hash, + JA4Hash: ja4Hash, // Internal use only - not serialized to LogRecord JA3: ja3, JA3Hash: ja3Hash, }, nil diff --git a/internal/fingerprint/engine_test.go b/internal/fingerprint/engine_test.go index af3e84f..7ac9c0b 100644 --- a/internal/fingerprint/engine_test.go +++ b/internal/fingerprint/engine_test.go @@ -45,3 +45,91 @@ func TestNewEngine(t *testing.T) { t.Error("NewEngine() returned nil") } } + +func TestFromClientHello_ValidPayload(t *testing.T) { + // Use a minimal valid TLS 1.2 ClientHello with extensions + // Build a proper ClientHello using the same structure as parser tests + clientHello := buildMinimalClientHelloForTest() + + ch := api.TLSClientHello{ + SrcIP: "192.168.1.100", + SrcPort: 54321, + DstIP: "10.0.0.1", + DstPort: 443, + Payload: clientHello, + } + + engine := NewEngine() + fp, err := engine.FromClientHello(ch) + + if err != nil { + t.Fatalf("FromClientHello() error = %v", err) + } + if fp == nil { + t.Fatal("FromClientHello() returned nil") + } + + // Verify JA4 is populated (format: t13d... or t12d...) + if fp.JA4 == "" { + t.Error("JA4 should not be empty") + } + + // JA4Hash is populated for internal use (but not serialized to LogRecord) + // It contains the hash portions of the JA4 string + if fp.JA4Hash == "" { + t.Error("JA4Hash should be populated for internal use") + } +} + +// buildMinimalClientHelloForTest creates a minimal valid TLS 1.2 ClientHello +func buildMinimalClientHelloForTest() []byte { + // Cipher suites (minimal set) + cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f} + // Compression methods (null only) + compressionMethods := []byte{0x01, 0x00} + // No extensions + extensions := []byte{} + extLen := len(extensions) + + // Build ClientHello handshake body + handshakeBody := []byte{ + 0x03, 0x03, // Version: TLS 1.2 + // Random (32 bytes) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, // Session ID length: 0 + } + + // Add cipher suites (with length prefix) + cipherSuiteLen := len(cipherSuites) + handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen)) + handshakeBody = append(handshakeBody, cipherSuites...) + + // Add compression methods (with length prefix) + handshakeBody = append(handshakeBody, compressionMethods...) + + // Add extensions (with length prefix) + handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen)) + handshakeBody = append(handshakeBody, extensions...) + + // Now build full handshake with type and length + handshakeLen := len(handshakeBody) + handshake := append([]byte{ + 0x01, // Handshake type: ClientHello + byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen), // Handshake length + }, handshakeBody...) + + // Build TLS record + recordLen := len(handshake) + record := make([]byte, 5+recordLen) + record[0] = 0x16 // Handshake + record[1] = 0x03 // Version: TLS 1.2 + record[2] = 0x03 + record[3] = byte(recordLen >> 8) + record[4] = byte(recordLen) + copy(record[5:], handshake) + + return record +} diff --git a/internal/output/writers.go b/internal/output/writers.go index 9faecf1..625173f 100644 --- a/internal/output/writers.go +++ b/internal/output/writers.go @@ -504,6 +504,7 @@ func NewBuilder() *BuilderImpl { } // NewFromConfig constructs writers from AppConfig +// Uses AsyncBuffer from OutputConfig if specified, otherwise uses DefaultQueueSize func (b *BuilderImpl) NewFromConfig(cfg api.AppConfig) (api.Writer, error) { multiWriter := NewMultiWriter() @@ -515,6 +516,12 @@ func (b *BuilderImpl) NewFromConfig(cfg api.AppConfig) (api.Writer, error) { var writer api.Writer var err error + // Determine queue size: use AsyncBuffer if specified, otherwise default + queueSize := DefaultQueueSize + if outputCfg.AsyncBuffer > 0 { + queueSize = outputCfg.AsyncBuffer + } + switch outputCfg.Type { case "stdout": writer = NewStdoutWriter() @@ -537,7 +544,7 @@ func (b *BuilderImpl) NewFromConfig(cfg api.AppConfig) (api.Writer, error) { if logLevel == "" { logLevel = "error" } - writer, err = NewUnixSocketWriterWithConfigAndLogLevel(socketPath, DefaultDialTimeout, DefaultWriteTimeout, DefaultQueueSize, logLevel) + writer, err = NewUnixSocketWriterWithConfigAndLogLevel(socketPath, DefaultDialTimeout, DefaultWriteTimeout, queueSize, logLevel) if err != nil { return nil, err } diff --git a/internal/output/writers_test.go b/internal/output/writers_test.go index 3c25c32..27b0380 100644 --- a/internal/output/writers_test.go +++ b/internal/output/writers_test.go @@ -258,6 +258,24 @@ func TestBuilder_NewFromConfig(t *testing.T) { }, wantErr: true, }, + { + name: "unix socket with custom AsyncBuffer", + config: api.AppConfig{ + Core: api.Config{ + Interface: "eth0", + ListenPorts: []uint16{443}, + }, + Outputs: []api.OutputConfig{ + { + Type: "unix_socket", + Enabled: true, + AsyncBuffer: 5000, + Params: map[string]string{"socket_path": "test.sock"}, + }, + }, + }, + wantErr: false, + }, } for _, tt := range tests { @@ -409,11 +427,17 @@ func TestLogRecordJSONSerialization(t *testing.T) { IPDF: true, TCPWindow: 65535, TCPOptions: "MSS,WS,SACK,TS", - JA4: "t13d1516h2_8daaf6152771_02cb136f2775", - JA4Hash: "8daaf6152771_02cb136f2775", - JA3: "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0", - JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d", - Timestamp: time.Now().UnixNano(), + // New fields per architecture.yml + ConnID: "flow-abc123", + SensorID: "sensor-01", + TLSVersion: "1.3", + SNI: "example.com", + ALPN: "h2", + // Fingerprints - note: JA4Hash is NOT in LogRecord per architecture + JA4: "t13d1516h2_8daaf6152771_02cb136f2775", + JA3: "771,4865-4866-4867,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0", + JA3Hash: "a0e6f06c7a6d15e5e3f0f0e6f06c7a6d", + Timestamp: time.Now().UnixNano(), } data, err := json.Marshal(rec) @@ -434,6 +458,15 @@ func TestLogRecordJSONSerialization(t *testing.T) { if got.JA4 != rec.JA4 { t.Errorf("JA4 = %v, want %v", got.JA4, rec.JA4) } + // Verify JA4Hash is NOT present (architecture decision) + // JA4Hash field doesn't exist in LogRecord anymore + // Verify new fields + if got.ConnID != rec.ConnID { + t.Errorf("ConnID = %v, want %v", got.ConnID, rec.ConnID) + } + if got.SNI != rec.SNI { + t.Errorf("SNI = %v, want %v", got.SNI, rec.SNI) + } } // Test to verify optional fields are omitted when empty diff --git a/internal/tlsparse/parser.go b/internal/tlsparse/parser.go index 9c815c7..7a2313e 100644 --- a/internal/tlsparse/parser.go +++ b/internal/tlsparse/parser.go @@ -12,6 +12,7 @@ import ( "github.com/google/gopacket" "github.com/google/gopacket/layers" + tlsfingerprint "github.com/psanford/tlsfingerprint" ) // ConnectionState represents the state of a TCP connection for TLS parsing @@ -220,15 +221,31 @@ func (p *ParserImpl) Process(pkt api.RawPacket) (*api.TLSClientHello, error) { flow.State = JA4_DONE flow.HelloBuffer = clientHello - return &api.TLSClientHello{ - SrcIP: srcIP, - SrcPort: srcPort, - DstIP: dstIP, - DstPort: dstPort, - Payload: clientHello, - IPMeta: ipMeta, - TCPMeta: tcpMeta, - }, nil + // Extract TLS extensions (SNI, ALPN, TLS version) + extInfo, _ := extractTLSExtensions(clientHello) + + // Generate ConnID from flow key + connID := key + + ch := &api.TLSClientHello{ + SrcIP: srcIP, + SrcPort: srcPort, + DstIP: dstIP, + DstPort: dstPort, + Payload: clientHello, + IPMeta: ipMeta, + TCPMeta: tcpMeta, + ConnID: connID, + SNI: extInfo.SNI, + ALPN: joinStringSlice(extInfo.ALPN, ","), + TLSVersion: extInfo.TLSVersion, + } + + // Calculate SynToCHMs if we have timing info + synToCH := uint32(time.Since(flow.CreatedAt).Milliseconds()) + ch.SynToCHMs = &synToCH + + return ch, nil } // Check for fragmented ClientHello (accumulate segments) @@ -257,15 +274,31 @@ func (p *ParserImpl) Process(pkt api.RawPacket) (*api.TLSClientHello, error) { // Complete ClientHello found flow.State = JA4_DONE - return &api.TLSClientHello{ - SrcIP: srcIP, - SrcPort: srcPort, - DstIP: dstIP, - DstPort: dstPort, - Payload: clientHello, - IPMeta: ipMeta, - TCPMeta: tcpMeta, - }, nil + // Extract TLS extensions (SNI, ALPN, TLS version) + extInfo, _ := extractTLSExtensions(clientHello) + + // Generate ConnID from flow key + connID := key + + ch := &api.TLSClientHello{ + SrcIP: srcIP, + SrcPort: srcPort, + DstIP: dstIP, + DstPort: dstPort, + Payload: clientHello, + IPMeta: ipMeta, + TCPMeta: tcpMeta, + ConnID: connID, + SNI: extInfo.SNI, + ALPN: joinStringSlice(extInfo.ALPN, ","), + TLSVersion: extInfo.TLSVersion, + } + + // Calculate SynToCHMs + synToCH := uint32(time.Since(flow.CreatedAt).Milliseconds()) + ch.SynToCHMs = &synToCH + + return ch, nil } } @@ -377,6 +410,13 @@ func extractTCPMeta(tcp *layers.TCP) api.TCPMeta { return meta } +// TLSExtensionInfo contains parsed TLS extension information +type TLSExtensionInfo struct { + SNI string + ALPN []string + TLSVersion string +} + // parseClientHello checks if the payload contains a TLS ClientHello and returns it func parseClientHello(payload []byte) ([]byte, error) { if len(payload) < 5 { @@ -419,6 +459,171 @@ func parseClientHello(payload []byte) ([]byte, error) { return payload[:5+recordLength], nil } +// extractTLSExtensions extracts SNI, ALPN, and TLS version from a ClientHello payload +// Uses tlsfingerprint library for ALPN and TLS version, manual parsing for SNI value +func extractTLSExtensions(payload []byte) (*TLSExtensionInfo, error) { + if len(payload) < 5 { + return nil, nil + } + + // TLS record layer + contentType := payload[0] + if contentType != 22 { + return nil, nil // Not a handshake + } + + version := binary.BigEndian.Uint16(payload[1:3]) + recordLength := int(binary.BigEndian.Uint16(payload[3:5])) + + if len(payload) < 5+recordLength { + return nil, nil // Incomplete record + } + + handshakePayload := payload[5 : 5+recordLength] + if len(handshakePayload) < 1 { + return nil, nil + } + + handshakeType := handshakePayload[0] + if handshakeType != 1 { + return nil, nil // Not a ClientHello + } + + info := &TLSExtensionInfo{} + + // Use tlsfingerprint to parse ALPN and TLS version + fp, err := tlsfingerprint.ParseClientHello(payload) + if err == nil && fp != nil { + // Extract ALPN protocols + if len(fp.ALPNProtocols) > 0 { + info.ALPN = fp.ALPNProtocols + } + // Extract TLS version + info.TLSVersion = tlsVersionToString(fp.Version) + } + + // If tlsfingerprint didn't provide version, fall back to record version + if info.TLSVersion == "" { + info.TLSVersion = tlsVersionToString(version) + } + + // Parse SNI manually (tlsfingerprint only provides HasSNI, not the value) + sniValue := extractSNIFromPayload(handshakePayload) + if sniValue != "" { + info.SNI = sniValue + } + + return info, nil +} + +// extractSNIFromPayload extracts the SNI value from a ClientHello handshake payload +// handshakePayload starts at the handshake type byte (0x01 for ClientHello) +func extractSNIFromPayload(handshakePayload []byte) string { + // handshakePayload structure: + // [0]: Handshake type (0x01 for ClientHello) + // [1:4]: Handshake length (3 bytes, big-endian) + // [4:6]: Version (2 bytes) + // [6:38]: Random (32 bytes) + // [38]: Session ID length + // ... + + if len(handshakePayload) < 40 { // type(1) + len(3) + version(2) + random(32) + sessionIDLen(1) + return "" + } + + // Start after type (1) + length (3) + version (2) + random (32) = 38 + offset := 38 + + // Session ID length (1 byte) + sessionIDLen := int(handshakePayload[offset]) + offset++ + + // Skip session ID + offset += sessionIDLen + + // Cipher suites length (2 bytes) + if offset+2 > len(handshakePayload) { + return "" + } + cipherSuiteLen := int(binary.BigEndian.Uint16(handshakePayload[offset : offset+2])) + offset += 2 + cipherSuiteLen + + // Compression methods length (1 byte) + if offset >= len(handshakePayload) { + return "" + } + compressionLen := int(handshakePayload[offset]) + offset++ + + // Skip compression methods + offset += compressionLen + + // Extensions length (2 bytes) - optional + if offset+2 > len(handshakePayload) { + return "" + } + extensionsLen := int(binary.BigEndian.Uint16(handshakePayload[offset : offset+2])) + offset += 2 + + if extensionsLen == 0 || offset+extensionsLen > len(handshakePayload) { + return "" + } + + extensionsEnd := offset + extensionsLen + + // Debug: log extension types found + _ = extensionsEnd // suppress unused warning in case we remove debug code + + // Parse extensions to find SNI (type 0) + for offset < extensionsEnd { + if offset+4 > len(handshakePayload) { + break + } + + extType := binary.BigEndian.Uint16(handshakePayload[offset : offset+2]) + extLen := int(binary.BigEndian.Uint16(handshakePayload[offset+2 : offset+4])) + offset += 4 + + if offset+extLen > len(handshakePayload) { + break + } + + extData := handshakePayload[offset : offset+extLen] + offset += extLen + + if extType == 0 && len(extData) >= 5 { // SNI extension + // SNI extension structure: + // - name_list_len (2 bytes) + // - name_type (1 byte) + // - name_len (2 bytes) + // - name (variable) + // Skip name_list_len (2), read name_type (1) + name_len (2) + nameLen := int(binary.BigEndian.Uint16(extData[3:5])) + if len(extData) >= 5+nameLen { + return string(extData[5 : 5+nameLen]) + } + } + } + + return "" +} + +// tlsVersionToString converts a TLS version number to a string +func tlsVersionToString(version uint16) string { + switch version { + case 0x0301: + return "1.0" + case 0x0302: + return "1.1" + case 0x0303: + return "1.2" + case 0x0304: + return "1.3" + default: + return "" + } +} + // IsClientHello checks if a payload contains a TLS ClientHello func IsClientHello(payload []byte) bool { if len(payload) < 6 { diff --git a/internal/tlsparse/parser_test.go b/internal/tlsparse/parser_test.go index 08d2828..e03017d 100644 --- a/internal/tlsparse/parser_test.go +++ b/internal/tlsparse/parser_test.go @@ -7,6 +7,7 @@ import ( "ja4sentinel/api" + tlsfingerprint "github.com/psanford/tlsfingerprint" "github.com/google/gopacket" "github.com/google/gopacket/layers" ) @@ -444,3 +445,272 @@ func buildRawPacket(t *testing.T, srcIP, dstIP string, srcPort, dstPort uint16, Timestamp: time.Now().UnixNano(), } } + +func TestTLSVersionToString(t *testing.T) { + tests := []struct { + version uint16 + want string + }{ + {0x0301, "1.0"}, + {0x0302, "1.1"}, + {0x0303, "1.2"}, + {0x0304, "1.3"}, + {0x0300, ""}, + {0x0305, ""}, + } + + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + got := tlsVersionToString(tt.version) + if got != tt.want { + t.Errorf("tlsVersionToString(%#x) = %v, want %v", tt.version, got, tt.want) + } + }) + } +} + +func TestExtractTLSExtensions(t *testing.T) { + tests := []struct { + name string + payload []byte + wantSNI string + wantALPN []string + wantVersion string + wantNil bool + }{ + { + name: "empty payload", + payload: []byte{}, + wantNil: true, + }, + { + name: "too short", + payload: []byte{0x16, 0x03, 0x03}, + wantNil: true, + }, + { + name: "TLS 1.2 ClientHello without extensions", + payload: createTLSClientHello(0x0303), + wantVersion: "1.2", + wantNil: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := extractTLSExtensions(tt.payload) + if err != nil { + t.Errorf("extractTLSExtensions() unexpected error = %v", err) + return + } + if (got == nil) != tt.wantNil { + t.Errorf("extractTLSExtensions() = %v, wantNil %v", got == nil, tt.wantNil) + return + } + if got != nil { + if got.TLSVersion != tt.wantVersion { + t.Errorf("TLSVersion = %v, want %v", got.TLSVersion, tt.wantVersion) + } + } + }) + } +} + +func TestParser_ExtractsTLSFields(t *testing.T) { + parser := NewParser() + defer parser.Close() + + // Create a minimal valid TLS 1.2 ClientHello with SNI and ALPN extensions + // This is a real-world-like ClientHello structure + clientHelloWithExt := createMinimalTLSClientHelloWithSNIAndALPN("example.com", []string{"h2", "http/1.1"}) + + // Debug: Check what extractTLSExtensions returns + extInfo, err := extractTLSExtensions(clientHelloWithExt) + if err != nil { + t.Logf("extractTLSExtensions error: %v", err) + } + if extInfo != nil { + t.Logf("extInfo: SNI=%q, ALPN=%v, Version=%q", extInfo.SNI, extInfo.ALPN, extInfo.TLSVersion) + } else { + t.Log("extInfo is nil") + } + + // Also test with tlsfingerprint directly + fp, err := tlsfingerprint.ParseClientHello(clientHelloWithExt) + if err != nil { + t.Logf("tlsfingerprint error: %v", err) + } else { + t.Logf("tlsfingerprint: ALPN=%v, Version=%#x, HasSNI=%v", fp.ALPNProtocols, fp.Version, fp.HasSNI) + } + + // Debug: print first bytes of ClientHello + t.Logf("ClientHello hex: % x", clientHelloWithExt[:min(50, len(clientHelloWithExt))]) + + srcIP := "192.168.1.100" + dstIP := "10.0.0.1" + srcPort := uint16(54321) + dstPort := uint16(443) + + pkt := buildRawPacket(t, srcIP, dstIP, srcPort, dstPort, clientHelloWithExt) + + result, err := parser.Process(pkt) + if err != nil { + t.Fatalf("Process() error = %v", err) + } + if result == nil { + t.Fatal("Process() should return TLSClientHello") + } + + // Verify new fields are populated + if result.SNI != "example.com" { + t.Errorf("SNI = %v, want example.com", result.SNI) + } + if result.ALPN == "" { + t.Error("ALPN should not be empty") + } + if result.TLSVersion != "1.2" { + t.Errorf("TLSVersion = %v, want 1.2", result.TLSVersion) + } + if result.ConnID == "" { + t.Error("ConnID should not be empty") + } + if result.SynToCHMs == nil { + t.Error("SynToCHMs should not be nil") + } +} + +// createMinimalTLSClientHelloWithSNIAndALPN creates a minimal but valid TLS 1.2 ClientHello +// with SNI and ALPN extensions +func createMinimalTLSClientHelloWithSNIAndALPN(sni string, alpnProtocols []string) []byte { + // Build SNI extension + sniExt := buildSNIExtension(sni) + + // Build ALPN extension + alpnExt := buildALPNExtension(alpnProtocols) + + // Build supported_versions extension (TLS 1.2) + supportedVersionsExt := []byte{0x00, 0x2b, 0x00, 0x03, 0x02, 0x03, 0x03} // type=43, len=3, TLS 1.2 + + // Combine extensions + extensions := append(sniExt, alpnExt...) + extensions = append(extensions, supportedVersionsExt...) + extLen := len(extensions) + + // Cipher suites (minimal set) + cipherSuites := []byte{0x00, 0x04, 0x13, 0x01, 0x13, 0x02, 0xc0, 0x2f} + // 4 cipher suites: TLS_AES_128_GCM_SHA256, TLS_AES_256_GCM_SHA384, TLS_CHACHA20_POLY1305_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 + + // Compression methods (null only) + compressionMethods := []byte{0x01, 0x00} + + // Build ClientHello handshake (without length header first) + handshakeBody := []byte{ + 0x03, 0x03, // Version: TLS 1.2 + // Random (32 bytes) + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, // Session ID length: 0 + } + + // Add cipher suites (with length prefix) + cipherSuiteLen := len(cipherSuites) + handshakeBody = append(handshakeBody, byte(cipherSuiteLen>>8), byte(cipherSuiteLen)) + handshakeBody = append(handshakeBody, cipherSuites...) + + // Add compression methods (with length prefix) + handshakeBody = append(handshakeBody, compressionMethods...) + + // Add extensions (with length prefix) + handshakeBody = append(handshakeBody, byte(extLen>>8), byte(extLen)) + handshakeBody = append(handshakeBody, extensions...) + + // Now build full handshake with type and length + handshakeLen := len(handshakeBody) + handshake := append([]byte{ + 0x01, // Handshake type: ClientHello + byte(handshakeLen >> 16), byte(handshakeLen >> 8), byte(handshakeLen), // Handshake length + }, handshakeBody...) + + // Build TLS record + recordLen := len(handshake) + record := make([]byte, 5+recordLen) + record[0] = 0x16 // Handshake + record[1] = 0x03 // Version: TLS 1.2 + record[2] = 0x03 + record[3] = byte(recordLen >> 8) + record[4] = byte(recordLen) + copy(record[5:], handshake) + + return record +} + +// buildSNIExtension builds a Server Name Indication extension +func buildSNIExtension(sni string) []byte { + nameLen := len(sni) + // SNI extension structure: + // - name_list_len (2 bytes): total length of all names + // - name_type (1 byte): always 0x00 for host_name + // - name_len (2 bytes): length of this name + // - name (variable): the actual hostname + nameListLen := 1 + 2 + nameLen // name_type + name_len + name + + // Extension data = name_list_len (2) + name_type (1) + name_len (2) + name + extDataLen := 2 + nameListLen + // Full extension = type (2) + length (2) + data (variable) + ext := make([]byte, 4+extDataLen) + ext[0] = 0x00 // Extension type: SNI (0) + ext[1] = 0x00 + ext[2] = byte(extDataLen >> 8) + ext[3] = byte(extDataLen) + ext[4] = byte(nameListLen >> 8) // name_list_len (high byte) + ext[5] = byte(nameListLen) // name_list_len (low byte) + ext[6] = 0x00 // name_type: host_name (0) + ext[7] = byte(nameLen >> 8) // name_len (high byte) + ext[8] = byte(nameLen) // name_len (low byte) + copy(ext[9:], sni) + + return ext +} + +// buildALPNExtension builds an Application-Layer Protocol Negotiation extension +func buildALPNExtension(protocols []string) []byte { + // Calculate ALPN data length + // ALPN data = alpn_list_len (2) + for each protocol: length (1) + data (variable) + alpnDataLen := 0 + for _, proto := range protocols { + alpnDataLen += 1 + len(proto) // length byte + protocol string + } + + // Extension data = alpn_list_len (2) + protocols + extDataLen := 2 + alpnDataLen + // Full extension = type (2) + length (2) + data (variable) + ext := make([]byte, 4+extDataLen) + ext[0] = 0x00 // Extension type: ALPN (16 = 0x10) + ext[1] = 0x10 + ext[2] = byte(extDataLen >> 8) + ext[3] = byte(extDataLen) + + // ALPN protocol list length (2 bytes) + ext[4] = byte(alpnDataLen >> 8) + ext[5] = byte(alpnDataLen) + + offset := 6 + for _, proto := range protocols { + ext[offset] = byte(len(proto)) + offset++ + copy(ext[offset:], proto) + offset += len(proto) + } + + return ext +} + +// min returns the minimum of two integers +func min(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/packaging/rpm/ja4sentinel.spec b/packaging/rpm/ja4sentinel.spec index 06e74ed..7de19cb 100644 --- a/packaging/rpm/ja4sentinel.spec +++ b/packaging/rpm/ja4sentinel.spec @@ -3,7 +3,7 @@ %if %{defined build_version} %define spec_version %{build_version} %else -%define spec_version 1.0.8 +%define spec_version 1.0.9 %endif Name: ja4sentinel @@ -117,6 +117,19 @@ fi %dir /var/run/logcorrelator %changelog +* Mon Mar 02 2026 Jacquin Antoine - 1.0.9-1 +- Add SNI (Server Name Indication) extraction from TLS ClientHello +- Add ALPN (Application-Layer Protocol Negotiation) extraction +- Add TLS version detection from ClientHello +- Add ConnID field for flow correlation +- Add SensorID field for multi-sensor deployments +- Add SynToCHMs timing field for behavioral detection +- Add AsyncBuffer configuration for output queue sizing +- Remove JA4Hash from LogRecord (JA4 format includes its own hash) +- Use tlsfingerprint library for ALPN and TLS version parsing +- Update architecture.yml compliance for all new fields +- Add unit tests for TLS extension extraction + * Sun Mar 01 2026 Jacquin Antoine - 1.0.8-1 - Add configurable log level (debug, info, warn, error) via config.yml - Add JA4SENTINEL_LOG_LEVEL environment variable support