feat: implémentation complète du pipeline JA4 + Docker + tests
Nouveaux modules: - cmd/ja4sentinel/main.go : point d'entrée avec pipeline capture→parse→fingerprint→output - internal/config/loader.go : chargement YAML + env (JA4SENTINEL_*) + validation - internal/tlsparse/parser.go : extraction ClientHello avec suivi d'état de flux (NEW/WAIT_CLIENT_HELLO/JA4_DONE) - internal/fingerprint/engine.go : génération JA4/JA3 via psanford/tlsfingerprint - internal/output/writers.go : StdoutWriter, FileWriter, UnixSocketWriter, MultiWriter Infrastructure: - Dockerfile (multi-stage), Dockerfile.dev, Dockerfile.test-server - Makefile (build, test, lint, docker-build-*) - docker-compose.test.yml pour tests d'intégration - README.md (276 lignes) avec architecture, config, exemples API (api/types.go): - Ajout Close() aux interfaces Capture et Parser - Ajout FlowTimeoutSec dans Config (défaut: 30s, env: JA4SENTINEL_FLOW_TIMEOUT) - ServiceLog: +Timestamp, +TraceID, +ConnID - LogRecord: champs flatten (ip_meta_*, tcp_meta_*, ja4*) - Helper NewLogRecord() pour conversion TLSClientHello+Fingerprints→LogRecord Architecture (architecture.yml): - Documentation module logging + interfaces LoggerFactory/Logger - Section service.systemd complète (unit, security, capabilities) - Section logging.strategy (JSON lines, champs, règles) - api.Config: +FlowTimeoutSec documenté Fixes/cleanup: - Suppression internal/api/types.go (consolidé dans api/types.go) - Correction imports logging (ja4sentinel/api) - .dockerignore / .gitignore - config.yml.example Tests: - Tous les modules ont leurs tests (*_test.go) - Tests unitaires : capture, config, fingerprint, output, tlsparse - Tests d'intégration via docker-compose.test.yml Build: - Binaires dans dist/ (make build → dist/ja4sentinel) - Docker runtime avec COPY --from=builder /app/dist/ Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
250
internal/output/writers.go
Normal file
250
internal/output/writers.go
Normal file
@ -0,0 +1,250 @@
|
||||
// Package output provides writers for ja4sentinel log records
|
||||
package output
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"sync"
|
||||
|
||||
"ja4sentinel/api"
|
||||
)
|
||||
|
||||
// StdoutWriter writes log records to stdout
|
||||
type StdoutWriter struct {
|
||||
encoder *json.Encoder
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// NewStdoutWriter creates a new stdout writer
|
||||
func NewStdoutWriter() *StdoutWriter {
|
||||
return &StdoutWriter{
|
||||
encoder: json.NewEncoder(os.Stdout),
|
||||
}
|
||||
}
|
||||
|
||||
// Write writes a log record to stdout
|
||||
func (w *StdoutWriter) Write(rec api.LogRecord) error {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
return w.encoder.Encode(rec)
|
||||
}
|
||||
|
||||
// Close closes the writer (no-op for stdout)
|
||||
func (w *StdoutWriter) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// FileWriter writes log records to a file
|
||||
type FileWriter struct {
|
||||
file *os.File
|
||||
encoder *json.Encoder
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// NewFileWriter creates a new file writer
|
||||
func NewFileWriter(path string) (*FileWriter, error) {
|
||||
file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open file %s: %w", path, err)
|
||||
}
|
||||
|
||||
return &FileWriter{
|
||||
file: file,
|
||||
encoder: json.NewEncoder(file),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Write writes a log record to the file
|
||||
func (w *FileWriter) Write(rec api.LogRecord) error {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
return w.encoder.Encode(rec)
|
||||
}
|
||||
|
||||
// Close closes the file
|
||||
func (w *FileWriter) Close() error {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
if w.file != nil {
|
||||
return w.file.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// UnixSocketWriter writes log records to a UNIX socket
|
||||
type UnixSocketWriter struct {
|
||||
socketPath string
|
||||
conn net.Conn
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// NewUnixSocketWriter creates a new UNIX socket writer
|
||||
func NewUnixSocketWriter(socketPath string) (*UnixSocketWriter, error) {
|
||||
w := &UnixSocketWriter{
|
||||
socketPath: socketPath,
|
||||
}
|
||||
|
||||
// Try to connect (socket may not exist yet)
|
||||
conn, err := net.Dial("unix", socketPath)
|
||||
if err != nil {
|
||||
// Socket doesn't exist yet, we'll try to connect on first write
|
||||
return w, nil
|
||||
}
|
||||
|
||||
w.conn = conn
|
||||
return w, nil
|
||||
}
|
||||
|
||||
// Write writes a log record to the UNIX socket
|
||||
func (w *UnixSocketWriter) Write(rec api.LogRecord) error {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
|
||||
// Connect if not already connected
|
||||
if w.conn == nil {
|
||||
conn, err := net.Dial("unix", w.socketPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to socket %s: %w", w.socketPath, err)
|
||||
}
|
||||
w.conn = conn
|
||||
}
|
||||
|
||||
data, err := json.Marshal(rec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to marshal record: %w", err)
|
||||
}
|
||||
|
||||
// Add newline for line-based protocols
|
||||
data = append(data, '\n')
|
||||
|
||||
_, err = w.conn.Write(data)
|
||||
if err != nil {
|
||||
// Connection failed, try to reconnect
|
||||
w.conn.Close()
|
||||
w.conn = nil
|
||||
return fmt.Errorf("failed to write to socket: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the UNIX socket connection
|
||||
func (w *UnixSocketWriter) Close() error {
|
||||
w.mutex.Lock()
|
||||
defer w.mutex.Unlock()
|
||||
if w.conn != nil {
|
||||
return w.conn.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MultiWriter combines multiple writers
|
||||
type MultiWriter struct {
|
||||
writers []api.Writer
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
// NewMultiWriter creates a new multi-writer
|
||||
func NewMultiWriter() *MultiWriter {
|
||||
return &MultiWriter{
|
||||
writers: make([]api.Writer, 0),
|
||||
}
|
||||
}
|
||||
|
||||
// Write writes a log record to all writers
|
||||
func (mw *MultiWriter) Write(rec api.LogRecord) error {
|
||||
mw.mutex.Lock()
|
||||
defer mw.mutex.Unlock()
|
||||
|
||||
var lastErr error
|
||||
for _, w := range mw.writers {
|
||||
if err := w.Write(rec); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Add adds a writer to the multi-writer
|
||||
func (mw *MultiWriter) Add(writer api.Writer) {
|
||||
mw.mutex.Lock()
|
||||
defer mw.mutex.Unlock()
|
||||
mw.writers = append(mw.writers, writer)
|
||||
}
|
||||
|
||||
// CloseAll closes all writers
|
||||
func (mw *MultiWriter) CloseAll() error {
|
||||
mw.mutex.Lock()
|
||||
defer mw.mutex.Unlock()
|
||||
|
||||
var lastErr error
|
||||
for _, w := range mw.writers {
|
||||
if closer, ok := w.(io.Closer); ok {
|
||||
if err := closer.Close(); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// BuilderImpl implements the api.Builder interface
|
||||
type BuilderImpl struct{}
|
||||
|
||||
// NewBuilder creates a new output builder
|
||||
func NewBuilder() *BuilderImpl {
|
||||
return &BuilderImpl{}
|
||||
}
|
||||
|
||||
// NewFromConfig constructs writers from AppConfig
|
||||
func (b *BuilderImpl) NewFromConfig(cfg api.AppConfig) (api.Writer, error) {
|
||||
multiWriter := NewMultiWriter()
|
||||
|
||||
for _, outputCfg := range cfg.Outputs {
|
||||
if !outputCfg.Enabled {
|
||||
continue
|
||||
}
|
||||
|
||||
var writer api.Writer
|
||||
var err error
|
||||
|
||||
switch outputCfg.Type {
|
||||
case "stdout":
|
||||
writer = NewStdoutWriter()
|
||||
case "file":
|
||||
path := outputCfg.Params["path"]
|
||||
if path == "" {
|
||||
return nil, fmt.Errorf("file output requires 'path' parameter")
|
||||
}
|
||||
writer, err = NewFileWriter(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
case "unix_socket":
|
||||
socketPath := outputCfg.Params["socket_path"]
|
||||
if socketPath == "" {
|
||||
return nil, fmt.Errorf("unix_socket output requires 'socket_path' parameter")
|
||||
}
|
||||
writer, err = NewUnixSocketWriter(socketPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown output type: %s", outputCfg.Type)
|
||||
}
|
||||
|
||||
multiWriter.Add(writer)
|
||||
}
|
||||
|
||||
// If no outputs configured, default to stdout
|
||||
if len(multiWriter.writers) == 0 {
|
||||
multiWriter.Add(NewStdoutWriter())
|
||||
}
|
||||
|
||||
return multiWriter, nil
|
||||
}
|
||||
235
internal/output/writers_test.go
Normal file
235
internal/output/writers_test.go
Normal file
@ -0,0 +1,235 @@
|
||||
package output
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"ja4sentinel/api"
|
||||
)
|
||||
|
||||
func TestStdoutWriter(t *testing.T) {
|
||||
// Capture stdout by replacing it temporarily
|
||||
oldStdout := os.Stdout
|
||||
r, w, _ := os.Pipe()
|
||||
os.Stdout = w
|
||||
|
||||
writer := NewStdoutWriter()
|
||||
rec := api.LogRecord{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 12345,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
JA4: "t12s0102ab_1234567890ab",
|
||||
}
|
||||
|
||||
err := writer.Write(rec)
|
||||
if err != nil {
|
||||
t.Errorf("Write() error = %v", err)
|
||||
}
|
||||
|
||||
w.Close()
|
||||
os.Stdout = oldStdout
|
||||
|
||||
var buf bytes.Buffer
|
||||
buf.ReadFrom(r)
|
||||
output := buf.String()
|
||||
|
||||
if output == "" {
|
||||
t.Error("Write() produced no output")
|
||||
}
|
||||
|
||||
// Verify it's valid JSON
|
||||
var result api.LogRecord
|
||||
if err := json.Unmarshal([]byte(output), &result); err != nil {
|
||||
t.Errorf("Output is not valid JSON: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFileWriter(t *testing.T) {
|
||||
// Create a temporary file
|
||||
tmpFile := "/tmp/ja4sentinel_test.log"
|
||||
defer os.Remove(tmpFile)
|
||||
|
||||
writer, err := NewFileWriter(tmpFile)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileWriter() error = %v", err)
|
||||
}
|
||||
defer writer.Close()
|
||||
|
||||
rec := api.LogRecord{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 12345,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
JA4: "t12s0102ab_1234567890ab",
|
||||
}
|
||||
|
||||
err = writer.Write(rec)
|
||||
if err != nil {
|
||||
t.Errorf("Write() error = %v", err)
|
||||
}
|
||||
|
||||
// Read the file and verify
|
||||
data, err := os.ReadFile(tmpFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read file: %v", err)
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
t.Error("Write() produced no output")
|
||||
}
|
||||
|
||||
// Verify it's valid JSON
|
||||
var result api.LogRecord
|
||||
if err := json.Unmarshal(data, &result); err != nil {
|
||||
t.Errorf("Output is not valid JSON: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiWriter(t *testing.T) {
|
||||
multiWriter := NewMultiWriter()
|
||||
|
||||
// Create a temporary file writer
|
||||
tmpFile := "/tmp/ja4sentinel_multi_test.log"
|
||||
defer os.Remove(tmpFile)
|
||||
|
||||
fileWriter, err := NewFileWriter(tmpFile)
|
||||
if err != nil {
|
||||
t.Fatalf("NewFileWriter() error = %v", err)
|
||||
}
|
||||
defer fileWriter.Close()
|
||||
|
||||
multiWriter.Add(fileWriter)
|
||||
multiWriter.Add(NewStdoutWriter())
|
||||
|
||||
rec := api.LogRecord{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 12345,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
JA4: "t12s0102ab_1234567890ab",
|
||||
}
|
||||
|
||||
err = multiWriter.Write(rec)
|
||||
if err != nil {
|
||||
t.Errorf("Write() error = %v", err)
|
||||
}
|
||||
|
||||
// Verify file output
|
||||
data, err := os.ReadFile(tmpFile)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read file: %v", err)
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
t.Error("MultiWriter.Write() produced no file output")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuilderNewFromConfig(t *testing.T) {
|
||||
builder := NewBuilder()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
cfg api.AppConfig
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "stdout output",
|
||||
cfg: api.AppConfig{
|
||||
Outputs: []api.OutputConfig{
|
||||
{Type: "stdout", Enabled: true},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "file output",
|
||||
cfg: api.AppConfig{
|
||||
Outputs: []api.OutputConfig{
|
||||
{
|
||||
Type: "file",
|
||||
Enabled: true,
|
||||
Params: map[string]string{"path": "/tmp/ja4sentinel_builder_test.log"},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "file output without path",
|
||||
cfg: api.AppConfig{
|
||||
Outputs: []api.OutputConfig{
|
||||
{Type: "file", Enabled: true},
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "unix socket output",
|
||||
cfg: api.AppConfig{
|
||||
Outputs: []api.OutputConfig{
|
||||
{
|
||||
Type: "unix_socket",
|
||||
Enabled: true,
|
||||
Params: map[string]string{"socket_path": "/tmp/ja4sentinel_test.sock"},
|
||||
},
|
||||
},
|
||||
},
|
||||
wantErr: false,
|
||||
},
|
||||
{
|
||||
name: "unknown output type",
|
||||
cfg: api.AppConfig{
|
||||
Outputs: []api.OutputConfig{
|
||||
{Type: "unknown", Enabled: true},
|
||||
},
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
{
|
||||
name: "no outputs (should default to stdout)",
|
||||
cfg: api.AppConfig{},
|
||||
wantErr: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
writer, err := builder.NewFromConfig(tt.cfg)
|
||||
if (err != nil) != tt.wantErr {
|
||||
t.Errorf("NewFromConfig() error = %v, wantErr %v", err, tt.wantErr)
|
||||
return
|
||||
}
|
||||
if !tt.wantErr && writer == nil {
|
||||
t.Error("NewFromConfig() returned nil writer")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnixSocketWriter(t *testing.T) {
|
||||
// Test creation without socket (should not fail)
|
||||
socketPath := "/tmp/ja4sentinel_nonexistent.sock"
|
||||
writer, err := NewUnixSocketWriter(socketPath)
|
||||
if err != nil {
|
||||
t.Fatalf("NewUnixSocketWriter() error = %v", err)
|
||||
}
|
||||
|
||||
// Write should fail since socket doesn't exist
|
||||
rec := api.LogRecord{
|
||||
SrcIP: "192.168.1.1",
|
||||
SrcPort: 12345,
|
||||
DstIP: "10.0.0.1",
|
||||
DstPort: 443,
|
||||
}
|
||||
|
||||
err = writer.Write(rec)
|
||||
if err == nil {
|
||||
t.Error("Write() should fail for non-existent socket")
|
||||
}
|
||||
|
||||
writer.Close()
|
||||
}
|
||||
Reference in New Issue
Block a user