code_public/logcorrelator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: renforcer corrélation A/B et sorties stdout/fichier
Some checks failed
Build and Test / test (push) Has been cancelled
Details
Build and Test / build (push) Has been cancelled
Details
Build and Test / docker (push) Has been cancelled
Details

Browse Source 
Co-authored-by: aider (openrouter/openai/gpt-5.3-codex) <aider@aider.chat>
This commit is contained in:
Jacquin Antoine
2026-03-01 12:10:17 +01:00
parent d3436f6245
commit 27c7659397
13 changed files with 441 additions and 259 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
internal/adapters/inbound/unixsocket/source.go
View File

@ -5,6 +5,7 @@ import (
"context"
"encoding/json"
"fmt"
"math"
"net"
"os"
"strconv"
@ -180,7 +181,7 @@ func (s *UnixSocketSource) readEvents(ctx context.Context, conn net.Conn, eventC
}
// Debug: log raw events
s.logger.Debugf("event received: source=%s src_ip=%s src_port=%d",
s.logger.Debugf("event received: source=%s src_ip=%s src_port=%d",
event.Source, event.SrcIP, event.SrcPort)
select {
@ -191,6 +192,21 @@ func (s *UnixSocketSource) readEvents(ctx context.Context, conn net.Conn, eventC
}
}
func resolveSource(sourceType string, headers map[string]string) domain.EventSource {
switch strings.ToLower(strings.TrimSpace(sourceType)) {
case "a", "apache", "http":
return domain.SourceA
case "b", "network", "net":
return domain.SourceB
default:
// fallback compat
if len(headers) > 0 {
return domain.SourceA
}
return domain.SourceB
}
}
func parseJSONEvent(data []byte, sourceType string) (*domain.NormalizedEvent, error) {
var raw map[string]any
if err := json.Unmarshal(data, &raw); err != nil {
@ -198,12 +214,29 @@ func parseJSONEvent(data []byte, sourceType string) (*domain.NormalizedEvent, er
}
event := &domain.NormalizedEvent{
Raw: raw,
Extra: make(map[string]any),
Raw: raw,
Extra: make(map[string]any),
Headers: make(map[string]string),
}
// Extract headers (header_* fields) first
for k, v := range raw {
if strings.HasPrefix(k, "header_") {
if sv, ok := v.(string); ok {
event.Headers[k[7:]] = sv
}
}
}
// Resolve source first (strict timestamp logic depends on source)
event.Source = resolveSource(sourceType, event.Headers)
// Extract and validate src_ip
if v, ok := getString(raw, "src_ip"); ok {
v = strings.TrimSpace(v)
if v == "" {
return nil, fmt.Errorf("src_ip cannot be empty")
}
event.SrcIP = v
} else {
return nil, fmt.Errorf("missing required field: src_ip")
@ -221,7 +254,7 @@ func parseJSONEvent(data []byte, sourceType string) (*domain.NormalizedEvent, er
// Extract dst_ip (optional)
if v, ok := getString(raw, "dst_ip"); ok {
event.DstIP = v
event.DstIP = strings.TrimSpace(v)
}
// Extract dst_port (optional)
@ -232,50 +265,23 @@ func parseJSONEvent(data []byte, sourceType string) (*domain.NormalizedEvent, er
event.DstPort = v
}
// Extract timestamp - try different fields
if ts, ok := getInt64(raw, "timestamp"); ok {
// Extract timestamp based on source contract
switch event.Source {
case domain.SourceA:
ts, ok := getInt64(raw, "timestamp")
if !ok {
return nil, fmt.Errorf("missing required numeric field: timestamp for source A")
}
// Assume nanoseconds
event.Timestamp = time.Unix(0, ts)
} else if tsStr, ok := getString(raw, "time"); ok {
if t, err := time.Parse(time.RFC3339, tsStr); err == nil {
event.Timestamp = t
}
} else if tsStr, ok := getString(raw, "timestamp"); ok {
if t, err := time.Parse(time.RFC3339, tsStr); err == nil {
event.Timestamp = t
}
}
if event.Timestamp.IsZero() {
case domain.SourceB:
// For network source, always use local reception time
event.Timestamp = time.Now()
}
// Extract headers (header_* fields)
event.Headers = make(map[string]string)
for k, v := range raw {
if len(k) > 7 && k[:7] == "header_" {
if sv, ok := v.(string); ok {
event.Headers[k[7:]] = sv
}
}
}
// Determine source based on explicit config or fallback to heuristic
switch sourceType {
case "A", "a", "apache", "http":
event.Source = domain.SourceA
case "B", "b", "network", "net":
event.Source = domain.SourceB
default:
// Fallback to heuristic detection for backward compatibility
if len(event.Headers) > 0 {
event.Source = domain.SourceA
} else {
event.Source = domain.SourceB
}
return nil, fmt.Errorf("unsupported source type: %s", event.Source)
}
// Extra fields (single pass)
// Extra fields
knownFields := map[string]bool{
"src_ip": true, "src_port": true, "dst_ip": true, "dst_port": true,
"timestamp": true, "time": true,
@ -306,6 +312,9 @@ func getInt(m map[string]any, key string) (int, bool) {
if v, ok := m[key]; ok {
switch val := v.(type) {
case float64:
if math.Trunc(val) != val {
return 0, false
}
return int(val), true
case int:
return val, true
@ -324,6 +333,9 @@ func getInt64(m map[string]any, key string) (int64, bool) {
if v, ok := m[key]; ok {
switch val := v.(type) {
case float64:
if math.Trunc(val) != val {
return 0, false
}
return int64(val), true
case int:
return int64(val), true

129
internal/adapters/inbound/unixsocket/source_test.go
View File

@ -41,6 +41,10 @@ func TestParseJSONEvent_Apache(t *testing.T) {
if event.Source != domain.SourceA {
t.Errorf("expected source A, got %s", event.Source)
}
expectedTs := time.Unix(0, 1704110400000000000)
if !event.Timestamp.Equal(expectedTs) {
t.Errorf("expected timestamp %v, got %v", expectedTs, event.Timestamp)
}
}
func TestParseJSONEvent_Network(t *testing.T) {
@ -49,12 +53,15 @@ func TestParseJSONEvent_Network(t *testing.T) {
"src_port": 8080,
"dst_ip": "10.0.0.1",
"dst_port": 443,
"timestamp": 1704110400000000000,
"ja3": "abc123def456",
"ja4": "xyz789",
"tcp_meta_flags": "SYN"
}`)
before := time.Now()
event, err := parseJSONEvent(data, "B")
after := time.Now()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
@ -68,6 +75,9 @@ func TestParseJSONEvent_Network(t *testing.T) {
if event.Source != domain.SourceB {
t.Errorf("expected source B, got %s", event.Source)
}
if event.Timestamp.Before(before.Add(-2*time.Second)) || event.Timestamp.After(after.Add(2*time.Second)) {
t.Errorf("expected network timestamp near now, got %v", event.Timestamp)
}
}
func TestParseJSONEvent_InvalidJSON(t *testing.T) {
@ -88,21 +98,35 @@ func TestParseJSONEvent_MissingFields(t *testing.T) {
}
}
func TestParseJSONEvent_StringTimestamp(t *testing.T) {
func TestParseJSONEvent_SourceARequiresNumericTimestamp(t *testing.T) {
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080,
"time": "2024-01-01T12:00:00Z"
}`)
event, err := parseJSONEvent(data, "")
_, err := parseJSONEvent(data, "A")
if err == nil {
t.Fatal("expected error for source A without numeric timestamp")
}
}
func TestParseJSONEvent_SourceBIgnoresPayloadTimestamp(t *testing.T) {
data := []byte(`{
"src_ip": "192.168.1.1",
"src_port": 8080,
"timestamp": 1
}`)
before := time.Now()
event, err := parseJSONEvent(data, "B")
after := time.Now()
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
expected := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
if !event.Timestamp.Equal(expected) {
t.Errorf("expected timestamp %v, got %v", expected, event.Timestamp)
if event.Timestamp.Before(before.Add(-2*time.Second)) || event.Timestamp.After(after.Add(2*time.Second)) {
t.Errorf("expected source B timestamp near now, got %v", event.Timestamp)
}
}
@ -114,40 +138,40 @@ func TestParseJSONEvent_ExplicitSourceType(t *testing.T) {
expected domain.EventSource
}{
{
name: "explicit A",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
name: "explicit A",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000}`,
sourceType: "A",
expected: domain.SourceA,
expected: domain.SourceA,
},
{
name: "explicit B",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
name: "explicit B",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
sourceType: "B",
expected: domain.SourceB,
expected: domain.SourceB,
},
{
name: "explicit apache",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
name: "explicit apache",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000}`,
sourceType: "apache",
expected: domain.SourceA,
expected: domain.SourceA,
},
{
name: "explicit network",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
name: "explicit network",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
sourceType: "network",
expected: domain.SourceB,
expected: domain.SourceB,
},
{
name: "auto-detect A with headers",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "header_host": "example.com"}`,
name: "auto-detect A with headers",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "timestamp": 1704110400000000000, "header_host": "example.com"}`,
sourceType: "",
expected: domain.SourceA,
expected: domain.SourceA,
},
{
name: "auto-detect B without headers",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "ja3": "abc"}`,
name: "auto-detect B without headers",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "ja3": "abc"}`,
sourceType: "",
expected: domain.SourceB,
expected: domain.SourceB,
},
}
@ -241,7 +265,7 @@ func TestGetInt(t *testing.T) {
expected int
ok bool
}{
{"float", 42, true},
{"float", 0, false},
{"int", 42, true},
{"int64", 42, true},
{"string", 42, true},
@ -278,7 +302,7 @@ func TestGetInt64(t *testing.T) {
expected int64
ok bool
}{
{"float", 42, true},
{"float", 0, false},
{"int", 42, true},
{"int64", 42, true},
{"string", 42, true},
@ -302,45 +326,52 @@ func TestGetInt64(t *testing.T) {
func TestParseJSONEvent_PortValidation(t *testing.T) {
tests := []struct {
name string
data string
wantErr bool
name string
data string
sourceType string
wantErr bool
}{
{
name: "valid src_port",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
wantErr: false,
name: "valid src_port",
data: `{"src_ip": "192.168.1.1", "src_port": 8080}`,
sourceType: "B",
wantErr: false,
},
{
name: "src_port zero",
data: `{"src_ip": "192.168.1.1", "src_port": 0}`,
wantErr: true,
name: "src_port zero",
data: `{"src_ip": "192.168.1.1", "src_port": 0}`,
sourceType: "B",
wantErr: true,
},
{
name: "src_port negative",
data: `{"src_ip": "192.168.1.1", "src_port": -1}`,
wantErr: true,
name: "src_port negative",
data: `{"src_ip": "192.168.1.1", "src_port": -1}`,
sourceType: "B",
wantErr: true,
},
{
name: "src_port too high",
data: `{"src_ip": "192.168.1.1", "src_port": 70000}`,
wantErr: true,
name: "src_port too high",
data: `{"src_ip": "192.168.1.1", "src_port": 70000}`,
sourceType: "B",
wantErr: true,
},
{
name: "valid dst_port zero",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 0}`,
wantErr: false,
name: "valid dst_port zero",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 0}`,
sourceType: "B",
wantErr: false,
},
{
name: "dst_port too high",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 70000}`,
wantErr: true,
name: "dst_port too high",
data: `{"src_ip": "192.168.1.1", "src_port": 8080, "dst_port": 70000}`,
sourceType: "B",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
_, err := parseJSONEvent([]byte(tt.data), "")
_, err := parseJSONEvent([]byte(tt.data), tt.sourceType)
if (err != nil) != tt.wantErr {
t.Errorf("parseJSONEvent() error = %v, wantErr %v", err, tt.wantErr)
}
@ -350,12 +381,12 @@ func TestParseJSONEvent_PortValidation(t *testing.T) {
func TestParseJSONEvent_TimestampFallback(t *testing.T) {
data := []byte(`{"src_ip": "192.168.1.1", "src_port": 8080}`)
event, err := parseJSONEvent(data, "")
event, err := parseJSONEvent(data, "B")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
// Should fallback to current time if no timestamp provided
// For source B, timestamp is reception time
if event.Timestamp.IsZero() {
t.Error("expected non-zero timestamp")
}

74
internal/adapters/outbound/clickhouse/sink.go
View File

@ -4,7 +4,9 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"net"
"strings"
"sync"
"time"
@ -115,35 +117,37 @@ func (s *ClickHouseSink) Name() string {
// Write adds a log to the buffer.
func (s *ClickHouseSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
s.mu.Lock()
defer s.mu.Unlock()
deadline := time.Now().Add(time.Duration(s.config.TimeoutMs) * time.Millisecond)
// Check buffer overflow
if len(s.buffer) >= s.config.MaxBufferSize {
if s.config.DropOnOverflow {
// Drop the log
for {
s.mu.Lock()
if len(s.buffer) < s.config.MaxBufferSize {
s.buffer = append(s.buffer, log)
if len(s.buffer) >= s.config.BatchSize {
select {
case s.flushChan <- struct{}{}:
default:
}
}
s.mu.Unlock()
return nil
}
// Block until space is available (with timeout)
drop := s.config.DropOnOverflow
s.mu.Unlock()
if drop {
return nil
}
if time.Now().After(deadline) {
return fmt.Errorf("buffer full, timeout exceeded")
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(time.Duration(s.config.TimeoutMs) * time.Millisecond):
return fmt.Errorf("buffer full, timeout exceeded")
case <-time.After(10 * time.Millisecond):
}
}
s.buffer = append(s.buffer, log)
// Trigger flush if batch is full
if len(s.buffer) >= s.config.BatchSize {
select {
case s.flushChan <- struct{}{}:
default:
}
}
return nil
}
// Flush flushes the buffer to ClickHouse.
@ -311,7 +315,33 @@ func isRetryableError(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
if errors.Is(err, context.Canceled) {
return false
}
var netErr net.Error
if errors.As(err, &netErr) {
if netErr.Timeout() {
return true
}
}
errStr := strings.ToLower(err.Error())
// Explicit non-retryable SQL/schema errors
if strings.Contains(errStr, "syntax error") ||
strings.Contains(errStr, "unknown table") ||
strings.Contains(errStr, "unknown column") ||
(strings.Contains(errStr, "table") && strings.Contains(errStr, "not found")) {
return false
}
// Fallback network/transient errors
retryableErrors := []string{
"connection refused",
"connection reset",
@ -319,11 +349,13 @@ func isRetryableError(err error) bool {
"temporary failure",
"network is unreachable",
"broken pipe",
"no route to host",
}
for _, re := range retryableErrors {
if strings.Contains(errStr, re) {
return true
}
}
return false
}

72
internal/adapters/outbound/file/sink.go
View File

@ -1,7 +1,6 @@
package file
import (
"bufio"
"context"
"encoding/json"
"fmt"
@ -30,7 +29,6 @@ type FileSink struct {
config Config
mu sync.Mutex
file *os.File
writer *bufio.Writer
}
// NewFileSink creates a new file sink.
@ -66,11 +64,12 @@ func (s *FileSink) Write(ctx context.Context, log domain.CorrelatedLog) error {
return fmt.Errorf("failed to marshal log: %w", err)
}
if _, err := s.writer.Write(data); err != nil {
return fmt.Errorf("failed to write log: %w", err)
line := append(data, '\n')
if _, err := s.file.Write(line); err != nil {
return fmt.Errorf("failed to write log line: %w", err)
}
if _, err := s.writer.WriteString("\n"); err != nil {
return fmt.Errorf("failed to write newline: %w", err)
if err := s.file.Sync(); err != nil {
return fmt.Errorf("failed to sync log line: %w", err)
}
return nil
@ -81,8 +80,8 @@ func (s *FileSink) Flush(ctx context.Context) error {
s.mu.Lock()
defer s.mu.Unlock()
if s.writer != nil {
return s.writer.Flush()
if s.file != nil {
return s.file.Sync()
}
return nil
}
@ -92,12 +91,6 @@ func (s *FileSink) Close() error {
s.mu.Lock()
defer s.mu.Unlock()
if s.writer != nil {
if err := s.writer.Flush(); err != nil {
return err
}
}
if s.file != nil {
return s.file.Close()
}
@ -122,47 +115,54 @@ func (s *FileSink) openFile() error {
}
s.file = file
s.writer = bufio.NewWriter(file)
return nil
}
// validateFilePath validates that the file path is safe and allowed.
func validateFilePath(path string) error {
if path == "" {
if strings.TrimSpace(path) == "" {
return fmt.Errorf("path cannot be empty")
}
// Clean the path
cleanPath := filepath.Clean(path)
// Ensure path is absolute or relative to allowed directories
allowedPrefixes := []string{
// Allow relative paths for testing/dev
if !filepath.IsAbs(cleanPath) {
return nil
}
absPath, err := filepath.Abs(cleanPath)
if err != nil {
return fmt.Errorf("failed to resolve absolute path: %w", err)
}
allowedRoots := []string{
"/var/log/logcorrelator",
"/var/log",
"/tmp",
}
// Check if path is in allowed directories
allowed := false
for _, prefix := range allowedPrefixes {
if strings.HasPrefix(cleanPath, prefix) {
allowed = true
break
for _, root := range allowedRoots {
absRoot, err := filepath.Abs(filepath.Clean(root))
if err != nil {
continue
}
}
if !allowed {
// Allow relative paths for testing
if !filepath.IsAbs(cleanPath) {
rel, err := filepath.Rel(absRoot, absPath)
if err != nil {
continue
}
if rel == "." {
return nil
}
if rel == ".." {
continue
}
if !strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
return nil
}
return fmt.Errorf("path must be in allowed directories: %v", allowedPrefixes)
}
// Check for path traversal
if strings.Contains(cleanPath, "..") {
return fmt.Errorf("path cannot contain '..'")
}
return nil
return fmt.Errorf("path must be under allowed directories: %v", allowedRoots)
}

37
internal/adapters/outbound/file/sink_test.go
View File

@ -44,6 +44,36 @@ func TestFileSink_Write(t *testing.T) {
}
}
func TestFileSink_WriteImmediatePersist_NoFlushNeeded(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
sink, err := NewFileSink(Config{Path: testPath})
if err != nil {
t.Fatalf("failed to create sink: %v", err)
}
defer sink.Close()
log := domain.CorrelatedLog{
SrcIP: "192.168.1.1",
SrcPort: 8080,
Correlated: true,
}
if err := sink.Write(context.Background(), log); err != nil {
t.Fatalf("failed to write: %v", err)
}
// Must be visible immediately without Flush()
data, err := os.ReadFile(testPath)
if err != nil {
t.Fatalf("failed to read file: %v", err)
}
if len(data) == 0 {
t.Error("expected data to be present immediately after Write without Flush")
}
}
func TestFileSink_MultipleWrites(t *testing.T) {
tmpDir := t.TempDir()
testPath := filepath.Join(tmpDir, "test.log")
@ -105,7 +135,7 @@ func TestFileSink_ValidateFilePath(t *testing.T) {
{"valid /var/log/logcorrelator", "/var/log/logcorrelator/test.log", false},
{"valid /var/log", "/var/log/test.log", false},
{"valid /tmp", "/tmp/test.log", false},
{"path traversal", "/var/log/../etc/passwd", true},
{"reject lookalike /var/logevil", "/var/logevil/test.log", true},
{"invalid directory", "/etc/logcorrelator/test.log", true},
{"relative path", "test.log", false}, // Allowed for testing
}
@ -137,9 +167,6 @@ func TestFileSink_OpenFile(t *testing.T) {
if sink.file == nil {
t.Error("expected file to be opened")
}
if sink.writer == nil {
t.Error("expected writer to be initialized")
}
}
func TestFileSink_WriteBeforeOpen(t *testing.T) {
@ -183,7 +210,7 @@ func TestFileSink_FlushBeforeOpen(t *testing.T) {
}
func TestFileSink_InvalidPath(t *testing.T) {
// Test with invalid path (path traversal)
// Test with invalid path (outside allowed directories)
_, err := NewFileSink(Config{Path: "/etc/../passwd"})
if err == nil {
t.Error("expected error for invalid path")