Initial commit: logcorrelator with unified packaging (DEB + RPM using fpm)

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>

internal/domain/correlated_log.go

@@ -0,0 +1,90 @@
+package domain
+
+import "time"
+
+// CorrelatedLog represents the output correlated log entry.
+type CorrelatedLog struct {
+	Timestamp    time.Time        `json:"timestamp"`
+	SrcIP        string           `json:"src_ip"`
+	SrcPort      int              `json:"src_port"`
+	DstIP        string           `json:"dst_ip,omitempty"`
+	DstPort      int              `json:"dst_port,omitempty"`
+	Correlated   bool             `json:"correlated"`
+	OrphanSide   string           `json:"orphan_side,omitempty"`
+	Apache       map[string]any   `json:"apache,omitempty"`
+	Network      map[string]any   `json:"network,omitempty"`
+	Extra        map[string]any   `json:"extra,omitempty"`
+}
+
+// NewCorrelatedLogFromEvent creates a correlated log from a single event (orphan).
+func NewCorrelatedLogFromEvent(event *NormalizedEvent, orphanSide string) CorrelatedLog {
+	return CorrelatedLog{
+		Timestamp:  event.Timestamp,
+		SrcIP:      event.SrcIP,
+		SrcPort:    event.SrcPort,
+		DstIP:      event.DstIP,
+		DstPort:    event.DstPort,
+		Correlated: false,
+		OrphanSide: orphanSide,
+		Apache:     extractApache(event),
+		Network:    extractNetwork(event),
+		Extra:      make(map[string]any),
+	}
+}
+
+// NewCorrelatedLog creates a correlated log from two matched events.
+func NewCorrelatedLog(apacheEvent, networkEvent *NormalizedEvent) CorrelatedLog {
+	ts := apacheEvent.Timestamp
+	if networkEvent.Timestamp.After(ts) {
+		ts = networkEvent.Timestamp
+	}
+
+	return CorrelatedLog{
+		Timestamp:  ts,
+		SrcIP:      apacheEvent.SrcIP,
+		SrcPort:    apacheEvent.SrcPort,
+		DstIP:      coalesceString(apacheEvent.DstIP, networkEvent.DstIP),
+		DstPort:    coalesceInt(apacheEvent.DstPort, networkEvent.DstPort),
+		Correlated: true,
+		OrphanSide: "",
+		Apache:     extractApache(apacheEvent),
+		Network:    extractNetwork(networkEvent),
+		Extra:      make(map[string]any),
+	}
+}
+
+func extractApache(e *NormalizedEvent) map[string]any {
+	if e.Source != SourceA {
+		return nil
+	}
+	result := make(map[string]any)
+	for k, v := range e.Raw {
+		result[k] = v
+	}
+	return result
+}
+
+func extractNetwork(e *NormalizedEvent) map[string]any {
+	if e.Source != SourceB {
+		return nil
+	}
+	result := make(map[string]any)
+	for k, v := range e.Raw {
+		result[k] = v
+	}
+	return result
+}
+
+func coalesceString(a, b string) string {
+	if a != "" {
+		return a
+	}
+	return b
+}
+
+func coalesceInt(a, b int) int {
+	if a != 0 {
+		return a
+	}
+	return b
+}

internal/domain/correlated_log_test.go

@@ -0,0 +1,115 @@
+package domain
+
+import (
+	"testing"
+	"time"
+)
+
+func TestNormalizedEvent_CorrelationKeyFull(t *testing.T) {
+	tests := []struct {
+		name     string
+		event    *NormalizedEvent
+		expected string
+	}{
+		{
+			name: "basic key",
+			event: &NormalizedEvent{
+				SrcIP:   "192.168.1.1",
+				SrcPort: 8080,
+			},
+			expected: "192.168.1.1:8080",
+		},
+		{
+			name: "different port",
+			event: &NormalizedEvent{
+				SrcIP:   "10.0.0.1",
+				SrcPort: 443,
+			},
+			expected: "10.0.0.1:443",
+		},
+		{
+			name: "port zero",
+			event: &NormalizedEvent{
+				SrcIP:   "127.0.0.1",
+				SrcPort: 0,
+			},
+			expected: "127.0.0.1:0",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			key := tt.event.CorrelationKeyFull()
+			if key != tt.expected {
+				t.Errorf("expected %s, got %s", tt.expected, key)
+			}
+		})
+	}
+}
+
+func TestNewCorrelatedLogFromEvent(t *testing.T) {
+	event := &NormalizedEvent{
+		Source:    SourceA,
+		Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+		DstIP:     "10.0.0.1",
+		DstPort:   80,
+		Raw: map[string]any{
+			"method": "GET",
+			"path":   "/api/test",
+		},
+	}
+
+	log := NewCorrelatedLogFromEvent(event, "A")
+
+	if log.Correlated {
+		t.Error("expected correlated to be false")
+	}
+	if log.OrphanSide != "A" {
+		t.Errorf("expected orphan_side A, got %s", log.OrphanSide)
+	}
+	if log.SrcIP != "192.168.1.1" {
+		t.Errorf("expected src_ip 192.168.1.1, got %s", log.SrcIP)
+	}
+	if log.Apache == nil {
+		t.Error("expected apache to be non-nil")
+	}
+}
+
+func TestNewCorrelatedLog(t *testing.T) {
+	apacheEvent := &NormalizedEvent{
+		Source:    SourceA,
+		Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC),
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+		DstIP:     "10.0.0.1",
+		DstPort:   80,
+		Raw:       map[string]any{"method": "GET"},
+	}
+
+	networkEvent := &NormalizedEvent{
+		Source:    SourceB,
+		Timestamp: time.Date(2024, 1, 1, 12, 0, 0, 500000000, time.UTC),
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+		DstIP:     "10.0.0.1",
+		DstPort:   80,
+		Raw:       map[string]any{"ja3": "abc123"},
+	}
+
+	log := NewCorrelatedLog(apacheEvent, networkEvent)
+
+	if !log.Correlated {
+		t.Error("expected correlated to be true")
+	}
+	if log.OrphanSide != "" {
+		t.Errorf("expected orphan_side to be empty, got %s", log.OrphanSide)
+	}
+	if log.Apache == nil {
+		t.Error("expected apache to be non-nil")
+	}
+	if log.Network == nil {
+		t.Error("expected network to be non-nil")
+	}
+}

internal/domain/correlation_service.go

@@ -0,0 +1,243 @@
+package domain
+
+import (
+	"container/list"
+	"sync"
+	"time"
+)
+
+const (
+	// DefaultMaxBufferSize is the default maximum number of events per buffer
+	DefaultMaxBufferSize = 10000
+)
+
+// CorrelationConfig holds the correlation configuration.
+type CorrelationConfig struct {
+	TimeWindow       time.Duration
+	ApacheAlwaysEmit bool
+	NetworkEmit      bool
+	MaxBufferSize    int // Maximum events to buffer per source
+}
+
+// CorrelationService handles the correlation logic between source A and B events.
+type CorrelationService struct {
+	config       CorrelationConfig
+	mu           sync.Mutex
+	bufferA      *eventBuffer
+	bufferB      *eventBuffer
+	pendingA     map[string]*list.Element // key -> list element containing NormalizedEvent
+	pendingB     map[string]*list.Element
+	timeProvider TimeProvider
+}
+
+type eventBuffer struct {
+	events *list.List
+}
+
+func newEventBuffer() *eventBuffer {
+	return &eventBuffer{
+		events: list.New(),
+	}
+}
+
+// TimeProvider abstracts time for testability.
+type TimeProvider interface {
+	Now() time.Time
+}
+
+// RealTimeProvider uses real system time.
+type RealTimeProvider struct{}
+
+func (p *RealTimeProvider) Now() time.Time {
+	return time.Now()
+}
+
+// NewCorrelationService creates a new correlation service.
+func NewCorrelationService(config CorrelationConfig, timeProvider TimeProvider) *CorrelationService {
+	if timeProvider == nil {
+		timeProvider = &RealTimeProvider{}
+	}
+	if config.MaxBufferSize <= 0 {
+		config.MaxBufferSize = DefaultMaxBufferSize
+	}
+	return &CorrelationService{
+		config:       config,
+		bufferA:      newEventBuffer(),
+		bufferB:      newEventBuffer(),
+		pendingA:     make(map[string]*list.Element),
+		pendingB:     make(map[string]*list.Element),
+		timeProvider: timeProvider,
+	}
+}
+
+// ProcessEvent processes an incoming event and returns correlated logs if matches are found.
+func (s *CorrelationService) ProcessEvent(event *NormalizedEvent) []CorrelatedLog {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	// Clean expired events first
+	s.cleanExpired()
+
+	// Check buffer overflow before adding
+	if s.isBufferFull(event.Source) {
+		// Buffer full, drop event or emit as orphan
+		if event.Source == SourceA && s.config.ApacheAlwaysEmit {
+			return []CorrelatedLog{NewCorrelatedLogFromEvent(event, "A")}
+		}
+		return nil
+	}
+
+	var results []CorrelatedLog
+
+	switch event.Source {
+	case SourceA:
+		results = s.processSourceA(event)
+	case SourceB:
+		results = s.processSourceB(event)
+	}
+
+	// Add the new event to the appropriate buffer
+	s.addEvent(event)
+
+	return results
+}
+
+func (s *CorrelationService) isBufferFull(source EventSource) bool {
+	switch source {
+	case SourceA:
+		return s.bufferA.events.Len() >= s.config.MaxBufferSize
+	case SourceB:
+		return s.bufferB.events.Len() >= s.config.MaxBufferSize
+	}
+	return false
+}
+
+func (s *CorrelationService) processSourceA(event *NormalizedEvent) []CorrelatedLog {
+	key := event.CorrelationKeyFull()
+
+	// Look for a matching B event
+	if elem, ok := s.pendingB[key]; ok {
+		bEvent := elem.Value.(*NormalizedEvent)
+		if s.eventsMatch(event, bEvent) {
+			// Found a match!
+			correlated := NewCorrelatedLog(event, bEvent)
+			s.bufferB.events.Remove(elem)
+			delete(s.pendingB, key)
+			return []CorrelatedLog{correlated}
+		}
+	}
+
+	// No match found
+	if s.config.ApacheAlwaysEmit {
+		orphan := NewCorrelatedLogFromEvent(event, "A")
+		return []CorrelatedLog{orphan}
+	}
+
+	// Keep in buffer for potential future match
+	return nil
+}
+
+func (s *CorrelationService) processSourceB(event *NormalizedEvent) []CorrelatedLog {
+	key := event.CorrelationKeyFull()
+
+	// Look for a matching A event
+	if elem, ok := s.pendingA[key]; ok {
+		aEvent := elem.Value.(*NormalizedEvent)
+		if s.eventsMatch(aEvent, event) {
+			// Found a match!
+			correlated := NewCorrelatedLog(aEvent, event)
+			s.bufferA.events.Remove(elem)
+			delete(s.pendingA, key)
+			return []CorrelatedLog{correlated}
+		}
+	}
+
+	// No match found - B is never emitted alone per spec
+	if s.config.NetworkEmit {
+		orphan := NewCorrelatedLogFromEvent(event, "B")
+		return []CorrelatedLog{orphan}
+	}
+
+	// Keep in buffer for potential future match (but won't be emitted alone)
+	return nil
+}
+
+func (s *CorrelationService) eventsMatch(a, b *NormalizedEvent) bool {
+	diff := a.Timestamp.Sub(b.Timestamp)
+	if diff < 0 {
+		diff = -diff
+	}
+	return diff <= s.config.TimeWindow
+}
+
+func (s *CorrelationService) addEvent(event *NormalizedEvent) {
+	key := event.CorrelationKeyFull()
+
+	switch event.Source {
+	case SourceA:
+		elem := s.bufferA.events.PushBack(event)
+		s.pendingA[key] = elem
+	case SourceB:
+		elem := s.bufferB.events.PushBack(event)
+		s.pendingB[key] = elem
+	}
+}
+
+func (s *CorrelationService) cleanExpired() {
+	now := s.timeProvider.Now()
+	cutoff := now.Add(-s.config.TimeWindow)
+
+	// Clean expired events from both buffers using shared logic
+	s.cleanBuffer(s.bufferA, s.pendingA, cutoff)
+	s.cleanBuffer(s.bufferB, s.pendingB, cutoff)
+}
+
+// cleanBuffer removes expired events from a buffer (shared logic for A and B).
+func (s *CorrelationService) cleanBuffer(buffer *eventBuffer, pending map[string]*list.Element, cutoff time.Time) {
+	for elem := buffer.events.Front(); elem != nil; {
+		event := elem.Value.(*NormalizedEvent)
+		if event.Timestamp.Before(cutoff) {
+			next := elem.Next()
+			key := event.CorrelationKeyFull()
+			buffer.events.Remove(elem)
+			if pending[key] == elem {
+				delete(pending, key)
+			}
+			elem = next
+		} else {
+			break // Events are ordered, so we can stop early
+		}
+	}
+}
+
+// Flush forces emission of remaining buffered events (for shutdown).
+func (s *CorrelationService) Flush() []CorrelatedLog {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+
+	var results []CorrelatedLog
+
+	// Emit remaining A events as orphans if configured
+	if s.config.ApacheAlwaysEmit {
+		for elem := s.bufferA.events.Front(); elem != nil; elem = elem.Next() {
+			event := elem.Value.(*NormalizedEvent)
+			orphan := NewCorrelatedLogFromEvent(event, "A")
+			results = append(results, orphan)
+		}
+	}
+
+	// Clear buffers
+	s.bufferA.events.Init()
+	s.bufferB.events.Init()
+	s.pendingA = make(map[string]*list.Element)
+	s.pendingB = make(map[string]*list.Element)
+
+	return results
+}
+
+// GetBufferSizes returns the current buffer sizes (for monitoring).
+func (s *CorrelationService) GetBufferSizes() (int, int) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	return s.bufferA.events.Len(), s.bufferB.events.Len()
+}

internal/domain/correlation_service_test.go

@@ -0,0 +1,153 @@
+package domain
+
+import (
+	"testing"
+	"time"
+)
+
+type mockTimeProvider struct {
+	now time.Time
+}
+
+func (m *mockTimeProvider) Now() time.Time {
+	return m.now
+}
+
+func TestCorrelationService_Match(t *testing.T) {
+	now := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
+	timeProvider := &mockTimeProvider{now: now}
+
+	config := CorrelationConfig{
+		TimeWindow:       time.Second,
+		ApacheAlwaysEmit: false, // Don't emit A immediately to test matching
+		NetworkEmit:      false,
+	}
+
+	svc := NewCorrelationService(config, timeProvider)
+
+	// Send Apache event (should be buffered, not emitted)
+	apacheEvent := &NormalizedEvent{
+		Source:    SourceA,
+		Timestamp: now,
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+		Raw:       map[string]any{"method": "GET"},
+	}
+
+	results := svc.ProcessEvent(apacheEvent)
+	if len(results) != 0 {
+		t.Fatalf("expected 0 results (buffered), got %d", len(results))
+	}
+
+	// Send matching Network event within window
+	networkEvent := &NormalizedEvent{
+		Source:    SourceB,
+		Timestamp: now.Add(500 * time.Millisecond),
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+		Raw:       map[string]any{"ja3": "abc"},
+	}
+
+	// This should match and return correlated log
+	results = svc.ProcessEvent(networkEvent)
+	if len(results) != 1 {
+		t.Errorf("expected 1 result (correlated), got %d", len(results))
+	} else if !results[0].Correlated {
+		t.Error("expected correlated result")
+	}
+}
+
+func TestCorrelationService_NoMatch_DifferentIP(t *testing.T) {
+	now := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
+	timeProvider := &mockTimeProvider{now: now}
+
+	config := CorrelationConfig{
+		TimeWindow:       time.Second,
+		ApacheAlwaysEmit: true,
+		NetworkEmit:      false,
+	}
+
+	svc := NewCorrelationService(config, timeProvider)
+
+	apacheEvent := &NormalizedEvent{
+		Source:    SourceA,
+		Timestamp: now,
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+	}
+
+	networkEvent := &NormalizedEvent{
+		Source:    SourceB,
+		Timestamp: now,
+		SrcIP:     "192.168.1.2", // Different IP
+		SrcPort:   8080,
+	}
+
+	svc.ProcessEvent(apacheEvent)
+	results := svc.ProcessEvent(networkEvent)
+
+	if len(results) != 0 {
+		t.Errorf("expected 0 results (different IP), got %d", len(results))
+	}
+}
+
+func TestCorrelationService_NoMatch_TimeWindowExceeded(t *testing.T) {
+	now := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
+	timeProvider := &mockTimeProvider{now: now}
+
+	config := CorrelationConfig{
+		TimeWindow:       time.Second,
+		ApacheAlwaysEmit: true,
+		NetworkEmit:      false,
+	}
+
+	svc := NewCorrelationService(config, timeProvider)
+
+	apacheEvent := &NormalizedEvent{
+		Source:    SourceA,
+		Timestamp: now,
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+	}
+
+	networkEvent := &NormalizedEvent{
+		Source:    SourceB,
+		Timestamp: now.Add(2 * time.Second), // Outside window
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+	}
+
+	svc.ProcessEvent(apacheEvent)
+	results := svc.ProcessEvent(networkEvent)
+
+	if len(results) != 0 {
+		t.Errorf("expected 0 results (time window exceeded), got %d", len(results))
+	}
+}
+
+func TestCorrelationService_Flush(t *testing.T) {
+	now := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
+	timeProvider := &mockTimeProvider{now: now}
+
+	config := CorrelationConfig{
+		TimeWindow:       time.Second,
+		ApacheAlwaysEmit: true,
+		NetworkEmit:      false,
+	}
+
+	svc := NewCorrelationService(config, timeProvider)
+
+	apacheEvent := &NormalizedEvent{
+		Source:    SourceA,
+		Timestamp: now,
+		SrcIP:     "192.168.1.1",
+		SrcPort:   8080,
+	}
+
+	svc.ProcessEvent(apacheEvent)
+
+	flushed := svc.Flush()
+	if len(flushed) != 1 {
+		t.Errorf("expected 1 flushed event, got %d", len(flushed))
+	}
+}

internal/domain/event.go

@@ -0,0 +1,37 @@
+package domain
+
+import (
+	"strconv"
+	"time"
+)
+
+// EventSource identifies the source of an event.
+type EventSource string
+
+const (
+	SourceA EventSource = "A" // Apache/HTTP source
+	SourceB EventSource = "B" // Network source
+)
+
+// NormalizedEvent represents a unified internal event from either source.
+type NormalizedEvent struct {
+	Source    EventSource
+	Timestamp time.Time
+	SrcIP     string
+	SrcPort   int
+	DstIP     string
+	DstPort   int
+	Headers   map[string]string
+	Extra     map[string]any
+	Raw       map[string]any // Original raw data
+}
+
+// CorrelationKey returns the key used for correlation (src_ip + src_port).
+func (e *NormalizedEvent) CorrelationKey() string {
+	return e.SrcIP + ":" + strconv.Itoa(e.SrcPort)
+}
+
+// CorrelationKeyFull returns a proper correlation key (alias for clarity).
+func (e *NormalizedEvent) CorrelationKeyFull() string {
+	return e.CorrelationKey()
+}