Initial commit: logcorrelator with unified packaging (DEB + RPM using fpm)
Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
334
internal/adapters/inbound/unixsocket/source.go
Normal file
334
internal/adapters/inbound/unixsocket/source.go
Normal file
@ -0,0 +1,334 @@
|
||||
package unixsocket
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/logcorrelator/logcorrelator/internal/domain"
|
||||
)
|
||||
|
||||
const (
|
||||
// Default socket file permissions (owner + group read/write)
|
||||
DefaultSocketPermissions os.FileMode = 0660
|
||||
// Maximum line size for JSON logs (1MB)
|
||||
MaxLineSize = 1024 * 1024
|
||||
// Maximum concurrent connections per socket
|
||||
MaxConcurrentConnections = 100
|
||||
// Rate limit: max events per second
|
||||
MaxEventsPerSecond = 10000
|
||||
)
|
||||
|
||||
// Config holds the Unix socket source configuration.
|
||||
type Config struct {
|
||||
Name string
|
||||
Path string
|
||||
}
|
||||
|
||||
// UnixSocketSource reads JSON events from a Unix socket.
|
||||
type UnixSocketSource struct {
|
||||
config Config
|
||||
mu sync.Mutex
|
||||
listener net.Listener
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
semaphore chan struct{} // Limit concurrent connections
|
||||
}
|
||||
|
||||
// NewUnixSocketSource creates a new Unix socket source.
|
||||
func NewUnixSocketSource(config Config) *UnixSocketSource {
|
||||
return &UnixSocketSource{
|
||||
config: config,
|
||||
done: make(chan struct{}),
|
||||
semaphore: make(chan struct{}, MaxConcurrentConnections),
|
||||
}
|
||||
}
|
||||
|
||||
// Name returns the source name.
|
||||
func (s *UnixSocketSource) Name() string {
|
||||
return s.config.Name
|
||||
}
|
||||
|
||||
// Start begins listening on the Unix socket.
|
||||
func (s *UnixSocketSource) Start(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) error {
|
||||
// Remove existing socket file if present
|
||||
if info, err := os.Stat(s.config.Path); err == nil {
|
||||
if info.Mode()&os.ModeSocket != 0 {
|
||||
if err := os.Remove(s.config.Path); err != nil {
|
||||
return fmt.Errorf("failed to remove existing socket: %w", err)
|
||||
}
|
||||
} else {
|
||||
return fmt.Errorf("path exists but is not a socket: %s", s.config.Path)
|
||||
}
|
||||
}
|
||||
|
||||
// Create listener
|
||||
listener, err := net.Listen("unix", s.config.Path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create unix socket listener: %w", err)
|
||||
}
|
||||
s.listener = listener
|
||||
|
||||
// Set permissions - fail if we can't
|
||||
if err := os.Chmod(s.config.Path, DefaultSocketPermissions); err != nil {
|
||||
listener.Close()
|
||||
os.Remove(s.config.Path)
|
||||
return fmt.Errorf("failed to set socket permissions: %w", err)
|
||||
}
|
||||
|
||||
s.wg.Add(1)
|
||||
go func() {
|
||||
defer s.wg.Done()
|
||||
s.acceptConnections(ctx, eventChan)
|
||||
}()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *UnixSocketSource) acceptConnections(ctx context.Context, eventChan chan<- *domain.NormalizedEvent) {
|
||||
for {
|
||||
select {
|
||||
case <-s.done:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
conn, err := s.listener.Accept()
|
||||
if err != nil {
|
||||
select {
|
||||
case <-s.done:
|
||||
return
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
// Check semaphore for connection limiting
|
||||
select {
|
||||
case s.semaphore <- struct{}{}:
|
||||
// Connection accepted
|
||||
default:
|
||||
// Too many connections, reject
|
||||
conn.Close()
|
||||
continue
|
||||
}
|
||||
|
||||
s.wg.Add(1)
|
||||
go func(c net.Conn) {
|
||||
defer s.wg.Done()
|
||||
defer func() { <-s.semaphore }()
|
||||
defer c.Close()
|
||||
s.readEvents(ctx, c, eventChan)
|
||||
}(conn)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *UnixSocketSource) readEvents(ctx context.Context, conn net.Conn, eventChan chan<- *domain.NormalizedEvent) {
|
||||
// Set read deadline to prevent hanging
|
||||
conn.SetReadDeadline(time.Now().Add(5 * time.Minute))
|
||||
|
||||
scanner := bufio.NewScanner(conn)
|
||||
// Increase buffer size limit to 1MB
|
||||
buf := make([]byte, 0, 4096)
|
||||
scanner.Buffer(buf, MaxLineSize)
|
||||
|
||||
for scanner.Scan() {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
line := scanner.Bytes()
|
||||
if len(line) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
event, err := parseJSONEvent(line)
|
||||
if err != nil {
|
||||
// Log parse errors but continue processing
|
||||
continue
|
||||
}
|
||||
|
||||
select {
|
||||
case eventChan <- event:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
// Connection error, log but don't crash
|
||||
}
|
||||
}
|
||||
|
||||
func parseJSONEvent(data []byte) (*domain.NormalizedEvent, error) {
|
||||
var raw map[string]any
|
||||
if err := json.Unmarshal(data, &raw); err != nil {
|
||||
return nil, fmt.Errorf("invalid JSON: %w", err)
|
||||
}
|
||||
|
||||
event := &domain.NormalizedEvent{
|
||||
Raw: raw,
|
||||
Extra: make(map[string]any),
|
||||
}
|
||||
|
||||
// Extract and validate src_ip
|
||||
if v, ok := getString(raw, "src_ip"); ok {
|
||||
event.SrcIP = v
|
||||
} else {
|
||||
return nil, fmt.Errorf("missing required field: src_ip")
|
||||
}
|
||||
|
||||
// Extract and validate src_port
|
||||
if v, ok := getInt(raw, "src_port"); ok {
|
||||
if v < 1 || v > 65535 {
|
||||
return nil, fmt.Errorf("src_port must be between 1 and 65535, got %d", v)
|
||||
}
|
||||
event.SrcPort = v
|
||||
} else {
|
||||
return nil, fmt.Errorf("missing required field: src_port")
|
||||
}
|
||||
|
||||
// Extract dst_ip (optional)
|
||||
if v, ok := getString(raw, "dst_ip"); ok {
|
||||
event.DstIP = v
|
||||
}
|
||||
|
||||
// Extract dst_port (optional)
|
||||
if v, ok := getInt(raw, "dst_port"); ok {
|
||||
if v < 0 || v > 65535 {
|
||||
return nil, fmt.Errorf("dst_port must be between 0 and 65535, got %d", v)
|
||||
}
|
||||
event.DstPort = v
|
||||
}
|
||||
|
||||
// Extract timestamp - try different fields
|
||||
if ts, ok := getInt64(raw, "timestamp"); ok {
|
||||
// Assume nanoseconds
|
||||
event.Timestamp = time.Unix(0, ts)
|
||||
} else if tsStr, ok := getString(raw, "time"); ok {
|
||||
if t, err := time.Parse(time.RFC3339, tsStr); err == nil {
|
||||
event.Timestamp = t
|
||||
}
|
||||
} else if tsStr, ok := getString(raw, "timestamp"); ok {
|
||||
if t, err := time.Parse(time.RFC3339, tsStr); err == nil {
|
||||
event.Timestamp = t
|
||||
}
|
||||
}
|
||||
|
||||
if event.Timestamp.IsZero() {
|
||||
event.Timestamp = time.Now()
|
||||
}
|
||||
|
||||
// Extract headers (header_* fields)
|
||||
event.Headers = make(map[string]string)
|
||||
for k, v := range raw {
|
||||
if len(k) > 7 && k[:7] == "header_" {
|
||||
if sv, ok := v.(string); ok {
|
||||
event.Headers[k[7:]] = sv
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Determine source based on fields present
|
||||
if len(event.Headers) > 0 {
|
||||
event.Source = domain.SourceA
|
||||
} else {
|
||||
event.Source = domain.SourceB
|
||||
}
|
||||
|
||||
// Extra fields (single pass)
|
||||
knownFields := map[string]bool{
|
||||
"src_ip": true, "src_port": true, "dst_ip": true, "dst_port": true,
|
||||
"timestamp": true, "time": true,
|
||||
}
|
||||
for k, v := range raw {
|
||||
if knownFields[k] {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(k, "header_") {
|
||||
continue
|
||||
}
|
||||
event.Extra[k] = v
|
||||
}
|
||||
|
||||
return event, nil
|
||||
}
|
||||
|
||||
func getString(m map[string]any, key string) (string, bool) {
|
||||
if v, ok := m[key]; ok {
|
||||
if s, ok := v.(string); ok {
|
||||
return s, true
|
||||
}
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
func getInt(m map[string]any, key string) (int, bool) {
|
||||
if v, ok := m[key]; ok {
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
return int(val), true
|
||||
case int:
|
||||
return val, true
|
||||
case int64:
|
||||
return int(val), true
|
||||
case string:
|
||||
if i, err := strconv.Atoi(val); err == nil {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func getInt64(m map[string]any, key string) (int64, bool) {
|
||||
if v, ok := m[key]; ok {
|
||||
switch val := v.(type) {
|
||||
case float64:
|
||||
return int64(val), true
|
||||
case int:
|
||||
return int64(val), true
|
||||
case int64:
|
||||
return val, true
|
||||
case string:
|
||||
if i, err := strconv.ParseInt(val, 10, 64); err == nil {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// Stop gracefully stops the source.
|
||||
func (s *UnixSocketSource) Stop() error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
close(s.done)
|
||||
|
||||
if s.listener != nil {
|
||||
s.listener.Close()
|
||||
}
|
||||
|
||||
s.wg.Wait()
|
||||
|
||||
// Clean up socket file
|
||||
if err := os.Remove(s.config.Path); err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("failed to remove socket file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
98
internal/adapters/inbound/unixsocket/source_test.go
Normal file
98
internal/adapters/inbound/unixsocket/source_test.go
Normal file
@ -0,0 +1,98 @@
|
||||
package unixsocket
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestParseJSONEvent_Apache(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"dst_ip": "10.0.0.1",
|
||||
"dst_port": 80,
|
||||
"timestamp": 1704110400000000000,
|
||||
"method": "GET",
|
||||
"path": "/api/test",
|
||||
"header_host": "example.com",
|
||||
"header_user_agent": "Mozilla/5.0"
|
||||
}`)
|
||||
|
||||
event, err := parseJSONEvent(data)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if event.SrcIP != "192.168.1.1" {
|
||||
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
|
||||
}
|
||||
if event.SrcPort != 8080 {
|
||||
t.Errorf("expected src_port 8080, got %d", event.SrcPort)
|
||||
}
|
||||
if event.Headers["host"] != "example.com" {
|
||||
t.Errorf("expected header host example.com, got %s", event.Headers["host"])
|
||||
}
|
||||
if event.Headers["user_agent"] != "Mozilla/5.0" {
|
||||
t.Errorf("expected header_user_agent Mozilla/5.0, got %s", event.Headers["user_agent"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_Network(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"dst_ip": "10.0.0.1",
|
||||
"dst_port": 443,
|
||||
"ja3": "abc123def456",
|
||||
"ja4": "xyz789",
|
||||
"tcp_meta_flags": "SYN"
|
||||
}`)
|
||||
|
||||
event, err := parseJSONEvent(data)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if event.SrcIP != "192.168.1.1" {
|
||||
t.Errorf("expected src_ip 192.168.1.1, got %s", event.SrcIP)
|
||||
}
|
||||
if event.Extra["ja3"] != "abc123def456" {
|
||||
t.Errorf("expected ja3 abc123def456, got %v", event.Extra["ja3"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_InvalidJSON(t *testing.T) {
|
||||
data := []byte(`{invalid json}`)
|
||||
|
||||
_, err := parseJSONEvent(data)
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid JSON")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_MissingFields(t *testing.T) {
|
||||
data := []byte(`{"other_field": "value"}`)
|
||||
|
||||
_, err := parseJSONEvent(data)
|
||||
if err == nil {
|
||||
t.Error("expected error for missing src_ip/src_port")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJSONEvent_StringTimestamp(t *testing.T) {
|
||||
data := []byte(`{
|
||||
"src_ip": "192.168.1.1",
|
||||
"src_port": 8080,
|
||||
"time": "2024-01-01T12:00:00Z"
|
||||
}`)
|
||||
|
||||
event, err := parseJSONEvent(data)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
expected := time.Date(2024, 1, 1, 12, 0, 0, 0, time.UTC)
|
||||
if !event.Timestamp.Equal(expected) {
|
||||
t.Errorf("expected timestamp %v, got %v", expected, event.Timestamp)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user