Initial commit: logcorrelator with unified packaging (DEB + RPM using fpm)
Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
340
internal/config/config.go
Normal file
340
internal/config/config.go
Normal file
@ -0,0 +1,340 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Config holds the complete application configuration.
|
||||
type Config struct {
|
||||
Service ServiceConfig
|
||||
Inputs InputsConfig
|
||||
Outputs OutputsConfig
|
||||
Correlation CorrelationConfig
|
||||
}
|
||||
|
||||
// ServiceConfig holds service-level configuration.
|
||||
type ServiceConfig struct {
|
||||
Name string
|
||||
Language string
|
||||
}
|
||||
|
||||
// InputsConfig holds input sources configuration.
|
||||
type InputsConfig struct {
|
||||
UnixSockets []UnixSocketConfig
|
||||
}
|
||||
|
||||
// UnixSocketConfig holds a Unix socket source configuration.
|
||||
type UnixSocketConfig struct {
|
||||
Name string
|
||||
Path string
|
||||
Format string
|
||||
}
|
||||
|
||||
// OutputsConfig holds output sinks configuration.
|
||||
type OutputsConfig struct {
|
||||
File FileOutputConfig
|
||||
ClickHouse ClickHouseOutputConfig
|
||||
Stdout StdoutOutputConfig
|
||||
}
|
||||
|
||||
// FileOutputConfig holds file sink configuration.
|
||||
type FileOutputConfig struct {
|
||||
Enabled bool
|
||||
Path string
|
||||
}
|
||||
|
||||
// ClickHouseOutputConfig holds ClickHouse sink configuration.
|
||||
type ClickHouseOutputConfig struct {
|
||||
Enabled bool
|
||||
DSN string
|
||||
Table string
|
||||
BatchSize int
|
||||
FlushIntervalMs int
|
||||
MaxBufferSize int
|
||||
DropOnOverflow bool
|
||||
AsyncInsert bool
|
||||
TimeoutMs int
|
||||
}
|
||||
|
||||
// StdoutOutputConfig holds stdout sink configuration.
|
||||
type StdoutOutputConfig struct {
|
||||
Enabled bool
|
||||
}
|
||||
|
||||
// CorrelationConfig holds correlation configuration.
|
||||
type CorrelationConfig struct {
|
||||
Key []string
|
||||
TimeWindow TimeWindowConfig
|
||||
OrphanPolicy OrphanPolicyConfig
|
||||
}
|
||||
|
||||
// TimeWindowConfig holds time window configuration.
|
||||
type TimeWindowConfig struct {
|
||||
Value int
|
||||
Unit string
|
||||
}
|
||||
|
||||
// OrphanPolicyConfig holds orphan event policy configuration.
|
||||
type OrphanPolicyConfig struct {
|
||||
ApacheAlwaysEmit bool
|
||||
NetworkEmit bool
|
||||
}
|
||||
|
||||
// Load loads configuration from a text file with directives.
|
||||
func Load(path string) (*Config, error) {
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to open config file: %w", err)
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
cfg := &Config{
|
||||
Service: ServiceConfig{
|
||||
Name: "logcorrelator",
|
||||
Language: "go",
|
||||
},
|
||||
Inputs: InputsConfig{
|
||||
UnixSockets: make([]UnixSocketConfig, 0),
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{
|
||||
Enabled: true,
|
||||
Path: "/var/log/logcorrelator/correlated.log",
|
||||
},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: false,
|
||||
BatchSize: 500,
|
||||
FlushIntervalMs: 200,
|
||||
MaxBufferSize: 5000,
|
||||
DropOnOverflow: true,
|
||||
AsyncInsert: true,
|
||||
TimeoutMs: 1000,
|
||||
},
|
||||
Stdout: StdoutOutputConfig{
|
||||
Enabled: false,
|
||||
},
|
||||
},
|
||||
Correlation: CorrelationConfig{
|
||||
Key: []string{"src_ip", "src_port"},
|
||||
TimeWindow: TimeWindowConfig{
|
||||
Value: 1,
|
||||
Unit: "s",
|
||||
},
|
||||
OrphanPolicy: OrphanPolicyConfig{
|
||||
ApacheAlwaysEmit: true,
|
||||
NetworkEmit: false,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
scanner := bufio.NewScanner(file)
|
||||
lineNum := 0
|
||||
|
||||
for scanner.Scan() {
|
||||
lineNum++
|
||||
line := strings.TrimSpace(scanner.Text())
|
||||
|
||||
// Skip empty lines and comments
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
|
||||
if err := parseDirective(cfg, line); err != nil {
|
||||
return nil, fmt.Errorf("line %d: %w", lineNum, err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to read config file: %w", err)
|
||||
}
|
||||
|
||||
if err := cfg.Validate(); err != nil {
|
||||
return nil, fmt.Errorf("invalid config: %w", err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
func parseDirective(cfg *Config, line string) error {
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) < 2 {
|
||||
return fmt.Errorf("invalid directive: %s", line)
|
||||
}
|
||||
|
||||
directive := parts[0]
|
||||
value := strings.Join(parts[1:], " ")
|
||||
|
||||
switch directive {
|
||||
case "service.name":
|
||||
cfg.Service.Name = value
|
||||
case "service.language":
|
||||
cfg.Service.Language = value
|
||||
|
||||
case "input.unix_socket":
|
||||
// Format: input.unix_socket <name> <path> [format]
|
||||
if len(parts) < 3 {
|
||||
return fmt.Errorf("input.unix_socket requires name and path")
|
||||
}
|
||||
format := "json"
|
||||
if len(parts) >= 4 {
|
||||
format = parts[3]
|
||||
}
|
||||
cfg.Inputs.UnixSockets = append(cfg.Inputs.UnixSockets, UnixSocketConfig{
|
||||
Name: parts[1],
|
||||
Path: parts[2],
|
||||
Format: format,
|
||||
})
|
||||
|
||||
case "output.file.enabled":
|
||||
enabled, err := parseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.file.enabled: %w", err)
|
||||
}
|
||||
cfg.Outputs.File.Enabled = enabled
|
||||
case "output.file.path":
|
||||
cfg.Outputs.File.Path = value
|
||||
|
||||
case "output.clickhouse.enabled":
|
||||
enabled, err := parseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.clickhouse.enabled: %w", err)
|
||||
}
|
||||
cfg.Outputs.ClickHouse.Enabled = enabled
|
||||
case "output.clickhouse.dsn":
|
||||
cfg.Outputs.ClickHouse.DSN = value
|
||||
case "output.clickhouse.table":
|
||||
cfg.Outputs.ClickHouse.Table = value
|
||||
case "output.clickhouse.batch_size":
|
||||
v, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.clickhouse.batch_size: %w", err)
|
||||
}
|
||||
cfg.Outputs.ClickHouse.BatchSize = v
|
||||
case "output.clickhouse.flush_interval_ms":
|
||||
v, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.clickhouse.flush_interval_ms: %w", err)
|
||||
}
|
||||
cfg.Outputs.ClickHouse.FlushIntervalMs = v
|
||||
case "output.clickhouse.max_buffer_size":
|
||||
v, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.clickhouse.max_buffer_size: %w", err)
|
||||
}
|
||||
cfg.Outputs.ClickHouse.MaxBufferSize = v
|
||||
case "output.clickhouse.drop_on_overflow":
|
||||
enabled, err := parseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.clickhouse.drop_on_overflow: %w", err)
|
||||
}
|
||||
cfg.Outputs.ClickHouse.DropOnOverflow = enabled
|
||||
case "output.clickhouse.async_insert":
|
||||
enabled, err := parseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.clickhouse.async_insert: %w", err)
|
||||
}
|
||||
cfg.Outputs.ClickHouse.AsyncInsert = enabled
|
||||
case "output.clickhouse.timeout_ms":
|
||||
v, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.clickhouse.timeout_ms: %w", err)
|
||||
}
|
||||
cfg.Outputs.ClickHouse.TimeoutMs = v
|
||||
|
||||
case "output.stdout.enabled":
|
||||
enabled, err := parseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for output.stdout.enabled: %w", err)
|
||||
}
|
||||
cfg.Outputs.Stdout.Enabled = enabled
|
||||
|
||||
case "correlation.key":
|
||||
cfg.Correlation.Key = strings.Split(value, ",")
|
||||
for i, k := range cfg.Correlation.Key {
|
||||
cfg.Correlation.Key[i] = strings.TrimSpace(k)
|
||||
}
|
||||
case "correlation.time_window.value":
|
||||
v, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for correlation.time_window.value: %w", err)
|
||||
}
|
||||
cfg.Correlation.TimeWindow.Value = v
|
||||
case "correlation.time_window.unit":
|
||||
cfg.Correlation.TimeWindow.Unit = value
|
||||
case "correlation.orphan_policy.apache_always_emit":
|
||||
enabled, err := parseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for correlation.orphan_policy.apache_always_emit: %w", err)
|
||||
}
|
||||
cfg.Correlation.OrphanPolicy.ApacheAlwaysEmit = enabled
|
||||
case "correlation.orphan_policy.network_emit":
|
||||
enabled, err := parseBool(value)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid value for correlation.orphan_policy.network_emit: %w", err)
|
||||
}
|
||||
cfg.Correlation.OrphanPolicy.NetworkEmit = enabled
|
||||
|
||||
default:
|
||||
return fmt.Errorf("unknown directive: %s", directive)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseBool(s string) (bool, error) {
|
||||
s = strings.ToLower(s)
|
||||
switch s {
|
||||
case "true", "yes", "1", "on":
|
||||
return true, nil
|
||||
case "false", "no", "0", "off":
|
||||
return false, nil
|
||||
default:
|
||||
return false, fmt.Errorf("invalid boolean value: %s", s)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate validates the configuration.
|
||||
func (c *Config) Validate() error {
|
||||
if len(c.Inputs.UnixSockets) < 2 {
|
||||
return fmt.Errorf("at least two unix socket inputs are required")
|
||||
}
|
||||
|
||||
if !c.Outputs.File.Enabled && !c.Outputs.ClickHouse.Enabled && !c.Outputs.Stdout.Enabled {
|
||||
return fmt.Errorf("at least one output must be enabled")
|
||||
}
|
||||
|
||||
if c.Outputs.ClickHouse.Enabled && c.Outputs.ClickHouse.DSN == "" {
|
||||
return fmt.Errorf("clickhouse DSN is required when enabled")
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GetTimeWindow returns the time window as a duration.
|
||||
func (c *CorrelationConfig) GetTimeWindow() time.Duration {
|
||||
value := c.TimeWindow.Value
|
||||
if value <= 0 {
|
||||
value = 1
|
||||
}
|
||||
|
||||
unit := c.TimeWindow.Unit
|
||||
if unit == "" {
|
||||
unit = "s"
|
||||
}
|
||||
|
||||
switch unit {
|
||||
case "ms", "millisecond", "milliseconds":
|
||||
return time.Duration(value) * time.Millisecond
|
||||
case "s", "second", "seconds":
|
||||
return time.Duration(value) * time.Second
|
||||
case "m", "minute", "minutes":
|
||||
return time.Duration(value) * time.Minute
|
||||
default:
|
||||
return time.Duration(value) * time.Second
|
||||
}
|
||||
}
|
||||
224
internal/config/config_test.go
Normal file
224
internal/config/config_test.go
Normal file
@ -0,0 +1,224 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLoad_ValidConfig(t *testing.T) {
|
||||
content := `
|
||||
# Test configuration
|
||||
service.name logcorrelator
|
||||
service.language go
|
||||
|
||||
input.unix_socket apache_source /var/run/logcorrelator/apache.sock json
|
||||
input.unix_socket network_source /var/run/logcorrelator/network.sock json
|
||||
|
||||
output.file.enabled true
|
||||
output.file.path /var/log/logcorrelator/correlated.log
|
||||
|
||||
output.clickhouse.enabled false
|
||||
output.clickhouse.dsn clickhouse://user:pass@localhost:9000/db
|
||||
output.clickhouse.table correlated_logs
|
||||
|
||||
correlation.key src_ip,src_port
|
||||
correlation.time_window.value 1
|
||||
correlation.time_window.unit s
|
||||
|
||||
correlation.orphan_policy.apache_always_emit true
|
||||
correlation.orphan_policy.network_emit false
|
||||
`
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "config.conf")
|
||||
if err := os.WriteFile(configPath, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Service.Name != "logcorrelator" {
|
||||
t.Errorf("expected service name logcorrelator, got %s", cfg.Service.Name)
|
||||
}
|
||||
if len(cfg.Inputs.UnixSockets) != 2 {
|
||||
t.Errorf("expected 2 unix sockets, got %d", len(cfg.Inputs.UnixSockets))
|
||||
}
|
||||
if !cfg.Outputs.File.Enabled {
|
||||
t.Error("expected file output enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_InvalidPath(t *testing.T) {
|
||||
_, err := Load("/nonexistent/path/config.conf")
|
||||
if err == nil {
|
||||
t.Error("expected error for nonexistent path")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_InvalidDirective(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "config.conf")
|
||||
content := `invalid.directive value`
|
||||
if err := os.WriteFile(configPath, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to write config: %v", err)
|
||||
}
|
||||
|
||||
_, err := Load(configPath)
|
||||
if err == nil {
|
||||
t.Error("expected error for invalid directive")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_Comments(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "config.conf")
|
||||
content := `
|
||||
# This is a comment
|
||||
service.name logcorrelator
|
||||
# Another comment
|
||||
input.unix_socket test /tmp/test.sock json
|
||||
input.unix_socket test2 /tmp/test2.sock json
|
||||
output.file.enabled true
|
||||
`
|
||||
if err := os.WriteFile(configPath, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Service.Name != "logcorrelator" {
|
||||
t.Errorf("expected service name logcorrelator, got %s", cfg.Service.Name)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_MinimumInputs(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Inputs: InputsConfig{
|
||||
UnixSockets: []UnixSocketConfig{
|
||||
{Name: "only_one", Path: "/tmp/test.sock"},
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Error("expected error for less than 2 inputs")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_AtLeastOneOutput(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Inputs: InputsConfig{
|
||||
UnixSockets: []UnixSocketConfig{
|
||||
{Name: "a", Path: "/tmp/a.sock"},
|
||||
{Name: "b", Path: "/tmp/b.sock"},
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
ClickHouse: ClickHouseOutputConfig{Enabled: false},
|
||||
Stdout: StdoutOutputConfig{Enabled: false},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Error("expected error for no outputs enabled")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTimeWindow(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config CorrelationConfig
|
||||
expected time.Duration
|
||||
}{
|
||||
{
|
||||
name: "seconds",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{Value: 1, Unit: "s"},
|
||||
},
|
||||
expected: time.Second,
|
||||
},
|
||||
{
|
||||
name: "milliseconds",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{Value: 500, Unit: "ms"},
|
||||
},
|
||||
expected: 500 * time.Millisecond,
|
||||
},
|
||||
{
|
||||
name: "minutes",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{Value: 2, Unit: "m"},
|
||||
},
|
||||
expected: 2 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "default",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{},
|
||||
},
|
||||
expected: time.Second,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tt.config.GetTimeWindow()
|
||||
if result != tt.expected {
|
||||
t.Errorf("expected %v, got %v", tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseBool(t *testing.T) {
|
||||
tests := []struct {
|
||||
input string
|
||||
expected bool
|
||||
hasError bool
|
||||
}{
|
||||
{"true", true, false},
|
||||
{"True", true, false},
|
||||
{"TRUE", true, false},
|
||||
{"yes", true, false},
|
||||
{"1", true, false},
|
||||
{"on", true, false},
|
||||
{"false", false, false},
|
||||
{"False", false, false},
|
||||
{"no", false, false},
|
||||
{"0", false, false},
|
||||
{"off", false, false},
|
||||
{"invalid", false, true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.input, func(t *testing.T) {
|
||||
result, err := parseBool(tt.input)
|
||||
if tt.hasError {
|
||||
if err == nil {
|
||||
t.Error("expected error, got nil")
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
t.Errorf("unexpected error: %v", err)
|
||||
}
|
||||
if result != tt.expected {
|
||||
t.Errorf("expected %v, got %v", tt.expected, result)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user