chore: version 1.0.6 - simplify YAML configuration
- Remove service.name and service.language (unused) - Remove enabled flags on outputs (presence = enabled) - Simplify correlation config: time_window_s (integer) instead of nested object - Simplify orphan_policy to emit_orphans boolean - Rename apache socket to http.socket - Add socket_permissions option for unix sockets (default: 0660) - Update tests for new configuration format Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
@ -3,6 +3,7 @@ package config
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -11,7 +12,6 @@ import (
|
||||
|
||||
// Config holds the complete application configuration.
|
||||
type Config struct {
|
||||
Service ServiceConfig `yaml:"service"`
|
||||
Inputs InputsConfig `yaml:"inputs"`
|
||||
Outputs OutputsConfig `yaml:"outputs"`
|
||||
Correlation CorrelationConfig `yaml:"correlation"`
|
||||
@ -30,23 +30,23 @@ type InputsConfig struct {
|
||||
|
||||
// UnixSocketConfig holds a Unix socket source configuration.
|
||||
type UnixSocketConfig struct {
|
||||
Name string `yaml:"name"`
|
||||
Path string `yaml:"path"`
|
||||
Format string `yaml:"format"`
|
||||
SourceType string `yaml:"source_type"` // "A" for Apache/HTTP, "B" for Network
|
||||
Name string `yaml:"name"`
|
||||
Path string `yaml:"path"`
|
||||
Format string `yaml:"format"`
|
||||
SourceType string `yaml:"source_type"` // "A" for Apache/HTTP, "B" for Network
|
||||
SocketPermissions string `yaml:"socket_permissions"` // octal string, e.g., "0660", "0666"
|
||||
}
|
||||
|
||||
// OutputsConfig holds output sinks configuration.
|
||||
type OutputsConfig struct {
|
||||
File FileOutputConfig `yaml:"file"`
|
||||
ClickHouse ClickHouseOutputConfig `yaml:"clickhouse"`
|
||||
Stdout StdoutOutputConfig `yaml:"stdout"`
|
||||
Stdout bool `yaml:"stdout"`
|
||||
}
|
||||
|
||||
// FileOutputConfig holds file sink configuration.
|
||||
type FileOutputConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Path string `yaml:"path"`
|
||||
Path string `yaml:"path"`
|
||||
}
|
||||
|
||||
// ClickHouseOutputConfig holds ClickHouse sink configuration.
|
||||
@ -63,27 +63,15 @@ type ClickHouseOutputConfig struct {
|
||||
}
|
||||
|
||||
// StdoutOutputConfig holds stdout sink configuration.
|
||||
// Deprecated: stdout is now a boolean flag in OutputsConfig.
|
||||
type StdoutOutputConfig struct {
|
||||
Enabled bool `yaml:"enabled"`
|
||||
}
|
||||
|
||||
// CorrelationConfig holds correlation configuration.
|
||||
type CorrelationConfig struct {
|
||||
Key []string `yaml:"key"`
|
||||
TimeWindow TimeWindowConfig `yaml:"time_window"`
|
||||
OrphanPolicy OrphanPolicyConfig `yaml:"orphan_policy"`
|
||||
}
|
||||
|
||||
// TimeWindowConfig holds time window configuration.
|
||||
type TimeWindowConfig struct {
|
||||
Value int `yaml:"value"`
|
||||
Unit string `yaml:"unit"`
|
||||
}
|
||||
|
||||
// OrphanPolicyConfig holds orphan event policy configuration.
|
||||
type OrphanPolicyConfig struct {
|
||||
ApacheAlwaysEmit bool `yaml:"apache_always_emit"`
|
||||
NetworkEmit bool `yaml:"network_emit"`
|
||||
TimeWindowS int `yaml:"time_window_s"`
|
||||
EmitOrphans bool `yaml:"emit_orphans"`
|
||||
}
|
||||
|
||||
// Load loads configuration from a YAML file.
|
||||
@ -109,17 +97,12 @@ func Load(path string) (*Config, error) {
|
||||
// defaultConfig returns a Config with default values.
|
||||
func defaultConfig() *Config {
|
||||
return &Config{
|
||||
Service: ServiceConfig{
|
||||
Name: "logcorrelator",
|
||||
Language: "go",
|
||||
},
|
||||
Inputs: InputsConfig{
|
||||
UnixSockets: make([]UnixSocketConfig, 0),
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{
|
||||
Enabled: true,
|
||||
Path: "/var/log/logcorrelator/correlated.log",
|
||||
Path: "/var/log/logcorrelator/correlated.log",
|
||||
},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: false,
|
||||
@ -130,20 +113,11 @@ func defaultConfig() *Config {
|
||||
AsyncInsert: true,
|
||||
TimeoutMs: 1000,
|
||||
},
|
||||
Stdout: StdoutOutputConfig{
|
||||
Enabled: false,
|
||||
},
|
||||
Stdout: false,
|
||||
},
|
||||
Correlation: CorrelationConfig{
|
||||
Key: []string{"src_ip", "src_port"},
|
||||
TimeWindow: TimeWindowConfig{
|
||||
Value: 1,
|
||||
Unit: "s",
|
||||
},
|
||||
OrphanPolicy: OrphanPolicyConfig{
|
||||
ApacheAlwaysEmit: true,
|
||||
NetworkEmit: false,
|
||||
},
|
||||
TimeWindowS: 1,
|
||||
EmitOrphans: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
@ -176,12 +150,20 @@ func (c *Config) Validate() error {
|
||||
seenPaths[input.Path] = struct{}{}
|
||||
}
|
||||
|
||||
if !c.Outputs.File.Enabled && !c.Outputs.ClickHouse.Enabled && !c.Outputs.Stdout.Enabled {
|
||||
return fmt.Errorf("at least one output must be enabled")
|
||||
// At least one output must be enabled
|
||||
hasOutput := false
|
||||
if c.Outputs.File.Path != "" {
|
||||
hasOutput = true
|
||||
}
|
||||
if c.Outputs.ClickHouse.Enabled {
|
||||
hasOutput = true
|
||||
}
|
||||
if c.Outputs.Stdout {
|
||||
hasOutput = true
|
||||
}
|
||||
|
||||
if c.Outputs.File.Enabled && strings.TrimSpace(c.Outputs.File.Path) == "" {
|
||||
return fmt.Errorf("file output path is required when file output is enabled")
|
||||
if !hasOutput {
|
||||
return fmt.Errorf("at least one output must be enabled (file, clickhouse, or stdout)")
|
||||
}
|
||||
|
||||
if c.Outputs.ClickHouse.Enabled {
|
||||
@ -202,11 +184,8 @@ func (c *Config) Validate() error {
|
||||
}
|
||||
}
|
||||
|
||||
if len(c.Correlation.Key) == 0 {
|
||||
return fmt.Errorf("correlation.key cannot be empty")
|
||||
}
|
||||
if c.Correlation.TimeWindow.Value <= 0 {
|
||||
return fmt.Errorf("correlation.time_window.value must be > 0")
|
||||
if c.Correlation.TimeWindowS <= 0 {
|
||||
return fmt.Errorf("correlation.time_window_s must be > 0")
|
||||
}
|
||||
|
||||
return nil
|
||||
@ -214,24 +193,25 @@ func (c *Config) Validate() error {
|
||||
|
||||
// GetTimeWindow returns the time window as a duration.
|
||||
func (c *CorrelationConfig) GetTimeWindow() time.Duration {
|
||||
value := c.TimeWindow.Value
|
||||
value := c.TimeWindowS
|
||||
if value <= 0 {
|
||||
value = 1
|
||||
}
|
||||
|
||||
unit := c.TimeWindow.Unit
|
||||
if unit == "" {
|
||||
unit = "s"
|
||||
}
|
||||
|
||||
switch unit {
|
||||
case "ms", "millisecond", "milliseconds":
|
||||
return time.Duration(value) * time.Millisecond
|
||||
case "s", "second", "seconds":
|
||||
return time.Duration(value) * time.Second
|
||||
case "m", "minute", "minutes":
|
||||
return time.Duration(value) * time.Minute
|
||||
default:
|
||||
return time.Duration(value) * time.Second
|
||||
}
|
||||
return time.Duration(value) * time.Second
|
||||
}
|
||||
|
||||
// GetSocketPermissions returns the socket permissions as os.FileMode.
|
||||
// Default is 0660 (owner + group read/write).
|
||||
func (c *UnixSocketConfig) GetSocketPermissions() os.FileMode {
|
||||
if c.SocketPermissions == "" {
|
||||
return 0660
|
||||
}
|
||||
|
||||
// Parse octal string (e.g., "0660", "660", "0666")
|
||||
perms, err := strconv.ParseUint(strings.TrimPrefix(c.SocketPermissions, "0"), 8, 32)
|
||||
if err != nil {
|
||||
return 0660
|
||||
}
|
||||
|
||||
return os.FileMode(perms)
|
||||
}
|
||||
|
||||
@ -9,38 +9,25 @@ import (
|
||||
|
||||
func TestLoad_ValidConfig(t *testing.T) {
|
||||
content := `
|
||||
service:
|
||||
name: logcorrelator
|
||||
language: go
|
||||
|
||||
inputs:
|
||||
unix_sockets:
|
||||
- name: apache_source
|
||||
path: /var/run/logcorrelator/apache.sock
|
||||
- name: http
|
||||
path: /var/run/logcorrelator/http.socket
|
||||
format: json
|
||||
- name: network_source
|
||||
path: /var/run/logcorrelator/network.sock
|
||||
- name: network
|
||||
path: /var/run/logcorrelator/network.socket
|
||||
format: json
|
||||
|
||||
outputs:
|
||||
file:
|
||||
enabled: true
|
||||
path: /var/log/logcorrelator/correlated.log
|
||||
clickhouse:
|
||||
enabled: false
|
||||
dsn: clickhouse://user:pass@localhost:9000/db
|
||||
table: correlated_logs
|
||||
|
||||
correlation:
|
||||
key:
|
||||
- src_ip
|
||||
- src_port
|
||||
time_window:
|
||||
value: 1
|
||||
unit: s
|
||||
orphan_policy:
|
||||
apache_always_emit: true
|
||||
network_emit: false
|
||||
time_window_s: 1
|
||||
emit_orphans: true
|
||||
`
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
@ -54,14 +41,11 @@ correlation:
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Service.Name != "logcorrelator" {
|
||||
t.Errorf("expected service name logcorrelator, got %s", cfg.Service.Name)
|
||||
}
|
||||
if len(cfg.Inputs.UnixSockets) != 2 {
|
||||
t.Errorf("expected 2 unix sockets, got %d", len(cfg.Inputs.UnixSockets))
|
||||
}
|
||||
if !cfg.Outputs.File.Enabled {
|
||||
t.Error("expected file output enabled")
|
||||
if cfg.Outputs.File.Path != "/var/log/logcorrelator/correlated.log" {
|
||||
t.Errorf("expected file path /var/log/logcorrelator/correlated.log, got %s", cfg.Outputs.File.Path)
|
||||
}
|
||||
}
|
||||
|
||||
@ -90,8 +74,6 @@ func TestLoad_DefaultValues(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "config.yml")
|
||||
content := `
|
||||
service:
|
||||
name: test-service
|
||||
inputs:
|
||||
unix_sockets:
|
||||
- name: a
|
||||
@ -100,7 +82,7 @@ inputs:
|
||||
path: /tmp/b.sock
|
||||
outputs:
|
||||
file:
|
||||
enabled: true
|
||||
path: /var/log/test.log
|
||||
`
|
||||
if err := os.WriteFile(configPath, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to write config: %v", err)
|
||||
@ -111,15 +93,12 @@ outputs:
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Service.Name != "test-service" {
|
||||
t.Errorf("expected service name test-service, got %s", cfg.Service.Name)
|
||||
}
|
||||
// Check defaults
|
||||
if cfg.Correlation.TimeWindow.Value != 1 {
|
||||
t.Errorf("expected default time window value 1, got %d", cfg.Correlation.TimeWindow.Value)
|
||||
if cfg.Correlation.TimeWindowS != 1 {
|
||||
t.Errorf("expected default time window value 1, got %d", cfg.Correlation.TimeWindowS)
|
||||
}
|
||||
if cfg.Correlation.OrphanPolicy.ApacheAlwaysEmit != true {
|
||||
t.Error("expected default apache_always_emit to be true")
|
||||
if cfg.Correlation.EmitOrphans != true {
|
||||
t.Error("expected default emit_orphans to be true")
|
||||
}
|
||||
}
|
||||
|
||||
@ -131,7 +110,7 @@ func TestValidate_MinimumInputs(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -150,9 +129,9 @@ func TestValidate_AtLeastOneOutput(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
File: FileOutputConfig{},
|
||||
ClickHouse: ClickHouseOutputConfig{Enabled: false},
|
||||
Stdout: StdoutOutputConfig{Enabled: false},
|
||||
Stdout: false,
|
||||
},
|
||||
}
|
||||
|
||||
@ -169,30 +148,23 @@ func TestGetTimeWindow(t *testing.T) {
|
||||
expected time.Duration
|
||||
}{
|
||||
{
|
||||
name: "seconds",
|
||||
name: "1 second",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{Value: 1, Unit: "s"},
|
||||
TimeWindowS: 1,
|
||||
},
|
||||
expected: time.Second,
|
||||
},
|
||||
{
|
||||
name: "milliseconds",
|
||||
name: "5 seconds",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{Value: 500, Unit: "ms"},
|
||||
TimeWindowS: 5,
|
||||
},
|
||||
expected: 500 * time.Millisecond,
|
||||
},
|
||||
{
|
||||
name: "minutes",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{Value: 2, Unit: "m"},
|
||||
},
|
||||
expected: 2 * time.Minute,
|
||||
expected: 5 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "default",
|
||||
config: CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{},
|
||||
TimeWindowS: 0,
|
||||
},
|
||||
expected: time.Second,
|
||||
},
|
||||
@ -217,7 +189,7 @@ func TestValidate_DuplicateNames(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -236,7 +208,7 @@ func TestValidate_DuplicatePaths(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -255,7 +227,7 @@ func TestValidate_EmptyName(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -274,7 +246,7 @@ func TestValidate_EmptyPath(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -293,7 +265,7 @@ func TestValidate_EmptyFilePath(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true, Path: ""},
|
||||
File: FileOutputConfig{Path: ""},
|
||||
},
|
||||
}
|
||||
|
||||
@ -312,7 +284,7 @@ func TestValidate_ClickHouseMissingDSN(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
File: FileOutputConfig{Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "",
|
||||
@ -336,7 +308,7 @@ func TestValidate_ClickHouseMissingTable(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
File: FileOutputConfig{Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -360,7 +332,7 @@ func TestValidate_ClickHouseInvalidBatchSize(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
File: FileOutputConfig{Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -385,7 +357,7 @@ func TestValidate_ClickHouseInvalidMaxBufferSize(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
File: FileOutputConfig{Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -411,7 +383,7 @@ func TestValidate_ClickHouseInvalidTimeout(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
File: FileOutputConfig{Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -437,33 +409,10 @@ func TestValidate_EmptyCorrelationKey(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
},
|
||||
Correlation: CorrelationConfig{
|
||||
Key: []string{},
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err == nil {
|
||||
t.Error("expected error for empty correlation key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_InvalidTimeWindow(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Inputs: InputsConfig{
|
||||
UnixSockets: []UnixSocketConfig{
|
||||
{Name: "a", Path: "/tmp/a.sock"},
|
||||
{Name: "b", Path: "/tmp/b.sock"},
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: true},
|
||||
},
|
||||
Correlation: CorrelationConfig{
|
||||
Key: []string{"src_ip", "src_port"},
|
||||
TimeWindow: TimeWindowConfig{Value: 0},
|
||||
TimeWindowS: 0,
|
||||
},
|
||||
}
|
||||
|
||||
@ -473,13 +422,55 @@ func TestValidate_InvalidTimeWindow(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestGetTimeWindow_UnknownUnit(t *testing.T) {
|
||||
config := CorrelationConfig{
|
||||
TimeWindow: TimeWindowConfig{Value: 5, Unit: "unknown"},
|
||||
func TestGetSocketPermissions(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
config UnixSocketConfig
|
||||
expected os.FileMode
|
||||
}{
|
||||
{
|
||||
name: "default",
|
||||
config: UnixSocketConfig{
|
||||
SocketPermissions: "",
|
||||
},
|
||||
expected: 0660,
|
||||
},
|
||||
{
|
||||
name: "explicit 0660",
|
||||
config: UnixSocketConfig{
|
||||
SocketPermissions: "0660",
|
||||
},
|
||||
expected: 0660,
|
||||
},
|
||||
{
|
||||
name: "explicit 0666",
|
||||
config: UnixSocketConfig{
|
||||
SocketPermissions: "0666",
|
||||
},
|
||||
expected: 0666,
|
||||
},
|
||||
{
|
||||
name: "without leading zero",
|
||||
config: UnixSocketConfig{
|
||||
SocketPermissions: "660",
|
||||
},
|
||||
expected: 0660,
|
||||
},
|
||||
{
|
||||
name: "invalid value",
|
||||
config: UnixSocketConfig{
|
||||
SocketPermissions: "invalid",
|
||||
},
|
||||
expected: 0660,
|
||||
},
|
||||
}
|
||||
result := config.GetTimeWindow()
|
||||
expected := 5 * time.Second // Should default to seconds
|
||||
if result != expected {
|
||||
t.Errorf("expected %v, got %v", expected, result)
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
result := tt.config.GetSocketPermissions()
|
||||
if result != tt.expected {
|
||||
t.Errorf("expected %o, got %o", tt.expected, result)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user