From 37f9c216722025c3822cbf6e6c7361ce2dbbdb39 Mon Sep 17 00:00:00 2001 From: Jacquin Antoine Date: Fri, 27 Feb 2026 15:51:25 +0100 Subject: [PATCH] feat: migrate configuration from custom format to YAML - Replace custom directive-based config parser with YAML using gopkg.in/yaml.v3 - Rename config.example.conf to config.example.yml with YAML syntax - Update default config path to /etc/logcorrelator/logcorrelator.yml - Update Dockerfile.package to copy YAML config files - Update packaging scripts to install logcorrelator.yml - Update architecture.yml to document YAML configuration - Add yaml.v3 dependency to go.mod Co-authored-by: Qwen-Coder --- Dockerfile.package | 16 +-- architecture.yml | 94 +++++++------- config.example.yml | 45 +++++++ go.mod | 2 + go.sum | 3 + internal/config/config.go | 258 ++++++++------------------------------ packaging/deb/postinst | 8 +- 7 files changed, 163 insertions(+), 263 deletions(-) create mode 100644 config.example.yml diff --git a/Dockerfile.package b/Dockerfile.package index 642c0f1..80707dd 100644 --- a/Dockerfile.package +++ b/Dockerfile.package @@ -47,15 +47,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Copy binary from builder COPY --from=builder /build/dist/logcorrelator /tmp/pkgroot/usr/bin/logcorrelator -COPY --from=builder /build/config.example.conf /tmp/pkgroot/etc/logcorrelator/logcorrelator.conf -COPY --from=builder /build/config.example.conf /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.conf.example +COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml +COPY --from=builder /build/config.example.yml /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.yml.example # Create directories and set permissions RUN mkdir -p /tmp/pkgroot/var/log/logcorrelator && \ mkdir -p /tmp/pkgroot/var/run/logcorrelator && \ chmod 755 /tmp/pkgroot/usr/bin/logcorrelator && \ - chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.conf && \ - chmod 640 /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.conf.example && \ + chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml && \ + chmod 640 /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.yml.example && \ chmod 755 /tmp/pkgroot/var/log/logcorrelator && \ chmod 755 /tmp/pkgroot/var/run/logcorrelator @@ -85,8 +85,8 @@ RUN mkdir -p /packages/deb && \ --after-remove /tmp/scripts/postrm \ -p /packages/deb/logcorrelator_${VERSION}_${ARCH}.deb \ usr/bin/logcorrelator \ - etc/logcorrelator/logcorrelator.conf \ - usr/share/logcorrelator/logcorrelator.conf.example \ + etc/logcorrelator/logcorrelator.yml \ + usr/share/logcorrelator/logcorrelator.yml.example \ var/log/logcorrelator \ var/run/logcorrelator @@ -108,8 +108,8 @@ RUN mkdir -p /packages/rpm && \ --after-remove /tmp/scripts/postrm \ -p /packages/rpm/logcorrelator-${VERSION}-1.x86_64.rpm \ usr/bin/logcorrelator \ - etc/logcorrelator/logcorrelator.conf \ - usr/share/logcorrelator/logcorrelator.conf.example \ + etc/logcorrelator/logcorrelator.yml \ + usr/share/logcorrelator/logcorrelator.yml.example \ var/log/logcorrelator \ var/run/logcorrelator diff --git a/architecture.yml b/architecture.yml index a391bb4..0b4c89b 100644 --- a/architecture.yml +++ b/architecture.yml @@ -21,11 +21,11 @@ runtime: unit_type: systemd description: > logcorrelator est livré sous forme de binaire autonome, exécuté comme un - service systemd. L’unité systemd assure le démarrage automatique au boot, - le redémarrage en cas de crash, et une intégration standard dans l’écosystème + service systemd. L'unité systemd assure le démarrage automatique au boot, + le redémarrage en cas de crash, et une intégration standard dans l'écosystème Linux (notamment sur Rocky Linux 8+). binary_path: /usr/bin/logcorrelator - config_path: /etc/logcorrelator/logcorrelator.toml + config_path: /etc/logcorrelator/logcorrelator.yml user: logcorrelator group: logcorrelator restart: on-failure @@ -40,7 +40,7 @@ runtime: Type=simple User=logcorrelator Group=logcorrelator - ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.toml + ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml Restart=on-failure RestartSec=5 @@ -68,51 +68,53 @@ runtime: de drop), ferme les connexions ClickHouse puis s’arrête. config: - format: toml - location: /etc/logcorrelator/logcorrelator.toml + format: yaml + location: /etc/logcorrelator/logcorrelator.yml description: > - Toute la configuration est centralisée dans un fichier TOML lisible, stocké - dans /etc/logcorrelator. Ni YAML ni JSON ne sont utilisés pour la config. + Toute la configuration est centralisée dans un fichier YAML lisible, + stocké dans /etc/logcorrelator. reload_strategy: restart_service example: | - [service] - name = "logcorrelator" - language = "go" + # Service configuration + service: + name: logcorrelator + language: go - [[inputs.unix_sockets]] - name = "apache_source" - path = "/var/run/logcorrelator/apache.sock" - format = "json" + # Input sources (at least 2 required) + inputs: + unix_sockets: + - name: apache_source + path: /var/run/logcorrelator/apache.sock + format: json + - name: network_source + path: /var/run/logcorrelator/network.sock + format: json - [[inputs.unix_sockets]] - name = "network_source" - path = "/var/run/logcorrelator/network.sock" - format = "json" + # File output + outputs: + file: + enabled: true + path: /var/log/logcorrelator/correlated.log - [outputs.file] - enabled = true - path = "/var/log/logcorrelator/correlated.log" + # ClickHouse output + outputs: + clickhouse: + enabled: false + dsn: clickhouse://user:pass@localhost:9000/db + table: correlated_logs_http_network + batch_size: 500 - [outputs.clickhouse] - enabled = true - dsn = "clickhouse://user:pass@host:9000/db" - table = "correlated_logs_http_network" - batch_size = 500 - flush_interval_ms = 200 - max_buffer_size = 5000 - drop_on_overflow = true - async_insert = true - - [correlation] - key = ["src_ip", "src_port"] - - [correlation.time_window] - value = 1 - unit = "s" - - [correlation.orphan_policy] - apache_always_emit = true - network_emit = false + # Correlation configuration + correlation: + key: + - src_ip + - src_port + time_window: + value: 1 + unit: s + orphan_policy: + apache_always_emit: true + network_emit: false inputs: description: > @@ -503,12 +505,12 @@ packaging: dest: /usr/bin/logcorrelator mode: "0755" config: - - source: config.example.conf - dest: /etc/logcorrelator/logcorrelator.conf + - source: config.example.yml + dest: /etc/logcorrelator/logcorrelator.yml mode: "0640" config_file: true - - source: config.example.conf - dest: /usr/share/logcorrelator/logcorrelator.conf.example + - source: config.example.yml + dest: /usr/share/logcorrelator/logcorrelator.yml.example mode: "0640" directories: - path: /var/log/logcorrelator diff --git a/config.example.yml b/config.example.yml new file mode 100644 index 0000000..59da8d7 --- /dev/null +++ b/config.example.yml @@ -0,0 +1,45 @@ +# logcorrelator configuration file +# Format: YAML + +service: + name: logcorrelator + language: go + +inputs: + unix_sockets: + - name: apache_source + path: /var/run/logcorrelator/apache.sock + format: json + - name: network_source + path: /var/run/logcorrelator/network.sock + format: json + +outputs: + file: + enabled: true + path: /var/log/logcorrelator/correlated.log + + clickhouse: + enabled: false + dsn: clickhouse://user:pass@localhost:9000/db + table: correlated_logs_http_network + batch_size: 500 + flush_interval_ms: 200 + max_buffer_size: 5000 + drop_on_overflow: true + async_insert: true + timeout_ms: 1000 + + stdout: + enabled: false + +correlation: + key: + - src_ip + - src_port + time_window: + value: 1 + unit: s + orphan_policy: + apache_always_emit: true + network_emit: false diff --git a/go.mod b/go.mod index 4ba01f4..8116323 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/logcorrelator/logcorrelator go 1.21 + +require gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index e69de29..4bc0337 100644 --- a/go.sum +++ b/go.sum @@ -0,0 +1,3 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go index b740bc2..59ff89c 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -1,99 +1,112 @@ package config import ( - "bufio" "fmt" "os" - "strconv" - "strings" "time" + + "gopkg.in/yaml.v3" ) // Config holds the complete application configuration. type Config struct { - Service ServiceConfig - Inputs InputsConfig - Outputs OutputsConfig - Correlation CorrelationConfig + Service ServiceConfig `yaml:"service"` + Inputs InputsConfig `yaml:"inputs"` + Outputs OutputsConfig `yaml:"outputs"` + Correlation CorrelationConfig `yaml:"correlation"` } // ServiceConfig holds service-level configuration. type ServiceConfig struct { - Name string - Language string + Name string `yaml:"name"` + Language string `yaml:"language"` } // InputsConfig holds input sources configuration. type InputsConfig struct { - UnixSockets []UnixSocketConfig + UnixSockets []UnixSocketConfig `yaml:"unix_sockets"` } // UnixSocketConfig holds a Unix socket source configuration. type UnixSocketConfig struct { - Name string - Path string - Format string + Name string `yaml:"name"` + Path string `yaml:"path"` + Format string `yaml:"format"` } // OutputsConfig holds output sinks configuration. type OutputsConfig struct { - File FileOutputConfig - ClickHouse ClickHouseOutputConfig - Stdout StdoutOutputConfig + File FileOutputConfig `yaml:"file"` + ClickHouse ClickHouseOutputConfig `yaml:"clickhouse"` + Stdout StdoutOutputConfig `yaml:"stdout"` } // FileOutputConfig holds file sink configuration. type FileOutputConfig struct { - Enabled bool - Path string + Enabled bool `yaml:"enabled"` + Path string `yaml:"path"` } // ClickHouseOutputConfig holds ClickHouse sink configuration. type ClickHouseOutputConfig struct { - Enabled bool - DSN string - Table string - BatchSize int - FlushIntervalMs int - MaxBufferSize int - DropOnOverflow bool - AsyncInsert bool - TimeoutMs int + Enabled bool `yaml:"enabled"` + DSN string `yaml:"dsn"` + Table string `yaml:"table"` + BatchSize int `yaml:"batch_size"` + FlushIntervalMs int `yaml:"flush_interval_ms"` + MaxBufferSize int `yaml:"max_buffer_size"` + DropOnOverflow bool `yaml:"drop_on_overflow"` + AsyncInsert bool `yaml:"async_insert"` + TimeoutMs int `yaml:"timeout_ms"` } // StdoutOutputConfig holds stdout sink configuration. type StdoutOutputConfig struct { - Enabled bool + Enabled bool `yaml:"enabled"` } // CorrelationConfig holds correlation configuration. type CorrelationConfig struct { - Key []string - TimeWindow TimeWindowConfig - OrphanPolicy OrphanPolicyConfig + Key []string `yaml:"key"` + TimeWindow TimeWindowConfig `yaml:"time_window"` + OrphanPolicy OrphanPolicyConfig `yaml:"orphan_policy"` } // TimeWindowConfig holds time window configuration. type TimeWindowConfig struct { - Value int - Unit string + Value int `yaml:"value"` + Unit string `yaml:"unit"` } // OrphanPolicyConfig holds orphan event policy configuration. type OrphanPolicyConfig struct { - ApacheAlwaysEmit bool - NetworkEmit bool + ApacheAlwaysEmit bool `yaml:"apache_always_emit"` + NetworkEmit bool `yaml:"network_emit"` } -// Load loads configuration from a text file with directives. +// Load loads configuration from a YAML file. func Load(path string) (*Config, error) { - file, err := os.Open(path) + data, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("failed to open config file: %w", err) + return nil, fmt.Errorf("failed to read config file: %w", err) } - defer file.Close() - cfg := &Config{ + cfg := defaultConfig() + + if err := yaml.Unmarshal(data, cfg); err != nil { + return nil, fmt.Errorf("failed to parse config file: %w", err) + } + + if err := cfg.Validate(); err != nil { + return nil, fmt.Errorf("invalid config: %w", err) + } + + return cfg, nil +} + +// defaultConfig returns a Config with default values. +func defaultConfig() *Config { + return &Config{ Service: ServiceConfig{ Name: "logcorrelator", Language: "go", @@ -131,171 +144,6 @@ func Load(path string) (*Config, error) { }, }, } - - scanner := bufio.NewScanner(file) - lineNum := 0 - - for scanner.Scan() { - lineNum++ - line := strings.TrimSpace(scanner.Text()) - - // Skip empty lines and comments - if line == "" || strings.HasPrefix(line, "#") { - continue - } - - if err := parseDirective(cfg, line); err != nil { - return nil, fmt.Errorf("line %d: %w", lineNum, err) - } - } - - if err := scanner.Err(); err != nil { - return nil, fmt.Errorf("failed to read config file: %w", err) - } - - if err := cfg.Validate(); err != nil { - return nil, fmt.Errorf("invalid config: %w", err) - } - - return cfg, nil -} - -func parseDirective(cfg *Config, line string) error { - parts := strings.Fields(line) - if len(parts) < 2 { - return fmt.Errorf("invalid directive: %s", line) - } - - directive := parts[0] - value := strings.Join(parts[1:], " ") - - switch directive { - case "service.name": - cfg.Service.Name = value - case "service.language": - cfg.Service.Language = value - - case "input.unix_socket": - // Format: input.unix_socket [format] - if len(parts) < 3 { - return fmt.Errorf("input.unix_socket requires name and path") - } - format := "json" - if len(parts) >= 4 { - format = parts[3] - } - cfg.Inputs.UnixSockets = append(cfg.Inputs.UnixSockets, UnixSocketConfig{ - Name: parts[1], - Path: parts[2], - Format: format, - }) - - case "output.file.enabled": - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("invalid value for output.file.enabled: %w", err) - } - cfg.Outputs.File.Enabled = enabled - case "output.file.path": - cfg.Outputs.File.Path = value - - case "output.clickhouse.enabled": - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("invalid value for output.clickhouse.enabled: %w", err) - } - cfg.Outputs.ClickHouse.Enabled = enabled - case "output.clickhouse.dsn": - cfg.Outputs.ClickHouse.DSN = value - case "output.clickhouse.table": - cfg.Outputs.ClickHouse.Table = value - case "output.clickhouse.batch_size": - v, err := strconv.Atoi(value) - if err != nil { - return fmt.Errorf("invalid value for output.clickhouse.batch_size: %w", err) - } - cfg.Outputs.ClickHouse.BatchSize = v - case "output.clickhouse.flush_interval_ms": - v, err := strconv.Atoi(value) - if err != nil { - return fmt.Errorf("invalid value for output.clickhouse.flush_interval_ms: %w", err) - } - cfg.Outputs.ClickHouse.FlushIntervalMs = v - case "output.clickhouse.max_buffer_size": - v, err := strconv.Atoi(value) - if err != nil { - return fmt.Errorf("invalid value for output.clickhouse.max_buffer_size: %w", err) - } - cfg.Outputs.ClickHouse.MaxBufferSize = v - case "output.clickhouse.drop_on_overflow": - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("invalid value for output.clickhouse.drop_on_overflow: %w", err) - } - cfg.Outputs.ClickHouse.DropOnOverflow = enabled - case "output.clickhouse.async_insert": - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("invalid value for output.clickhouse.async_insert: %w", err) - } - cfg.Outputs.ClickHouse.AsyncInsert = enabled - case "output.clickhouse.timeout_ms": - v, err := strconv.Atoi(value) - if err != nil { - return fmt.Errorf("invalid value for output.clickhouse.timeout_ms: %w", err) - } - cfg.Outputs.ClickHouse.TimeoutMs = v - - case "output.stdout.enabled": - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("invalid value for output.stdout.enabled: %w", err) - } - cfg.Outputs.Stdout.Enabled = enabled - - case "correlation.key": - cfg.Correlation.Key = strings.Split(value, ",") - for i, k := range cfg.Correlation.Key { - cfg.Correlation.Key[i] = strings.TrimSpace(k) - } - case "correlation.time_window.value": - v, err := strconv.Atoi(value) - if err != nil { - return fmt.Errorf("invalid value for correlation.time_window.value: %w", err) - } - cfg.Correlation.TimeWindow.Value = v - case "correlation.time_window.unit": - cfg.Correlation.TimeWindow.Unit = value - case "correlation.orphan_policy.apache_always_emit": - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("invalid value for correlation.orphan_policy.apache_always_emit: %w", err) - } - cfg.Correlation.OrphanPolicy.ApacheAlwaysEmit = enabled - case "correlation.orphan_policy.network_emit": - enabled, err := parseBool(value) - if err != nil { - return fmt.Errorf("invalid value for correlation.orphan_policy.network_emit: %w", err) - } - cfg.Correlation.OrphanPolicy.NetworkEmit = enabled - - default: - return fmt.Errorf("unknown directive: %s", directive) - } - - return nil -} - -func parseBool(s string) (bool, error) { - s = strings.ToLower(s) - switch s { - case "true", "yes", "1", "on": - return true, nil - case "false", "no", "0", "off": - return false, nil - default: - return false, fmt.Errorf("invalid boolean value: %s", s) - } } // Validate validates the configuration. diff --git a/packaging/deb/postinst b/packaging/deb/postinst index 212c942..30134ad 100644 --- a/packaging/deb/postinst +++ b/packaging/deb/postinst @@ -37,10 +37,10 @@ case "$1" in chmod 750 /etc/logcorrelator # Install default config if it doesn't exist - if [ ! -f /etc/logcorrelator/logcorrelator.conf ]; then - cp /usr/share/logcorrelator/logcorrelator.conf.example /etc/logcorrelator/logcorrelator.conf - chown logcorrelator:logcorrelator /etc/logcorrelator/logcorrelator.conf - chmod 640 /etc/logcorrelator/logcorrelator.conf + if [ ! -f /etc/logcorrelator/logcorrelator.yml ]; then + cp /usr/share/logcorrelator/logcorrelator.yml.example /etc/logcorrelator/logcorrelator.yml + chown logcorrelator:logcorrelator /etc/logcorrelator/logcorrelator.yml + chmod 640 /etc/logcorrelator/logcorrelator.yml fi # Enable and start the service (if running in a real system, not container)