code_public/logcorrelator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: migrate configuration from custom format to YAML

Browse Source 
- Replace custom directive-based config parser with YAML using gopkg.in/yaml.v3
- Rename config.example.conf to config.example.yml with YAML syntax
- Update default config path to /etc/logcorrelator/logcorrelator.yml
- Update Dockerfile.package to copy YAML config files
- Update packaging scripts to install logcorrelator.yml
- Update architecture.yml to document YAML configuration
- Add yaml.v3 dependency to go.mod

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
2026-02-27 15:51:25 +01:00
parent 0d84a1284f
commit 37f9c21672
7 changed files with 163 additions and 263 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
Dockerfile.package
View File

@ -47,15 +47,15 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
# Copy binary from builder
COPY --from=builder /build/dist/logcorrelator /tmp/pkgroot/usr/bin/logcorrelator
COPY --from=builder /build/config.example.conf /tmp/pkgroot/etc/logcorrelator/logcorrelator.conf
COPY --from=builder /build/config.example.conf /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.conf.example
COPY --from=builder /build/config.example.yml /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml
COPY --from=builder /build/config.example.yml /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.yml.example
# Create directories and set permissions
RUN mkdir -p /tmp/pkgroot/var/log/logcorrelator && \
mkdir -p /tmp/pkgroot/var/run/logcorrelator && \
chmod 755 /tmp/pkgroot/usr/bin/logcorrelator && \
chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.conf && \
chmod 640 /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.conf.example && \
chmod 640 /tmp/pkgroot/etc/logcorrelator/logcorrelator.yml && \
chmod 640 /tmp/pkgroot/usr/share/logcorrelator/logcorrelator.yml.example && \
chmod 755 /tmp/pkgroot/var/log/logcorrelator && \
chmod 755 /tmp/pkgroot/var/run/logcorrelator
@ -85,8 +85,8 @@ RUN mkdir -p /packages/deb && \
--after-remove /tmp/scripts/postrm \
-p /packages/deb/logcorrelator_${VERSION}_${ARCH}.deb \
usr/bin/logcorrelator \
etc/logcorrelator/logcorrelator.conf \
usr/share/logcorrelator/logcorrelator.conf.example \
etc/logcorrelator/logcorrelator.yml \
usr/share/logcorrelator/logcorrelator.yml.example \
var/log/logcorrelator \
var/run/logcorrelator
@ -108,8 +108,8 @@ RUN mkdir -p /packages/rpm && \
--after-remove /tmp/scripts/postrm \
-p /packages/rpm/logcorrelator-${VERSION}-1.x86_64.rpm \
usr/bin/logcorrelator \
etc/logcorrelator/logcorrelator.conf \
usr/share/logcorrelator/logcorrelator.conf.example \
etc/logcorrelator/logcorrelator.yml \
usr/share/logcorrelator/logcorrelator.yml.example \
var/log/logcorrelator \
var/run/logcorrelator

94
architecture.yml
View File

@ -21,11 +21,11 @@ runtime:
unit_type: systemd
description: >
logcorrelator est livré sous forme de binaire autonome, exécuté comme un
service systemd. L’unité systemd assure le démarrage automatique au boot,
le redémarrage en cas de crash, et une intégration standard dans l’écosystème
service systemd. L'unité systemd assure le démarrage automatique au boot,
le redémarrage en cas de crash, et une intégration standard dans l'écosystème
Linux (notamment sur Rocky Linux 8+).
binary_path: /usr/bin/logcorrelator
config_path: /etc/logcorrelator/logcorrelator.toml
config_path: /etc/logcorrelator/logcorrelator.yml
user: logcorrelator
group: logcorrelator
restart: on-failure
@ -40,7 +40,7 @@ runtime:
Type=simple
User=logcorrelator
Group=logcorrelator
ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.toml
ExecStart=/usr/bin/logcorrelator -config /etc/logcorrelator/logcorrelator.yml
Restart=on-failure
RestartSec=5
@ -68,51 +68,53 @@ runtime:
de drop), ferme les connexions ClickHouse puis s’arrête.
config:
format: toml
location: /etc/logcorrelator/logcorrelator.toml
format: yaml
location: /etc/logcorrelator/logcorrelator.yml
description: >
Toute la configuration est centralisée dans un fichier TOML lisible, stocké
dans /etc/logcorrelator. Ni YAML ni JSON ne sont utilisés pour la config.
Toute la configuration est centralisée dans un fichier YAML lisible,
stocké dans /etc/logcorrelator.
reload_strategy: restart_service
example: |
[service]
name = "logcorrelator"
language = "go"
# Service configuration
service:
name: logcorrelator
language: go
[[inputs.unix_sockets]]
name = "apache_source"
path = "/var/run/logcorrelator/apache.sock"
format = "json"
# Input sources (at least 2 required)
inputs:
unix_sockets:
- name: apache_source
path: /var/run/logcorrelator/apache.sock
format: json
- name: network_source
path: /var/run/logcorrelator/network.sock
format: json
[[inputs.unix_sockets]]
name = "network_source"
path = "/var/run/logcorrelator/network.sock"
format = "json"
# File output
outputs:
file:
enabled: true
path: /var/log/logcorrelator/correlated.log
[outputs.file]
enabled = true
path = "/var/log/logcorrelator/correlated.log"
# ClickHouse output
outputs:
clickhouse:
enabled: false
dsn: clickhouse://user:pass@localhost:9000/db
table: correlated_logs_http_network
batch_size: 500
[outputs.clickhouse]
enabled = true
dsn = "clickhouse://user:pass@host:9000/db"
table = "correlated_logs_http_network"
batch_size = 500
flush_interval_ms = 200
max_buffer_size = 5000
drop_on_overflow = true
async_insert = true
[correlation]
key = ["src_ip", "src_port"]
[correlation.time_window]
value = 1
unit = "s"
[correlation.orphan_policy]
apache_always_emit = true
network_emit = false
# Correlation configuration
correlation:
key:
- src_ip
- src_port
time_window:
value: 1
unit: s
orphan_policy:
apache_always_emit: true
network_emit: false
inputs:
description: >
@ -503,12 +505,12 @@ packaging:
dest: /usr/bin/logcorrelator
mode: "0755"
config:
- source: config.example.conf
dest: /etc/logcorrelator/logcorrelator.conf
- source: config.example.yml
dest: /etc/logcorrelator/logcorrelator.yml
mode: "0640"
config_file: true
- source: config.example.conf
dest: /usr/share/logcorrelator/logcorrelator.conf.example
- source: config.example.yml
dest: /usr/share/logcorrelator/logcorrelator.yml.example
mode: "0640"
directories:
- path: /var/log/logcorrelator

45
config.example.yml Normal file
View File

@ -0,0 +1,45 @@
# logcorrelator configuration file
# Format: YAML
service:
name: logcorrelator
language: go
inputs:
unix_sockets:
- name: apache_source
path: /var/run/logcorrelator/apache.sock
format: json
- name: network_source
path: /var/run/logcorrelator/network.sock
format: json
outputs:
file:
enabled: true
path: /var/log/logcorrelator/correlated.log
clickhouse:
enabled: false
dsn: clickhouse://user:pass@localhost:9000/db
table: correlated_logs_http_network
batch_size: 500
flush_interval_ms: 200
max_buffer_size: 5000
drop_on_overflow: true
async_insert: true
timeout_ms: 1000
stdout:
enabled: false
correlation:
key:
- src_ip
- src_port
time_window:
value: 1
unit: s
orphan_policy:
apache_always_emit: true
network_emit: false

2
go.mod
View File

@ -1,3 +1,5 @@
module github.com/logcorrelator/logcorrelator
go 1.21
require gopkg.in/yaml.v3 v3.0.1 // indirect

3
go.sum
View File

@ -0,0 +1,3 @@
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

258
internal/config/config.go
View File

@ -1,99 +1,112 @@
package config
import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
"time"
"gopkg.in/yaml.v3"
)
// Config holds the complete application configuration.
type Config struct {
Service ServiceConfig
Inputs InputsConfig
Outputs OutputsConfig
Correlation CorrelationConfig
Service ServiceConfig `yaml:"service"`
Inputs InputsConfig `yaml:"inputs"`
Outputs OutputsConfig `yaml:"outputs"`
Correlation CorrelationConfig `yaml:"correlation"`
}
// ServiceConfig holds service-level configuration.
type ServiceConfig struct {
Name string
Language string
Name string `yaml:"name"`
Language string `yaml:"language"`
}
// InputsConfig holds input sources configuration.
type InputsConfig struct {
UnixSockets []UnixSocketConfig
UnixSockets []UnixSocketConfig `yaml:"unix_sockets"`
}
// UnixSocketConfig holds a Unix socket source configuration.
type UnixSocketConfig struct {
Name string
Path string
Format string
Name string `yaml:"name"`
Path string `yaml:"path"`
Format string `yaml:"format"`
}
// OutputsConfig holds output sinks configuration.
type OutputsConfig struct {
File FileOutputConfig
ClickHouse ClickHouseOutputConfig
Stdout StdoutOutputConfig
File FileOutputConfig `yaml:"file"`
ClickHouse ClickHouseOutputConfig `yaml:"clickhouse"`
Stdout StdoutOutputConfig `yaml:"stdout"`
}
// FileOutputConfig holds file sink configuration.
type FileOutputConfig struct {
Enabled bool
Path string
Enabled bool `yaml:"enabled"`
Path string `yaml:"path"`
}
// ClickHouseOutputConfig holds ClickHouse sink configuration.
type ClickHouseOutputConfig struct {
Enabled bool
DSN string
Table string
BatchSize int
FlushIntervalMs int
MaxBufferSize int
DropOnOverflow bool
AsyncInsert bool
TimeoutMs int
Enabled bool `yaml:"enabled"`
DSN string `yaml:"dsn"`
Table string `yaml:"table"`
BatchSize int `yaml:"batch_size"`
FlushIntervalMs int `yaml:"flush_interval_ms"`
MaxBufferSize int `yaml:"max_buffer_size"`
DropOnOverflow bool `yaml:"drop_on_overflow"`
AsyncInsert bool `yaml:"async_insert"`
TimeoutMs int `yaml:"timeout_ms"`
}
// StdoutOutputConfig holds stdout sink configuration.
type StdoutOutputConfig struct {
Enabled bool
Enabled bool `yaml:"enabled"`
}
// CorrelationConfig holds correlation configuration.
type CorrelationConfig struct {
Key []string
TimeWindow TimeWindowConfig
OrphanPolicy OrphanPolicyConfig
Key []string `yaml:"key"`
TimeWindow TimeWindowConfig `yaml:"time_window"`
OrphanPolicy OrphanPolicyConfig `yaml:"orphan_policy"`
}
// TimeWindowConfig holds time window configuration.
type TimeWindowConfig struct {
Value int
Unit string
Value int `yaml:"value"`
Unit string `yaml:"unit"`
}
// OrphanPolicyConfig holds orphan event policy configuration.
type OrphanPolicyConfig struct {
ApacheAlwaysEmit bool
NetworkEmit bool
ApacheAlwaysEmit bool `yaml:"apache_always_emit"`
NetworkEmit bool `yaml:"network_emit"`
}
// Load loads configuration from a text file with directives.
// Load loads configuration from a YAML file.
func Load(path string) (*Config, error) {
file, err := os.Open(path)
data, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("failed to open config file: %w", err)
return nil, fmt.Errorf("failed to read config file: %w", err)
}
defer file.Close()
cfg := &Config{
cfg := defaultConfig()
if err := yaml.Unmarshal(data, cfg); err != nil {
return nil, fmt.Errorf("failed to parse config file: %w", err)
}
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
// defaultConfig returns a Config with default values.
func defaultConfig() *Config {
return &Config{
Service: ServiceConfig{
Name: "logcorrelator",
Language: "go",
@ -131,171 +144,6 @@ func Load(path string) (*Config, error) {
},
},
}
scanner := bufio.NewScanner(file)
lineNum := 0
for scanner.Scan() {
lineNum++
line := strings.TrimSpace(scanner.Text())
// Skip empty lines and comments
if line == "" || strings.HasPrefix(line, "#") {
continue
}
if err := parseDirective(cfg, line); err != nil {
return nil, fmt.Errorf("line %d: %w", lineNum, err)
}
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("failed to read config file: %w", err)
}
if err := cfg.Validate(); err != nil {
return nil, fmt.Errorf("invalid config: %w", err)
}
return cfg, nil
}
func parseDirective(cfg *Config, line string) error {
parts := strings.Fields(line)
if len(parts) < 2 {
return fmt.Errorf("invalid directive: %s", line)
}
directive := parts[0]
value := strings.Join(parts[1:], " ")
switch directive {
case "service.name":
cfg.Service.Name = value
case "service.language":
cfg.Service.Language = value
case "input.unix_socket":
// Format: input.unix_socket <name> <path> [format]
if len(parts) < 3 {
return fmt.Errorf("input.unix_socket requires name and path")
}
format := "json"
if len(parts) >= 4 {
format = parts[3]
}
cfg.Inputs.UnixSockets = append(cfg.Inputs.UnixSockets, UnixSocketConfig{
Name: parts[1],
Path: parts[2],
Format: format,
})
case "output.file.enabled":
enabled, err := parseBool(value)
if err != nil {
return fmt.Errorf("invalid value for output.file.enabled: %w", err)
}
cfg.Outputs.File.Enabled = enabled
case "output.file.path":
cfg.Outputs.File.Path = value
case "output.clickhouse.enabled":
enabled, err := parseBool(value)
if err != nil {
return fmt.Errorf("invalid value for output.clickhouse.enabled: %w", err)
}
cfg.Outputs.ClickHouse.Enabled = enabled
case "output.clickhouse.dsn":
cfg.Outputs.ClickHouse.DSN = value
case "output.clickhouse.table":
cfg.Outputs.ClickHouse.Table = value
case "output.clickhouse.batch_size":
v, err := strconv.Atoi(value)
if err != nil {
return fmt.Errorf("invalid value for output.clickhouse.batch_size: %w", err)
}
cfg.Outputs.ClickHouse.BatchSize = v
case "output.clickhouse.flush_interval_ms":
v, err := strconv.Atoi(value)
if err != nil {
return fmt.Errorf("invalid value for output.clickhouse.flush_interval_ms: %w", err)
}
cfg.Outputs.ClickHouse.FlushIntervalMs = v
case "output.clickhouse.max_buffer_size":
v, err := strconv.Atoi(value)
if err != nil {
return fmt.Errorf("invalid value for output.clickhouse.max_buffer_size: %w", err)
}
cfg.Outputs.ClickHouse.MaxBufferSize = v
case "output.clickhouse.drop_on_overflow":
enabled, err := parseBool(value)
if err != nil {
return fmt.Errorf("invalid value for output.clickhouse.drop_on_overflow: %w", err)
}
cfg.Outputs.ClickHouse.DropOnOverflow = enabled
case "output.clickhouse.async_insert":
enabled, err := parseBool(value)
if err != nil {
return fmt.Errorf("invalid value for output.clickhouse.async_insert: %w", err)
}
cfg.Outputs.ClickHouse.AsyncInsert = enabled
case "output.clickhouse.timeout_ms":
v, err := strconv.Atoi(value)
if err != nil {
return fmt.Errorf("invalid value for output.clickhouse.timeout_ms: %w", err)
}
cfg.Outputs.ClickHouse.TimeoutMs = v
case "output.stdout.enabled":
enabled, err := parseBool(value)
if err != nil {
return fmt.Errorf("invalid value for output.stdout.enabled: %w", err)
}
cfg.Outputs.Stdout.Enabled = enabled
case "correlation.key":
cfg.Correlation.Key = strings.Split(value, ",")
for i, k := range cfg.Correlation.Key {
cfg.Correlation.Key[i] = strings.TrimSpace(k)
}
case "correlation.time_window.value":
v, err := strconv.Atoi(value)
if err != nil {
return fmt.Errorf("invalid value for correlation.time_window.value: %w", err)
}
cfg.Correlation.TimeWindow.Value = v
case "correlation.time_window.unit":
cfg.Correlation.TimeWindow.Unit = value
case "correlation.orphan_policy.apache_always_emit":
enabled, err := parseBool(value)
if err != nil {
return fmt.Errorf("invalid value for correlation.orphan_policy.apache_always_emit: %w", err)
}
cfg.Correlation.OrphanPolicy.ApacheAlwaysEmit = enabled
case "correlation.orphan_policy.network_emit":
enabled, err := parseBool(value)
if err != nil {
return fmt.Errorf("invalid value for correlation.orphan_policy.network_emit: %w", err)
}
cfg.Correlation.OrphanPolicy.NetworkEmit = enabled
default:
return fmt.Errorf("unknown directive: %s", directive)
}
return nil
}
func parseBool(s string) (bool, error) {
s = strings.ToLower(s)
switch s {
case "true", "yes", "1", "on":
return true, nil
case "false", "no", "0", "off":
return false, nil
default:
return false, fmt.Errorf("invalid boolean value: %s", s)
}
}
// Validate validates the configuration.

8
packaging/deb/postinst
View File

@ -37,10 +37,10 @@ case "$1" in
chmod 750 /etc/logcorrelator
# Install default config if it doesn't exist
if [ ! -f /etc/logcorrelator/logcorrelator.conf ]; then
cp /usr/share/logcorrelator/logcorrelator.conf.example /etc/logcorrelator/logcorrelator.conf
chown logcorrelator:logcorrelator /etc/logcorrelator/logcorrelator.conf
chmod 640 /etc/logcorrelator/logcorrelator.conf
if [ ! -f /etc/logcorrelator/logcorrelator.yml ]; then
cp /usr/share/logcorrelator/logcorrelator.yml.example /etc/logcorrelator/logcorrelator.yml
chown logcorrelator:logcorrelator /etc/logcorrelator/logcorrelator.yml
chmod 640 /etc/logcorrelator/logcorrelator.yml
fi
# Enable and start the service (if running in a real system, not container)