Feat: Détection menaces HTTP via vues ClickHouse + simplification shutdown
Nouvelles vues de détection (sql/views.sql) : - Identification hosts par IP/JA4 (view_host_identification, view_host_ja4_anomalies) - Détection brute force POST et query params variables - Header fingerprinting (ordre, headers modernes manquants, Sec-CH-UA) - ALPN mismatch detection (h2 déclaré mais HTTP/1.1 parlé) - Rate limiting & burst detection (50 req/min, 20 req/10s) - Path enumeration/scanning (paths sensibles) - Payload attacks (SQLi, XSS, path traversal) - JA4 botnet detection (même fingerprint sur 20+ IPs) - Correlation quality (orphan ratio >80%) ClickHouse (sql/init.sql) : - Compression ZSTD(3) sur champs texte (path, query, headers, ja3/ja4) - TTL automatique : 1 jour (raw) + 7 jours (http_logs) - Paramètre ttl_only_drop_parts = 1 Shutdown simplifié (internal/app/orchestrator.go) : - Suppression ShutdownTimeout et logique de flush/attente - Stop() = cancel() + Close() uniquement - systemd TimeoutStopSec gère l'arrêt forcé si besoin File output toggle (internal/config/*.go) : - Ajout champ Enabled dans FileOutputConfig - Le sink fichier n'est créé que si enabled && path != '' - Tests : TestValidate_FileOutputDisabled, TestLoadConfig_FileOutputDisabled RPM packaging (packaging/rpm/logcorrelator.spec) : - Changelog 1.1.18 → 1.1.22 - Suppression logcorrelator-tmpfiles.conf (redondant RuntimeDirectory=) Nettoyage : - idees.txt → idees/ (dossier) - Suppression 91.224.92.185.txt (logs exemple) Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
@ -69,7 +69,8 @@ type OutputsConfig struct {
|
||||
|
||||
// FileOutputConfig holds file sink configuration.
|
||||
type FileOutputConfig struct {
|
||||
Path string `yaml:"path"`
|
||||
Enabled bool `yaml:"enabled"`
|
||||
Path string `yaml:"path"`
|
||||
}
|
||||
|
||||
// ClickHouseOutputConfig holds ClickHouse sink configuration.
|
||||
@ -182,7 +183,8 @@ func defaultConfig() *Config {
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{
|
||||
Path: "/var/log/logcorrelator/correlated.log",
|
||||
Enabled: true,
|
||||
Path: "/var/log/logcorrelator/correlated.log",
|
||||
},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: false,
|
||||
@ -232,7 +234,7 @@ func (c *Config) Validate() error {
|
||||
|
||||
// At least one output must be enabled
|
||||
hasOutput := false
|
||||
if c.Outputs.File.Path != "" {
|
||||
if c.Outputs.File.Enabled && c.Outputs.File.Path != "" {
|
||||
hasOutput = true
|
||||
}
|
||||
if c.Outputs.ClickHouse.Enabled {
|
||||
|
||||
@ -47,6 +47,9 @@ correlation:
|
||||
if cfg.Outputs.File.Path != "/var/log/logcorrelator/correlated.log" {
|
||||
t.Errorf("expected file path /var/log/logcorrelator/correlated.log, got %s", cfg.Outputs.File.Path)
|
||||
}
|
||||
if !cfg.Outputs.File.Enabled {
|
||||
t.Error("expected file output to be enabled by default when path is set")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad_InvalidPath(t *testing.T) {
|
||||
@ -110,7 +113,7 @@ func TestValidate_MinimumInputs(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
File: FileOutputConfig{Enabled: true, Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -129,7 +132,7 @@ func TestValidate_AtLeastOneOutput(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{},
|
||||
File: FileOutputConfig{Enabled: false},
|
||||
ClickHouse: ClickHouseOutputConfig{Enabled: false},
|
||||
Stdout: StdoutOutputConfig{Enabled: false},
|
||||
},
|
||||
@ -189,7 +192,7 @@ func TestValidate_DuplicateNames(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
File: FileOutputConfig{Enabled: true, Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -208,7 +211,7 @@ func TestValidate_DuplicatePaths(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
File: FileOutputConfig{Enabled: true, Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -227,7 +230,7 @@ func TestValidate_EmptyName(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
File: FileOutputConfig{Enabled: true, Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -246,7 +249,7 @@ func TestValidate_EmptyPath(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
File: FileOutputConfig{Enabled: true, Path: "/var/log/test.log"},
|
||||
},
|
||||
}
|
||||
|
||||
@ -265,7 +268,7 @@ func TestValidate_EmptyFilePath(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: ""},
|
||||
File: FileOutputConfig{Enabled: true, Path: ""},
|
||||
},
|
||||
}
|
||||
|
||||
@ -284,7 +287,7 @@ func TestValidate_ClickHouseMissingDSN(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: ""},
|
||||
File: FileOutputConfig{Enabled: true, Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "",
|
||||
@ -308,7 +311,7 @@ func TestValidate_ClickHouseMissingTable(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: ""},
|
||||
File: FileOutputConfig{Enabled: true, Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -332,7 +335,7 @@ func TestValidate_ClickHouseInvalidBatchSize(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: ""},
|
||||
File: FileOutputConfig{Enabled: true, Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -357,7 +360,7 @@ func TestValidate_ClickHouseInvalidMaxBufferSize(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: ""},
|
||||
File: FileOutputConfig{Enabled: true, Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -383,7 +386,7 @@ func TestValidate_ClickHouseInvalidTimeout(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: ""},
|
||||
File: FileOutputConfig{Enabled: true, Path: ""},
|
||||
ClickHouse: ClickHouseOutputConfig{
|
||||
Enabled: true,
|
||||
DSN: "clickhouse://localhost:9000/db",
|
||||
@ -409,7 +412,7 @@ func TestValidate_EmptyCorrelationKey(t *testing.T) {
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Path: "/var/log/test.log"},
|
||||
File: FileOutputConfig{Enabled: true, Path: "/var/log/test.log"},
|
||||
},
|
||||
Correlation: CorrelationConfig{
|
||||
TimeWindowS: 0,
|
||||
@ -938,3 +941,70 @@ correlation:
|
||||
t.Error("expected error for ClickHouse enabled without table")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidate_FileOutputDisabled(t *testing.T) {
|
||||
cfg := &Config{
|
||||
Inputs: InputsConfig{
|
||||
UnixSockets: []UnixSocketConfig{
|
||||
{Name: "a", Path: "/tmp/a.sock"},
|
||||
{Name: "b", Path: "/tmp/b.sock"},
|
||||
},
|
||||
},
|
||||
Outputs: OutputsConfig{
|
||||
File: FileOutputConfig{Enabled: false, Path: "/var/log/test.log"},
|
||||
ClickHouse: ClickHouseOutputConfig{Enabled: false},
|
||||
Stdout: StdoutOutputConfig{Enabled: true},
|
||||
},
|
||||
Correlation: CorrelationConfig{
|
||||
TimeWindowS: 1,
|
||||
},
|
||||
}
|
||||
|
||||
err := cfg.Validate()
|
||||
if err != nil {
|
||||
t.Errorf("expected no error when file is disabled but stdout is enabled, got: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadConfig_FileOutputDisabled(t *testing.T) {
|
||||
content := `
|
||||
inputs:
|
||||
unix_sockets:
|
||||
- name: http
|
||||
path: /var/run/logcorrelator/http.socket
|
||||
- name: network
|
||||
path: /var/run/logcorrelator/network.socket
|
||||
|
||||
outputs:
|
||||
file:
|
||||
enabled: false
|
||||
path: /var/log/logcorrelator/correlated.log
|
||||
stdout:
|
||||
enabled: true
|
||||
|
||||
correlation:
|
||||
time_window_s: 1
|
||||
emit_orphans: true
|
||||
`
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
configPath := filepath.Join(tmpDir, "config.yml")
|
||||
if err := os.WriteFile(configPath, []byte(content), 0644); err != nil {
|
||||
t.Fatalf("failed to write config: %v", err)
|
||||
}
|
||||
|
||||
cfg, err := Load(configPath)
|
||||
if err != nil {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
|
||||
if cfg.Outputs.File.Enabled {
|
||||
t.Error("expected file output to be disabled")
|
||||
}
|
||||
if cfg.Outputs.File.Path != "/var/log/logcorrelator/correlated.log" {
|
||||
t.Errorf("expected file path to be preserved, got %s", cfg.Outputs.File.Path)
|
||||
}
|
||||
if !cfg.Outputs.Stdout.Enabled {
|
||||
t.Error("expected stdout output to be enabled")
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user