feat: release v1.0.3 with flattened JSON output structure

- breaking: remove apache and network subdivisions from JSON output
- feat: all log fields now merged into single-level JSON structure
- feat: custom MarshalJSON() implementation for flat output
- chore: update ClickHouse schema to use single fields JSON column
- docs: update CHANGELOG.md and README.md with v1.0.3 changes
- build: bump version to 1.0.3 in build.sh and RPM spec

Migration notes:
- Existing ClickHouse tables need schema migration to use fields JSON column
- Replace apache JSON and network JSON columns with fields JSON column

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
Jacquin Antoine
2026-02-28 22:26:20 +01:00
parent 180c57c35b
commit 514cb553ef
7 changed files with 89 additions and 43 deletions

View File

@ -1,19 +1,49 @@
package domain
import "time"
import (
"encoding/json"
"time"
)
// CorrelatedLog represents the output correlated log entry.
// All fields are flattened into a single-level structure.
type CorrelatedLog struct {
Timestamp time.Time `json:"timestamp"`
SrcIP string `json:"src_ip"`
SrcPort int `json:"src_port"`
DstIP string `json:"dst_ip,omitempty"`
DstPort int `json:"dst_port,omitempty"`
Correlated bool `json:"correlated"`
OrphanSide string `json:"orphan_side,omitempty"`
Apache map[string]any `json:"apache,omitempty"`
Network map[string]any `json:"network,omitempty"`
Extra map[string]any `json:"extra,omitempty"`
Timestamp time.Time `json:"timestamp"`
SrcIP string `json:"src_ip"`
SrcPort int `json:"src_port"`
DstIP string `json:"dst_ip,omitempty"`
DstPort int `json:"dst_port,omitempty"`
Correlated bool `json:"correlated"`
OrphanSide string `json:"orphan_side,omitempty"`
Fields map[string]any `json:"-"` // Additional fields, merged at marshal time
}
// MarshalJSON implements custom JSON marshaling to flatten the structure.
func (c CorrelatedLog) MarshalJSON() ([]byte, error) {
// Create a flat map with all fields
flat := make(map[string]any)
// Add core fields
flat["timestamp"] = c.Timestamp
flat["src_ip"] = c.SrcIP
flat["src_port"] = c.SrcPort
if c.DstIP != "" {
flat["dst_ip"] = c.DstIP
}
if c.DstPort != 0 {
flat["dst_port"] = c.DstPort
}
flat["correlated"] = c.Correlated
if c.OrphanSide != "" {
flat["orphan_side"] = c.OrphanSide
}
// Merge additional fields
for k, v := range c.Fields {
flat[k] = v
}
return json.Marshal(flat)
}
// NewCorrelatedLogFromEvent creates a correlated log from a single event (orphan).
@ -26,9 +56,7 @@ func NewCorrelatedLogFromEvent(event *NormalizedEvent, orphanSide string) Correl
DstPort: event.DstPort,
Correlated: false,
OrphanSide: orphanSide,
Apache: extractApache(event),
Network: extractNetwork(event),
Extra: make(map[string]any),
Fields: extractFields(event),
}
}
@ -47,16 +75,11 @@ func NewCorrelatedLog(apacheEvent, networkEvent *NormalizedEvent) CorrelatedLog
DstPort: coalesceInt(apacheEvent.DstPort, networkEvent.DstPort),
Correlated: true,
OrphanSide: "",
Apache: extractApache(apacheEvent),
Network: extractNetwork(networkEvent),
Extra: make(map[string]any),
Fields: mergeFields(apacheEvent, networkEvent),
}
}
func extractApache(e *NormalizedEvent) map[string]any {
if e.Source != SourceA {
return nil
}
func extractFields(e *NormalizedEvent) map[string]any {
result := make(map[string]any)
for k, v := range e.Raw {
result[k] = v
@ -64,12 +87,13 @@ func extractApache(e *NormalizedEvent) map[string]any {
return result
}
func extractNetwork(e *NormalizedEvent) map[string]any {
if e.Source != SourceB {
return nil
}
func mergeFields(a, b *NormalizedEvent) map[string]any {
result := make(map[string]any)
for k, v := range e.Raw {
// Merge fields from both events
for k, v := range a.Raw {
result[k] = v
}
for k, v := range b.Raw {
result[k] = v
}
return result

View File

@ -72,8 +72,8 @@ func TestNewCorrelatedLogFromEvent(t *testing.T) {
if log.SrcIP != "192.168.1.1" {
t.Errorf("expected src_ip 192.168.1.1, got %s", log.SrcIP)
}
if log.Apache == nil {
t.Error("expected apache to be non-nil")
if log.Fields == nil {
t.Error("expected fields to be non-nil")
}
}
@ -106,10 +106,7 @@ func TestNewCorrelatedLog(t *testing.T) {
if log.OrphanSide != "" {
t.Errorf("expected orphan_side to be empty, got %s", log.OrphanSide)
}
if log.Apache == nil {
t.Error("expected apache to be non-nil")
}
if log.Network == nil {
t.Error("expected network to be non-nil")
if log.Fields == nil {
t.Error("expected fields to be non-nil")
}
}