code_public/ja4sentinel
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.1.11: Fix exclude_source_ips config loading and debug logging

Browse Source 
Major fixes:
- Add exclude_source_ips to mergeConfigs() - config file values now properly loaded
- Add validation for exclude_source_ips (IP/CIDR format validation)
- Remove JA4SENTINEL_LOG_LEVEL env var from systemd service
- Config file log_level now respected without env override

Debug logging improvements:
- Log IP filter entries at startup (debug mode)
- Track filtered packet count with atomic counter
- Display filter statistics at shutdown via GetFilterStats()
- New debug logs in tlsparse component

Testing:
- Add 6 new unit tests for exclude_source_ips and log_level config loading
- Test mergeConfigs() behavior with empty/override values
- Test validation of invalid IPs and CIDR ranges

Documentation:
- Update architecture.yml with ipfilter module
- Document config loading priority and notes
- Update api.Config fields (LocalIPs, ExcludeSourceIPs, LogLevel)

Files changed:
- internal/config/loader.go (merge, validation, helpers)
- internal/config/loader_test.go (6 new tests)
- internal/tlsparse/parser.go (GetFilterStats, counter)
- cmd/ja4sentinel/main.go (debug logging)
- packaging/systemd/ja4sentinel.service (remove env var)
- architecture.yml (ipfilter module, config_loading section)

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
This commit is contained in:
toto
2026-03-04 15:55:00 +01:00
parent 952701d4da
commit bd45344d19
7 changed files with 426 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

70
internal/config/loader.go
View File

@ -175,6 +175,11 @@ func mergeConfigs(base, override api.AppConfig) api.AppConfig {
result.Core.LogLevel = override.Core.LogLevel
}
// Merge exclude_source_ips (override takes precedence)
if len(override.Core.ExcludeSourceIPs) > 0 {
result.Core.ExcludeSourceIPs = override.Core.ExcludeSourceIPs
}
if len(override.Outputs) > 0 {
result.Outputs = override.Outputs
}
@ -218,6 +223,27 @@ func (l *LoaderImpl) validate(config api.AppConfig) error {
}
}
// Validate exclude_source_ips (if provided)
if len(config.Core.ExcludeSourceIPs) > 0 {
for i, ip := range config.Core.ExcludeSourceIPs {
if ip == "" {
return fmt.Errorf("exclude_source_ips[%d]: entry cannot be empty", i)
}
// Basic validation: check if it looks like an IP or CIDR
if !strings.Contains(ip, "/") {
// Single IP - basic check
if !isValidIP(ip) {
return fmt.Errorf("exclude_source_ips[%d]: invalid IP address %q", i, ip)
}
} else {
// CIDR - basic check
if !isValidCIDR(ip) {
return fmt.Errorf("exclude_source_ips[%d]: invalid CIDR %q", i, ip)
}
}
}
}
allowedTypes := map[string]struct{}{
"stdout": {},
"file": {},
@ -257,3 +283,47 @@ func ToJSON(config api.AppConfig) string {
}
return string(data)
}
// isValidIP checks if a string is a valid IP address
func isValidIP(ip string) bool {
if ip == "" {
return false
}
// Simple validation: check if it contains only valid IP characters
for _, ch := range ip {
if !((ch >= '0' && ch <= '9') || ch == '.') {
// Could be IPv6
if ch == ':' {
return true // Accept IPv6 without detailed validation
}
return false
}
}
return true
}
// isValidCIDR checks if a string is a valid CIDR notation
func isValidCIDR(cidr string) bool {
if cidr == "" {
return false
}
parts := strings.Split(cidr, "/")
if len(parts) != 2 {
return false
}
// Check IP part
if !isValidIP(parts[0]) {
return false
}
// Check prefix length
prefix, err := strconv.Atoi(parts[1])
if err != nil {
return false
}
if strings.Contains(parts[0], ":") {
// IPv6
return prefix >= 0 && prefix <= 128
}
// IPv4
return prefix >= 0 && prefix <= 32
}

259
internal/config/loader_test.go
View File

@ -741,3 +741,262 @@ func TestMergeConfigs_OutputMerge(t *testing.T) {
t.Errorf("Merged Outputs[0].Type = %q, want 'file'", result.Outputs[0].Type)
}
}
// TestMergeConfigs_ExcludeSourceIPs tests that exclude_source_ips is properly merged
func TestMergeConfigs_ExcludeSourceIPs(t *testing.T) {
base := api.AppConfig{
Core: api.Config{
Interface: "eth0",
ListenPorts: []uint16{443},
ExcludeSourceIPs: []string{}, // Empty base
},
}
override := api.AppConfig{
Core: api.Config{
ExcludeSourceIPs: []string{"10.0.0.0/8", "192.168.1.1"},
},
}
result := mergeConfigs(base, override)
if len(result.Core.ExcludeSourceIPs) != 2 {
t.Errorf("Merged ExcludeSourceIPs length = %d, want 2", len(result.Core.ExcludeSourceIPs))
}
if result.Core.ExcludeSourceIPs[0] != "10.0.0.0/8" {
t.Errorf("Merged ExcludeSourceIPs[0] = %q, want '10.0.0.0/8'", result.Core.ExcludeSourceIPs[0])
}
if result.Core.ExcludeSourceIPs[1] != "192.168.1.1" {
t.Errorf("Merged ExcludeSourceIPs[1] = %q, want '192.168.1.1'", result.Core.ExcludeSourceIPs[1])
}
}
// TestMergeConfigs_ExcludeSourceIPs_EmptyOverride tests that empty override doesn't replace
func TestMergeConfigs_ExcludeSourceIPs_EmptyOverride(t *testing.T) {
base := api.AppConfig{
Core: api.Config{
Interface: "eth0",
ListenPorts: []uint16{443},
ExcludeSourceIPs: []string{"10.0.0.0/8"},
},
}
override := api.AppConfig{
Core: api.Config{
ExcludeSourceIPs: []string{}, // Empty override
},
}
result := mergeConfigs(base, override)
// Empty override should NOT replace base
if len(result.Core.ExcludeSourceIPs) != 1 {
t.Errorf("Merged ExcludeSourceIPs length = %d, want 1", len(result.Core.ExcludeSourceIPs))
}
if result.Core.ExcludeSourceIPs[0] != "10.0.0.0/8" {
t.Errorf("Merged ExcludeSourceIPs[0] = %q, want '10.0.0.0/8'", result.Core.ExcludeSourceIPs[0])
}
}
// TestLoadFromFile_ExcludeSourceIPs tests loading exclude_source_ips from YAML file
func TestLoadFromFile_ExcludeSourceIPs(t *testing.T) {
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "config.yml")
yamlContent := `
core:
interface: eth0
listen_ports:
- 443
exclude_source_ips:
- "10.0.0.0/8"
- "192.168.1.100"
log_level: debug
outputs:
- type: stdout
enabled: true
`
if err := os.WriteFile(configPath, []byte(yamlContent), 0600); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
loader := NewLoader(configPath)
cfg, err := loader.Load()
if err != nil {
t.Fatalf("Load() error = %v", err)
}
if len(cfg.Core.ExcludeSourceIPs) != 2 {
t.Errorf("ExcludeSourceIPs length = %d, want 2", len(cfg.Core.ExcludeSourceIPs))
}
if cfg.Core.ExcludeSourceIPs[0] != "10.0.0.0/8" {
t.Errorf("ExcludeSourceIPs[0] = %q, want '10.0.0.0/8'", cfg.Core.ExcludeSourceIPs[0])
}
if cfg.Core.ExcludeSourceIPs[1] != "192.168.1.100" {
t.Errorf("ExcludeSourceIPs[1] = %q, want '192.168.1.100'", cfg.Core.ExcludeSourceIPs[1])
}
if cfg.Core.LogLevel != "debug" {
t.Errorf("LogLevel = %q, want 'debug'", cfg.Core.LogLevel)
}
}
// TestLoadFromFile_LogLevel tests loading log_level from YAML file
func TestLoadFromFile_LogLevel(t *testing.T) {
tmpDir := t.TempDir()
configPath := filepath.Join(tmpDir, "config.yml")
tests := []struct {
name string
logLevel string
want string
wantErr bool
}{
{"debug level", "debug", "debug", false},
{"info level", "info", "info", false},
{"warn level", "warn", "warn", false},
{"error level", "error", "error", false},
{"invalid level", "invalid", "", true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
yamlContent := `
core:
interface: eth0
listen_ports:
- 443
log_level: ` + tt.logLevel + `
outputs:
- type: stdout
enabled: true
`
if err := os.WriteFile(configPath, []byte(yamlContent), 0600); err != nil {
t.Fatalf("WriteFile() error = %v", err)
}
loader := NewLoader(configPath)
cfg, err := loader.Load()
if (err != nil) != tt.wantErr {
t.Fatalf("Load() error = %v, wantErr %v", err, tt.wantErr)
}
if !tt.wantErr && cfg.Core.LogLevel != tt.want {
t.Errorf("LogLevel = %q, want %q", cfg.Core.LogLevel, tt.want)
}
})
}
}
// TestValidate_ExcludeSourceIPs tests validation of exclude_source_ips entries
func TestValidate_ExcludeSourceIPs(t *testing.T) {
loader := &LoaderImpl{}
tests := []struct {
name string
ips []string
wantErr bool
errContains string
}{
{
name: "empty list",
ips: []string{},
wantErr: false,
},
{
name: "valid single IP",
ips: []string{"192.168.1.1"},
wantErr: false,
},
{
name: "valid CIDR",
ips: []string{"10.0.0.0/8"},
wantErr: false,
},
{
name: "multiple valid entries",
ips: []string{"10.0.0.0/8", "192.168.1.1", "172.16.0.0/12"},
wantErr: false,
},
{
name: "empty entry",
ips: []string{""},
wantErr: true,
errContains: "entry cannot be empty",
},
{
name: "invalid IP",
ips: []string{"999.999.999.999"},
wantErr: true,
errContains: "invalid IP address",
},
{
name: "invalid CIDR format",
ips: []string{"10.0.0.0/33"},
wantErr: true,
errContains: "invalid CIDR",
},
{
name: "invalid CIDR syntax",
ips: []string{"10.0.0.0/"},
wantErr: true,
errContains: "invalid CIDR",
},
{
name: "mixed valid and invalid",
ips: []string{"10.0.0.0/8", "invalid"},
wantErr: true,
errContains: "invalid",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
cfg := api.AppConfig{
Core: api.Config{
Interface: "eth0",
ListenPorts: []uint16{443},
ExcludeSourceIPs: tt.ips,
},
Outputs: []api.OutputConfig{
{Type: "stdout", Enabled: true},
},
}
err := loader.validate(cfg)
if (err != nil) != tt.wantErr {
t.Fatalf("validate() error = %v, wantErr %v", err, tt.wantErr)
}
if tt.wantErr && tt.errContains != "" {
if err == nil || !strings.Contains(err.Error(), tt.errContains) {
t.Errorf("validate() error = %v, should contain %q", err, tt.errContains)
}
}
})
}
}
// TestLoadFromEnv_ExcludeSourceIPs tests that exclude_source_ips can be set via env (future feature)
// Currently exclude_source_ips is NOT loaded from env, only from config file
func TestLoadFromEnv_ExcludeSourceIPs_NotSupported(t *testing.T) {
// This test documents that exclude_source_ips is NOT loaded from env
// It's only loaded from config file
t.Setenv("JA4SENTINEL_LOG_LEVEL", "debug")
loader := NewLoader("")
cfg, err := loader.Load()
if err != nil {
t.Fatalf("Load() error = %v", err)
}
// exclude_source_ips should be empty (not loaded from env)
if len(cfg.Core.ExcludeSourceIPs) != 0 {
t.Errorf("ExcludeSourceIPs should be empty from env, got %v", cfg.Core.ExcludeSourceIPs)
}
// But log_level should be loaded from env
if cfg.Core.LogLevel != "debug" {
t.Errorf("LogLevel = %q, want 'debug'", cfg.Core.LogLevel)
}
}

13
internal/tlsparse/parser.go
View File

@ -6,6 +6,7 @@ import (
"fmt"
"strings"
"sync"
"sync/atomic"
"time"
"ja4sentinel/api"
@ -65,6 +66,7 @@ type ParserImpl struct {
maxTrackedFlows int
maxHelloBufferBytes int
sourceIPFilter *ipfilter.Filter
filteredCount uint64 // Counter for filtered packets (debug)
}
// NewParser creates a new TLS parser with connection state tracking
@ -98,9 +100,11 @@ func NewParserWithTimeoutAndFilter(timeout time.Duration, excludeSourceIPs []str
flowTimeout: timeout,
cleanupDone: make(chan struct{}),
cleanupClose: make(chan struct{}),
closeOnce: sync.Once{},
maxTrackedFlows: DefaultMaxTrackedFlows,
maxHelloBufferBytes: DefaultMaxHelloBufferBytes,
sourceIPFilter: filter,
filteredCount: 0,
}
go p.cleanupLoop()
return p
@ -243,6 +247,7 @@ func (p *ParserImpl) Process(pkt api.RawPacket) (*api.TLSClientHello, error) {
// Check if source IP should be excluded
if p.sourceIPFilter != nil && p.sourceIPFilter.ShouldExclude(srcIP) {
atomic.AddUint64(&p.filteredCount, 1)
return nil, nil // Source IP is excluded
}
@ -406,6 +411,14 @@ func (p *ParserImpl) getOrCreateFlow(key string, srcIP string, srcPort uint16, d
return flow
}
// GetFilterStats returns statistics about the IP filter (for debug/monitoring)
func (p *ParserImpl) GetFilterStats() (filteredCount uint64, hasFilter bool) {
if p.sourceIPFilter == nil {
return 0, false
}
return atomic.LoadUint64(&p.filteredCount), true
}
// Close cleans up the parser and stops background goroutines
func (p *ParserImpl) Close() error {
p.closeOnce.Do(func() {