feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized

Services:
- ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap)
- logcorrelator: JA4 log correlation engine (Go, ClickHouse)
- mod_reqin_log: Apache module (C, JSON request logging)
- bot_detector: ML bot detection pipeline (Python)
- dashboard: FastAPI/Streamlit analytics UI (Python)

Shared libraries:
- shared/go/ja4common: logger, config, shutdown, ipfilter (Go module)
- shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package)
- shared/clickhouse/: canonical SQL migrations (10 files)

Build & packaging:
- Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10)
- go.work workspace linking sentinel, correlator, ja4common
- Makefile with test-all, build-all, rpm-* targets

Fixes applied:
- go.work: 1.21 → 1.24.6 (required by sentinel)
- correlator Dockerfiles: golang:1.21 → golang:1.24
- replace directives in go.mod for ja4common local path
- pyproject.toml: setuptools.backends → setuptools.build_meta
- Removed static libpcap linking (unavailable on Rocky 9)
- Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32)
- Rewrote corrupted test files (logger_test.go × 2)

Test coverage:
- correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%)
- sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse)

Documentation:
- README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 16:42:59 +02:00
commit d469e39da7
278 changed files with 1621301 additions and 0 deletions

View File

@ -0,0 +1,139 @@
package logger
import (
"strings"
"testing"
)
func TestParseLogLevel(t *testing.T) {
tests := []struct {
input string
want LogLevel
}{
{"debug", DEBUG},
{"DEBUG", DEBUG},
{"info", INFO},
{"INFO", INFO},
{"warn", WARN},
{"WARN", WARN},
{"warning", WARN},
{"WARNING", WARN},
{"error", ERROR},
{"ERROR", ERROR},
{"invalid", INFO},
{"", INFO},
}
for _, tt := range tests {
got := ParseLogLevel(tt.input)
if got != tt.want {
t.Errorf("ParseLogLevel(%q) = %v, want %v", tt.input, got, tt.want)
}
}
}
func TestLogger_LevelFiltering(t *testing.T) {
tests := []struct {
loggerLevel string
logLevel LogLevel
shouldLog bool
}{
{"debug", DEBUG, true},
{"debug", INFO, true},
{"info", DEBUG, false},
{"info", INFO, true},
{"warn", INFO, false},
{"warn", WARN, true},
{"error", WARN, false},
{"error", ERROR, true},
}
for _, tt := range tests {
l := NewWithLevel("test", tt.loggerLevel)
got := l.ShouldLog(tt.logLevel)
if got != tt.shouldLog {
t.Errorf("level=%s ShouldLog(%v)=%v want %v", tt.loggerLevel, tt.logLevel, got, tt.shouldLog)
}
}
}
func TestLogger_WithFields(t *testing.T) {
l := New("test")
l2 := l.WithFields(map[string]any{"key": "value", "n": 42})
if l2 == l {
t.Error("WithFields should return a new Logger")
}
if len(l2.fields) != 2 {
t.Errorf("expected 2 fields, got %d", len(l2.fields))
}
}
func TestLogger_SetLevel(t *testing.T) {
l := New("test")
if l.minLevel != INFO {
t.Errorf("default level should be INFO, got %v", l.minLevel)
}
l.SetLevel("debug")
if l.minLevel != DEBUG {
t.Errorf("level after SetLevel(debug) should be DEBUG, got %v", l.minLevel)
}
}
func TestComponentLogger_Interface(t *testing.T) {
cl := NewComponentLogger("debug")
// Verify it implements the component-based interface by calling all methods
cl.Debug("component", "debug msg", nil)
cl.Info("component", "info msg", map[string]string{"key": "val"})
cl.Warn("component", "warn msg", nil)
cl.Error("component", "error msg", map[string]string{"err": "test"})
cl.Log("component", "info", "log msg", nil)
}
func TestComponentLogger_LevelFiltering(t *testing.T) {
cl := NewComponentLogger("warn")
// At warn level, debug and info should be filtered
if cl.Logger.ShouldLog(DEBUG) {
t.Error("DEBUG should be filtered at warn level")
}
if cl.Logger.ShouldLog(INFO) {
t.Error("INFO should be filtered at warn level")
}
if !cl.Logger.ShouldLog(WARN) {
t.Error("WARN should pass at warn level")
}
}
func TestLogger_LogLevelString(t *testing.T) {
tests := []struct {
level LogLevel
want string
}{
{DEBUG, "DEBUG"},
{INFO, "INFO"},
{WARN, "WARN"},
{ERROR, "ERROR"},
}
for _, tt := range tests {
if got := tt.level.String(); got != tt.want {
t.Errorf("%v.String() = %q, want %q", tt.level, got, tt.want)
}
}
}
func TestLogger_EmitContainsLevel(t *testing.T) {
// Use a custom logger that captures output
var buf strings.Builder
l := New("myservice")
l.logger.SetOutput(&buf)
l.SetLevel("debug")
l.Info("hello from info")
if !strings.Contains(buf.String(), "INFO") {
t.Errorf("expected INFO in output, got: %s", buf.String())
}
buf.Reset()
l.Debug("hello from debug")
if !strings.Contains(buf.String(), "DEBUG") {
t.Errorf("expected DEBUG in output, got: %s", buf.String())
}
}