feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized

Services:
- ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap)
- logcorrelator: JA4 log correlation engine (Go, ClickHouse)
- mod_reqin_log: Apache module (C, JSON request logging)
- bot_detector: ML bot detection pipeline (Python)
- dashboard: FastAPI/Streamlit analytics UI (Python)

Shared libraries:
- shared/go/ja4common: logger, config, shutdown, ipfilter (Go module)
- shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package)
- shared/clickhouse/: canonical SQL migrations (10 files)

Build & packaging:
- Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10)
- go.work workspace linking sentinel, correlator, ja4common
- Makefile with test-all, build-all, rpm-* targets

Fixes applied:
- go.work: 1.21 → 1.24.6 (required by sentinel)
- correlator Dockerfiles: golang:1.21 → golang:1.24
- replace directives in go.mod for ja4common local path
- pyproject.toml: setuptools.backends → setuptools.build_meta
- Removed static libpcap linking (unavailable on Rocky 9)
- Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32)
- Rewrote corrupted test files (logger_test.go × 2)

Test coverage:
- correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%)
- sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse)

Documentation:
- README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 16:42:59 +02:00
commit d469e39da7
278 changed files with 1621301 additions and 0 deletions

View File

@ -0,0 +1,139 @@
package config
import (
"os"
"path/filepath"
"testing"
)
type testConfig struct {
Host string `yaml:"host" env:"HOST"`
Port int `yaml:"port" env:"PORT"`
TLS bool `yaml:"tls" env:"TLS"`
Tags []string `yaml:"tags" env:"TAGS"`
}
func TestLoadYAML_Basic(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "test.yml")
content := `
host: myhost
port: 9000
tls: true
tags:
- a
- b
`
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
t.Fatal(err)
}
cfg, err := LoadYAML[testConfig](path, false)
if err != nil {
t.Fatalf("LoadYAML error: %v", err)
}
if cfg.Host != "myhost" {
t.Errorf("Host = %q, want %q", cfg.Host, "myhost")
}
if cfg.Port != 9000 {
t.Errorf("Port = %d, want 9000", cfg.Port)
}
if !cfg.TLS {
t.Error("TLS should be true")
}
if len(cfg.Tags) != 2 {
t.Errorf("Tags len = %d, want 2", len(cfg.Tags))
}
}
func TestLoadYAML_Optional_MissingFile(t *testing.T) {
cfg, err := LoadYAML[testConfig]("/nonexistent/path.yml", true)
if err != nil {
t.Fatalf("optional missing file should not error: %v", err)
}
if cfg.Host != "" {
t.Errorf("zero value expected, got host=%q", cfg.Host)
}
}
func TestLoadYAML_Required_MissingFile(t *testing.T) {
_, err := LoadYAML[testConfig]("/nonexistent/path.yml", false)
if err == nil {
t.Error("expected error for missing required file")
}
}
func TestLoadYAML_EmptyPath_Optional(t *testing.T) {
cfg, err := LoadYAML[testConfig]("", true)
if err != nil {
t.Fatalf("empty optional path should not error: %v", err)
}
_ = cfg
}
func TestOverrideFromEnv_String(t *testing.T) {
t.Setenv("HOST", "envhost")
cfg := testConfig{Host: "original"}
if err := OverrideFromEnv(&cfg, ""); err != nil {
t.Fatal(err)
}
if cfg.Host != "envhost" {
t.Errorf("Host = %q, want envhost", cfg.Host)
}
}
func TestOverrideFromEnv_Int(t *testing.T) {
t.Setenv("PORT", "8080")
cfg := testConfig{Port: 1234}
if err := OverrideFromEnv(&cfg, ""); err != nil {
t.Fatal(err)
}
if cfg.Port != 8080 {
t.Errorf("Port = %d, want 8080", cfg.Port)
}
}
func TestOverrideFromEnv_Bool(t *testing.T) {
t.Setenv("TLS", "false")
cfg := testConfig{TLS: true}
if err := OverrideFromEnv(&cfg, ""); err != nil {
t.Fatal(err)
}
if cfg.TLS {
t.Error("TLS should be false after env override")
}
}
func TestOverrideFromEnv_Slice(t *testing.T) {
t.Setenv("TAGS", "x,y,z")
cfg := testConfig{}
if err := OverrideFromEnv(&cfg, ""); err != nil {
t.Fatal(err)
}
if len(cfg.Tags) != 3 || cfg.Tags[0] != "x" {
t.Errorf("Tags = %v, want [x y z]", cfg.Tags)
}
}
func TestOverrideFromEnv_WithPrefix(t *testing.T) {
t.Setenv("APP_HOST", "prefixed")
cfg := testConfig{Host: "original"}
if err := OverrideFromEnv(&cfg, "APP"); err != nil {
t.Fatal(err)
}
if cfg.Host != "prefixed" {
t.Errorf("Host = %q, want prefixed", cfg.Host)
}
}
func TestOverrideFromEnv_NoEnvSet_NoChange(t *testing.T) {
os.Unsetenv("HOST")
os.Unsetenv("PORT")
cfg := testConfig{Host: "keep", Port: 42}
if err := OverrideFromEnv(&cfg, ""); err != nil {
t.Fatal(err)
}
if cfg.Host != "keep" || cfg.Port != 42 {
t.Errorf("unset env should not change values: host=%q port=%d", cfg.Host, cfg.Port)
}
}