feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized
Services: - ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap) - logcorrelator: JA4 log correlation engine (Go, ClickHouse) - mod_reqin_log: Apache module (C, JSON request logging) - bot_detector: ML bot detection pipeline (Python) - dashboard: FastAPI/Streamlit analytics UI (Python) Shared libraries: - shared/go/ja4common: logger, config, shutdown, ipfilter (Go module) - shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package) - shared/clickhouse/: canonical SQL migrations (10 files) Build & packaging: - Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10) - go.work workspace linking sentinel, correlator, ja4common - Makefile with test-all, build-all, rpm-* targets Fixes applied: - go.work: 1.21 → 1.24.6 (required by sentinel) - correlator Dockerfiles: golang:1.21 → golang:1.24 - replace directives in go.mod for ja4common local path - pyproject.toml: setuptools.backends → setuptools.build_meta - Removed static libpcap linking (unavailable on Rocky 9) - Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32) - Rewrote corrupted test files (logger_test.go × 2) Test coverage: - correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%) - sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse) Documentation: - README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
114
shared/go/ja4common/config/loader.go
Normal file
114
shared/go/ja4common/config/loader.go
Normal file
@ -0,0 +1,114 @@
|
||||
// Package config provides generic YAML config loading with env override support.
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// LoadYAML reads a YAML file at path and unmarshals it into T.
|
||||
// If path is empty or the file does not exist and optional is true, the zero value of T is returned.
|
||||
func LoadYAML[T any](path string, optional bool) (T, error) {
|
||||
var zero T
|
||||
if path == "" {
|
||||
if optional {
|
||||
return zero, nil
|
||||
}
|
||||
return zero, fmt.Errorf("config path is empty")
|
||||
}
|
||||
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if optional && os.IsNotExist(err) {
|
||||
return zero, nil
|
||||
}
|
||||
return zero, fmt.Errorf("reading config file %q: %w", path, err)
|
||||
}
|
||||
|
||||
var cfg T
|
||||
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
||||
return zero, fmt.Errorf("parsing config file %q: %w", path, err)
|
||||
}
|
||||
|
||||
return cfg, nil
|
||||
}
|
||||
|
||||
// OverrideFromEnv applies environment variable overrides to a struct using struct tags.
|
||||
// Tag format: env:"ENV_VAR_NAME"
|
||||
// Supports field types: string, int, bool, []string (comma-separated).
|
||||
// envPrefix is prepended to tag values if non-empty (e.g. envPrefix="APP" + tag="PORT" → "APP_PORT").
|
||||
func OverrideFromEnv[T any](cfg *T, envPrefix string) error {
|
||||
return overrideStruct(reflect.ValueOf(cfg).Elem(), envPrefix)
|
||||
}
|
||||
|
||||
func overrideStruct(v reflect.Value, envPrefix string) error {
|
||||
t := v.Type()
|
||||
for i := 0; i < t.NumField(); i++ {
|
||||
field := t.Field(i)
|
||||
fv := v.Field(i)
|
||||
|
||||
if !fv.CanSet() {
|
||||
continue
|
||||
}
|
||||
|
||||
// Recurse into embedded/nested structs
|
||||
if fv.Kind() == reflect.Struct {
|
||||
if err := overrideStruct(fv, envPrefix); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
envTag := field.Tag.Get("env")
|
||||
if envTag == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
envKey := envTag
|
||||
if envPrefix != "" {
|
||||
envKey = envPrefix + "_" + envTag
|
||||
}
|
||||
|
||||
val := os.Getenv(envKey)
|
||||
if val == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
switch fv.Kind() {
|
||||
case reflect.String:
|
||||
fv.SetString(val)
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
n, err := strconv.ParseInt(val, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("env %s: cannot parse %q as int: %w", envKey, val, err)
|
||||
}
|
||||
fv.SetInt(n)
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
n, err := strconv.ParseUint(val, 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("env %s: cannot parse %q as uint: %w", envKey, val, err)
|
||||
}
|
||||
fv.SetUint(n)
|
||||
case reflect.Bool:
|
||||
b, err := strconv.ParseBool(val)
|
||||
if err != nil {
|
||||
return fmt.Errorf("env %s: cannot parse %q as bool: %w", envKey, val, err)
|
||||
}
|
||||
fv.SetBool(b)
|
||||
case reflect.Slice:
|
||||
if fv.Type().Elem().Kind() == reflect.String {
|
||||
parts := strings.Split(val, ",")
|
||||
for i, p := range parts {
|
||||
parts[i] = strings.TrimSpace(p)
|
||||
}
|
||||
fv.Set(reflect.ValueOf(parts))
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
139
shared/go/ja4common/config/loader_test.go
Normal file
139
shared/go/ja4common/config/loader_test.go
Normal file
@ -0,0 +1,139 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
type testConfig struct {
|
||||
Host string `yaml:"host" env:"HOST"`
|
||||
Port int `yaml:"port" env:"PORT"`
|
||||
TLS bool `yaml:"tls" env:"TLS"`
|
||||
Tags []string `yaml:"tags" env:"TAGS"`
|
||||
}
|
||||
|
||||
func TestLoadYAML_Basic(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.yml")
|
||||
content := `
|
||||
host: myhost
|
||||
port: 9000
|
||||
tls: true
|
||||
tags:
|
||||
- a
|
||||
- b
|
||||
`
|
||||
if err := os.WriteFile(path, []byte(content), 0600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
cfg, err := LoadYAML[testConfig](path, false)
|
||||
if err != nil {
|
||||
t.Fatalf("LoadYAML error: %v", err)
|
||||
}
|
||||
if cfg.Host != "myhost" {
|
||||
t.Errorf("Host = %q, want %q", cfg.Host, "myhost")
|
||||
}
|
||||
if cfg.Port != 9000 {
|
||||
t.Errorf("Port = %d, want 9000", cfg.Port)
|
||||
}
|
||||
if !cfg.TLS {
|
||||
t.Error("TLS should be true")
|
||||
}
|
||||
if len(cfg.Tags) != 2 {
|
||||
t.Errorf("Tags len = %d, want 2", len(cfg.Tags))
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadYAML_Optional_MissingFile(t *testing.T) {
|
||||
cfg, err := LoadYAML[testConfig]("/nonexistent/path.yml", true)
|
||||
if err != nil {
|
||||
t.Fatalf("optional missing file should not error: %v", err)
|
||||
}
|
||||
if cfg.Host != "" {
|
||||
t.Errorf("zero value expected, got host=%q", cfg.Host)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadYAML_Required_MissingFile(t *testing.T) {
|
||||
_, err := LoadYAML[testConfig]("/nonexistent/path.yml", false)
|
||||
if err == nil {
|
||||
t.Error("expected error for missing required file")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadYAML_EmptyPath_Optional(t *testing.T) {
|
||||
cfg, err := LoadYAML[testConfig]("", true)
|
||||
if err != nil {
|
||||
t.Fatalf("empty optional path should not error: %v", err)
|
||||
}
|
||||
_ = cfg
|
||||
}
|
||||
|
||||
func TestOverrideFromEnv_String(t *testing.T) {
|
||||
t.Setenv("HOST", "envhost")
|
||||
cfg := testConfig{Host: "original"}
|
||||
if err := OverrideFromEnv(&cfg, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Host != "envhost" {
|
||||
t.Errorf("Host = %q, want envhost", cfg.Host)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverrideFromEnv_Int(t *testing.T) {
|
||||
t.Setenv("PORT", "8080")
|
||||
cfg := testConfig{Port: 1234}
|
||||
if err := OverrideFromEnv(&cfg, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Port != 8080 {
|
||||
t.Errorf("Port = %d, want 8080", cfg.Port)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverrideFromEnv_Bool(t *testing.T) {
|
||||
t.Setenv("TLS", "false")
|
||||
cfg := testConfig{TLS: true}
|
||||
if err := OverrideFromEnv(&cfg, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.TLS {
|
||||
t.Error("TLS should be false after env override")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverrideFromEnv_Slice(t *testing.T) {
|
||||
t.Setenv("TAGS", "x,y,z")
|
||||
cfg := testConfig{}
|
||||
if err := OverrideFromEnv(&cfg, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cfg.Tags) != 3 || cfg.Tags[0] != "x" {
|
||||
t.Errorf("Tags = %v, want [x y z]", cfg.Tags)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverrideFromEnv_WithPrefix(t *testing.T) {
|
||||
t.Setenv("APP_HOST", "prefixed")
|
||||
cfg := testConfig{Host: "original"}
|
||||
if err := OverrideFromEnv(&cfg, "APP"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Host != "prefixed" {
|
||||
t.Errorf("Host = %q, want prefixed", cfg.Host)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOverrideFromEnv_NoEnvSet_NoChange(t *testing.T) {
|
||||
os.Unsetenv("HOST")
|
||||
os.Unsetenv("PORT")
|
||||
cfg := testConfig{Host: "keep", Port: 42}
|
||||
if err := OverrideFromEnv(&cfg, ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if cfg.Host != "keep" || cfg.Port != 42 {
|
||||
t.Errorf("unset env should not change values: host=%q port=%d", cfg.Host, cfg.Port)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user