feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized

Services:
- ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap)
- logcorrelator: JA4 log correlation engine (Go, ClickHouse)
- mod_reqin_log: Apache module (C, JSON request logging)
- bot_detector: ML bot detection pipeline (Python)
- dashboard: FastAPI/Streamlit analytics UI (Python)

Shared libraries:
- shared/go/ja4common: logger, config, shutdown, ipfilter (Go module)
- shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package)
- shared/clickhouse/: canonical SQL migrations (10 files)

Build & packaging:
- Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10)
- go.work workspace linking sentinel, correlator, ja4common
- Makefile with test-all, build-all, rpm-* targets

Fixes applied:
- go.work: 1.21 → 1.24.6 (required by sentinel)
- correlator Dockerfiles: golang:1.21 → golang:1.24
- replace directives in go.mod for ja4common local path
- pyproject.toml: setuptools.backends → setuptools.build_meta
- Removed static libpcap linking (unavailable on Rocky 9)
- Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32)
- Rewrote corrupted test files (logger_test.go × 2)

Test coverage:
- correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%)
- sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse)

Documentation:
- README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 16:42:59 +02:00
commit d469e39da7
278 changed files with 1621301 additions and 0 deletions

View File

@ -0,0 +1,114 @@
// Package config provides generic YAML config loading with env override support.
package config
import (
"fmt"
"os"
"reflect"
"strconv"
"strings"
"gopkg.in/yaml.v3"
)
// LoadYAML reads a YAML file at path and unmarshals it into T.
// If path is empty or the file does not exist and optional is true, the zero value of T is returned.
func LoadYAML[T any](path string, optional bool) (T, error) {
var zero T
if path == "" {
if optional {
return zero, nil
}
return zero, fmt.Errorf("config path is empty")
}
data, err := os.ReadFile(path)
if err != nil {
if optional && os.IsNotExist(err) {
return zero, nil
}
return zero, fmt.Errorf("reading config file %q: %w", path, err)
}
var cfg T
if err := yaml.Unmarshal(data, &cfg); err != nil {
return zero, fmt.Errorf("parsing config file %q: %w", path, err)
}
return cfg, nil
}
// OverrideFromEnv applies environment variable overrides to a struct using struct tags.
// Tag format: env:"ENV_VAR_NAME"
// Supports field types: string, int, bool, []string (comma-separated).
// envPrefix is prepended to tag values if non-empty (e.g. envPrefix="APP" + tag="PORT" → "APP_PORT").
func OverrideFromEnv[T any](cfg *T, envPrefix string) error {
return overrideStruct(reflect.ValueOf(cfg).Elem(), envPrefix)
}
func overrideStruct(v reflect.Value, envPrefix string) error {
t := v.Type()
for i := 0; i < t.NumField(); i++ {
field := t.Field(i)
fv := v.Field(i)
if !fv.CanSet() {
continue
}
// Recurse into embedded/nested structs
if fv.Kind() == reflect.Struct {
if err := overrideStruct(fv, envPrefix); err != nil {
return err
}
continue
}
envTag := field.Tag.Get("env")
if envTag == "" {
continue
}
envKey := envTag
if envPrefix != "" {
envKey = envPrefix + "_" + envTag
}
val := os.Getenv(envKey)
if val == "" {
continue
}
switch fv.Kind() {
case reflect.String:
fv.SetString(val)
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
n, err := strconv.ParseInt(val, 10, 64)
if err != nil {
return fmt.Errorf("env %s: cannot parse %q as int: %w", envKey, val, err)
}
fv.SetInt(n)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
n, err := strconv.ParseUint(val, 10, 64)
if err != nil {
return fmt.Errorf("env %s: cannot parse %q as uint: %w", envKey, val, err)
}
fv.SetUint(n)
case reflect.Bool:
b, err := strconv.ParseBool(val)
if err != nil {
return fmt.Errorf("env %s: cannot parse %q as bool: %w", envKey, val, err)
}
fv.SetBool(b)
case reflect.Slice:
if fv.Type().Elem().Kind() == reflect.String {
parts := strings.Split(val, ",")
for i, p := range parts {
parts[i] = strings.TrimSpace(p)
}
fv.Set(reflect.ValueOf(parts))
}
}
}
return nil
}