Services: - ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap) - logcorrelator: JA4 log correlation engine (Go, ClickHouse) - mod_reqin_log: Apache module (C, JSON request logging) - bot_detector: ML bot detection pipeline (Python) - dashboard: FastAPI/Streamlit analytics UI (Python) Shared libraries: - shared/go/ja4common: logger, config, shutdown, ipfilter (Go module) - shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package) - shared/clickhouse/: canonical SQL migrations (10 files) Build & packaging: - Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10) - go.work workspace linking sentinel, correlator, ja4common - Makefile with test-all, build-all, rpm-* targets Fixes applied: - go.work: 1.21 → 1.24.6 (required by sentinel) - correlator Dockerfiles: golang:1.21 → golang:1.24 - replace directives in go.mod for ja4common local path - pyproject.toml: setuptools.backends → setuptools.build_meta - Removed static libpcap linking (unavailable on Rocky 9) - Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32) - Rewrote corrupted test files (logger_test.go × 2) Test coverage: - correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%) - sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse) Documentation: - README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
115 lines
2.8 KiB
Go
115 lines
2.8 KiB
Go
// Package config provides generic YAML config loading with env override support.
|
|
package config
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"reflect"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"gopkg.in/yaml.v3"
|
|
)
|
|
|
|
// LoadYAML reads a YAML file at path and unmarshals it into T.
|
|
// If path is empty or the file does not exist and optional is true, the zero value of T is returned.
|
|
func LoadYAML[T any](path string, optional bool) (T, error) {
|
|
var zero T
|
|
if path == "" {
|
|
if optional {
|
|
return zero, nil
|
|
}
|
|
return zero, fmt.Errorf("config path is empty")
|
|
}
|
|
|
|
data, err := os.ReadFile(path)
|
|
if err != nil {
|
|
if optional && os.IsNotExist(err) {
|
|
return zero, nil
|
|
}
|
|
return zero, fmt.Errorf("reading config file %q: %w", path, err)
|
|
}
|
|
|
|
var cfg T
|
|
if err := yaml.Unmarshal(data, &cfg); err != nil {
|
|
return zero, fmt.Errorf("parsing config file %q: %w", path, err)
|
|
}
|
|
|
|
return cfg, nil
|
|
}
|
|
|
|
// OverrideFromEnv applies environment variable overrides to a struct using struct tags.
|
|
// Tag format: env:"ENV_VAR_NAME"
|
|
// Supports field types: string, int, bool, []string (comma-separated).
|
|
// envPrefix is prepended to tag values if non-empty (e.g. envPrefix="APP" + tag="PORT" → "APP_PORT").
|
|
func OverrideFromEnv[T any](cfg *T, envPrefix string) error {
|
|
return overrideStruct(reflect.ValueOf(cfg).Elem(), envPrefix)
|
|
}
|
|
|
|
func overrideStruct(v reflect.Value, envPrefix string) error {
|
|
t := v.Type()
|
|
for i := 0; i < t.NumField(); i++ {
|
|
field := t.Field(i)
|
|
fv := v.Field(i)
|
|
|
|
if !fv.CanSet() {
|
|
continue
|
|
}
|
|
|
|
// Recurse into embedded/nested structs
|
|
if fv.Kind() == reflect.Struct {
|
|
if err := overrideStruct(fv, envPrefix); err != nil {
|
|
return err
|
|
}
|
|
continue
|
|
}
|
|
|
|
envTag := field.Tag.Get("env")
|
|
if envTag == "" {
|
|
continue
|
|
}
|
|
|
|
envKey := envTag
|
|
if envPrefix != "" {
|
|
envKey = envPrefix + "_" + envTag
|
|
}
|
|
|
|
val := os.Getenv(envKey)
|
|
if val == "" {
|
|
continue
|
|
}
|
|
|
|
switch fv.Kind() {
|
|
case reflect.String:
|
|
fv.SetString(val)
|
|
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
|
n, err := strconv.ParseInt(val, 10, 64)
|
|
if err != nil {
|
|
return fmt.Errorf("env %s: cannot parse %q as int: %w", envKey, val, err)
|
|
}
|
|
fv.SetInt(n)
|
|
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
|
n, err := strconv.ParseUint(val, 10, 64)
|
|
if err != nil {
|
|
return fmt.Errorf("env %s: cannot parse %q as uint: %w", envKey, val, err)
|
|
}
|
|
fv.SetUint(n)
|
|
case reflect.Bool:
|
|
b, err := strconv.ParseBool(val)
|
|
if err != nil {
|
|
return fmt.Errorf("env %s: cannot parse %q as bool: %w", envKey, val, err)
|
|
}
|
|
fv.SetBool(b)
|
|
case reflect.Slice:
|
|
if fv.Type().Elem().Kind() == reflect.String {
|
|
parts := strings.Split(val, ",")
|
|
for i, p := range parts {
|
|
parts[i] = strings.TrimSpace(p)
|
|
}
|
|
fv.Set(reflect.ValueOf(parts))
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|