feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized
Services: - ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap) - logcorrelator: JA4 log correlation engine (Go, ClickHouse) - mod_reqin_log: Apache module (C, JSON request logging) - bot_detector: ML bot detection pipeline (Python) - dashboard: FastAPI/Streamlit analytics UI (Python) Shared libraries: - shared/go/ja4common: logger, config, shutdown, ipfilter (Go module) - shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package) - shared/clickhouse/: canonical SQL migrations (10 files) Build & packaging: - Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10) - go.work workspace linking sentinel, correlator, ja4common - Makefile with test-all, build-all, rpm-* targets Fixes applied: - go.work: 1.21 → 1.24.6 (required by sentinel) - correlator Dockerfiles: golang:1.21 → golang:1.24 - replace directives in go.mod for ja4common local path - pyproject.toml: setuptools.backends → setuptools.build_meta - Removed static libpcap linking (unavailable on Rocky 9) - Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32) - Rewrote corrupted test files (logger_test.go × 2) Test coverage: - correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%) - sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse) Documentation: - README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
357
services/sentinel/cmd/ja4sentinel/main.go
Normal file
357
services/sentinel/cmd/ja4sentinel/main.go
Normal file
@ -0,0 +1,357 @@
|
||||
// Package main provides the entry point for ja4sentinel
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/coreos/go-systemd/v22/daemon"
|
||||
"github.com/antitbone/ja4/sentinel/api"
|
||||
"github.com/antitbone/ja4/sentinel/internal/capture"
|
||||
"github.com/antitbone/ja4/sentinel/internal/config"
|
||||
"github.com/antitbone/ja4/sentinel/internal/fingerprint"
|
||||
"github.com/antitbone/ja4/sentinel/internal/logging"
|
||||
"github.com/antitbone/ja4/sentinel/internal/output"
|
||||
"github.com/antitbone/ja4/sentinel/internal/tlsparse"
|
||||
)
|
||||
|
||||
var (
|
||||
// Version information (set via ldflags)
|
||||
Version = "1.1.15"
|
||||
BuildTime = "unknown"
|
||||
GitCommit = "unknown"
|
||||
)
|
||||
|
||||
func main() {
|
||||
// Parse command-line flags
|
||||
configPath := flag.String("config", "", "Path to configuration file (YAML)")
|
||||
version := flag.Bool("version", false, "Show version information")
|
||||
flag.Parse()
|
||||
|
||||
if *version {
|
||||
fmt.Printf("ja4sentinel version %s (built %s, commit %s)\n", Version, BuildTime, GitCommit)
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
// Load configuration
|
||||
cfgLoader := config.NewLoader(*configPath)
|
||||
appConfig, err := cfgLoader.Load()
|
||||
if err != nil {
|
||||
// Create logger with default level for error reporting
|
||||
loggerFactory := &logging.LoggerFactory{}
|
||||
appLogger := loggerFactory.NewDefaultLogger()
|
||||
appLogger.Error("main", "Failed to load configuration", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Create logger factory with configured log level
|
||||
loggerFactory := &logging.LoggerFactory{}
|
||||
appLogger := loggerFactory.NewLogger(appConfig.Core.LogLevel)
|
||||
|
||||
appLogger.Info("main", "Starting ja4sentinel", map[string]string{
|
||||
"version": Version,
|
||||
"build_time": BuildTime,
|
||||
"git_commit": GitCommit,
|
||||
})
|
||||
|
||||
appLogger.Info("main", "Configuration loaded", map[string]string{
|
||||
"interface": appConfig.Core.Interface,
|
||||
"listen_ports": formatPorts(appConfig.Core.ListenPorts),
|
||||
"log_level": appConfig.Core.LogLevel,
|
||||
})
|
||||
|
||||
// Create context with cancellation for graceful shutdown
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
// Signal readiness to systemd
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyReady); err != nil {
|
||||
appLogger.Warn("main", "Failed to send READY notification to systemd", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Start watchdog goroutine if enabled
|
||||
watchdogInterval, err := daemon.SdWatchdogEnabled(false)
|
||||
if err != nil {
|
||||
appLogger.Warn("main", "Failed to check watchdog status", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
if watchdogInterval > 0 {
|
||||
appLogger.Info("main", "systemd watchdog enabled", map[string]string{
|
||||
"interval": watchdogInterval.String(),
|
||||
})
|
||||
go func() {
|
||||
ticker := time.NewTicker(watchdogInterval / 2)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyWatchdog); err != nil {
|
||||
appLogger.Warn("main", "Failed to send WATCHDOG notification", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// Setup signal handling for shutdown and log rotation
|
||||
sigChan := make(chan os.Signal, 1)
|
||||
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP)
|
||||
|
||||
// Create pipeline components
|
||||
captureEngine := capture.New()
|
||||
parser := tlsparse.NewParserWithTimeoutAndFilter(
|
||||
time.Duration(appConfig.Core.FlowTimeoutSec)*time.Second,
|
||||
appConfig.Core.ExcludeSourceIPs,
|
||||
)
|
||||
fingerprintEngine := fingerprint.NewEngine()
|
||||
|
||||
// Log exclusion configuration with debug details
|
||||
if len(appConfig.Core.ExcludeSourceIPs) > 0 {
|
||||
appLogger.Info("main", "Source IP exclusion enabled", map[string]string{
|
||||
"exclude_count": fmt.Sprintf("%d", len(appConfig.Core.ExcludeSourceIPs)),
|
||||
"exclude_ips": strings.Join(appConfig.Core.ExcludeSourceIPs, ", "),
|
||||
})
|
||||
appLogger.Debug("tlsparse", "IP filter configured", map[string]string{
|
||||
"filter_entries": strings.Join(appConfig.Core.ExcludeSourceIPs, ", "),
|
||||
})
|
||||
} else {
|
||||
appLogger.Debug("tlsparse", "IP filter disabled (no exclusions configured)", nil)
|
||||
}
|
||||
|
||||
// Log filter stats at startup (debug mode)
|
||||
filteredCount, hasFilter := parser.GetFilterStats()
|
||||
if hasFilter {
|
||||
appLogger.Debug("tlsparse", "IP filter initialized", map[string]string{
|
||||
"filtered_packets": fmt.Sprintf("%d", filteredCount),
|
||||
})
|
||||
}
|
||||
|
||||
// Create output builder with error callback for socket connection errors
|
||||
outputBuilder := output.NewBuilder().WithErrorCallback(func(socketPath string, err error, attempt int) {
|
||||
appLogger.Error("output", "UNIX socket connection failed", map[string]string{
|
||||
"socket_path": socketPath,
|
||||
"error": err.Error(),
|
||||
"attempt": fmt.Sprintf("%d", attempt),
|
||||
})
|
||||
})
|
||||
|
||||
outputWriter, err := outputBuilder.NewFromConfig(appConfig)
|
||||
if err != nil {
|
||||
appLogger.Error("main", "Failed to create output writer", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Create channel for raw packets (configurable buffer size)
|
||||
bufferSize := appConfig.Core.PacketBufferSize
|
||||
if bufferSize <= 0 {
|
||||
bufferSize = 1000 // Default fallback
|
||||
}
|
||||
packetChan := make(chan api.RawPacket, bufferSize)
|
||||
|
||||
// Start capture goroutine
|
||||
captureErrChan := make(chan error, 1)
|
||||
go func() {
|
||||
appLogger.Info("capture", "Starting packet capture", map[string]string{
|
||||
"interface": appConfig.Core.Interface,
|
||||
})
|
||||
err := captureEngine.Run(appConfig.Core, packetChan)
|
||||
close(packetChan) // Close channel to signal packet processor to shut down
|
||||
captureErrChan <- err
|
||||
}()
|
||||
|
||||
// Log capture diagnostics after a short delay to allow initialization
|
||||
go func() {
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
ifName, localIPs, bpfFilter, linkType := captureEngine.GetDiagnostics()
|
||||
appLogger.Debug("capture", "Capture initialized", map[string]string{
|
||||
"interface": ifName,
|
||||
"link_type": fmt.Sprintf("%d", linkType),
|
||||
"local_ips": strings.Join(localIPs, ", "),
|
||||
"bpf_filter": bpfFilter,
|
||||
})
|
||||
}()
|
||||
|
||||
// Process packets
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
appLogger.Info("main", "Packet processor shutting down", nil)
|
||||
return
|
||||
case pkt, ok := <-packetChan:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
// Parse TLS ClientHello
|
||||
clientHello, err := parser.Process(pkt)
|
||||
if err != nil {
|
||||
appLogger.Warn("tlsparse", "Failed to parse TLS ClientHello", map[string]string{
|
||||
"error": err.Error(),
|
||||
"packet_len": fmt.Sprintf("%d", len(pkt.Data)),
|
||||
"link_type": fmt.Sprintf("%d", pkt.LinkType),
|
||||
"timestamp": fmt.Sprintf("%d", pkt.Timestamp),
|
||||
})
|
||||
continue
|
||||
}
|
||||
if clientHello == nil {
|
||||
continue // Not a TLS ClientHello packet
|
||||
}
|
||||
|
||||
appLogger.Debug("tlsparse", "ClientHello extracted", map[string]string{
|
||||
"src_ip": clientHello.SrcIP,
|
||||
"src_port": fmt.Sprintf("%d", clientHello.SrcPort),
|
||||
"dst_ip": clientHello.DstIP,
|
||||
"dst_port": fmt.Sprintf("%d", clientHello.DstPort),
|
||||
})
|
||||
|
||||
// Generate fingerprints
|
||||
fingerprints, err := fingerprintEngine.FromClientHello(*clientHello)
|
||||
if err != nil {
|
||||
appLogger.Warn("fingerprint", "Failed to generate fingerprints", map[string]string{
|
||||
"error": err.Error(),
|
||||
"src_ip": clientHello.SrcIP,
|
||||
"src_port": fmt.Sprintf("%d", clientHello.SrcPort),
|
||||
"dst_ip": clientHello.DstIP,
|
||||
"dst_port": fmt.Sprintf("%d", clientHello.DstPort),
|
||||
"conn_id": clientHello.ConnID,
|
||||
"payload_len": fmt.Sprintf("%d", len(clientHello.Payload)),
|
||||
"sni": clientHello.SNI,
|
||||
"tls_version": clientHello.TLSVersion,
|
||||
"alpn": clientHello.ALPN,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
appLogger.Debug("fingerprint", "Fingerprints generated", map[string]string{
|
||||
"src_ip": clientHello.SrcIP,
|
||||
"ja4": fingerprints.JA4,
|
||||
})
|
||||
|
||||
// Create log record
|
||||
logRecord := api.NewLogRecord(*clientHello, fingerprints)
|
||||
|
||||
// Write output
|
||||
if err := outputWriter.Write(logRecord); err != nil {
|
||||
appLogger.Error("output", "Failed to write log record", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
// Wait for shutdown signal or capture error
|
||||
for {
|
||||
select {
|
||||
case sig := <-sigChan:
|
||||
switch sig {
|
||||
case syscall.SIGHUP:
|
||||
// Handle log rotation - reopen output files
|
||||
appLogger.Info("main", "Received SIGHUP, reopening log files", nil)
|
||||
if mw, ok := outputWriter.(api.Reopenable); ok {
|
||||
if err := mw.Reopen(); err != nil {
|
||||
appLogger.Error("main", "Failed to reopen log files", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
} else {
|
||||
appLogger.Info("main", "Log files reopened successfully", nil)
|
||||
}
|
||||
} else {
|
||||
appLogger.Warn("main", "Output writer does not support log rotation", nil)
|
||||
}
|
||||
case syscall.SIGINT, syscall.SIGTERM:
|
||||
appLogger.Info("main", "Received shutdown signal", map[string]string{
|
||||
"signal": sig.String(),
|
||||
})
|
||||
goto shutdown
|
||||
}
|
||||
case err := <-captureErrChan:
|
||||
if err != nil {
|
||||
appLogger.Error("capture", "Capture engine failed", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
goto shutdown
|
||||
}
|
||||
}
|
||||
|
||||
shutdown:
|
||||
// Graceful shutdown
|
||||
appLogger.Info("main", "Shutting down...", nil)
|
||||
|
||||
// Signal stopping to systemd
|
||||
if _, err := daemon.SdNotify(false, daemon.SdNotifyStopping); err != nil {
|
||||
appLogger.Warn("main", "Failed to send STOPPING notification to systemd", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
cancel()
|
||||
|
||||
// Close components
|
||||
if err := captureEngine.Close(); err != nil {
|
||||
appLogger.Error("main", "Failed to close capture engine", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
if err := parser.Close(); err != nil {
|
||||
appLogger.Error("main", "Failed to close parser", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
|
||||
// Log final filter stats
|
||||
filteredCount, hasFilter = parser.GetFilterStats()
|
||||
if hasFilter {
|
||||
appLogger.Info("tlsparse", "IP filter statistics", map[string]string{
|
||||
"total_filtered_packets": fmt.Sprintf("%d", filteredCount),
|
||||
})
|
||||
}
|
||||
|
||||
if mw, ok := outputWriter.(interface{ CloseAll() error }); ok {
|
||||
if err := mw.CloseAll(); err != nil {
|
||||
appLogger.Error("main", "Failed to close output writers", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
} else if closer, ok := outputWriter.(interface{ Close() error }); ok {
|
||||
if err := closer.Close(); err != nil {
|
||||
appLogger.Error("main", "Failed to close output writer", map[string]string{
|
||||
"error": err.Error(),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
appLogger.Info("main", "ja4sentinel stopped", nil)
|
||||
}
|
||||
|
||||
// formatPorts formats a slice of ports as a comma-separated string
|
||||
func formatPorts(ports []uint16) string {
|
||||
if len(ports) == 0 {
|
||||
return ""
|
||||
}
|
||||
result := fmt.Sprintf("%d", ports[0])
|
||||
for _, port := range ports[1:] {
|
||||
result += fmt.Sprintf(",%d", port)
|
||||
}
|
||||
return result
|
||||
}
|
||||
Reference in New Issue
Block a user