feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized

Services:
- ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap)
- logcorrelator: JA4 log correlation engine (Go, ClickHouse)
- mod_reqin_log: Apache module (C, JSON request logging)
- bot_detector: ML bot detection pipeline (Python)
- dashboard: FastAPI/Streamlit analytics UI (Python)

Shared libraries:
- shared/go/ja4common: logger, config, shutdown, ipfilter (Go module)
- shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package)
- shared/clickhouse/: canonical SQL migrations (10 files)

Build & packaging:
- Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10)
- go.work workspace linking sentinel, correlator, ja4common
- Makefile with test-all, build-all, rpm-* targets

Fixes applied:
- go.work: 1.21 → 1.24.6 (required by sentinel)
- correlator Dockerfiles: golang:1.21 → golang:1.24
- replace directives in go.mod for ja4common local path
- pyproject.toml: setuptools.backends → setuptools.build_meta
- Removed static libpcap linking (unavailable on Rocky 9)
- Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32)
- Rewrote corrupted test files (logger_test.go × 2)

Test coverage:
- correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%)
- sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse)

Documentation:
- README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 16:42:59 +02:00
commit d469e39da7
278 changed files with 1621301 additions and 0 deletions

View File

@ -0,0 +1,333 @@
/*
* test_config_parsing.c - Unit tests for configuration parsing
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include <string.h>
#include <stdio.h>
#include <stdlib.h>
#include <errno.h>
#include <limits.h>
/* Default configuration values */
#define DEFAULT_MAX_HEADERS 10
#define DEFAULT_MAX_HEADER_VALUE_LEN 256
#define DEFAULT_RECONNECT_INTERVAL 10
#define DEFAULT_ERROR_REPORT_INTERVAL 10
#define MAX_SOCKET_PATH_LEN 108
/* Mock configuration structure */
typedef struct {
int enabled;
const char *socket_path;
int max_headers;
int max_header_value_len;
int reconnect_interval;
int error_report_interval;
} mock_config_t;
/* Mock parsing functions */
static int parse_enabled(const char *value)
{
if (strcasecmp(value, "on") == 0 || strcmp(value, "1") == 0) {
return 1;
}
return 0;
}
static const char *parse_socket_path(const char *value)
{
if (value == NULL || strlen(value) == 0) {
return NULL;
}
if (strlen(value) >= MAX_SOCKET_PATH_LEN) {
return NULL;
}
return value;
}
static int parse_int_strict(const char *value, int *result)
{
char *endptr = NULL;
long val;
if (value == NULL || *value == '\0' || result == NULL) {
return -1;
}
errno = 0;
val = strtol(value, &endptr, 10);
if (errno != 0 || endptr == value || *endptr != '\0' || val < INT_MIN || val > INT_MAX) {
return -1;
}
*result = (int)val;
return 0;
}
static int parse_max_headers(const char *value, int *result)
{
if (parse_int_strict(value, result) != 0 || *result < 0) {
return -1;
}
return 0;
}
static int parse_interval(const char *value, int *result)
{
if (parse_int_strict(value, result) != 0 || *result < 0) {
return -1;
}
return 0;
}
static int parse_max_header_value_len(const char *value, int *result)
{
if (parse_int_strict(value, result) != 0 || *result < 1) {
return -1;
}
return 0;
}
/* Test: Parse enabled On */
static void test_parse_enabled_on(void **state)
{
(void)state;
assert_int_equal(parse_enabled("On"), 1);
assert_int_equal(parse_enabled("on"), 1);
assert_int_equal(parse_enabled("ON"), 1);
assert_int_equal(parse_enabled("1"), 1);
}
/* Test: Parse enabled Off */
static void test_parse_enabled_off(void **state)
{
(void)state;
assert_int_equal(parse_enabled("Off"), 0);
assert_int_equal(parse_enabled("off"), 0);
assert_int_equal(parse_enabled("OFF"), 0);
assert_int_equal(parse_enabled("0"), 0);
}
/* Test: Parse socket path valid */
static void test_parse_socket_path_valid(void **state)
{
(void)state;
const char *result = parse_socket_path("/var/run/logcorrelator/http.socket");
assert_string_equal(result, "/var/run/logcorrelator/http.socket");
}
/* Test: Parse socket path empty */
static void test_parse_socket_path_empty(void **state)
{
(void)state;
const char *result = parse_socket_path("");
assert_null(result);
}
/* Test: Parse socket path NULL */
static void test_parse_socket_path_null(void **state)
{
(void)state;
const char *result = parse_socket_path(NULL);
assert_null(result);
}
/* Test: Parse socket path max length valid */
static void test_parse_socket_path_max_len_valid(void **state)
{
(void)state;
char path[MAX_SOCKET_PATH_LEN];
memset(path, 'a', MAX_SOCKET_PATH_LEN - 1);
path[MAX_SOCKET_PATH_LEN - 1] = '\0';
assert_non_null(parse_socket_path(path));
}
/* Test: Parse socket path max length invalid */
static void test_parse_socket_path_max_len_invalid(void **state)
{
(void)state;
char path[MAX_SOCKET_PATH_LEN + 1];
memset(path, 'b', MAX_SOCKET_PATH_LEN);
path[MAX_SOCKET_PATH_LEN] = '\0';
assert_null(parse_socket_path(path));
}
/* Test: Parse max headers valid */
static void test_parse_max_headers_valid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("10", &result), 0);
assert_int_equal(result, 10);
assert_int_equal(parse_max_headers("0", &result), 0);
assert_int_equal(result, 0);
assert_int_equal(parse_max_headers("100", &result), 0);
assert_int_equal(result, 100);
}
/* Test: Parse max headers invalid */
static void test_parse_max_headers_invalid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("-1", &result), -1);
assert_int_equal(parse_max_headers("abc", &result), -1);
assert_int_equal(parse_max_headers("10abc", &result), -1);
}
/* Test: Parse reconnect interval valid */
static void test_parse_reconnect_interval_valid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_interval("10", &result), 0);
assert_int_equal(result, 10);
assert_int_equal(parse_interval("0", &result), 0);
assert_int_equal(result, 0);
assert_int_equal(parse_interval("60", &result), 0);
assert_int_equal(result, 60);
}
/* Test: Parse reconnect interval invalid */
static void test_parse_reconnect_interval_invalid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_interval("-5", &result), -1);
assert_int_equal(parse_interval("abc", &result), -1);
assert_int_equal(parse_interval("10abc", &result), -1);
}
/* Test: Parse max header value length valid */
static void test_parse_max_header_value_len_valid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_header_value_len("1", &result), 0);
assert_int_equal(result, 1);
assert_int_equal(parse_max_header_value_len("256", &result), 0);
assert_int_equal(result, 256);
}
/* Test: Parse max header value length invalid */
static void test_parse_max_header_value_len_invalid(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_header_value_len("0", &result), -1);
assert_int_equal(parse_max_header_value_len("-1", &result), -1);
assert_int_equal(parse_max_header_value_len("10abc", &result), -1);
}
/* Test: strict numeric parsing invalid suffix for all int directives */
static void test_strict_numeric_invalid_suffix_all(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("10abc", &result), -1);
assert_int_equal(parse_interval("10abc", &result), -1);
assert_int_equal(parse_max_header_value_len("10abc", &result), -1);
}
/* Test: Default configuration values */
static void test_default_config_values(void **state)
{
(void)state;
assert_int_equal(DEFAULT_MAX_HEADERS, 10);
assert_int_equal(DEFAULT_MAX_HEADER_VALUE_LEN, 256);
assert_int_equal(DEFAULT_RECONNECT_INTERVAL, 10);
assert_int_equal(DEFAULT_ERROR_REPORT_INTERVAL, 10);
}
/* Test: Configuration validation - enabled requires socket */
static void test_config_validation_enabled_requires_socket(void **state)
{
int enabled = 1;
const char *socket = "/var/run/socket";
(void)state;
assert_true(enabled == 0 || socket != NULL);
socket = NULL;
assert_false(enabled == 0 || socket != NULL);
}
/* Test: Configuration validation - enabled with empty socket is invalid */
static void test_config_validation_enabled_with_empty_socket(void **state)
{
int enabled = 1;
const char *socket = parse_socket_path("");
(void)state;
assert_false(enabled == 0 || socket != NULL);
}
/* Test: Header value length validation */
static void test_header_value_len_validation(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_header_value_len("1", &result), 0);
assert_true(result >= 1);
assert_int_equal(parse_max_header_value_len("0", &result), -1);
}
/* Test: Large but valid values */
static void test_large_valid_values(void **state)
{
int result;
(void)state;
assert_int_equal(parse_max_headers("1000000", &result), 0);
assert_int_equal(result, 1000000);
assert_int_equal(parse_interval("86400", &result), 0);
assert_int_equal(result, 86400);
}
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_parse_enabled_on),
cmocka_unit_test(test_parse_enabled_off),
cmocka_unit_test(test_parse_socket_path_valid),
cmocka_unit_test(test_parse_socket_path_empty),
cmocka_unit_test(test_parse_socket_path_null),
cmocka_unit_test(test_parse_socket_path_max_len_valid),
cmocka_unit_test(test_parse_socket_path_max_len_invalid),
cmocka_unit_test(test_parse_max_headers_valid),
cmocka_unit_test(test_parse_max_headers_invalid),
cmocka_unit_test(test_parse_reconnect_interval_valid),
cmocka_unit_test(test_parse_reconnect_interval_invalid),
cmocka_unit_test(test_parse_max_header_value_len_valid),
cmocka_unit_test(test_parse_max_header_value_len_invalid),
cmocka_unit_test(test_strict_numeric_invalid_suffix_all),
cmocka_unit_test(test_default_config_values),
cmocka_unit_test(test_config_validation_enabled_requires_socket),
cmocka_unit_test(test_config_validation_enabled_with_empty_socket),
cmocka_unit_test(test_header_value_len_validation),
cmocka_unit_test(test_large_valid_values),
};
return cmocka_run_group_tests(tests, NULL, NULL);
}

View File

@ -0,0 +1,226 @@
/*
* test_header_handling.c - Unit tests for header handling (truncation and limits)
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include <string.h>
#include <stdio.h>
#include <apr_strings.h>
#include <apr_tables.h>
#include <apr_pools.h>
#include <apr_general.h>
/* Mock header truncation function */
static char *truncate_header_value(apr_pool_t *pool, const char *value, int max_len)
{
if (value == NULL) {
return NULL;
}
size_t len = strlen(value);
if ((int)len > max_len) {
return apr_pstrmemdup(pool, value, max_len);
}
return apr_pstrdup(pool, value);
}
/* Mock header matching function */
static int header_name_matches(const char *configured, const char *actual)
{
return strcasecmp(configured, actual) == 0;
}
/* Test: Header value within limit */
static void test_header_truncation_within_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "short value";
char *result = truncate_header_value(pool, value, 256);
assert_string_equal(result, "short value");
apr_pool_destroy(pool);
}
/* Test: Header value exactly at limit */
static void test_header_truncation_exact_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "exactly10c";
char *result = truncate_header_value(pool, value, 10);
assert_string_equal(result, "exactly10c");
apr_pool_destroy(pool);
}
/* Test: Header value exceeds limit */
static void test_header_truncation_exceeds_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "this is a very long header value that should be truncated";
char *result = truncate_header_value(pool, value, 15);
assert_string_equal(result, "this is a very ");
assert_int_equal(strlen(result), 15);
apr_pool_destroy(pool);
}
/* Test: Header value with limit of 1 */
static void test_header_truncation_limit_one(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "abc";
char *result = truncate_header_value(pool, value, 1);
assert_string_equal(result, "a");
apr_pool_destroy(pool);
}
/* Test: NULL header value */
static void test_header_truncation_null(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
char *result = truncate_header_value(pool, NULL, 256);
assert_null(result);
apr_pool_destroy(pool);
}
/* Test: Empty header value */
static void test_header_truncation_empty(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "";
char *result = truncate_header_value(pool, value, 256);
assert_string_equal(result, "");
apr_pool_destroy(pool);
}
/* Test: Header name matching (case-insensitive) */
static void test_header_name_matching_case_insensitive(void **state)
{
assert_true(header_name_matches("X-Request-Id", "x-request-id"));
assert_true(header_name_matches("user-agent", "User-Agent"));
assert_true(header_name_matches("HOST", "host"));
}
/* Test: Header name matching (different headers) */
static void test_header_name_matching_different(void **state)
{
assert_false(header_name_matches("X-Request-Id", "X-Trace-Id"));
assert_false(header_name_matches("Host", "User-Agent"));
}
/* Test: Multiple headers with limit */
static void test_header_count_limit(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
/* Simulate configured headers */
const char *configured[] = {"X-Request-Id", "X-Trace-Id", "User-Agent", "Referer"};
int configured_count = 4;
int max_headers = 2;
/* Simulate present headers */
const char *present[] = {"X-Request-Id", "User-Agent", "Referer"};
int present_count = 3;
int logged_count = 0;
for (int i = 0; i < configured_count && logged_count < max_headers; i++) {
for (int j = 0; j < present_count; j++) {
if (header_name_matches(configured[i], present[j])) {
logged_count++;
break;
}
}
}
assert_int_equal(logged_count, 2);
apr_pool_destroy(pool);
}
/* Test: Header value with special JSON characters */
static void test_header_value_json_special(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "test\"value\\with\tspecial";
char *truncated = truncate_header_value(pool, value, 256);
/* Truncation should preserve the value */
assert_string_equal(truncated, "test\"value\\with\tspecial");
apr_pool_destroy(pool);
}
/* Test: Unicode in header value (UTF-8) */
static void test_header_value_unicode(void **state)
{
apr_pool_t *pool;
apr_pool_create(&pool, NULL);
const char *value = "Mozilla/5.0 (compatible; 日本語)";
char *result = truncate_header_value(pool, value, 50);
/* Should be truncated but valid */
assert_non_null(result);
assert_true(strlen(result) <= 50);
apr_pool_destroy(pool);
}
static int group_setup(void **state)
{
(void)state;
return apr_initialize();
}
static int group_teardown(void **state)
{
(void)state;
apr_terminate();
return 0;
}
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_header_truncation_within_limit),
cmocka_unit_test(test_header_truncation_exact_limit),
cmocka_unit_test(test_header_truncation_exceeds_limit),
cmocka_unit_test(test_header_truncation_limit_one),
cmocka_unit_test(test_header_truncation_null),
cmocka_unit_test(test_header_truncation_empty),
cmocka_unit_test(test_header_name_matching_case_insensitive),
cmocka_unit_test(test_header_name_matching_different),
cmocka_unit_test(test_header_count_limit),
cmocka_unit_test(test_header_value_json_special),
cmocka_unit_test(test_header_value_unicode),
};
return cmocka_run_group_tests(tests, group_setup, group_teardown);
}

View File

@ -0,0 +1,266 @@
/*
* test_json_serialization.c - Unit tests for JSON serialization
*/
#include <stdarg.h>
#include <stddef.h>
#include <setjmp.h>
#include <cmocka.h>
#include <string.h>
#include <stdio.h>
#include <apr_pools.h>
#include <apr_strings.h>
#include <apr_time.h>
#include <apr_lib.h>
#include <apr_general.h>
typedef struct {
char *data;
size_t len;
size_t cap;
apr_pool_t *pool;
} testbuf_t;
static void testbuf_init(testbuf_t *buf, apr_pool_t *pool, size_t initial_capacity)
{
buf->pool = pool;
buf->cap = initial_capacity;
buf->len = 0;
buf->data = apr_palloc(pool, initial_capacity);
buf->data[0] = '\0';
}
static void testbuf_append(testbuf_t *buf, const char *str, size_t len)
{
if (str == NULL) {
return;
}
if (len == (size_t)-1) {
len = strlen(str);
}
if (buf->len + len + 1 > buf->cap) {
size_t new_cap = (buf->len + len + 1) * 2;
char *new_data = apr_palloc(buf->pool, new_cap);
memcpy(new_data, buf->data, buf->len + 1); /* Copy including null terminator */
buf->data = new_data;
buf->cap = new_cap;
}
memcpy(buf->data + buf->len, str, len);
buf->len += len;
buf->data[buf->len] = '\0';
}
static void testbuf_append_char(testbuf_t *buf, char c)
{
if (buf->len + 2 > buf->cap) {
size_t new_cap = (buf->cap * 2);
char *new_data = apr_palloc(buf->pool, new_cap);
memcpy(new_data, buf->data, buf->len + 1);
buf->data = new_data;
buf->cap = new_cap;
}
buf->data[buf->len++] = c;
buf->data[buf->len] = '\0';
}
/* Mock JSON string escaping function for testing */
static void append_json_string(testbuf_t *buf, const char *str)
{
if (str == NULL) {
return;
}
for (const char *p = str; *p; p++) {
char c = *p;
switch (c) {
case '"': testbuf_append(buf, "\\\"", 2); break;
case '\\': testbuf_append(buf, "\\\\", 2); break;
case '\b': testbuf_append(buf, "\\b", 2); break;
case '\f': testbuf_append(buf, "\\f", 2); break;
case '\n': testbuf_append(buf, "\\n", 2); break;
case '\r': testbuf_append(buf, "\\r", 2); break;
case '\t': testbuf_append(buf, "\\t", 2); break;
default:
if ((unsigned char)c < 0x20) {
char unicode[8];
apr_snprintf(unicode, sizeof(unicode), "\\u%04x", (unsigned char)c);
testbuf_append(buf, unicode, (size_t)-1);
} else {
testbuf_append_char(buf, c);
}
break;
}
}
}
/* Test: Empty string */
static void test_json_escape_empty_string(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "");
assert_string_equal(buf.data, "");
apr_pool_destroy(pool);
}
/* Test: Simple string without special characters */
static void test_json_escape_simple_string(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "hello world");
assert_string_equal(buf.data, "hello world");
apr_pool_destroy(pool);
}
/* Test: String with double quotes */
static void test_json_escape_quotes(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "hello \"world\"");
assert_string_equal(buf.data, "hello \\\"world\\\"");
apr_pool_destroy(pool);
}
/* Test: String with backslashes */
static void test_json_escape_backslashes(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "path\\to\\file");
assert_string_equal(buf.data, "path\\\\to\\\\file");
apr_pool_destroy(pool);
}
/* Test: String with newlines and tabs */
static void test_json_escape_newlines_tabs(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, "line1\nline2\ttab");
assert_string_equal(buf.data, "line1\\nline2\\ttab");
apr_pool_destroy(pool);
}
/* Test: String with control characters */
static void test_json_escape_control_chars(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
/* Test with bell character (0x07) - use octal literal */
append_json_string(&buf, "test\007bell");
/* Should contain unicode escape for bell (0x07) */
assert_true(strstr(buf.data, "\\u0007") != NULL);
apr_pool_destroy(pool);
}
/* Test: NULL string */
static void test_json_escape_null_string(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 256);
append_json_string(&buf, NULL);
assert_string_equal(buf.data, "");
apr_pool_destroy(pool);
}
/* Test: Complex user agent string */
static void test_json_escape_user_agent(void **state)
{
apr_pool_t *pool;
testbuf_t buf;
const char *ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) \"Test\"";
(void)state;
apr_pool_create(&pool, NULL);
testbuf_init(&buf, pool, 512);
append_json_string(&buf, ua);
assert_true(strstr(buf.data, "\\\"Test\\\"") != NULL);
apr_pool_destroy(pool);
}
static int group_setup(void **state)
{
(void)state;
return apr_initialize();
}
static int group_teardown(void **state)
{
(void)state;
apr_terminate();
return 0;
}
int main(void)
{
const struct CMUnitTest tests[] = {
cmocka_unit_test(test_json_escape_empty_string),
cmocka_unit_test(test_json_escape_simple_string),
cmocka_unit_test(test_json_escape_quotes),
cmocka_unit_test(test_json_escape_backslashes),
cmocka_unit_test(test_json_escape_newlines_tabs),
cmocka_unit_test(test_json_escape_control_chars),
cmocka_unit_test(test_json_escape_null_string),
cmocka_unit_test(test_json_escape_user_agent),
};
return cmocka_run_group_tests(tests, group_setup, group_teardown);
}

File diff suppressed because it is too large Load Diff