feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized

Services:
- ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap)
- logcorrelator: JA4 log correlation engine (Go, ClickHouse)
- mod_reqin_log: Apache module (C, JSON request logging)
- bot_detector: ML bot detection pipeline (Python)
- dashboard: FastAPI/Streamlit analytics UI (Python)

Shared libraries:
- shared/go/ja4common: logger, config, shutdown, ipfilter (Go module)
- shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package)
- shared/clickhouse/: canonical SQL migrations (10 files)

Build & packaging:
- Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10)
- go.work workspace linking sentinel, correlator, ja4common
- Makefile with test-all, build-all, rpm-* targets

Fixes applied:
- go.work: 1.21 → 1.24.6 (required by sentinel)
- correlator Dockerfiles: golang:1.21 → golang:1.24
- replace directives in go.mod for ja4common local path
- pyproject.toml: setuptools.backends → setuptools.build_meta
- Removed static libpcap linking (unavailable on Rocky 9)
- Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32)
- Rewrote corrupted test files (logger_test.go × 2)

Test coverage:
- correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%)
- sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse)

Documentation:
- README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations)

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-07 16:42:59 +02:00
commit d469e39da7
278 changed files with 1621301 additions and 0 deletions

View File

@ -0,0 +1,185 @@
import { useEffect, useState } from 'react';
interface UserAgentData {
value: string;
count: number;
percentage: number;
classification: 'normal' | 'bot' | 'script';
}
interface UserAgentAnalysis {
ip_user_agents: UserAgentData[];
ja4_user_agents: UserAgentData[];
bot_percentage: number;
alert: boolean;
}
interface UserAgentAnalysisProps {
ip: string;
}
export function UserAgentAnalysis({ ip }: UserAgentAnalysisProps) {
const [data, setData] = useState<UserAgentAnalysis | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
const [showAllIpUA, setShowAllIpUA] = useState(false);
const [showAllJa4UA, setShowAllJa4UA] = useState(false);
useEffect(() => {
const fetchUserAgentAnalysis = async () => {
setLoading(true);
try {
const response = await fetch(`/api/analysis/${encodeURIComponent(ip)}/user-agents`);
if (!response.ok) throw new Error('Erreur chargement User-Agents');
const result = await response.json();
setData(result);
} catch (err) {
setError(err instanceof Error ? err.message : 'Erreur inconnue');
} finally {
setLoading(false);
}
};
fetchUserAgentAnalysis();
}, [ip]);
if (loading) {
return (
<div className="bg-background-secondary rounded-lg p-6">
<div className="text-center text-text-secondary">Chargement...</div>
</div>
);
}
if (error || !data) {
return (
<div className="bg-background-secondary rounded-lg p-6">
<div className="text-center text-text-secondary">User-Agents non disponibles</div>
</div>
);
}
const getClassificationBadge = (classification: string) => {
switch (classification) {
case 'normal':
return <span className="bg-threat-low/20 text-threat-low px-2 py-0.5 rounded text-xs whitespace-nowrap"> Normal</span>;
case 'bot':
return <span className="bg-threat-medium/20 text-threat-medium px-2 py-0.5 rounded text-xs whitespace-nowrap"> Bot</span>;
case 'script':
return <span className="bg-threat-high/20 text-threat-high px-2 py-0.5 rounded text-xs whitespace-nowrap"> Script</span>;
default:
return null;
}
};
const INITIAL_COUNT = 5;
return (
<div className="bg-background-secondary rounded-lg p-6">
<div className="flex items-center justify-between mb-4">
<h3 className="text-lg font-medium text-text-primary">4. USER-AGENT ANALYSIS</h3>
{data.alert && (
<span className="bg-threat-high text-white px-3 py-1 rounded text-xs font-medium">
{data.bot_percentage.toFixed(0)}% bots/scripts
</span>
)}
</div>
<div className="grid grid-cols-1 md:grid-cols-2 gap-6">
{/* User-Agents pour cette IP */}
<div>
<div className="text-sm text-text-secondary mb-3">
User-Agents pour cette IP ({data.ip_user_agents.length})
</div>
<div className="space-y-2">
{(showAllIpUA ? data.ip_user_agents : data.ip_user_agents.slice(0, INITIAL_COUNT)).map((ua, idx) => (
<div key={idx} className="bg-background-card rounded-lg p-3 space-y-2">
<div className="flex items-start justify-between gap-2">
<div className="text-text-primary text-xs font-mono break-all flex-1 leading-relaxed">
{ua.value}
</div>
{getClassificationBadge(ua.classification)}
</div>
<div className="flex items-center gap-2">
<div className="text-text-secondary text-xs">{ua.count} requêtes</div>
<div className="text-text-secondary text-xs">{ua.percentage.toFixed(1)}%</div>
</div>
</div>
))}
{data.ip_user_agents.length === 0 && (
<div className="text-text-secondary text-sm">Aucun User-Agent trouvé</div>
)}
</div>
{data.ip_user_agents.length > INITIAL_COUNT && (
<button
onClick={() => setShowAllIpUA(v => !v)}
className="mt-3 w-full text-xs text-accent-primary hover:text-accent-primary/80 transition-colors"
>
{showAllIpUA
? '↑ Réduire'
: `↓ Voir les ${data.ip_user_agents.length - INITIAL_COUNT} autres`}
</button>
)}
</div>
{/* User-Agents pour le JA4 */}
<div>
<div className="text-sm text-text-secondary mb-3">
User-Agents pour le JA4 (toutes IPs)
</div>
<div className="space-y-2">
{(showAllJa4UA ? data.ja4_user_agents : data.ja4_user_agents.slice(0, INITIAL_COUNT)).map((ua, idx) => (
<div key={idx} className="bg-background-card rounded-lg p-3 space-y-2">
<div className="flex items-start justify-between gap-2">
<div className="text-text-primary text-xs font-mono break-all flex-1 leading-relaxed">
{ua.value}
</div>
{getClassificationBadge(ua.classification)}
</div>
<div className="flex items-center gap-2">
<div className="text-text-secondary text-xs">{ua.count} IPs</div>
<div className="text-text-secondary text-xs">{ua.percentage.toFixed(1)}%</div>
</div>
</div>
))}
</div>
{data.ja4_user_agents.length > INITIAL_COUNT && (
<button
onClick={() => setShowAllJa4UA(v => !v)}
className="mt-3 w-full text-xs text-accent-primary hover:text-accent-primary/80 transition-colors"
>
{showAllJa4UA
? '↑ Réduire'
: `↓ Voir les ${data.ja4_user_agents.length - INITIAL_COUNT} autres`}
</button>
)}
</div>
</div>
{/* Stats bots */}
<div className="mt-6">
<div className="flex items-center justify-between mb-2">
<div className="text-sm text-text-secondary">Pourcentage de bots/scripts</div>
<div className={`text-lg font-bold ${data.bot_percentage > 20 ? 'text-threat-high' : 'text-text-primary'}`}>
{data.bot_percentage.toFixed(1)}%
</div>
</div>
<div className="w-full bg-background-card rounded-full h-3">
<div
className={`h-3 rounded-full transition-all ${
data.bot_percentage > 50 ? 'bg-threat-high' :
data.bot_percentage > 20 ? 'bg-threat-medium' :
'bg-threat-low'
}`}
style={{ width: `${Math.min(data.bot_percentage, 100)}%` }}
/>
</div>
{data.bot_percentage > 20 && (
<div className="mt-2 text-threat-high text-sm">
ALERT: {data.bot_percentage.toFixed(0)}% d'UAs bots/scripts
</div>
)}
</div>
</div>
);
}