feat: ja4-platform monorepo — 5 services unified, tests & RPM builds standardized
Services: - ja4sentinel: TLS/JA4 fingerprint capture daemon (Go, libpcap) - logcorrelator: JA4 log correlation engine (Go, ClickHouse) - mod_reqin_log: Apache module (C, JSON request logging) - bot_detector: ML bot detection pipeline (Python) - dashboard: FastAPI/Streamlit analytics UI (Python) Shared libraries: - shared/go/ja4common: logger, config, shutdown, ipfilter (Go module) - shared/python/ja4_common: ClickHouseClient, ClickHouseSettings (Python package) - shared/clickhouse/: canonical SQL migrations (10 files) Build & packaging: - Unified 3-stage Dockerfile.package for Go RPMs (el8/el9/el10) - go.work workspace linking sentinel, correlator, ja4common - Makefile with test-all, build-all, rpm-* targets Fixes applied: - go.work: 1.21 → 1.24.6 (required by sentinel) - correlator Dockerfiles: golang:1.21 → golang:1.24 - replace directives in go.mod for ja4common local path - pyproject.toml: setuptools.backends → setuptools.build_meta - Removed static libpcap linking (unavailable on Rocky 9) - Fixed data races in output/writers_test.go (sync.Mutex + atomic.Int32) - Rewrote corrupted test files (logger_test.go × 2) Test coverage: - correlator: 67.1% total (unixsocket 80.5%, config 91.7%, app 83.3%, multi 87.7%, stdout 100%) - sentinel: all 10 packages pass (api, capture, config, fingerprint, ipfilter, logging, output, tlsparse) Documentation: - README.md + docs/ (architecture, development, 5 services, shared libs, DB schema & migrations) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
305
services/dashboard/frontend/src/utils/STIXExporter.ts
Normal file
305
services/dashboard/frontend/src/utils/STIXExporter.ts
Normal file
@ -0,0 +1,305 @@
|
||||
/**
|
||||
* Export STIX 2.1 pour Threat Intelligence
|
||||
* Format standard pour l'échange d'informations de cybermenaces
|
||||
*/
|
||||
|
||||
interface STIXIndicator {
|
||||
id: string;
|
||||
type: string;
|
||||
spec_version: string;
|
||||
created: string;
|
||||
modified: string;
|
||||
name: string;
|
||||
description: string;
|
||||
pattern: string;
|
||||
pattern_type: string;
|
||||
valid_from: string;
|
||||
labels: string[];
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
interface STIXObservables {
|
||||
id: string;
|
||||
type: string;
|
||||
spec_version: string;
|
||||
value?: string;
|
||||
hashes?: {
|
||||
MD5?: string;
|
||||
'SHA-256'?: string;
|
||||
};
|
||||
}
|
||||
|
||||
interface STIXBundle {
|
||||
type: string;
|
||||
id: string;
|
||||
objects: (STIXIndicator | STIXObservables)[];
|
||||
}
|
||||
|
||||
export class STIXExporter {
|
||||
/**
|
||||
* Génère un bundle STIX 2.1 à partir d'une liste d'IPs
|
||||
*/
|
||||
static exportIPs(ips: string[], metadata: {
|
||||
label: string;
|
||||
tags: string[];
|
||||
confidence: number;
|
||||
analyst: string;
|
||||
comment: string;
|
||||
}): STIXBundle {
|
||||
const now = new Date().toISOString();
|
||||
const objects: (STIXIndicator | STIXObservables)[] = [];
|
||||
|
||||
// Identity (organisation SOC)
|
||||
objects.push({
|
||||
id: `identity--${this.generateUUID()}`,
|
||||
type: 'identity',
|
||||
spec_version: '2.1',
|
||||
name: 'SOC Bot Detector',
|
||||
identity_class: 'system',
|
||||
created: now,
|
||||
modified: now
|
||||
} as any);
|
||||
|
||||
// Create indicators and observables for each IP
|
||||
ips.forEach((ip) => {
|
||||
const indicatorId = `indicator--${this.generateUUID()}`;
|
||||
const observableId = `ipv4-addr--${this.generateUUID()}`;
|
||||
|
||||
// STIX Indicator
|
||||
objects.push({
|
||||
id: indicatorId,
|
||||
type: 'indicator',
|
||||
spec_version: '2.1',
|
||||
created: now,
|
||||
modified: now,
|
||||
name: `Malicious IP - ${ip}`,
|
||||
description: `${metadata.comment} | Tags: ${metadata.tags.join(', ')} | Analyst: ${metadata.analyst}`,
|
||||
pattern: `[ipv4-addr:value = '${ip}']`,
|
||||
pattern_type: 'stix',
|
||||
valid_from: now,
|
||||
labels: [...metadata.tags, metadata.label],
|
||||
confidence: Math.round(metadata.confidence * 100),
|
||||
created_by_ref: objects[0].id,
|
||||
object_marking_refs: [`marking-definition--${this.generateUUID()}`]
|
||||
} as STIXIndicator);
|
||||
|
||||
// STIX Observable (IPv4 Address)
|
||||
objects.push({
|
||||
id: observableId,
|
||||
type: 'ipv4-addr',
|
||||
spec_version: '2.1',
|
||||
value: ip,
|
||||
object_marking_refs: [`marking-definition--${this.generateUUID()}`]
|
||||
} as STIXObservables);
|
||||
|
||||
// Relationship between indicator and observable
|
||||
objects.push({
|
||||
id: `relationship--${this.generateUUID()}`,
|
||||
type: 'relationship',
|
||||
spec_version: '2.1',
|
||||
created: now,
|
||||
modified: now,
|
||||
relationship_type: 'indicates',
|
||||
source_ref: indicatorId,
|
||||
target_ref: observableId,
|
||||
description: 'Indicator indicates malicious IP address'
|
||||
} as any);
|
||||
});
|
||||
|
||||
// Marking Definition (TLP:AMBER)
|
||||
objects.push({
|
||||
id: 'marking-definition--78ca4366-f5b8-4764-83f7-34ce38198e27',
|
||||
type: 'marking-definition',
|
||||
spec_version: '2.1',
|
||||
name: 'TLP:AMBER',
|
||||
created: '2017-01-20T00:00:00.000Z',
|
||||
definition_type: 'statement',
|
||||
definition: { statement: 'This information is TLP:AMBER' }
|
||||
} as any);
|
||||
|
||||
return {
|
||||
type: 'bundle',
|
||||
id: `bundle--${this.generateUUID()}`,
|
||||
objects
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Génère un bundle STIX pour un incident complet
|
||||
*/
|
||||
static exportIncident(incident: {
|
||||
id: string;
|
||||
subnet: string;
|
||||
ips: string[];
|
||||
ja4?: string;
|
||||
severity: string;
|
||||
first_seen: string;
|
||||
last_seen: string;
|
||||
description: string;
|
||||
tags: string[];
|
||||
}): STIXBundle {
|
||||
const now = new Date().toISOString();
|
||||
const objects: any[] = [];
|
||||
|
||||
// Identity
|
||||
objects.push({
|
||||
id: `identity--${this.generateUUID()}`,
|
||||
type: 'identity',
|
||||
spec_version: '2.1',
|
||||
name: 'SOC Bot Detector',
|
||||
identity_class: 'system',
|
||||
created: now,
|
||||
modified: now
|
||||
});
|
||||
|
||||
// Incident
|
||||
objects.push({
|
||||
id: `incident--${this.generateUUID()}`,
|
||||
type: 'incident',
|
||||
spec_version: '2.1',
|
||||
created: now,
|
||||
modified: now,
|
||||
name: `Bot Detection Incident ${incident.id}`,
|
||||
description: incident.description,
|
||||
objective: 'Detect and classify bot activity',
|
||||
first_seen: incident.first_seen,
|
||||
last_seen: incident.last_seen,
|
||||
status: 'active',
|
||||
labels: [...incident.tags, incident.severity]
|
||||
});
|
||||
|
||||
// Campaign (for the attack pattern)
|
||||
objects.push({
|
||||
id: `campaign--${this.generateUUID()}`,
|
||||
type: 'campaign',
|
||||
spec_version: '2.1',
|
||||
created: now,
|
||||
modified: now,
|
||||
name: `Bot Campaign - ${incident.subnet}`,
|
||||
description: `Automated bot activity from subnet ${incident.subnet}`,
|
||||
first_seen: incident.first_seen,
|
||||
last_seen: incident.last_seen,
|
||||
labels: incident.tags
|
||||
});
|
||||
|
||||
// Relationship: Campaign uses Attack Pattern
|
||||
objects.push({
|
||||
id: `relationship--${this.generateUUID()}`,
|
||||
type: 'relationship',
|
||||
spec_version: '2.1',
|
||||
created: now,
|
||||
modified: now,
|
||||
relationship_type: 'related-to',
|
||||
source_ref: objects[objects.length - 1].id, // campaign
|
||||
target_ref: objects[objects.length - 2].id // incident
|
||||
});
|
||||
|
||||
// Add indicators for each IP
|
||||
incident.ips.slice(0, 100).forEach(ip => {
|
||||
const indicatorId = `indicator--${this.generateUUID()}`;
|
||||
|
||||
objects.push({
|
||||
id: indicatorId,
|
||||
type: 'indicator',
|
||||
spec_version: '2.1',
|
||||
created: now,
|
||||
modified: now,
|
||||
name: `Malicious IP - ${ip}`,
|
||||
description: `Part of incident ${incident.id}`,
|
||||
pattern: `[ipv4-addr:value = '${ip}']`,
|
||||
pattern_type: 'stix',
|
||||
valid_from: now,
|
||||
labels: incident.tags,
|
||||
confidence: 80
|
||||
});
|
||||
|
||||
// Relationship: Incident indicates IP
|
||||
objects.push({
|
||||
id: `relationship--${this.generateUUID()}`,
|
||||
type: 'relationship',
|
||||
spec_version: '2.1',
|
||||
created: now,
|
||||
modified: now,
|
||||
relationship_type: 'related-to',
|
||||
source_ref: objects[objects.length - 2].id, // incident
|
||||
target_ref: indicatorId
|
||||
});
|
||||
});
|
||||
|
||||
return {
|
||||
type: 'bundle',
|
||||
id: `bundle--${this.generateUUID()}`,
|
||||
objects
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Télécharge le bundle STIX
|
||||
*/
|
||||
static download(bundle: STIXBundle, filename?: string): void {
|
||||
const json = JSON.stringify(bundle, null, 2);
|
||||
const blob = new Blob([json], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename || `stix_export_${Date.now()}.json`;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
document.body.removeChild(a);
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
/**
|
||||
* Génère un UUID v4
|
||||
*/
|
||||
private static generateUUID(): string {
|
||||
return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, c => {
|
||||
const r = Math.random() * 16 | 0;
|
||||
const v = c === 'x' ? r : (r & 0x3 | 0x8);
|
||||
return v.toString(16);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Export au format MISP (alternative à STIX)
|
||||
*/
|
||||
static exportMISP(ips: string[], metadata: any): object {
|
||||
return {
|
||||
response: {
|
||||
Event: {
|
||||
id: this.generateUUID(),
|
||||
orgc: 'SOC Bot Detector',
|
||||
date: new Date().toISOString().split('T')[0],
|
||||
threat_level_id: metadata.label === 'malicious' ? '1' :
|
||||
metadata.label === 'suspicious' ? '2' : '3',
|
||||
analysis: '2', // Completed
|
||||
info: `Bot Detection: ${metadata.comment}`,
|
||||
uuid: this.generateUUID(),
|
||||
Attribute: ips.map((ip) => ({
|
||||
type: 'ip-dst',
|
||||
category: 'Network activity',
|
||||
value: ip,
|
||||
to_ids: true,
|
||||
uuid: this.generateUUID(),
|
||||
timestamp: Math.floor(Date.now() / 1000),
|
||||
comment: `${metadata.tags.join(', ')} | Confidence: ${metadata.confidence}`
|
||||
})),
|
||||
Tag: metadata.tags.map((tag: string) => ({
|
||||
name: tag,
|
||||
colour: this.getTagColor(tag)
|
||||
}))
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static getTagColor(tag: string): string {
|
||||
// Generate consistent colors for tags
|
||||
const colors = [
|
||||
'#ff6b6b', '#4ecdc4', '#45b7d1', '#96ceb4',
|
||||
'#ffeaa7', '#dfe6e9', '#fd79a8', '#a29bfe'
|
||||
];
|
||||
const hash = tag.split('').reduce((acc, char) => acc + char.charCodeAt(0), 0);
|
||||
return colors[hash % colors.length];
|
||||
}
|
||||
}
|
||||
36
services/dashboard/frontend/src/utils/classifications.ts
Normal file
36
services/dashboard/frontend/src/utils/classifications.ts
Normal file
@ -0,0 +1,36 @@
|
||||
/**
|
||||
* Tags prédéfinis pour la classification SOC.
|
||||
*
|
||||
* Utilisé par BulkClassification, CorrelationSummary, JA4CorrelationSummary.
|
||||
* Ajouter de nouveaux tags ici pour les propager partout.
|
||||
*/
|
||||
export const PREDEFINED_TAGS: readonly string[] = [
|
||||
'scraping',
|
||||
'bot-network',
|
||||
'scanner',
|
||||
'bruteforce',
|
||||
'data-exfil',
|
||||
'ddos',
|
||||
'spam',
|
||||
'proxy',
|
||||
'tor',
|
||||
'vpn',
|
||||
'hosting-asn',
|
||||
'distributed',
|
||||
'ja4-rotation',
|
||||
'ua-rotation',
|
||||
'country-cn',
|
||||
'country-us',
|
||||
'country-ru',
|
||||
];
|
||||
|
||||
/**
|
||||
* Tags supplémentaires spécifiques aux fingerprints JA4.
|
||||
* S'étend de PREDEFINED_TAGS.
|
||||
*/
|
||||
export const PREDEFINED_TAGS_JA4: readonly string[] = [
|
||||
...PREDEFINED_TAGS,
|
||||
'known-bot',
|
||||
'crawler',
|
||||
'search-engine',
|
||||
];
|
||||
11
services/dashboard/frontend/src/utils/countryUtils.ts
Normal file
11
services/dashboard/frontend/src/utils/countryUtils.ts
Normal file
@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Convertit un code pays ISO 3166-1 alpha-2 en emoji drapeau.
|
||||
* Utilise les Regional Indicator Symbols Unicode (U+1F1E6…U+1F1FF).
|
||||
* Retourne 🌐 pour les codes invalides ou vides.
|
||||
*/
|
||||
export function getCountryFlag(code: string): string {
|
||||
if (!code || code.length !== 2) return '🌐';
|
||||
return code
|
||||
.toUpperCase()
|
||||
.replace(/./g, (char) => String.fromCodePoint(char.charCodeAt(0) + 127397));
|
||||
}
|
||||
86
services/dashboard/frontend/src/utils/dateUtils.ts
Normal file
86
services/dashboard/frontend/src/utils/dateUtils.ts
Normal file
@ -0,0 +1,86 @@
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
// Utilitaires de formatage des dates et des nombres
|
||||
//
|
||||
// Les dates sont stockées en UTC dans ClickHouse (sans suffixe TZ).
|
||||
// Ces fonctions les convertissent dans le fuseau horaire local du navigateur
|
||||
// et utilisent la locale du navigateur pour l'affichage.
|
||||
// ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Normalise une chaîne datetime ClickHouse (sans TZ) en Date UTC.
|
||||
* ClickHouse retourne "2024-01-15 14:32:00" → on force Z pour UTC.
|
||||
*/
|
||||
function parseUTC(iso: string): Date {
|
||||
if (!iso) return new Date(NaN);
|
||||
// Déjà un ISO complet avec TZ → pas de modification
|
||||
if (iso.endsWith('Z') || iso.includes('+')) return new Date(iso);
|
||||
// "2024-01-15 14:32:00" ou "2024-01-15T14:32:00" → forcer UTC
|
||||
const normalized = iso.replace(' ', 'T');
|
||||
return new Date(normalized + 'Z');
|
||||
}
|
||||
|
||||
/**
|
||||
* Formate une date/heure complète dans la locale et le fuseau du navigateur.
|
||||
* Exemple : "15/01/2024, 15:32" (fr) ou "1/15/2024, 3:32 PM" (en-US)
|
||||
*/
|
||||
export function formatDate(iso: string): string {
|
||||
const d = parseUTC(iso);
|
||||
if (isNaN(d.getTime())) return iso;
|
||||
return d.toLocaleString(navigator.language || undefined, {
|
||||
year: 'numeric',
|
||||
month: '2-digit',
|
||||
day: '2-digit',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Formate une date courte (jour/mois heure:min) pour les tableaux.
|
||||
* Exemple : "15/01 15:32"
|
||||
*/
|
||||
export function formatDateShort(iso: string): string {
|
||||
const d = parseUTC(iso);
|
||||
if (isNaN(d.getTime())) return iso;
|
||||
return d.toLocaleString(navigator.language || undefined, {
|
||||
day: '2-digit',
|
||||
month: '2-digit',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Formate uniquement la partie date.
|
||||
* Exemple : "15/01/2024"
|
||||
*/
|
||||
export function formatDateOnly(iso: string): string {
|
||||
const d = parseUTC(iso);
|
||||
if (isNaN(d.getTime())) return iso;
|
||||
return d.toLocaleDateString(navigator.language || undefined, {
|
||||
year: 'numeric',
|
||||
month: '2-digit',
|
||||
day: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Formate uniquement l'heure (heure:min).
|
||||
* Exemple : "15:32"
|
||||
*/
|
||||
export function formatTimeOnly(iso: string): string {
|
||||
const d = parseUTC(iso);
|
||||
if (isNaN(d.getTime())) return iso;
|
||||
return d.toLocaleTimeString(navigator.language || undefined, {
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Formate un nombre entier avec séparateurs de milliers selon la locale du navigateur.
|
||||
* Exemple : 1234567 → "1 234 567" (fr) ou "1,234,567" (en-US)
|
||||
*/
|
||||
export function formatNumber(n: number): string {
|
||||
return n.toLocaleString(navigator.language || undefined);
|
||||
}
|
||||
Reference in New Issue
Block a user