code_public/dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
df0fe1387bc7d11aec54c016bd536c444abcb389
dashboard/test_dashboard_entities.sql
SOC Analyst a61828d1e7 Initial commit: Bot Detector Dashboard for SOC Incident Response 
🛡️ Dashboard complet pour l'analyse et la classification des menaces

Fonctionnalités principales:
- Visualisation des détections en temps réel (24h)
- Investigation multi-entités (IP, JA4, ASN, Host, User-Agent)
- Analyse de corrélation pour classification SOC
- Clustering automatique par subnet/JA4/UA
- Export des classifications pour ML

Composants:
- Backend: FastAPI (Python) + ClickHouse
- Frontend: React + TypeScript + TailwindCSS
- 6 routes API: metrics, detections, variability, attributes, analysis, entities
- 7 types d'entités investigables

Documentation ajoutée:
- NAVIGATION_GRAPH.md: Graph complet de navigation
- SOC_OPTIMIZATION_PROPOSAL.md: Proposition d'optimisation pour SOC
  • Réduction de 7 à 2 clics pour classification
  • Nouvelle vue /incidents clusterisée
  • Panel latéral d'investigation
  • Quick Search (Cmd+K)
  • Timeline interactive
  • Graph de corrélations

Sécurité:
- .gitignore configuré (exclut .env, secrets, node_modules)
- Credentials dans .env (à ne pas committer)

⚠️ Audit sécurité réalisé - Voir recommandations dans SOC_OPTIMIZATION_PROPOSAL.md

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
2026-03-14 21:33:55 +01:00

432 lines
15 KiB
SQL
Raw Blame History

-- =============================================================================
-- TESTS DE VALIDATION - Dashboard Entities
-- =============================================================================
-- Exécuter chaque bloc pour valider le fonctionnement complet
-- =============================================================================
USE mabase_prod;
-- =============================================================================
-- TEST 1 : Vérifier que la table existe
-- =============================================================================
SELECT 'TEST 1: Table exists' AS test;
SELECT name, engine
FROM system.tables
WHERE database = 'mabase_prod'
AND name = 'view_dashboard_entities';
-- =============================================================================
-- TEST 2 : Vérifier que la vue matérialisée existe
-- =============================================================================
SELECT 'TEST 2: Materialized View exists' AS test;
SELECT name, engine
FROM system.tables
WHERE database = 'mabase_prod'
AND name = 'view_dashboard_entities_mv';
-- =============================================================================
-- TEST 3 : Vérifier le schéma de la table
-- =============================================================================
SELECT 'TEST 3: Table schema' AS test;
DESCRIBE TABLE mabase_prod.view_dashboard_entities;
-- =============================================================================
-- TEST 4 : Compter les entités par type
-- =============================================================================
SELECT 'TEST 4: Count by entity type' AS test;
SELECT
entity_type,
count() AS total_entities,
sum(requests) AS total_requests
FROM mabase_prod.view_dashboard_entities
GROUP BY entity_type
ORDER BY total_entities DESC;
-- =============================================================================
-- TEST 5 : Vérifier les données IP
-- =============================================================================
SELECT 'TEST 5: IP entities sample' AS test;
SELECT
entity_value AS ip,
requests,
unique_ips,
arraySlice(user_agents, 1, 3) AS sample_user_agents,
arraySlice(paths, 1, 5) AS sample_paths,
arraySlice(asns, 1, 2) AS sample_asns,
arraySlice(countries, 1, 2) AS sample_countries
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'ip'
ORDER BY requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 6 : Vérifier les données JA4
-- =============================================================================
SELECT 'TEST 6: JA4 entities sample' AS test;
SELECT
entity_value AS ja4,
requests,
unique_ips,
arraySlice(user_agents, 1, 3) AS sample_user_agents,
arraySlice(countries, 1, 2) AS sample_countries
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'ja4'
ORDER BY requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 7 : Vérifier les données User-Agent
-- =============================================================================
SELECT 'TEST 7: User-Agent entities sample' AS test;
SELECT
entity_value AS user_agent,
requests,
unique_ips,
arraySlice(paths, 1, 3) AS sample_paths
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'user_agent'
ORDER BY requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 8 : Vérifier les données Client Header
-- =============================================================================
SELECT 'TEST 8: Client Header entities sample' AS test;
SELECT
entity_value AS client_header,
requests,
unique_ips
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'client_header'
ORDER BY requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 9 : Vérifier les données Host
-- =============================================================================
SELECT 'TEST 9: Host entities sample' AS test;
SELECT
entity_value AS host,
requests,
unique_ips,
arraySlice(user_agents, 1, 3) AS sample_user_agents
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'host'
ORDER BY requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 10 : Vérifier les données Path
-- =============================================================================
SELECT 'TEST 10: Path entities sample' AS test;
SELECT
entity_value AS path,
requests,
unique_ips,
arraySlice(user_agents, 1, 2) AS sample_user_agents
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'path'
ORDER BY requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 11 : Vérifier les données Query Param
-- =============================================================================
SELECT 'TEST 11: Query Param entities sample' AS test;
SELECT
entity_value AS query_params,
requests,
unique_ips
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'query_param'
ORDER BY requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 12 : Vérifier les ASN (ne doit pas être vide)
-- =============================================================================
SELECT 'TEST 12: ASN data validation' AS test;
SELECT
entity_type,
countIf(length(asns) > 0 AND asns[1] != '') AS with_asn,
countIf(length(asns) = 0 OR asns[1] = '') AS without_asn
FROM mabase_prod.view_dashboard_entities
GROUP BY entity_type;
-- =============================================================================
-- TEST 13 : Vérifier les Countries (ne doit pas être vide)
-- =============================================================================
SELECT 'TEST 13: Country data validation' AS test;
SELECT
entity_type,
countIf(length(countries) > 0 AND countries[1] != '') AS with_country,
countIf(length(countries) = 0 OR countries[1] = '') AS without_country
FROM mabase_prod.view_dashboard_entities
GROUP BY entity_type;
-- =============================================================================
-- TEST 14 : Top 10 des IPs les plus actives
-- =============================================================================
SELECT 'TEST 14: Top 10 IPs' AS test;
SELECT
entity_value AS ip,
sum(requests) AS total_requests,
sum(unique_ips) AS total_unique_ips
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'ip'
GROUP BY entity_value
ORDER BY total_requests DESC
LIMIT 10;
-- =============================================================================
-- TEST 15 : Top 10 des JA4 les plus actifs
-- =============================================================================
SELECT 'TEST 15: Top 10 JA4' AS test;
SELECT
entity_value AS ja4,
sum(requests) AS total_requests,
sum(unique_ips) AS total_unique_ips
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'ja4'
GROUP BY entity_value
ORDER BY total_requests DESC
LIMIT 10;
-- =============================================================================
-- TEST 16 : Top 10 des User-Agents
-- =============================================================================
SELECT 'TEST 16: Top 10 User-Agents' AS test;
SELECT
entity_value AS user_agent,
sum(requests) AS total_requests
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'user_agent'
GROUP BY entity_value
ORDER BY total_requests DESC
LIMIT 10;
-- =============================================================================
-- TEST 17 : Top 10 des Hosts
-- =============================================================================
SELECT 'TEST 17: Top 10 Hosts' AS test;
SELECT
entity_value AS host,
sum(requests) AS total_requests
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'host'
GROUP BY entity_value
ORDER BY total_requests DESC
LIMIT 10;
-- =============================================================================
-- TEST 18 : Top 10 des Paths
-- =============================================================================
SELECT 'TEST 18: Top 10 Paths' AS test;
SELECT
entity_value AS path,
sum(requests) AS total_requests
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'path'
GROUP BY entity_value
ORDER BY total_requests DESC
LIMIT 10;
-- =============================================================================
-- TEST 19 : Activité par date (derniers 7 jours)
-- =============================================================================
SELECT 'TEST 19: Activity by date (last 7 days)' AS test;
SELECT
log_date,
entity_type,
sum(requests) AS total_requests,
sum(unique_ips) AS total_unique_ips
FROM mabase_prod.view_dashboard_entities
WHERE log_date >= today() - INTERVAL 7 DAY
GROUP BY log_date, entity_type
ORDER BY log_date DESC, entity_type;
-- =============================================================================
-- TEST 20 : Requête de corrélation IP + JA4 + User-Agent
-- =============================================================================
SELECT 'TEST 20: Correlation IP + JA4 + User-Agent' AS test;
SELECT
ip.entity_value AS ip,
ip.ja4,
arraySlice(ip.user_agents, 1, 3) AS user_agents,
ip.requests AS ip_requests
FROM mabase_prod.view_dashboard_entities AS ip
WHERE ip.entity_type = 'ip'
ORDER BY ip.requests DESC
LIMIT 5;
-- =============================================================================
-- TEST 21 : Vérifier les types de données
-- =============================================================================
SELECT 'TEST 21: Data types validation' AS test;
SELECT
'requests type' AS check,
toTypeName(requests) AS type
FROM mabase_prod.view_dashboard_entities
LIMIT 1
UNION ALL
SELECT
'unique_ips type',
toTypeName(unique_ips)
FROM mabase_prod.view_dashboard_entities
LIMIT 1
UNION ALL
SELECT
'user_agents type',
toTypeName(user_agents)
FROM mabase_prod.view_dashboard_entities
LIMIT 1
UNION ALL
SELECT
'asns type',
toTypeName(asns)
FROM mabase_prod.view_dashboard_entities
LIMIT 1
UNION ALL
SELECT
'countries type',
toTypeName(countries)
FROM mabase_prod.view_dashboard_entities
LIMIT 1;
-- =============================================================================
-- TEST 22 : Vérifier qu'il n'y a pas de valeurs NULL critiques
-- =============================================================================
SELECT 'TEST 22: NULL check' AS test;
SELECT
'entity_type' AS field,
countIf(entity_type IS NULL) AS null_count
FROM mabase_prod.view_dashboard_entities
UNION ALL
SELECT
'entity_value',
countIf(entity_value IS NULL)
FROM mabase_prod.view_dashboard_entities
UNION ALL
SELECT
'src_ip',
countIf(src_ip IS NULL)
FROM mabase_prod.view_dashboard_entities
UNION ALL
SELECT
'log_date',
countIf(log_date IS NULL)
FROM mabase_prod.view_dashboard_entities
UNION ALL
SELECT
'requests',
countIf(requests IS NULL)
FROM mabase_prod.view_dashboard_entities;
-- =============================================================================
-- TEST 23 : Stats globales
-- =============================================================================
SELECT 'TEST 23: Global stats' AS test;
SELECT
count() AS total_rows,
countDistinct(entity_value) AS unique_entities,
sum(requests) AS total_requests,
sum(unique_ips) AS total_unique_ips,
min(log_date) AS min_date,
max(log_date) AS max_date
FROM mabase_prod.view_dashboard_entities;
-- =============================================================================
-- TEST 24 : Vérifier les index
-- =============================================================================
SELECT 'TEST 24: Index check' AS test;
SELECT
name,
type,
expr,
granularity
FROM system.data_skipping_indices
WHERE table = 'view_dashboard_entities'
AND database = 'mabase_prod';
-- =============================================================================
-- TEST 25 : Performance test - Requête avec filtre
-- =============================================================================
SELECT 'TEST 25: Performance test' AS test;
SELECT
entity_type,
count() AS count,
sum(requests) AS sum_requests
FROM mabase_prod.view_dashboard_entities
WHERE entity_type IN ('ip', 'ja4')
AND log_date >= today() - INTERVAL 1 DAY
GROUP BY entity_type;
-- =============================================================================
-- TEST 26 : Vérifier la TTL (date d'expiration)
-- =============================================================================
SELECT 'TEST 26: TTL check' AS test;
SELECT
name,
engine,
ttl_expression
FROM system.tables
WHERE database = 'mabase_prod'
AND name = 'view_dashboard_entities';
-- =============================================================================
-- TEST 27 : Distinct values par entité
-- =============================================================================
SELECT 'TEST 27: Distinct values per entity' AS test;
SELECT
entity_type,
countDistinct(entity_value) AS distinct_values,
countDistinct(src_ip) AS distinct_src_ips,
countDistinct(ja4) AS distinct_ja4,
countDistinct(host) AS distinct_hosts
FROM mabase_prod.view_dashboard_entities
GROUP BY entity_type;
-- =============================================================================
-- TEST 28 : Query params avec plusieurs paramètres
-- =============================================================================
SELECT 'TEST 28: Multi query params' AS test;
SELECT
entity_value AS query_params,
requests,
length(splitByString(',', entity_value)) AS param_count
FROM mabase_prod.view_dashboard_entities
WHERE entity_type = 'query_param'
ORDER BY param_count DESC, requests DESC
LIMIT 10;
-- =============================================================================
-- TEST 29 : Countries distribution
-- =============================================================================
SELECT 'TEST 29: Countries distribution' AS test;
SELECT
arrayJoin(countries) AS country,
count() AS occurrences
FROM mabase_prod.view_dashboard_entities
WHERE length(countries) > 0 AND countries[1] != ''
GROUP BY country
ORDER BY occurrences DESC
LIMIT 15;
-- =============================================================================
-- TEST 30 : ASN distribution
-- =============================================================================
SELECT 'TEST 30: ASN distribution' AS test;
SELECT
arrayJoin(asns) AS asn,
count() AS occurrences
FROM mabase_prod.view_dashboard_entities
WHERE length(asns) > 0 AND asns[1] != ''
GROUP BY asn
ORDER BY occurrences DESC
LIMIT 15;
-- =============================================================================
-- FIN DES TESTS
-- =============================================================================
SELECT '=== ALL TESTS COMPLETED ===' AS status;
Reference in New Issue View Git Blame Copy Permalink