9ea36ad22e
Schema cleanup:
- Remove anubis_ua_rules table stub from 03_anubis_tables.sql
- Remove anubis_ua_rules from bot-detector deploy_schema.sql
- Remove UA seed step from clickhouse-init.sh (no more REGEXP_TREE dependency)
- Drop dict_anubis_ua, dict_anubis_country, anubis_ua_rules, anubis_country_rules
New scripts:
- scripts/init-stack.sh: comprehensive ClickHouse init (13 SQL files + migrations
+ validation + cleanup of obsolete tables). Supports --reset, --import-prod.
- scripts/import-prod-data.sh: imports pre-exported prod data (Native format)
with dynamic date shift (max(time) → now). Supports --shift, --no-truncate.
- scripts/data/prod-export/: directory for cached Native format exports
Makefile targets: init-stack, import-prod-data, init-and-import
Tested: init-stack.sh passes all 13 SQL + 7 critical tables + 7 dicts
import-prod-data.sh: 3M rows in ~37s with auto date shift
Dashboard: 55 routes OK, bot-detector: 36/36 tests pass
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
252 lines
11 KiB
Bash
Executable File
252 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# init-stack.sh — Initialisation complète de la stack ClickHouse pour ja4-platform
|
|
#
|
|
# Ce script exécute l'ensemble du schéma SQL, charge les données CSV de
|
|
# référence et vérifie que tous les composants sont opérationnels.
|
|
# Il est utilisé par les tests d'intégration et pour la mise en place de
|
|
# l'environnement de développement.
|
|
#
|
|
# Usage:
|
|
# ./scripts/init-stack.sh # init dev stack
|
|
# ./scripts/init-stack.sh --container my-ch-1 # conteneur spécifique
|
|
# ./scripts/init-stack.sh --user admin --pass X # credentials spécifiques
|
|
# ./scripts/init-stack.sh --import-prod # init + import données prod
|
|
# ./scripts/init-stack.sh --reset # DROP databases, recréer tout
|
|
#
|
|
# Variables d'environnement :
|
|
# DEV_CONTAINER Nom du conteneur ClickHouse (défaut: integration-clickhouse-1)
|
|
# DEV_USER Utilisateur ClickHouse (défaut: default)
|
|
# DEV_PASSWORD Mot de passe ClickHouse (défaut: vide)
|
|
# CLICKHOUSE_DB_LOGS Base de données logs (défaut: ja4_logs)
|
|
# CLICKHOUSE_DB_PROC Base de données processing (défaut: ja4_processing)
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
|
|
# ── Configuration ────────────────────────────────────────────────────────────
|
|
DEV_CONTAINER="${DEV_CONTAINER:-integration-clickhouse-1}"
|
|
DEV_USER="${DEV_USER:-default}"
|
|
DEV_PASSWORD="${DEV_PASSWORD:-}"
|
|
DB_LOGS="${CLICKHOUSE_DB_LOGS:-ja4_logs}"
|
|
DB_PROC="${CLICKHOUSE_DB_PROC:-ja4_processing}"
|
|
IMPORT_PROD=false
|
|
RESET=false
|
|
|
|
# ── Parsing des arguments ────────────────────────────────────────────────────
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--container) DEV_CONTAINER="$2"; shift 2 ;;
|
|
--user) DEV_USER="$2"; shift 2 ;;
|
|
--pass) DEV_PASSWORD="$2"; shift 2 ;;
|
|
--import-prod) IMPORT_PROD=true; shift ;;
|
|
--reset) RESET=true; shift ;;
|
|
-h|--help)
|
|
sed -n '2,/^# =====/{ /^# =====/d; s/^# \?//p; }' "$0"
|
|
exit 0
|
|
;;
|
|
*) echo "Option inconnue : $1"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
SQL_DIR="${REPO_ROOT}/shared/clickhouse"
|
|
|
|
# ── Couleurs ─────────────────────────────────────────────────────────────────
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
log() { echo -e "${CYAN}[init]${NC} $(date '+%H:%M:%S') $*"; }
|
|
ok() { echo -e "${GREEN} ✓ $*${NC}"; }
|
|
err() { echo -e "${RED} ✗ $*${NC}" >&2; exit 1; }
|
|
|
|
# ── Requêteur CH ─────────────────────────────────────────────────────────────
|
|
ch() {
|
|
local args=("--query" "$1")
|
|
if [[ -n "${DEV_PASSWORD}" ]]; then
|
|
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
|
fi
|
|
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}"
|
|
}
|
|
|
|
ch_multiquery() {
|
|
local args=("--multiquery")
|
|
if [[ -n "${DEV_PASSWORD}" ]]; then
|
|
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
|
fi
|
|
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}" <<< "$1"
|
|
}
|
|
|
|
ch_insert_native() {
|
|
# $1 = table, stdin = Native data
|
|
local args=("--query" "INSERT INTO $1 FORMAT Native")
|
|
if [[ -n "${DEV_PASSWORD}" ]]; then
|
|
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
|
fi
|
|
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}"
|
|
}
|
|
|
|
# ── Vérification du conteneur ────────────────────────────────────────────────
|
|
log "Vérification du conteneur ${DEV_CONTAINER}…"
|
|
if ! docker exec "${DEV_CONTAINER}" clickhouse-client --query "SELECT 1" > /dev/null 2>&1; then
|
|
err "Le conteneur ${DEV_CONTAINER} n'est pas accessible"
|
|
fi
|
|
ok "Conteneur ${DEV_CONTAINER} accessible"
|
|
|
|
# ── Reset optionnel ──────────────────────────────────────────────────────────
|
|
if [ "${RESET}" = true ]; then
|
|
log "Reset demandé — suppression des bases de données…"
|
|
ch "DROP DATABASE IF EXISTS ${DB_LOGS}" 2>/dev/null || true
|
|
ch "DROP DATABASE IF EXISTS ${DB_PROC}" 2>/dev/null || true
|
|
ok "Bases ${DB_LOGS} et ${DB_PROC} supprimées"
|
|
fi
|
|
|
|
# ── Exécution des fichiers SQL ───────────────────────────────────────────────
|
|
SQL_FILES=(
|
|
00_database.sql
|
|
01_raw_tables.sql
|
|
02_dictionaries.sql
|
|
03_anubis_tables.sql
|
|
04_mv_http_logs.sql
|
|
05_aggregation_tables.sql
|
|
06_ml_tables.sql
|
|
07_ai_features_view.sql
|
|
08_users.sql
|
|
09_audit_table.sql
|
|
10_perf_indexes.sql
|
|
11_views.sql
|
|
12_thesis_features.sql
|
|
)
|
|
|
|
log "Application du schéma SQL (${#SQL_FILES[@]} fichiers)…"
|
|
ERRORS=0
|
|
|
|
for f in "${SQL_FILES[@]}"; do
|
|
filepath="${SQL_DIR}/${f}"
|
|
if [[ ! -f "${filepath}" ]]; then
|
|
echo " WARN: ${f} non trouvé, ignoré" >&2
|
|
continue
|
|
fi
|
|
|
|
# Substitution des noms de bases et des credentials
|
|
SQL_PATCHED=$(sed \
|
|
-e "s/ja4_logs/${DB_LOGS}/g" \
|
|
-e "s/ja4_processing/${DB_PROC}/g" \
|
|
-e "s/USER 'admin'/USER '${DEV_USER}'/g" \
|
|
-e "s/PASSWORD 'CHANGE_ME'/PASSWORD '${DEV_PASSWORD}'/g" \
|
|
-e "s/PASSWORD 'ChangeMe'/PASSWORD '${DEV_PASSWORD}'/g" \
|
|
"${filepath}")
|
|
|
|
# 10_perf_indexes.sql peut échouer si les index existent déjà
|
|
if [[ "${f}" == 10_* ]]; then
|
|
if ch_multiquery "${SQL_PATCHED}" 2>/dev/null; then
|
|
ok "${f}"
|
|
else
|
|
echo " ⚠ ${f} (erreurs ignorées — index déjà existants)"
|
|
fi
|
|
else
|
|
if ch_multiquery "${SQL_PATCHED}" 2>/dev/null; then
|
|
ok "${f}"
|
|
else
|
|
echo " ✗ ${f} — ERREUR" >&2
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "${ERRORS}" -gt 0 ]; then
|
|
err "${ERRORS} fichier(s) SQL en erreur"
|
|
fi
|
|
|
|
# ── Migrations post-schéma (colonnes manquantes sur DB existante) ────────────
|
|
log "Application des migrations post-schéma…"
|
|
MIGRATIONS=(
|
|
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_xff SimpleAggregateFunction(sum, UInt64)"
|
|
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_unusual_ct SimpleAggregateFunction(sum, UInt64)"
|
|
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_non_std_port SimpleAggregateFunction(sum, UInt64)"
|
|
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_login_post SimpleAggregateFunction(sum, UInt64)"
|
|
"ALTER TABLE ${DB_PROC}.agg_header_fingerprint_1h ADD COLUMN IF NOT EXISTS sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8)"
|
|
)
|
|
for mig in "${MIGRATIONS[@]}"; do
|
|
ch "${mig}" 2>/dev/null || true
|
|
done
|
|
ok "Migrations appliquées"
|
|
|
|
# ── Nettoyage des tables Anubis obsolètes (UA, Country) ─────────────────────
|
|
log "Nettoyage des tables Anubis obsolètes…"
|
|
ch "DROP DICTIONARY IF EXISTS ${DB_PROC}.dict_anubis_ua" 2>/dev/null || true
|
|
ch "DROP DICTIONARY IF EXISTS ${DB_PROC}.dict_anubis_country" 2>/dev/null || true
|
|
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_ua_rules" 2>/dev/null || true
|
|
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_country_rules" 2>/dev/null || true
|
|
ok "Tables obsolètes supprimées"
|
|
|
|
# ── Vérification du schéma ───────────────────────────────────────────────────
|
|
log "Vérification du schéma…"
|
|
|
|
TABLE_COUNT=$(ch "SELECT count() FROM system.tables WHERE database IN ('${DB_LOGS}','${DB_PROC}')")
|
|
DICT_COUNT=$(ch "SELECT count() FROM system.dictionaries WHERE database='${DB_PROC}'")
|
|
VIEW_COUNT=$(ch "SELECT count() FROM system.tables WHERE database='${DB_PROC}' AND engine='View'")
|
|
MV_COUNT=$(ch "SELECT count() FROM system.tables WHERE database IN ('${DB_LOGS}','${DB_PROC}') AND engine='MaterializedView'")
|
|
|
|
ok "Tables: ${TABLE_COUNT} | Dictionnaires: ${DICT_COUNT} | Vues: ${VIEW_COUNT} | MVs: ${MV_COUNT}"
|
|
|
|
# Vérification des tables critiques
|
|
CRITICAL_TABLES=(
|
|
"${DB_LOGS}.http_logs_raw"
|
|
"${DB_LOGS}.http_logs"
|
|
"${DB_PROC}.ml_detected_anomalies"
|
|
"${DB_PROC}.ml_all_scores"
|
|
"${DB_PROC}.agg_host_ip_ja4_1h"
|
|
"${DB_PROC}.anubis_ip_rules"
|
|
"${DB_PROC}.anubis_asn_rules"
|
|
)
|
|
for t in "${CRITICAL_TABLES[@]}"; do
|
|
db="${t%%.*}"
|
|
tbl="${t##*.}"
|
|
EXISTS=$(ch "SELECT count() FROM system.tables WHERE database='${db}' AND name='${tbl}'" 2>/dev/null || echo "0")
|
|
if [ "${EXISTS}" = "1" ]; then
|
|
ok " ${t}"
|
|
else
|
|
err " Table manquante : ${t}"
|
|
fi
|
|
done
|
|
|
|
# Vérification des dictionnaires critiques
|
|
CRITICAL_DICTS=(
|
|
"dict_anubis_ip"
|
|
"dict_anubis_asn"
|
|
"dict_iplocate_asn"
|
|
"dict_bot_ip"
|
|
"dict_bot_ja4"
|
|
"dict_browser_ja4"
|
|
"dict_asn_reputation"
|
|
)
|
|
for d in "${CRITICAL_DICTS[@]}"; do
|
|
STATUS=$(ch "SELECT status FROM system.dictionaries WHERE database='${DB_PROC}' AND name='${d}'" 2>/dev/null || echo "MISSING")
|
|
if [ "${STATUS}" = "LOADED" ] || [ "${STATUS}" = "NOT_LOADED" ]; then
|
|
ok " ${d} (${STATUS})"
|
|
else
|
|
echo " ⚠ Dictionnaire ${d}: ${STATUS}"
|
|
fi
|
|
done
|
|
|
|
# ── Import des données prod (optionnel) ──────────────────────────────────────
|
|
if [ "${IMPORT_PROD}" = true ]; then
|
|
IMPORT_SCRIPT="${SCRIPT_DIR}/import-prod-data.sh"
|
|
if [[ -x "${IMPORT_SCRIPT}" ]]; then
|
|
log "Lancement de l'import des données prod…"
|
|
"${IMPORT_SCRIPT}" --container "${DEV_CONTAINER}"
|
|
else
|
|
echo " ⚠ Script d'import non trouvé : ${IMPORT_SCRIPT}"
|
|
fi
|
|
fi
|
|
|
|
# ── Résultat ─────────────────────────────────────────────────────────────────
|
|
log "════════════════════════════════════════════════════"
|
|
log " Initialisation terminée"
|
|
log " Bases : ${DB_LOGS}, ${DB_PROC}"
|
|
log " Tables: ${TABLE_COUNT} | Dicts: ${DICT_COUNT} | MVs: ${MV_COUNT}"
|
|
log "════════════════════════════════════════════════════"
|