bc11cfa8eb
Root cause: CREATE TABLE IF NOT EXISTS is a no-op on existing tables, so stale schemas miss new columns. Views (07+) then fail with UNKNOWN_IDENTIFIER errors. Fix: split SQL execution into 3 phases: Phase 1: databases, raw tables, dictionaries (00-04) Phase 2: DROP all derived tables (agg_*, ml_*) — safe, repopulated by MVs Phase 3: recreate derived tables + views with full current schema (05-12) This removes the incomplete inline migrations and makes the script truly idempotent regardless of prior schema version. Tested: fresh --reset, existing stale DB, idempotent re-run. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
290 lines
12 KiB
Bash
Executable File
290 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# =============================================================================
|
|
# init-stack.sh — Initialisation complète de la stack ClickHouse pour ja4-platform
|
|
#
|
|
# Ce script exécute l'ensemble du schéma SQL, charge les données CSV de
|
|
# référence et vérifie que tous les composants sont opérationnels.
|
|
# Il est utilisé par les tests d'intégration et pour la mise en place de
|
|
# l'environnement de développement.
|
|
#
|
|
# Usage:
|
|
# ./scripts/init-stack.sh # init dev stack
|
|
# ./scripts/init-stack.sh --container my-ch-1 # conteneur spécifique
|
|
# ./scripts/init-stack.sh --user admin --pass X # credentials spécifiques
|
|
# ./scripts/init-stack.sh --import-prod # init + import données prod
|
|
# ./scripts/init-stack.sh --reset # DROP databases, recréer tout
|
|
#
|
|
# Variables d'environnement :
|
|
# DEV_CONTAINER Nom du conteneur ClickHouse (défaut: integration-clickhouse-1)
|
|
# DEV_USER Utilisateur ClickHouse (défaut: default)
|
|
# DEV_PASSWORD Mot de passe ClickHouse (défaut: vide)
|
|
# CLICKHOUSE_DB_LOGS Base de données logs (défaut: ja4_logs)
|
|
# CLICKHOUSE_DB_PROC Base de données processing (défaut: ja4_processing)
|
|
# =============================================================================
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
|
|
|
# ── Configuration ────────────────────────────────────────────────────────────
|
|
DEV_CONTAINER="${DEV_CONTAINER:-integration-clickhouse-1}"
|
|
DEV_USER="${DEV_USER:-default}"
|
|
DEV_PASSWORD="${DEV_PASSWORD:-}"
|
|
DB_LOGS="${CLICKHOUSE_DB_LOGS:-ja4_logs}"
|
|
DB_PROC="${CLICKHOUSE_DB_PROC:-ja4_processing}"
|
|
IMPORT_PROD=false
|
|
RESET=false
|
|
|
|
# ── Parsing des arguments ────────────────────────────────────────────────────
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--container) DEV_CONTAINER="$2"; shift 2 ;;
|
|
--user) DEV_USER="$2"; shift 2 ;;
|
|
--pass) DEV_PASSWORD="$2"; shift 2 ;;
|
|
--import-prod) IMPORT_PROD=true; shift ;;
|
|
--reset) RESET=true; shift ;;
|
|
-h|--help)
|
|
sed -n '2,/^# =====/{ /^# =====/d; s/^# \?//p; }' "$0"
|
|
exit 0
|
|
;;
|
|
*) echo "Option inconnue : $1"; exit 1 ;;
|
|
esac
|
|
done
|
|
|
|
SQL_DIR="${REPO_ROOT}/shared/clickhouse"
|
|
|
|
# ── Couleurs ─────────────────────────────────────────────────────────────────
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
CYAN='\033[0;36m'
|
|
NC='\033[0m'
|
|
log() { echo -e "${CYAN}[init]${NC} $(date '+%H:%M:%S') $*"; }
|
|
ok() { echo -e "${GREEN} ✓ $*${NC}"; }
|
|
err() { echo -e "${RED} ✗ $*${NC}" >&2; exit 1; }
|
|
|
|
# ── Requêteur CH ─────────────────────────────────────────────────────────────
|
|
ch() {
|
|
local args=("--query" "$1")
|
|
if [[ -n "${DEV_PASSWORD}" ]]; then
|
|
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
|
fi
|
|
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}"
|
|
}
|
|
|
|
ch_multiquery() {
|
|
local args=("--multiquery")
|
|
if [[ -n "${DEV_PASSWORD}" ]]; then
|
|
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
|
fi
|
|
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}" <<< "$1"
|
|
}
|
|
|
|
ch_insert_native() {
|
|
# $1 = table, stdin = Native data
|
|
local args=("--query" "INSERT INTO $1 FORMAT Native")
|
|
if [[ -n "${DEV_PASSWORD}" ]]; then
|
|
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
|
fi
|
|
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}"
|
|
}
|
|
|
|
# ── Vérification du conteneur ────────────────────────────────────────────────
|
|
log "Vérification du conteneur ${DEV_CONTAINER}…"
|
|
if ! docker exec "${DEV_CONTAINER}" clickhouse-client --query "SELECT 1" > /dev/null 2>&1; then
|
|
err "Le conteneur ${DEV_CONTAINER} n'est pas accessible"
|
|
fi
|
|
ok "Conteneur ${DEV_CONTAINER} accessible"
|
|
|
|
# ── Reset optionnel ──────────────────────────────────────────────────────────
|
|
if [ "${RESET}" = true ]; then
|
|
log "Reset demandé — suppression des bases de données…"
|
|
ch "DROP DATABASE IF EXISTS ${DB_LOGS}" 2>/dev/null || true
|
|
ch "DROP DATABASE IF EXISTS ${DB_PROC}" 2>/dev/null || true
|
|
ok "Bases ${DB_LOGS} et ${DB_PROC} supprimées"
|
|
fi
|
|
|
|
# ── Exécution des fichiers SQL ───────────────────────────────────────────────
|
|
# Phase 1 : bases + tables persistantes + dictionnaires + MVs primaires (00-04)
|
|
# Phase 2 : drop/recreate des tables dérivées (agg_*, ml_*) pour garantir le schéma
|
|
# Phase 3 : tables dérivées + vues (05-12)
|
|
# Cette approche garantit que les vues (07+) trouvent toutes les colonnes attendues,
|
|
# même si le schéma a évolué depuis la dernière initialisation.
|
|
|
|
SQL_PHASE1=(
|
|
00_database.sql
|
|
01_raw_tables.sql
|
|
02_dictionaries.sql
|
|
03_anubis_tables.sql
|
|
04_mv_http_logs.sql
|
|
)
|
|
|
|
SQL_PHASE3=(
|
|
05_aggregation_tables.sql
|
|
06_ml_tables.sql
|
|
07_ai_features_view.sql
|
|
08_users.sql
|
|
09_audit_table.sql
|
|
10_perf_indexes.sql
|
|
11_views.sql
|
|
12_thesis_features.sql
|
|
)
|
|
|
|
ALL_SQL=("${SQL_PHASE1[@]}" "${SQL_PHASE3[@]}")
|
|
log "Application du schéma SQL (${#ALL_SQL[@]} fichiers)…"
|
|
ERRORS=0
|
|
|
|
# Fonction commune d'exécution SQL avec substitution
|
|
run_sql_file() {
|
|
local f="$1"
|
|
local filepath="${SQL_DIR}/${f}"
|
|
if [[ ! -f "${filepath}" ]]; then
|
|
echo " WARN: ${f} non trouvé, ignoré" >&2
|
|
return 0
|
|
fi
|
|
local SQL_PATCHED
|
|
SQL_PATCHED=$(sed \
|
|
-e "s/ja4_logs/${DB_LOGS}/g" \
|
|
-e "s/ja4_processing/${DB_PROC}/g" \
|
|
-e "s/USER 'admin'/USER '${DEV_USER}'/g" \
|
|
-e "s/PASSWORD 'CHANGE_ME'/PASSWORD '${DEV_PASSWORD}'/g" \
|
|
-e "s/PASSWORD 'ChangeMe'/PASSWORD '${DEV_PASSWORD}'/g" \
|
|
"${filepath}")
|
|
|
|
if [[ "${f}" == 10_* ]]; then
|
|
if ch_multiquery "${SQL_PATCHED}" 2>/dev/null; then
|
|
ok "${f}"
|
|
else
|
|
echo " ⚠ ${f} (erreurs ignorées — index déjà existants)"
|
|
fi
|
|
else
|
|
if ch_multiquery "${SQL_PATCHED}" 2>/dev/null; then
|
|
ok "${f}"
|
|
else
|
|
echo " ✗ ${f} — ERREUR" >&2
|
|
ERRORS=$((ERRORS + 1))
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# Phase 1 : bases, tables persistantes, dictionnaires
|
|
for f in "${SQL_PHASE1[@]}"; do
|
|
run_sql_file "${f}"
|
|
done
|
|
|
|
# Phase 2 : drop tables dérivées (repopulées automatiquement par les MVs)
|
|
# Ceci garantit que 05/06/12 recréent les tables avec TOUTES les colonnes du
|
|
# schéma actuel, même si une version antérieure était déjà déployée.
|
|
log "Nettoyage des tables dérivées (agg_*, ml_*) pour garantir le schéma…"
|
|
DERIVED_MVS=(
|
|
"${DB_PROC}.mv_agg_host_ip_ja4_1h"
|
|
"${DB_PROC}.mv_agg_header_fingerprint_1h"
|
|
"${DB_PROC}.mv_agg_path_sequences_1h"
|
|
"${DB_PROC}.mv_agg_request_timing_1h"
|
|
"${DB_PROC}.mv_agg_ip_behavior_1h"
|
|
"${DB_PROC}.mv_agg_resource_cascade_1h"
|
|
)
|
|
DERIVED_TABLES=(
|
|
"${DB_PROC}.agg_host_ip_ja4_1h"
|
|
"${DB_PROC}.agg_header_fingerprint_1h"
|
|
"${DB_PROC}.agg_path_sequences_1h"
|
|
"${DB_PROC}.agg_request_timing_1h"
|
|
"${DB_PROC}.agg_ip_behavior_1h"
|
|
"${DB_PROC}.agg_resource_cascade_1h"
|
|
"${DB_PROC}.ml_detected_anomalies"
|
|
"${DB_PROC}.ml_all_scores"
|
|
)
|
|
for mv in "${DERIVED_MVS[@]}"; do
|
|
ch "DROP VIEW IF EXISTS ${mv}" 2>/dev/null || true
|
|
done
|
|
for tbl in "${DERIVED_TABLES[@]}"; do
|
|
ch "DROP TABLE IF EXISTS ${tbl}" 2>/dev/null || true
|
|
done
|
|
ok "Tables dérivées nettoyées"
|
|
|
|
# Phase 3 : tables dérivées + vues (schéma complet garanti)
|
|
for f in "${SQL_PHASE3[@]}"; do
|
|
run_sql_file "${f}"
|
|
done
|
|
|
|
if [ "${ERRORS}" -gt 0 ]; then
|
|
err "${ERRORS} fichier(s) SQL en erreur"
|
|
fi
|
|
|
|
# ── Nettoyage post-schéma (tables Anubis obsolètes) ──────────────────────────
|
|
# Note : les migrations inline ne sont plus nécessaires — les tables dérivées
|
|
# sont DROP+CREATE en phase 2/3, garantissant le schéma complet.
|
|
log "Nettoyage des tables Anubis obsolètes…"
|
|
ch "DROP DICTIONARY IF EXISTS ${DB_PROC}.dict_anubis_ua" 2>/dev/null || true
|
|
ch "DROP DICTIONARY IF EXISTS ${DB_PROC}.dict_anubis_country" 2>/dev/null || true
|
|
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_ua_rules" 2>/dev/null || true
|
|
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_country_rules" 2>/dev/null || true
|
|
ok "Tables obsolètes supprimées"
|
|
|
|
# ── Vérification du schéma ───────────────────────────────────────────────────
|
|
log "Vérification du schéma…"
|
|
|
|
TABLE_COUNT=$(ch "SELECT count() FROM system.tables WHERE database IN ('${DB_LOGS}','${DB_PROC}')")
|
|
DICT_COUNT=$(ch "SELECT count() FROM system.dictionaries WHERE database='${DB_PROC}'")
|
|
VIEW_COUNT=$(ch "SELECT count() FROM system.tables WHERE database='${DB_PROC}' AND engine='View'")
|
|
MV_COUNT=$(ch "SELECT count() FROM system.tables WHERE database IN ('${DB_LOGS}','${DB_PROC}') AND engine='MaterializedView'")
|
|
|
|
ok "Tables: ${TABLE_COUNT} | Dictionnaires: ${DICT_COUNT} | Vues: ${VIEW_COUNT} | MVs: ${MV_COUNT}"
|
|
|
|
# Vérification des tables critiques
|
|
CRITICAL_TABLES=(
|
|
"${DB_LOGS}.http_logs_raw"
|
|
"${DB_LOGS}.http_logs"
|
|
"${DB_PROC}.ml_detected_anomalies"
|
|
"${DB_PROC}.ml_all_scores"
|
|
"${DB_PROC}.agg_host_ip_ja4_1h"
|
|
"${DB_PROC}.anubis_ip_rules"
|
|
"${DB_PROC}.anubis_asn_rules"
|
|
)
|
|
for t in "${CRITICAL_TABLES[@]}"; do
|
|
db="${t%%.*}"
|
|
tbl="${t##*.}"
|
|
EXISTS=$(ch "SELECT count() FROM system.tables WHERE database='${db}' AND name='${tbl}'" 2>/dev/null || echo "0")
|
|
if [ "${EXISTS}" = "1" ]; then
|
|
ok " ${t}"
|
|
else
|
|
err " Table manquante : ${t}"
|
|
fi
|
|
done
|
|
|
|
# Vérification des dictionnaires critiques
|
|
CRITICAL_DICTS=(
|
|
"dict_anubis_ip"
|
|
"dict_anubis_asn"
|
|
"dict_iplocate_asn"
|
|
"dict_bot_ip"
|
|
"dict_bot_ja4"
|
|
"dict_browser_ja4"
|
|
"dict_asn_reputation"
|
|
)
|
|
for d in "${CRITICAL_DICTS[@]}"; do
|
|
STATUS=$(ch "SELECT status FROM system.dictionaries WHERE database='${DB_PROC}' AND name='${d}'" 2>/dev/null || echo "MISSING")
|
|
if [ "${STATUS}" = "LOADED" ] || [ "${STATUS}" = "NOT_LOADED" ]; then
|
|
ok " ${d} (${STATUS})"
|
|
else
|
|
echo " ⚠ Dictionnaire ${d}: ${STATUS}"
|
|
fi
|
|
done
|
|
|
|
# ── Import des données prod (optionnel) ──────────────────────────────────────
|
|
if [ "${IMPORT_PROD}" = true ]; then
|
|
IMPORT_SCRIPT="${SCRIPT_DIR}/import-prod-data.sh"
|
|
if [[ -x "${IMPORT_SCRIPT}" ]]; then
|
|
log "Lancement de l'import des données prod…"
|
|
"${IMPORT_SCRIPT}" --container "${DEV_CONTAINER}"
|
|
else
|
|
echo " ⚠ Script d'import non trouvé : ${IMPORT_SCRIPT}"
|
|
fi
|
|
fi
|
|
|
|
# ── Résultat ─────────────────────────────────────────────────────────────────
|
|
log "════════════════════════════════════════════════════"
|
|
log " Initialisation terminée"
|
|
log " Bases : ${DB_LOGS}, ${DB_PROC}"
|
|
log " Tables: ${TABLE_COUNT} | Dicts: ${DICT_COUNT} | MVs: ${MV_COUNT}"
|
|
log "════════════════════════════════════════════════════"
|