fix(init-stack): pre-drop mv_http_logs + http_logs before schema apply

Ensure h2 columns are always included on fresh init. Also add migration
loop for fleet_detections and ml_performance_metrics tables.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
2026-04-10 01:00:04 +02:00
parent b409a70970
commit 040437921c
2 changed files with 39 additions and 2 deletions

View File

@ -56,10 +56,12 @@ SQL_DIR="${REPO_ROOT}/shared/clickhouse"
# ── Couleurs ─────────────────────────────────────────────────────────────────
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[0;33m'
CYAN='\033[0;36m'
NC='\033[0m'
log() { echo -e "${CYAN}[init]${NC} $(date '+%H:%M:%S') $*"; }
ok() { echo -e "${GREEN}$*${NC}"; }
warn() { echo -e "${YELLOW}$*${NC}"; }
err() { echo -e "${RED}$*${NC}" >&2; exit 1; }
# ── Requêteur CH ─────────────────────────────────────────────────────────────
@ -166,6 +168,12 @@ run_sql_file() {
fi
}
# Pré-Phase 1 : drop mv_http_logs + http_logs pour garantir la recréation avec le
# schéma complet (y compris les colonnes h2_* ajoutées à l'étape 1).
# Ces tables sont vides pendant le développement/test — safe to drop.
ch "DROP VIEW IF EXISTS ${DB_LOGS}.mv_http_logs" 2>/dev/null || true
ch "DROP TABLE IF EXISTS ${DB_LOGS}.http_logs" 2>/dev/null || true
# Phase 1 : bases, tables persistantes, dictionnaires
for f in "${SQL_PHASE1[@]}"; do
run_sql_file "${f}"
@ -220,6 +228,33 @@ ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_ua_rules" 2>/dev/null || true
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_country_rules" 2>/dev/null || true
ok "Tables obsolètes supprimées"
# ── Migrations additionnelles (fleet_detections, ml_performance_metrics) ─────
log "Application des migrations additionnelles…"
MIGRATIONS_DIR="${SCRIPT_DIR}/../services/correlator/sql/migrations"
if [ -d "${MIGRATIONS_DIR}" ]; then
for mig in "${MIGRATIONS_DIR}"/0[0-9]_*.sql; do
[ -f "${mig}" ] || continue
fname=$(basename "${mig}")
SQL_MIG=$(sed \
-e "s/ja4_logs/${DB_LOGS}/g" \
-e "s/ja4_processing/${DB_PROC}/g" \
"${mig}")
if ch_multiquery "${SQL_MIG}" 2>/dev/null; then
ok " migration/${fname}"
else
# Réessayer pour capturer l'erreur réelle
mig_err=$(ch_multiquery "${SQL_MIG}" 2>&1 || true)
if echo "${mig_err}" | grep -qiE "ALREADY_EXISTS|already exists|Object .* already exists"; then
ok " migration/${fname} (déjà existant)"
else
warn " migration/${fname}${mig_err}"
fi
fi
done
else
warn "Répertoire migrations introuvable : ${MIGRATIONS_DIR}"
fi
# ── Vérification du schéma ───────────────────────────────────────────────────
log "Vérification du schéma…"
@ -239,6 +274,8 @@ CRITICAL_TABLES=(
"${DB_PROC}.agg_host_ip_ja4_1h"
"${DB_PROC}.anubis_ip_rules"
"${DB_PROC}.anubis_asn_rules"
"${DB_PROC}.fleet_detections"
"${DB_PROC}.ml_performance_metrics"
)
for t in "${CRITICAL_TABLES[@]}"; do
db="${t%%.*}"