fix(init-stack): pre-drop mv_http_logs + http_logs before schema apply
Ensure h2 columns are always included on fresh init. Also add migration loop for fleet_detections and ml_performance_metrics tables. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
@ -56,10 +56,12 @@ SQL_DIR="${REPO_ROOT}/shared/clickhouse"
|
||||
# ── Couleurs ─────────────────────────────────────────────────────────────────
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[0;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
log() { echo -e "${CYAN}[init]${NC} $(date '+%H:%M:%S') $*"; }
|
||||
ok() { echo -e "${GREEN} ✓ $*${NC}"; }
|
||||
warn() { echo -e "${YELLOW} ⚠ $*${NC}"; }
|
||||
err() { echo -e "${RED} ✗ $*${NC}" >&2; exit 1; }
|
||||
|
||||
# ── Requêteur CH ─────────────────────────────────────────────────────────────
|
||||
@ -166,6 +168,12 @@ run_sql_file() {
|
||||
fi
|
||||
}
|
||||
|
||||
# Pré-Phase 1 : drop mv_http_logs + http_logs pour garantir la recréation avec le
|
||||
# schéma complet (y compris les colonnes h2_* ajoutées à l'étape 1).
|
||||
# Ces tables sont vides pendant le développement/test — safe to drop.
|
||||
ch "DROP VIEW IF EXISTS ${DB_LOGS}.mv_http_logs" 2>/dev/null || true
|
||||
ch "DROP TABLE IF EXISTS ${DB_LOGS}.http_logs" 2>/dev/null || true
|
||||
|
||||
# Phase 1 : bases, tables persistantes, dictionnaires
|
||||
for f in "${SQL_PHASE1[@]}"; do
|
||||
run_sql_file "${f}"
|
||||
@ -220,6 +228,33 @@ ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_ua_rules" 2>/dev/null || true
|
||||
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_country_rules" 2>/dev/null || true
|
||||
ok "Tables obsolètes supprimées"
|
||||
|
||||
# ── Migrations additionnelles (fleet_detections, ml_performance_metrics) ─────
|
||||
log "Application des migrations additionnelles…"
|
||||
MIGRATIONS_DIR="${SCRIPT_DIR}/../services/correlator/sql/migrations"
|
||||
if [ -d "${MIGRATIONS_DIR}" ]; then
|
||||
for mig in "${MIGRATIONS_DIR}"/0[0-9]_*.sql; do
|
||||
[ -f "${mig}" ] || continue
|
||||
fname=$(basename "${mig}")
|
||||
SQL_MIG=$(sed \
|
||||
-e "s/ja4_logs/${DB_LOGS}/g" \
|
||||
-e "s/ja4_processing/${DB_PROC}/g" \
|
||||
"${mig}")
|
||||
if ch_multiquery "${SQL_MIG}" 2>/dev/null; then
|
||||
ok " migration/${fname}"
|
||||
else
|
||||
# Réessayer pour capturer l'erreur réelle
|
||||
mig_err=$(ch_multiquery "${SQL_MIG}" 2>&1 || true)
|
||||
if echo "${mig_err}" | grep -qiE "ALREADY_EXISTS|already exists|Object .* already exists"; then
|
||||
ok " migration/${fname} (déjà existant)"
|
||||
else
|
||||
warn " migration/${fname} — ${mig_err}"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
else
|
||||
warn "Répertoire migrations introuvable : ${MIGRATIONS_DIR}"
|
||||
fi
|
||||
|
||||
# ── Vérification du schéma ───────────────────────────────────────────────────
|
||||
log "Vérification du schéma…"
|
||||
|
||||
@ -239,6 +274,8 @@ CRITICAL_TABLES=(
|
||||
"${DB_PROC}.agg_host_ip_ja4_1h"
|
||||
"${DB_PROC}.anubis_ip_rules"
|
||||
"${DB_PROC}.anubis_asn_rules"
|
||||
"${DB_PROC}.fleet_detections"
|
||||
"${DB_PROC}.ml_performance_metrics"
|
||||
)
|
||||
for t in "${CRITICAL_TABLES[@]}"; do
|
||||
db="${t%%.*}"
|
||||
|
||||
Reference in New Issue
Block a user