From 040437921cd459caf8892ee141c27387b43f182e Mon Sep 17 00:00:00 2001 From: toto Date: Fri, 10 Apr 2026 01:00:04 +0200 Subject: [PATCH] fix(init-stack): pre-drop mv_http_logs + http_logs before schema apply Ensure h2 columns are always included on fresh init. Also add migration loop for fleet_detections and ml_performance_metrics tables. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- scripts/init-stack.sh | 37 +++++++++++++++++++++++++++ shared/clickhouse/04_mv_http_logs.sql | 4 +-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/scripts/init-stack.sh b/scripts/init-stack.sh index 3a87753..e8f69cd 100755 --- a/scripts/init-stack.sh +++ b/scripts/init-stack.sh @@ -56,10 +56,12 @@ SQL_DIR="${REPO_ROOT}/shared/clickhouse" # ── Couleurs ───────────────────────────────────────────────────────────────── RED='\033[0;31m' GREEN='\033[0;32m' +YELLOW='\033[0;33m' CYAN='\033[0;36m' NC='\033[0m' log() { echo -e "${CYAN}[init]${NC} $(date '+%H:%M:%S') $*"; } ok() { echo -e "${GREEN} ✓ $*${NC}"; } +warn() { echo -e "${YELLOW} ⚠ $*${NC}"; } err() { echo -e "${RED} ✗ $*${NC}" >&2; exit 1; } # ── Requêteur CH ───────────────────────────────────────────────────────────── @@ -166,6 +168,12 @@ run_sql_file() { fi } +# Pré-Phase 1 : drop mv_http_logs + http_logs pour garantir la recréation avec le +# schéma complet (y compris les colonnes h2_* ajoutées à l'étape 1). +# Ces tables sont vides pendant le développement/test — safe to drop. +ch "DROP VIEW IF EXISTS ${DB_LOGS}.mv_http_logs" 2>/dev/null || true +ch "DROP TABLE IF EXISTS ${DB_LOGS}.http_logs" 2>/dev/null || true + # Phase 1 : bases, tables persistantes, dictionnaires for f in "${SQL_PHASE1[@]}"; do run_sql_file "${f}" @@ -220,6 +228,33 @@ ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_ua_rules" 2>/dev/null || true ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_country_rules" 2>/dev/null || true ok "Tables obsolètes supprimées" +# ── Migrations additionnelles (fleet_detections, ml_performance_metrics) ───── +log "Application des migrations additionnelles…" +MIGRATIONS_DIR="${SCRIPT_DIR}/../services/correlator/sql/migrations" +if [ -d "${MIGRATIONS_DIR}" ]; then + for mig in "${MIGRATIONS_DIR}"/0[0-9]_*.sql; do + [ -f "${mig}" ] || continue + fname=$(basename "${mig}") + SQL_MIG=$(sed \ + -e "s/ja4_logs/${DB_LOGS}/g" \ + -e "s/ja4_processing/${DB_PROC}/g" \ + "${mig}") + if ch_multiquery "${SQL_MIG}" 2>/dev/null; then + ok " migration/${fname}" + else + # Réessayer pour capturer l'erreur réelle + mig_err=$(ch_multiquery "${SQL_MIG}" 2>&1 || true) + if echo "${mig_err}" | grep -qiE "ALREADY_EXISTS|already exists|Object .* already exists"; then + ok " migration/${fname} (déjà existant)" + else + warn " migration/${fname} — ${mig_err}" + fi + fi + done +else + warn "Répertoire migrations introuvable : ${MIGRATIONS_DIR}" +fi + # ── Vérification du schéma ─────────────────────────────────────────────────── log "Vérification du schéma…" @@ -239,6 +274,8 @@ CRITICAL_TABLES=( "${DB_PROC}.agg_host_ip_ja4_1h" "${DB_PROC}.anubis_ip_rules" "${DB_PROC}.anubis_asn_rules" + "${DB_PROC}.fleet_detections" + "${DB_PROC}.ml_performance_metrics" ) for t in "${CRITICAL_TABLES[@]}"; do db="${t%%.*}" diff --git a/shared/clickhouse/04_mv_http_logs.sql b/shared/clickhouse/04_mv_http_logs.sql index 6342a72..16d0f1e 100644 --- a/shared/clickhouse/04_mv_http_logs.sql +++ b/shared/clickhouse/04_mv_http_logs.sql @@ -90,8 +90,8 @@ CREATE TABLE IF NOT EXISTS ja4_logs.http_logs `anubis_bot_category` LowCardinality(String) DEFAULT '', -- Fingerprint HTTP/2 passif (mod_reqin_log connection filter) - `h2_fingerprint` String CODEC(ZSTD(3)) DEFAULT '', - `h2_settings_fp` String CODEC(ZSTD(3)) DEFAULT '', + `h2_fingerprint` String DEFAULT '' CODEC(ZSTD(3)), + `h2_settings_fp` String DEFAULT '' CODEC(ZSTD(3)), `h2_window_update` UInt32 DEFAULT 0, `h2_pseudo_order` LowCardinality(String) DEFAULT '',