feat(scripts): complete stack init + prod data import with date shift
Schema cleanup:
- Remove anubis_ua_rules table stub from 03_anubis_tables.sql
- Remove anubis_ua_rules from bot-detector deploy_schema.sql
- Remove UA seed step from clickhouse-init.sh (no more REGEXP_TREE dependency)
- Drop dict_anubis_ua, dict_anubis_country, anubis_ua_rules, anubis_country_rules
New scripts:
- scripts/init-stack.sh: comprehensive ClickHouse init (13 SQL files + migrations
+ validation + cleanup of obsolete tables). Supports --reset, --import-prod.
- scripts/import-prod-data.sh: imports pre-exported prod data (Native format)
with dynamic date shift (max(time) → now). Supports --shift, --no-truncate.
- scripts/data/prod-export/: directory for cached Native format exports
Makefile targets: init-stack, import-prod-data, init-and-import
Tested: init-stack.sh passes all 13 SQL + 7 critical tables + 7 dicts
import-prod-data.sh: 3M rows in ~37s with auto date shift
Dashboard: 55 routes OK, bot-detector: 36/36 tests pass
Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
This commit is contained in:
toto
251
scripts/init-stack.sh
Executable file
251
scripts/init-stack.sh
Executable file
@ -0,0 +1,251 @@
|
||||
#!/usr/bin/env bash
|
||||
# =============================================================================
|
||||
# init-stack.sh — Initialisation complète de la stack ClickHouse pour ja4-platform
|
||||
#
|
||||
# Ce script exécute l'ensemble du schéma SQL, charge les données CSV de
|
||||
# référence et vérifie que tous les composants sont opérationnels.
|
||||
# Il est utilisé par les tests d'intégration et pour la mise en place de
|
||||
# l'environnement de développement.
|
||||
#
|
||||
# Usage:
|
||||
# ./scripts/init-stack.sh # init dev stack
|
||||
# ./scripts/init-stack.sh --container my-ch-1 # conteneur spécifique
|
||||
# ./scripts/init-stack.sh --user admin --pass X # credentials spécifiques
|
||||
# ./scripts/init-stack.sh --import-prod # init + import données prod
|
||||
# ./scripts/init-stack.sh --reset # DROP databases, recréer tout
|
||||
#
|
||||
# Variables d'environnement :
|
||||
# DEV_CONTAINER Nom du conteneur ClickHouse (défaut: integration-clickhouse-1)
|
||||
# DEV_USER Utilisateur ClickHouse (défaut: default)
|
||||
# DEV_PASSWORD Mot de passe ClickHouse (défaut: vide)
|
||||
# CLICKHOUSE_DB_LOGS Base de données logs (défaut: ja4_logs)
|
||||
# CLICKHOUSE_DB_PROC Base de données processing (défaut: ja4_processing)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
# ── Configuration ────────────────────────────────────────────────────────────
|
||||
DEV_CONTAINER="${DEV_CONTAINER:-integration-clickhouse-1}"
|
||||
DEV_USER="${DEV_USER:-default}"
|
||||
DEV_PASSWORD="${DEV_PASSWORD:-}"
|
||||
DB_LOGS="${CLICKHOUSE_DB_LOGS:-ja4_logs}"
|
||||
DB_PROC="${CLICKHOUSE_DB_PROC:-ja4_processing}"
|
||||
IMPORT_PROD=false
|
||||
RESET=false
|
||||
|
||||
# ── Parsing des arguments ────────────────────────────────────────────────────
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--container) DEV_CONTAINER="$2"; shift 2 ;;
|
||||
--user) DEV_USER="$2"; shift 2 ;;
|
||||
--pass) DEV_PASSWORD="$2"; shift 2 ;;
|
||||
--import-prod) IMPORT_PROD=true; shift ;;
|
||||
--reset) RESET=true; shift ;;
|
||||
-h|--help)
|
||||
sed -n '2,/^# =====/{ /^# =====/d; s/^# \?//p; }' "$0"
|
||||
exit 0
|
||||
;;
|
||||
*) echo "Option inconnue : $1"; exit 1 ;;
|
||||
esac
|
||||
done
|
||||
|
||||
SQL_DIR="${REPO_ROOT}/shared/clickhouse"
|
||||
|
||||
# ── Couleurs ─────────────────────────────────────────────────────────────────
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m'
|
||||
log() { echo -e "${CYAN}[init]${NC} $(date '+%H:%M:%S') $*"; }
|
||||
ok() { echo -e "${GREEN} ✓ $*${NC}"; }
|
||||
err() { echo -e "${RED} ✗ $*${NC}" >&2; exit 1; }
|
||||
|
||||
# ── Requêteur CH ─────────────────────────────────────────────────────────────
|
||||
ch() {
|
||||
local args=("--query" "$1")
|
||||
if [[ -n "${DEV_PASSWORD}" ]]; then
|
||||
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
||||
fi
|
||||
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}"
|
||||
}
|
||||
|
||||
ch_multiquery() {
|
||||
local args=("--multiquery")
|
||||
if [[ -n "${DEV_PASSWORD}" ]]; then
|
||||
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
||||
fi
|
||||
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}" <<< "$1"
|
||||
}
|
||||
|
||||
ch_insert_native() {
|
||||
# $1 = table, stdin = Native data
|
||||
local args=("--query" "INSERT INTO $1 FORMAT Native")
|
||||
if [[ -n "${DEV_PASSWORD}" ]]; then
|
||||
args+=("--user" "${DEV_USER}" "--password" "${DEV_PASSWORD}")
|
||||
fi
|
||||
docker exec -i "${DEV_CONTAINER}" clickhouse-client "${args[@]}"
|
||||
}
|
||||
|
||||
# ── Vérification du conteneur ────────────────────────────────────────────────
|
||||
log "Vérification du conteneur ${DEV_CONTAINER}…"
|
||||
if ! docker exec "${DEV_CONTAINER}" clickhouse-client --query "SELECT 1" > /dev/null 2>&1; then
|
||||
err "Le conteneur ${DEV_CONTAINER} n'est pas accessible"
|
||||
fi
|
||||
ok "Conteneur ${DEV_CONTAINER} accessible"
|
||||
|
||||
# ── Reset optionnel ──────────────────────────────────────────────────────────
|
||||
if [ "${RESET}" = true ]; then
|
||||
log "Reset demandé — suppression des bases de données…"
|
||||
ch "DROP DATABASE IF EXISTS ${DB_LOGS}" 2>/dev/null || true
|
||||
ch "DROP DATABASE IF EXISTS ${DB_PROC}" 2>/dev/null || true
|
||||
ok "Bases ${DB_LOGS} et ${DB_PROC} supprimées"
|
||||
fi
|
||||
|
||||
# ── Exécution des fichiers SQL ───────────────────────────────────────────────
|
||||
SQL_FILES=(
|
||||
00_database.sql
|
||||
01_raw_tables.sql
|
||||
02_dictionaries.sql
|
||||
03_anubis_tables.sql
|
||||
04_mv_http_logs.sql
|
||||
05_aggregation_tables.sql
|
||||
06_ml_tables.sql
|
||||
07_ai_features_view.sql
|
||||
08_users.sql
|
||||
09_audit_table.sql
|
||||
10_perf_indexes.sql
|
||||
11_views.sql
|
||||
12_thesis_features.sql
|
||||
)
|
||||
|
||||
log "Application du schéma SQL (${#SQL_FILES[@]} fichiers)…"
|
||||
ERRORS=0
|
||||
|
||||
for f in "${SQL_FILES[@]}"; do
|
||||
filepath="${SQL_DIR}/${f}"
|
||||
if [[ ! -f "${filepath}" ]]; then
|
||||
echo " WARN: ${f} non trouvé, ignoré" >&2
|
||||
continue
|
||||
fi
|
||||
|
||||
# Substitution des noms de bases et des credentials
|
||||
SQL_PATCHED=$(sed \
|
||||
-e "s/ja4_logs/${DB_LOGS}/g" \
|
||||
-e "s/ja4_processing/${DB_PROC}/g" \
|
||||
-e "s/USER 'admin'/USER '${DEV_USER}'/g" \
|
||||
-e "s/PASSWORD 'CHANGE_ME'/PASSWORD '${DEV_PASSWORD}'/g" \
|
||||
-e "s/PASSWORD 'ChangeMe'/PASSWORD '${DEV_PASSWORD}'/g" \
|
||||
"${filepath}")
|
||||
|
||||
# 10_perf_indexes.sql peut échouer si les index existent déjà
|
||||
if [[ "${f}" == 10_* ]]; then
|
||||
if ch_multiquery "${SQL_PATCHED}" 2>/dev/null; then
|
||||
ok "${f}"
|
||||
else
|
||||
echo " ⚠ ${f} (erreurs ignorées — index déjà existants)"
|
||||
fi
|
||||
else
|
||||
if ch_multiquery "${SQL_PATCHED}" 2>/dev/null; then
|
||||
ok "${f}"
|
||||
else
|
||||
echo " ✗ ${f} — ERREUR" >&2
|
||||
ERRORS=$((ERRORS + 1))
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "${ERRORS}" -gt 0 ]; then
|
||||
err "${ERRORS} fichier(s) SQL en erreur"
|
||||
fi
|
||||
|
||||
# ── Migrations post-schéma (colonnes manquantes sur DB existante) ────────────
|
||||
log "Application des migrations post-schéma…"
|
||||
MIGRATIONS=(
|
||||
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_xff SimpleAggregateFunction(sum, UInt64)"
|
||||
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_unusual_ct SimpleAggregateFunction(sum, UInt64)"
|
||||
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_non_std_port SimpleAggregateFunction(sum, UInt64)"
|
||||
"ALTER TABLE ${DB_PROC}.agg_host_ip_ja4_1h ADD COLUMN IF NOT EXISTS count_login_post SimpleAggregateFunction(sum, UInt64)"
|
||||
"ALTER TABLE ${DB_PROC}.agg_header_fingerprint_1h ADD COLUMN IF NOT EXISTS sec_ch_mobile_mismatch SimpleAggregateFunction(max, UInt8)"
|
||||
)
|
||||
for mig in "${MIGRATIONS[@]}"; do
|
||||
ch "${mig}" 2>/dev/null || true
|
||||
done
|
||||
ok "Migrations appliquées"
|
||||
|
||||
# ── Nettoyage des tables Anubis obsolètes (UA, Country) ─────────────────────
|
||||
log "Nettoyage des tables Anubis obsolètes…"
|
||||
ch "DROP DICTIONARY IF EXISTS ${DB_PROC}.dict_anubis_ua" 2>/dev/null || true
|
||||
ch "DROP DICTIONARY IF EXISTS ${DB_PROC}.dict_anubis_country" 2>/dev/null || true
|
||||
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_ua_rules" 2>/dev/null || true
|
||||
ch "DROP TABLE IF EXISTS ${DB_PROC}.anubis_country_rules" 2>/dev/null || true
|
||||
ok "Tables obsolètes supprimées"
|
||||
|
||||
# ── Vérification du schéma ───────────────────────────────────────────────────
|
||||
log "Vérification du schéma…"
|
||||
|
||||
TABLE_COUNT=$(ch "SELECT count() FROM system.tables WHERE database IN ('${DB_LOGS}','${DB_PROC}')")
|
||||
DICT_COUNT=$(ch "SELECT count() FROM system.dictionaries WHERE database='${DB_PROC}'")
|
||||
VIEW_COUNT=$(ch "SELECT count() FROM system.tables WHERE database='${DB_PROC}' AND engine='View'")
|
||||
MV_COUNT=$(ch "SELECT count() FROM system.tables WHERE database IN ('${DB_LOGS}','${DB_PROC}') AND engine='MaterializedView'")
|
||||
|
||||
ok "Tables: ${TABLE_COUNT} | Dictionnaires: ${DICT_COUNT} | Vues: ${VIEW_COUNT} | MVs: ${MV_COUNT}"
|
||||
|
||||
# Vérification des tables critiques
|
||||
CRITICAL_TABLES=(
|
||||
"${DB_LOGS}.http_logs_raw"
|
||||
"${DB_LOGS}.http_logs"
|
||||
"${DB_PROC}.ml_detected_anomalies"
|
||||
"${DB_PROC}.ml_all_scores"
|
||||
"${DB_PROC}.agg_host_ip_ja4_1h"
|
||||
"${DB_PROC}.anubis_ip_rules"
|
||||
"${DB_PROC}.anubis_asn_rules"
|
||||
)
|
||||
for t in "${CRITICAL_TABLES[@]}"; do
|
||||
db="${t%%.*}"
|
||||
tbl="${t##*.}"
|
||||
EXISTS=$(ch "SELECT count() FROM system.tables WHERE database='${db}' AND name='${tbl}'" 2>/dev/null || echo "0")
|
||||
if [ "${EXISTS}" = "1" ]; then
|
||||
ok " ${t}"
|
||||
else
|
||||
err " Table manquante : ${t}"
|
||||
fi
|
||||
done
|
||||
|
||||
# Vérification des dictionnaires critiques
|
||||
CRITICAL_DICTS=(
|
||||
"dict_anubis_ip"
|
||||
"dict_anubis_asn"
|
||||
"dict_iplocate_asn"
|
||||
"dict_bot_ip"
|
||||
"dict_bot_ja4"
|
||||
"dict_browser_ja4"
|
||||
"dict_asn_reputation"
|
||||
)
|
||||
for d in "${CRITICAL_DICTS[@]}"; do
|
||||
STATUS=$(ch "SELECT status FROM system.dictionaries WHERE database='${DB_PROC}' AND name='${d}'" 2>/dev/null || echo "MISSING")
|
||||
if [ "${STATUS}" = "LOADED" ] || [ "${STATUS}" = "NOT_LOADED" ]; then
|
||||
ok " ${d} (${STATUS})"
|
||||
else
|
||||
echo " ⚠ Dictionnaire ${d}: ${STATUS}"
|
||||
fi
|
||||
done
|
||||
|
||||
# ── Import des données prod (optionnel) ──────────────────────────────────────
|
||||
if [ "${IMPORT_PROD}" = true ]; then
|
||||
IMPORT_SCRIPT="${SCRIPT_DIR}/import-prod-data.sh"
|
||||
if [[ -x "${IMPORT_SCRIPT}" ]]; then
|
||||
log "Lancement de l'import des données prod…"
|
||||
"${IMPORT_SCRIPT}" --container "${DEV_CONTAINER}"
|
||||
else
|
||||
echo " ⚠ Script d'import non trouvé : ${IMPORT_SCRIPT}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ── Résultat ─────────────────────────────────────────────────────────────────
|
||||
log "════════════════════════════════════════════════════"
|
||||
log " Initialisation terminée"
|
||||
log " Bases : ${DB_LOGS}, ${DB_PROC}"
|
||||
log " Tables: ${TABLE_COUNT} | Dicts: ${DICT_COUNT} | MVs: ${MV_COUNT}"
|
||||
log "════════════════════════════════════════════════════"
|
||||
Reference in New Issue
Block a user