refactor(anubis): simplify to IP/CIDR + ASN only, remove UA and Country rules

- Remove UA regex extraction (extract_ua_regex, _extract_ua_from_all/any)
- Remove Country rule collection from parse_bot_policies_inline
- Simplify fetch_rules.py: collect_all_rules returns (ip_rules, asn_rules)
- Remove insert_ua_rules and insert_country_rules functions
- reload_dicts now only reloads dict_anubis_ip + dict_anubis_asn
- Simplify CASE blocks in 04_mv_http_logs.sql, 07_ai_features_view.sql,
  view_ai_features_anubis.sql, mv_http_logs.sql: IP > ASN (was 5-level
  UA+IP > UA > IP > ASN > Country cascade)
- Remove dict_anubis_country + dict_anubis_ua from 03_anubis_tables.sql
  (UA table kept as stub for REGEXP_TREE catch-all compatibility)
- Remove anubis_country_rules table from schema
- Remove Anubis UA and Country tabs from dashboard reflists page
- Remove anubis_ua_rules/country_rules from API reflist queries
- deploy_schema.sql simplified from 339 to 122 lines
- 764 lines removed across 9 files

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

services/dashboard/backend/routes/api.py

@@ -1499,10 +1499,8 @@ _REFLIST_SORT = {
     "browser_ja4": {"ja4", "browser_family", "tls_library"},
     "asn_reputation": {"src_asn", "label"},
     "iplocate_asn": {"asn", "country_code", "name", "network"},
-    "anubis_ua_rules": {"id", "regexp", "bot_name", "action", "category"},
     "anubis_ip_rules": {"prefix", "bot_name", "action", "category"},
     "anubis_asn_rules": {"asn", "bot_name", "action", "category"},
-    "anubis_country_rules": {"country_code", "bot_name", "action", "category"},
 }
 
 _REFLIST_SEARCH_COLS: dict[str, list[str]] = {
@@ -1511,10 +1509,8 @@ _REFLIST_SEARCH_COLS: dict[str, list[str]] = {
     "browser_ja4": ["ja4", "browser_family", "tls_library", "context"],
     "asn_reputation": ["toString(src_asn)", "label"],
     "iplocate_asn": ["network", "toString(asn)", "country_code", "name"],
-    "anubis_ua_rules": ["regexp", "bot_name", "action", "category"],
     "anubis_ip_rules": ["prefix", "bot_name", "action", "category"],
     "anubis_asn_rules": ["toString(asn)", "bot_name", "action", "category"],
-    "anubis_country_rules": ["country_code", "bot_name", "action", "category"],
 }
 
 
@@ -1532,22 +1528,12 @@ _REFLIST_QUERIES: dict[str, str] = {
         f"SELECT network, asn, country_code, name "
         f"FROM dictionary('{_DB}.dict_iplocate_asn')"
     ),
-    "anubis_ua_rules": (
-        f"SELECT id, parent_id, regexp, "
-        f"  arrayElement(values, indexOf(keys, 'bot_name')) AS bot_name, "
-        f"  arrayElement(values, indexOf(keys, 'action'))   AS action, "
-        f"  arrayElement(values, indexOf(keys, 'category')) AS category "
-        f"FROM {_DB}.anubis_ua_rules"
-    ),
     "anubis_ip_rules": (
         f"SELECT prefix, bot_name, action, category FROM {_DB}.anubis_ip_rules"
     ),
     "anubis_asn_rules": (
         f"SELECT asn, bot_name, action, category FROM {_DB}.anubis_asn_rules"
     ),
-    "anubis_country_rules": (
-        f"SELECT country_code, bot_name, action, category FROM {_DB}.anubis_country_rules"
-    ),
 }
 
 

services/dashboard/backend/templates/reflists.html

@@ -17,7 +17,7 @@ Listes de référence
             <tr><td class="pr-2 font-mono text-brand-400">browser_ja4</td><td>Fingerprints JA4 navigateurs → famille, lib TLS</td></tr>
             <tr><td class="pr-2 font-mono text-brand-400">asn_reputation</td><td>ASN → label de réputation (isp, datacenter, cdn…)</td></tr>
             <tr><td class="pr-2 font-mono text-brand-400">iplocate_asn</td><td>IP/CIDR → ASN, pays, nom (géolocalisation)</td></tr>
-            <tr><td class="pr-2 font-mono text-brand-400">anubis_*</td><td>Règles Anubis : UA, IP, ASN, pays (filtrage crawlers)</td></tr>
+            <tr><td class="pr-2 font-mono text-brand-400">anubis_*</td><td>Règles Anubis : IP/CIDR et ASN (filtrage crawlers)</td></tr>
         </table>
     </div>
 </div>
@@ -55,10 +55,8 @@ Listes de référence
     <button class="tab-btn" data-tab="browser_ja4">🌐 Browser JA4</button>
     <button class="tab-btn" data-tab="asn_reputation">🏢 ASN Réputation</button>
     <button class="tab-btn" data-tab="iplocate_asn">🌍 IPLocate</button>
-    <button class="tab-btn" data-tab="anubis_ua_rules">🕷 Anubis UA</button>
     <button class="tab-btn" data-tab="anubis_ip_rules">🕷 Anubis IP</button>
     <button class="tab-btn" data-tab="anubis_asn_rules">🕷 Anubis ASN</button>
-    <button class="tab-btn" data-tab="anubis_country_rules">🕷 Anubis Pays</button>
 </div>
 
 <!-- Controls row -->
@@ -141,10 +139,8 @@ const LIST_DOC = {
     browser_ja4:'Fingerprints JA4 de navigateurs légitimes avec famille (Chromium, Firefox, Safari…), librairie TLS et contexte. Source : dict_browser_ja4.',
     asn_reputation:'Réputation de chaque ASN (isp, datacenter, cdn, hosting, education…). Source : dict_asn_reputation. Utilisé pour la baseline ISP humaine.',
     iplocate_asn:'Géolocalisation IP→ASN : réseau, numéro ASN, code pays, nom opérateur. Source : dict_iplocate_asn (IP_TRIE, ~714K entrées).',
-    anubis_ua_rules:'Règles Anubis de détection par User-Agent (REGEXP_TREE). Chaque règle associe un pattern regex à un bot_name et une action (ALLOW/DENY/WEIGH).',
     anubis_ip_rules:'Règles Anubis de détection par plage IP (IP_TRIE). Associe des CIDR à des bots connus avec action de filtrage.',
     anubis_asn_rules:'Règles Anubis par ASN : certains ASN sont associés à des botnets ou services automatisés connus.',
-    anubis_country_rules:'Règles Anubis par pays : politique de filtrage par code pays (ex: bloquer le trafic de certaines régions).',
 };
 
 // Colonnes par liste
@@ -154,10 +150,8 @@ const COLUMNS = {
     browser_ja4:[{k:'ja4',f:'JA4'},{k:'browser_family',f:'Famille'},{k:'tls_library',f:'Lib TLS'},{k:'context',f:'Contexte'}],
     asn_reputation:[{k:'src_asn',f:'ASN'},{k:'label',f:'Réputation'}],
     iplocate_asn:[{k:'network',f:'Réseau'},{k:'asn',f:'ASN'},{k:'country_code',f:'Pays'},{k:'name',f:'Nom'}],
-    anubis_ua_rules:[{k:'id',f:'ID'},{k:'regexp',f:'Regex'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
     anubis_ip_rules:[{k:'prefix',f:'IP/CIDR'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
     anubis_asn_rules:[{k:'asn',f:'ASN'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
-    anubis_country_rules:[{k:'country_code',f:'Pays'},{k:'bot_name',f:'Bot'},{k:'action',f:'Action'},{k:'category',f:'Catégorie'}],
 };
 
 // Tab click