#!/usr/bin/env python3
"""
Minute-cadence poll for AI-monitor health and HAProxy stats.

Runs non-interactively from cron. Never long-running — one poll cycle per
invocation to stay in line with WHP's cron-driven pattern (see also
haproxy-watchdog.sh).

Sources:
  - https://localhost:8443/api/health.php?check=all&detailed=1
  - http://localhost:8404/stats;csv

On a healthy→warning/critical transition for any subsystem or backend, fetches
relevant context, calls AI for diagnosis, and inserts (or increments) an Issue
via whp_monitor_core.
"""
import csv
import io
import json
import os
import re
import shlex
import subprocess
import sys
import time
import sqlite3
import urllib3

# Suppress localhost self-signed cert warning for the health.php poll.
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from whp_monitor_core import (
    connect_mysql, connect_sqlite, compute_signature, ten_minute_bucket,
    insert_or_increment_issue, mark_auto_resolved, call_ai,
)

import requests

HEALTH_URL = 'https://localhost:8443/api/health.php?check=all&detailed=1'
# HAProxy stats bind to 127.0.0.1:8404 INSIDE the haproxy-manager container
# and are not exposed to the host. Fetch via `docker exec` instead.
HAPROXY_STATS_CMD = ['docker', 'exec', 'haproxy-manager',
                     'curl', '-s', 'http://127.0.0.1:8404/stats;csv']


def ensure_sqlite_schema(conn):
    """Create health_state, backend_state, poll_heartbeat, poll_cursor,
    health_transitions tables. Also runs an idempotent ALTER for the
    severity_level column added later for tiered disk alerting."""
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS health_state (
            subsystem TEXT PRIMARY KEY,
            status TEXT NOT NULL,
            detail_json TEXT,
            transitioned_at INTEGER NOT NULL
        );
        CREATE TABLE IF NOT EXISTS backend_state (
            backend_name TEXT PRIMARY KEY,
            status TEXT NOT NULL,
            transitioned_at INTEGER NOT NULL
        );
        CREATE TABLE IF NOT EXISTS poll_heartbeat (
            source TEXT PRIMARY KEY,
            last_run_at INTEGER NOT NULL,
            last_status TEXT NOT NULL,
            last_error  TEXT
        );
        CREATE TABLE IF NOT EXISTS poll_cursor (
            key   TEXT PRIMARY KEY,
            value INTEGER NOT NULL
        );
        -- Row-per-event log of subsystem state changes. Powers the AI Monitor
        -- dashboard timeline so we can correlate with external alerts (e.g.
        -- Uptime Kuma) and see what the panel did about each event.
        CREATE TABLE IF NOT EXISTS health_transitions (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            subsystem TEXT NOT NULL,           -- e.g. 'cpu', 'disk:root', 'haproxy'
            prev_status TEXT NOT NULL,
            new_status TEXT NOT NULL,
            prev_severity INTEGER DEFAULT 0,   -- disk tier 0-6; 0 for non-tiered subsystems
            new_severity INTEGER DEFAULT 0,
            detail_json TEXT,                  -- raw subsystem snapshot at transition
            ai_diagnosis TEXT,                 -- AI-extracted diagnosis (warning/critical only)
            mysql_signature VARCHAR(64),       -- joins to whp.ai_monitor_issues.signature
            transitioned_at INTEGER NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_ht_subsystem_time ON health_transitions(subsystem, transitioned_at);
        CREATE INDEX IF NOT EXISTS idx_ht_time ON health_transitions(transitioned_at);
        -- Rate-limit state for auto-restart of customer site containers. One row
        -- per container; the window resets once RESTART_WINDOW_SECONDS elapses
        -- since window_started_at so a flapping site can't be restarted forever.
        CREATE TABLE IF NOT EXISTS site_recovery (
            container_name    TEXT PRIMARY KEY,
            attempts          INTEGER NOT NULL DEFAULT 0,
            window_started_at INTEGER NOT NULL,
            last_attempt_at   INTEGER NOT NULL,
            last_result       TEXT
        );
        -- Active PHP-probe state. Tracks consecutive backend failures so a single
        -- transient 5xx doesn't trigger a restart (require N in a row).
        CREATE TABLE IF NOT EXISTS site_php_probe (
            container_name       TEXT PRIMARY KEY,
            consecutive_failures INTEGER NOT NULL DEFAULT 0,
            last_code            TEXT,
            last_checked_at      INTEGER NOT NULL
        );
    """)
    # Idempotent column add (SQLite has no IF NOT EXISTS for columns; check
    # PRAGMA first).
    cols = {row[1] for row in conn.execute("PRAGMA table_info(health_state)").fetchall()}
    if 'severity_level' not in cols:
        try:
            conn.execute("ALTER TABLE health_state ADD COLUMN severity_level INTEGER DEFAULT 0")
        except sqlite3.OperationalError:
            pass  # raced with another invocation
    conn.commit()


def write_heartbeat(conn, source: str, status: str, error: str = None):
    """Stamp a heartbeat row so the AI Monitor admin page can show
    'last health poll: Ns ago' without having to invent it from log
    transitions. Called at the end of every poll cycle, success or not."""
    conn.execute(
        "INSERT OR REPLACE INTO poll_heartbeat (source, last_run_at, last_status, last_error) "
        "VALUES (?, ?, ?, ?)",
        (source, int(time.time()), status, error))
    conn.commit()


# ---------- Health API polling ----------

HEALTH_SUBSYSTEMS = ['cpu', 'connections', 'disk', 'swap', 'docker', 'haproxy']

CATEGORY_BY_SUBSYSTEM = {
    'cpu': 'cpu_pressure',
    'swap': 'swap_pressure',
    'disk': 'disk_full',
    'connections': 'connection_storm',
    'docker': 'docker_degraded',
    'haproxy': 'haproxy_unhealthy',
}


def disk_tier(usage_pct: float) -> int:
    """Map a partition's usage_percent to a 0-6 severity tier.

    0  = healthy (<80)        -- no alert
    1  = warning  (>=80)
    2  = warning  (>=85)
    3  = warning  (>=90)
    4  = critical (>=95)
    5  = critical (>=99)
    6  = critical (==100)     -- system at risk of collapse
    """
    if usage_pct >= 100: return 6
    if usage_pct >= 99:  return 5
    if usage_pct >= 95:  return 4
    if usage_pct >= 90:  return 3
    if usage_pct >= 85:  return 2
    if usage_pct >= 80:  return 1
    return 0


def status_from_tier(tier: int) -> str:
    if tier >= 4: return 'critical'
    if tier >= 1: return 'warning'
    return 'healthy'


def record_transition(sqlite_conn, subsystem: str,
                      prev_status: str, new_status: str,
                      prev_tier: int, new_tier: int,
                      detail: dict, ai_diagnosis: str = None,
                      signature: str = None):
    """Persist a row-per-transition log entry. Caller is responsible for
    the surrounding INSERT OR REPLACE on health_state."""
    sqlite_conn.execute(
        "INSERT INTO health_transitions "
        "(subsystem, prev_status, new_status, prev_severity, new_severity, "
        " detail_json, ai_diagnosis, mysql_signature, transitioned_at) "
        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
        (subsystem, prev_status, new_status, prev_tier, new_tier,
         json.dumps(detail), ai_diagnosis, signature, int(time.time())))
    sqlite_conn.commit()


def poll_health(sqlite_conn, mysql_conn):
    try:
        r = requests.get(HEALTH_URL, verify=False, timeout=15)
        r.raise_for_status()
        data = r.json()
    except Exception as e:
        # Don't error-loop on transient failures. Log once and return.
        print(f"[health] poll failed: {e}", flush=True)
        return

    checks = data.get('data', {}).get('checks', {})
    for sub in HEALTH_SUBSYSTEMS:
        sub_data = checks.get(sub) or {}
        status = sub_data.get('status', 'unknown')
        if status not in ('healthy', 'warning', 'critical', 'error'):
            continue

        # Disk: evaluate each partition independently with 6-tier severity.
        # A 1->2 tier change inside 'warning' is still a transition we want
        # logged (and notified on) -- the existing status comparison would
        # miss it.
        if sub == 'disk':
            filesystems = sub_data.get('filesystems') or {}
            for partition, fs in filesystems.items():
                pct = float(fs.get('usage_percent') or 0)
                evaluate_disk_partition(sqlite_conn, mysql_conn, partition, fs, pct)
            continue

        # Non-disk subsystems use the original status-only model.
        evaluate_simple_subsystem(sqlite_conn, mysql_conn, sub, status, sub_data)


def evaluate_simple_subsystem(sqlite_conn, mysql_conn, sub, new_status, sub_data):
    last = sqlite_conn.execute(
        "SELECT status FROM health_state WHERE subsystem=?", (sub,)).fetchone()
    last_status = last['status'] if last else 'healthy'
    if new_status == last_status:
        return

    sig = (compute_signature(kind='health', subsystem=sub, severity=new_status)
           if new_status in ('warning', 'critical') else None)

    sqlite_conn.execute(
        "INSERT OR REPLACE INTO health_state (subsystem, status, detail_json, transitioned_at, severity_level) "
        "VALUES (?, ?, ?, ?, 0)",
        (sub, new_status, json.dumps(sub_data), int(time.time())))
    sqlite_conn.commit()

    if new_status == 'healthy':
        for sev in ('critical', 'warning'):
            recovered_sig = compute_signature(kind='health', subsystem=sub, severity=sev)
            mark_auto_resolved(mysql_conn, recovered_sig)
        record_transition(sqlite_conn, sub, last_status, new_status, 0, 0, sub_data)
        return

    if new_status in ('warning', 'critical'):
        ai_text = handle_health_transition(sqlite_conn, mysql_conn, sub, new_status, sub_data)
        record_transition(sqlite_conn, sub, last_status, new_status,
                          0, 0, sub_data, ai_text, sig)


def evaluate_disk_partition(sqlite_conn, mysql_conn, partition, fs, pct):
    """Per-partition tracking with 6-tier severity. Each 5% crossing on the
    way up is a transition; downward movement still updates state but only
    counts as a recovery once we cross back below 80% (tier 0).

    Signature is partition-scoped (not severity-scoped) so the issue row
    in MySQL is reused across tier bumps -- 'one issue per partition,
    severity climbs as the partition fills.'"""
    sub_key = f'disk:{partition}'
    new_tier = disk_tier(pct)
    new_status = status_from_tier(new_tier)
    signature = compute_signature(kind='health', subsystem=sub_key, severity='all')  # tier-agnostic

    last = sqlite_conn.execute(
        "SELECT status, severity_level FROM health_state WHERE subsystem=?",
        (sub_key,)).fetchone()
    last_status = last['status'] if last else 'healthy'
    last_tier = (last['severity_level'] if last else 0) or 0

    if new_tier == last_tier and new_status == last_status:
        return  # genuinely unchanged

    sqlite_conn.execute(
        "INSERT OR REPLACE INTO health_state "
        "(subsystem, status, detail_json, transitioned_at, severity_level) "
        "VALUES (?, ?, ?, ?, ?)",
        (sub_key, new_status, json.dumps(fs), int(time.time()), new_tier))
    sqlite_conn.commit()

    # Recovery: dropped to tier 0. Mark the open issue auto-resolved.
    if new_tier == 0:
        mark_auto_resolved(mysql_conn, signature)
        record_transition(sqlite_conn, sub_key, last_status, new_status,
                          last_tier, new_tier, fs, signature=signature)
        return

    # Downward but still elevated (e.g. 5->4): just log the transition;
    # don't re-call AI, don't re-notify.
    if new_tier < last_tier:
        record_transition(sqlite_conn, sub_key, last_status, new_status,
                          last_tier, new_tier, fs, signature=signature)
        return

    # Upward: enrich, ask AI for diagnosis, write/update the issue.
    detail = dict(fs)
    detail['partition'] = partition
    detail['severity_level'] = new_tier
    ai_text = handle_health_transition(sqlite_conn, mysql_conn, sub_key,
                                       new_status, detail)
    record_transition(sqlite_conn, sub_key, last_status, new_status,
                      last_tier, new_tier, fs, ai_text, signature)


def handle_health_transition(sqlite_conn, mysql_conn, subsystem, severity, detail):
    """Run the AI diagnosis + persist (or update) the issue row. Returns the
    AI diagnosis text so callers can also stamp it on the timeline row."""
    # Strip 'disk:partition' suffix when looking up category/context — both
    # are keyed by base subsystem.
    base = subsystem.split(':', 1)[0]
    category = CATEGORY_BY_SUBSYSTEM.get(base, 'general')

    # Disk uses a partition-scoped signature so all six tier bumps share one
    # MySQL issue row that simply has its severity bumped.
    if base == 'disk':
        signature = compute_signature(kind='health', subsystem=subsystem, severity='all')
    else:
        signature = compute_signature(kind='health', subsystem=subsystem, severity=severity)

    context = gather_health_context(base)
    snapshot = {'health_detail': detail, 'context': context}

    # Ask the AI for both a free-text diagnosis AND a structured action
    # block. The structured part (confidence + recommended_target) feeds
    # the auto-remediation gate.
    #
    # The architecture preface is critical: without it the AI defaults to
    # "user X is using a lot of memory -> restart X's PHP-FPM process" or
    # "kill user X's apache workers", which is wrong on this platform.
    # Customer workloads run inside Docker containers (one per site,
    # named `<safe-domain>-NN`) -- the host's httpd/php-fpm exclusively
    # serve the control panel itself, NOT customer sites. So when a per-
    # user resource issue is identified, the right remediation target is
    # ALWAYS the offending site's container, never a host service.
    architecture = (
        "ARCHITECTURE PREFACE (read carefully -- this is a hosting platform with a specific layout):\n"
        "- The host runs a few system services (httpd, php-fpm, proftpd, mysql, postgresql, haproxy-manager) "
        "and a Docker engine.\n"
        "- httpd + php-fpm on the host serve ONLY the WHP control panel itself. They do NOT serve customer sites.\n"
        "- Every customer site runs as a dedicated Docker container, named `<safe-domain>-NN` "
        "(e.g. `example.com-01`, `wildcard.foo.com-01`). The container holds that one site's PHP/Node/static "
        "workload. Customer system users (the ones in /etc/passwd with home `/docker/users/<user>`) own the files "
        "and are the cgroup leader inside their containers.\n"
        "- A user-driven resource problem (high CPU, swap, memory by some username) is therefore a CONTAINER "
        "problem, not a host-service problem. The correct remediation is `docker restart <container>` for the "
        "site(s) owned by that user. NEVER recommend restarting host httpd or php-fpm to fix a customer issue -- "
        "that would only bounce the control panel and leave the offending container untouched.\n"
        "- haproxy-manager is the only customer-traffic-related host-side container; bouncing it should only be "
        "recommended for haproxy-specific issues, not generic resource pressure.\n"
        "- READ THE iostat BLOCK BEFORE RECOMMENDING ACTION: a high %iowait (>30%) means the system is "
        "disk-bound rather than CPU-bound. That is often a running backup (mysqldump, pg_dump, rsync) "
        "saturating the same disk that customer workloads share. In that case, restarting the customer "
        "container will NOT help; the right diagnosis is 'wait for backup to finish' or 'pause the backup' -- "
        "use confidence=low and target='' so the engine doesn't auto-restart anything. A high per-device "
        "%util on a single disk while iowait is also high points at that disk's saturation.\n"
    )
    prompt = (
        f"{architecture}\n"
        f"Subsystem '{subsystem}' transitioned to '{severity}' on a WHP hosting server.\n\n"
        f"Health snapshot:\n{json.dumps(detail, indent=2)}\n\n"
        f"Context:\n{context[:3500]}\n\n"
        f"Diagnose the most likely root cause and suggest one concrete remediation. "
        f"Be concise (under 300 words). When the issue points at a specific user's workload, look up that "
        f"user's container(s) in the PID->container mapping in the context block and name the container.\n\n"
        f"After your prose, append a single line in this exact JSON format:\n"
        f'  REMEDIATION_HINT={{"confidence": "high|medium|low", "target": "<container or service name or empty>"}}\n'
        f"Use confidence=high ONLY when you are certain the named target is the cause AND restarting "
        f"it will resolve the issue. The target MUST be a docker container name (e.g. `example.com-01`) when "
        f"the cause is a customer workload, NOT a host service name. Leave target empty when you're unsure."
    )
    diagnosis = ''
    confidence = ''
    ai_target = ''
    try:
        ai = call_ai(prompt, tier='basic', max_tokens=600)
        diagnosis = ai['content']
        # Pull the trailing REMEDIATION_HINT line out of the response.
        m = re.search(r'REMEDIATION_HINT=({[^\n]+})', diagnosis)
        if m:
            try:
                hint = json.loads(m.group(1))
                confidence = (hint.get('confidence') or '').lower()
                ai_target = (hint.get('target') or '').strip()
            except Exception:
                pass
    except Exception as e:
        diagnosis = f"AI call failed: {e}"

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=None,
        subscription_id=None,
        username=None,
        container_name=None,
        severity=severity,
        category=category,
        subsystem=subsystem,
        source='health_api',
        title=f"{subsystem} {severity}",
        diagnosis=diagnosis,
        recommendation='',
        raw_log_excerpt=None,
        context_data=snapshot,
    )

    # Phase 2: try auto-remediation. The shared guard refuses by default
    # (dry_run=true and enabled=false in shipped config) and only fires
    # when category, allowlist, cooldown, backup-protection, and confidence
    # gates all line up.
    try:
        cfg = _load_remediation_config()
        if cfg.get('enabled'):
            from whp_monitor_core import attempt_remediation
            outcome = attempt_remediation(
                sqlite_conn=sqlite_conn,
                mysql_conn=mysql_conn,
                category=category,
                signature=signature,
                ai_target=ai_target,
                ai_confidence=confidence,
                cooldown_minutes=cfg.get('cooldown_minutes', 60),
                require_high_confidence=cfg.get('require_high_confidence', True),
                dry_run=cfg.get('dry_run', True),
                backup_load_multiplier=cfg.get('backup_load_multiplier', 4.0),
                backup_swap_threshold=cfg.get('backup_swap_threshold', 90.0),
                allowed_containers=cfg.get('allowed_containers', []),
                allowed_services=cfg.get('allowed_services', []),
            )
            print(f"[remediation] {subsystem}: {outcome.get('reason')}", flush=True)
    except Exception as rem_e:
        print(f"[remediation] error for {subsystem}: {rem_e}", flush=True)

    return diagnosis


def _load_remediation_config() -> dict:
    """Read the [remediation] section out of /etc/whp/ai-monitor.conf for
    the poll-monitor's auto-remediation gate. Returns parsed values with
    safe defaults so a missing config never causes a crash."""
    import configparser
    cp = configparser.ConfigParser()
    cp.read('/etc/whp/ai-monitor.conf')
    sec = 'remediation'
    if not cp.has_section(sec):
        return {'enabled': False}
    def b(k, d=False):
        try: return cp.getboolean(sec, k)
        except Exception: return d
    def f(k, d=0.0):
        try: return float(cp.get(sec, k))
        except Exception: return d
    def i(k, d=0):
        try: return int(cp.get(sec, k))
        except Exception: return d
    def lst(k):
        raw = cp.get(sec, k, fallback='') or ''
        return [s.strip() for s in raw.split(',') if s.strip()]
    return {
        'enabled': b('enabled', False),
        'dry_run': b('dry_run', True),
        'require_high_confidence': b('require_high_confidence', True),
        'cooldown_minutes': i('cooldown_minutes', 60),
        'backup_load_multiplier': f('backup_load_multiplier', 4.0),
        'backup_swap_threshold': f('backup_swap_threshold', 90.0),
        'allowed_containers': lst('allowed_containers'),
        'allowed_services': lst('allowed_services'),
    }


def whp_iostat_snapshot() -> str:
    """Run iostat for one second and return a compact summary. Combines
    a CPU breakdown (user/system/iowait/idle) with per-device read/write
    utilization. Lets the AI tell apart "CPU-bound user workload" from
    "everyone blocked on a backup rsyncing the same disk".

    sysstat may not be installed yet on older boxes -- the install-sysstat
    update script ships it. Fall back to a noop note when missing.
    """
    if subprocess.run(["which", "iostat"], capture_output=True).returncode != 0:
        return "(iostat unavailable -- install sysstat to surface iowait + disk utilization)"
    try:
        # `iostat -xm 1 2` prints since-boot averages first, then a
        # 1s-interval sample. We want the second block.
        out = subprocess.run(
            "iostat -xm 1 2 2>/dev/null | awk 'BEGIN{n=0} /^avg-cpu:/ {n++} n==2'",
            shell=True, capture_output=True, timeout=8, text=True,
        )
        return (out.stdout or '').strip() or "(iostat returned empty)"
    except Exception as e:
        return f"(iostat collection failed: {e})"


def whp_pid_to_container_lines() -> str:
    """For each running container, list its top-level PID + name. The AI
    uses this to map a high-CPU/memory PID it sees in `top` or `ps` back
    to the docker container that should be restarted, rather than
    recommending a host service bounce. Output is a compact two-column
    table: PID  CONTAINER."""
    cmd = (
        "echo 'TOP_PID  CONTAINER  IMAGE'; "
        "docker ps --format '{{.ID}} {{.Names}} {{.Image}}' | "
        "while read cid name image; do "
        "  pid=$(docker inspect --format '{{.State.Pid}}' \"$cid\" 2>/dev/null); "
        "  printf '%-7s %-30s %s\\n' \"$pid\" \"$name\" \"$image\"; "
        "done"
    )
    out = subprocess.run(cmd, shell=True, capture_output=True, timeout=15, text=True)
    return (out.stdout or '').strip()


def whp_build_pid_container_map() -> dict:
    """Build a {host_pid: container_name} dict covering every process in
    every running container (not just PID 1). Used to annotate top/ps
    output so the AI sees the container next to the offending row instead
    of having to infer it from the username.

    Mechanism: read /proc/<pid>/cgroup for each container's PID-list
    sourced from `docker top`. The cgroup-file approach also works
    (`/proc/<pid>/cgroup` carries the docker container ID), but
    `docker top` gives us a complete enumeration in one command per
    container which is faster than open-reading thousands of /proc dirs.
    """
    mapping = {}
    try:
        ids_proc = subprocess.run(
            "docker ps --format '{{.ID}} {{.Names}}'",
            shell=True, capture_output=True, timeout=10, text=True,
        )
        for line in (ids_proc.stdout or '').splitlines():
            parts = line.strip().split(None, 1)
            if len(parts) != 2:
                continue
            cid, cname = parts
            try:
                top = subprocess.run(
                    f"docker top {cid} -eo pid 2>/dev/null",
                    shell=True, capture_output=True, timeout=8, text=True,
                )
                for raw in (top.stdout or '').splitlines()[1:]:  # skip header
                    p = raw.strip()
                    if p.isdigit():
                        mapping[int(p)] = cname
            except Exception:
                continue
    except Exception:
        pass
    return mapping


def whp_top_processes_by_container(pmap: dict, kind: str = 'cpu', limit: int = 15) -> str:
    """Return the top-N processes ranked by CPU or RSS, with a CONTAINER
    column appended. `kind`='cpu' sorts by --sort=-pcpu, anything else
    by --sort=-rss. The container value is "host" for processes outside
    any docker cgroup."""
    sort_key = '-pcpu' if kind == 'cpu' else '-rss'
    cmd = f"ps axo pid,user,pcpu,pmem,rss,comm --sort={sort_key} 2>/dev/null | head -{limit + 1}"
    try:
        out = subprocess.run(cmd, shell=True, capture_output=True, timeout=8, text=True)
    except Exception as e:
        return f"(top processes collection failed: {e})"
    lines = (out.stdout or '').splitlines()
    if not lines:
        return ''
    out_lines = [lines[0].rstrip() + '  CONTAINER']
    for line in lines[1:]:
        cols = line.split(None, 5)  # pid user %cpu %mem rss comm-rest
        if not cols or not cols[0].isdigit():
            out_lines.append(line)
            continue
        pid = int(cols[0])
        ctr = pmap.get(pid, 'host')
        out_lines.append(line.rstrip() + f"  {ctr}")
    return '\n'.join(out_lines)


def gather_health_context(subsystem: str) -> str:
    """Return up to 3KB of subsystem-specific diagnostic context."""
    commands = {
        # CPU + swap: list top processes AND the PID->container mapping
        # so the AI can resolve a noisy username/PID to a docker container
        # name (the correct remediation target on this platform). Without
        # the mapping the AI tends to recommend host service restarts.
        'cpu':         "top -bn1 | head -20",
        'swap':        "ps axo pid,user,rss,comm --sort=-rss | head -10",
        # Disk: per-partition usage, top space hogs, and cleanup candidates.
        # The probe lists rotated logs (>30d), large Docker container json
        # logs, and the journal size so the AI can see exactly which paths
        # are eating space and recommend specific cleanups (the user reviews
        # and clicks-to-execute -- we don't auto-delete).
        'disk':        ("df -h; echo ---; "
                        "echo 'TOP SPACE HOGS:'; "
                        "du -hx --max-depth=2 /docker /var 2>/dev/null | sort -rh | head -20; "
                        "echo ---; "
                        "echo 'ROTATED LOGS >30d (>10MB):'; "
                        "find /var/log /docker/users -type f \\( -name '*.gz' -o -name '*.[0-9]' -o -name '*.[0-9].gz' \\) "
                        "    -mtime +30 -size +10M -printf '%s %p\\n' 2>/dev/null | sort -rn | head -20; "
                        "echo ---; "
                        "echo 'DOCKER CONTAINER JSON LOGS (>100MB):'; "
                        "find /var/lib/docker/containers -name '*-json.log' -size +100M -printf '%s %p\\n' 2>/dev/null | sort -rn | head -10; "
                        "echo ---; "
                        "echo 'JOURNAL SIZE:'; "
                        "journalctl --disk-usage 2>/dev/null"),
        'connections': "ss -s",
        'docker':      "docker ps --format 'table {{.Names}}\\t{{.Status}}' | head -20",
        'haproxy':     "docker logs haproxy-manager --tail 50 2>&1",
    }
    cmd = commands.get(subsystem)
    if not cmd:
        return ''
    try:
        out = subprocess.run(cmd, shell=True, capture_output=True, timeout=10, text=True)
        body = (out.stdout + out.stderr)[:3000]
    except Exception as e:
        return f"(context collection failed: {e})"

    # Append a top-processes-by-container table for the subsystems where
    # it matters. Each row already shows which container the PID belongs
    # to, so the AI doesn't have to infer "high RSS by user X must mean
    # container Y" -- it sees the answer in the row.
    if subsystem in ('cpu', 'swap', 'connections'):
        try:
            pmap = whp_build_pid_container_map()
            kind = 'cpu' if subsystem in ('cpu', 'connections') else 'rss'
            top = whp_top_processes_by_container(pmap, kind=kind, limit=15)
            if top:
                body += "\n\n--- top processes (CONTAINER column = `host` means a host-side "\
                        "system process; otherwise the docker container that owns the PID) ---\n" + top
        except Exception:
            pass
    # iostat for cpu/swap/disk -- the iowait column is the key signal for
    # "is this disk-bound rather than CPU-bound" and the per-device utilization
    # surfaces a runaway backup or a single device saturating.
    if subsystem in ('cpu', 'swap', 'disk'):
        try:
            ios = whp_iostat_snapshot()
            if ios:
                body += "\n\n--- iostat -xm 1 (live sample; high %iowait = disk-bound, " \
                        "not CPU-bound; high %util on a device = saturation) ---\n" + ios
        except Exception:
            pass
    return body[:4000]


# ---------- HAProxy stats polling ----------

def poll_haproxy_stats(sqlite_conn, mysql_conn):
    try:
        out = subprocess.run(HAPROXY_STATS_CMD, capture_output=True, timeout=10,
                             text=True, check=True)
        text = out.stdout
        if not text:
            print("[haproxy-stats] empty response from docker exec", flush=True)
            return
    except Exception as e:
        print(f"[haproxy-stats] poll failed: {e}", flush=True)
        return

    reader = csv.reader(io.StringIO(text.lstrip('# ')))
    header = next(reader, None)
    if not header:
        return
    try:
        pxname_i = header.index('pxname')
        svname_i = header.index('svname')
        status_i = header.index('status')
    except ValueError:
        print("[haproxy-stats] unexpected CSV header", flush=True)
        return

    for row in reader:
        if len(row) <= max(pxname_i, svname_i, status_i):
            continue
        pxname = row[pxname_i]
        svname = row[svname_i]
        status = row[status_i]
        # svname 'BACKEND' and 'FRONTEND' are aggregates, skip.
        if svname in ('BACKEND', 'FRONTEND'):
            continue
        if not status:
            continue

        backend_key = f"{pxname}/{svname}"
        last = sqlite_conn.execute(
            "SELECT status FROM backend_state WHERE backend_name=?",
            (backend_key,)).fetchone()
        last_status = last['status'] if last else 'UP'

        if status == last_status:
            continue

        sqlite_conn.execute(
            "INSERT OR REPLACE INTO backend_state (backend_name, status, transitioned_at) "
            "VALUES (?, ?, ?)",
            (backend_key, status, int(time.time())))
        sqlite_conn.commit()

        # Normalize HAProxy statuses.
        # HAProxy returns compound statuses like 'MAINT (resolution)' or
        # 'DOWN 1/3' — match by prefix rather than exact string.
        def _is_down(s):
            return s.startswith(('DOWN', 'NOLB', 'MAINT'))
        is_down_now = _is_down(status)
        was_down = _is_down(last_status)

        if is_down_now and not was_down:
            handle_backend_down(sqlite_conn, mysql_conn, pxname, svname, status)
        elif not is_down_now and was_down:
            handle_backend_up(mysql_conn, pxname, svname)


def resolve_backend_to_site(mysql_conn, svname: str):
    """
    Backend container name follows 'domain.tld-NN' convention.
    Returns (site_id, primary_domain, subscription_id, username)
    or (None, svname, None, None).

    Subdomain sites (e.g. transcribe.shadowdao.com) share the apex row
    in whp.domains with their parent site; the specific FQDN lives in
    whp.site_domains. Try site_domains first, fall back to the apex
    linkage for legacy sites that predate site_domains.
    """
    m = re.match(r'^(.+)-\d{1,2}$', svname)
    if not m:
        return (None, svname, None, None)
    domain = m.group(1)
    # Container names use wildcard.x.com; DB stores *.x.com. Reverse the
    # translation so the lookup hits the subscribed domain.
    if domain.startswith('wildcard.'):
        domain = '*.' + domain[len('wildcard.'):]
    with mysql_conn.cursor() as cur:
        # Preferred: exact FQDN match in site_domains.
        cur.execute("""
            SELECT s.id AS site_id, s.username,
                   COALESCE(sdp.fqdn, d.domain_name) AS primary_domain,
                   sub.id AS subscription_id
            FROM whp.site_domains sd
            JOIN whp.sites s ON s.id = sd.site_id
            JOIN whp.domains d ON d.id = s.primary_domain_id
            LEFT JOIN whp.site_domains sdp
                   ON sdp.site_id = s.id AND sdp.is_primary = 1
            LEFT JOIN whp.ai_monitor_subscriptions sub
                   ON sub.site_id = s.id AND sub.status = 'active'
            WHERE sd.fqdn = %s
            LIMIT 1
        """, (domain,))
        row = cur.fetchone()
        if not row:
            # Fallback: apex registered in whp.domains (legacy sites).
            cur.execute("""
                SELECT s.id AS site_id, s.username,
                       d.domain_name AS primary_domain,
                       sub.id AS subscription_id
                FROM whp.domains d
                JOIN whp.sites s ON s.primary_domain_id = d.id
                LEFT JOIN whp.ai_monitor_subscriptions sub
                       ON sub.site_id = s.id AND sub.status = 'active'
                WHERE d.domain_name = %s
                LIMIT 1
            """, (domain,))
            row = cur.fetchone()
    if not row:
        return (None, domain, None, None)
    return (row['site_id'], row['primary_domain'],
            row['subscription_id'], row['username'])


def handle_backend_down(sqlite_conn, mysql_conn, pxname, svname, raw_status):
    site_id, domain, subscription_id, username = resolve_backend_to_site(mysql_conn, svname)
    bucket = ten_minute_bucket(int(time.time()))
    signature = compute_signature(
        kind='site',
        site_id=site_id if site_id else 0,
        category='site_down',
        bucket=bucket)

    # Pull container logs.
    container = svname
    try:
        out = subprocess.run(
            f"docker logs {shlex.quote(container)} --since 5m 2>&1 | tail -100",
            shell=True, capture_output=True, timeout=15, text=True)
        logs = out.stdout[:5000]
    except Exception as e:
        logs = f"(log fetch failed: {e})"

    prompt = (
        f"HAProxy backend '{pxname}/{svname}' transitioned to {raw_status} on a WHP hosting server.\n\n"
        f"Last 100 log lines from container '{container}':\n{logs}\n\n"
        f"Diagnose the most likely cause of the backend going down and suggest a remediation. "
        f"Be concise (under 300 words)."
    )
    try:
        ai = call_ai(prompt, tier='basic', max_tokens=600)
        diagnosis = ai['content']
    except Exception as e:
        diagnosis = f"AI call failed: {e}"

    # Attempt auto-recovery (suspension-aware, rate-capped). Record the outcome
    # alongside the diagnosis so the action is visible in the Issues tracker.
    recovery = attempt_site_recovery(
        sqlite_conn, mysql_conn, svname, site_id, domain,
        reason=f"backend {raw_status}")
    diagnosis = f"{diagnosis}\n\n[auto-recovery] {recovery}"

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=site_id,
        subscription_id=subscription_id,
        username=username,
        container_name=svname,
        severity='critical',
        category='site_down',
        subsystem=None,
        source='haproxy_stats',
        title=f"Backend DOWN: {domain} ({svname})",
        diagnosis=diagnosis,
        recommendation='',
        raw_log_excerpt=logs,
        context_data={'pxname': pxname, 'svname': svname,
                      'haproxy_status': raw_status, 'auto_recovery': recovery},
    )


def handle_backend_up(mysql_conn, pxname, svname):
    # On UP transition, close any open site_down Issue for this site/container.
    # Lookup by (site_id, container_name) — not by signature, because the original
    # Down event's 10-minute bucket may have rolled over by the time recovery
    # is observed.
    site_id, _domain, _sub, _username = resolve_backend_to_site(mysql_conn, svname)
    with mysql_conn.cursor() as cur:
        if site_id:
            cur.execute(
                "UPDATE whp.ai_monitor_issues "
                "SET status = 'fixed', "
                "    auto_resolved_at = COALESCE(auto_resolved_at, NOW()), "
                "    status_changed_by = 'system', "
                "    status_changed_at = NOW() "
                "WHERE site_id=%s AND category='site_down' AND status='open' "
                "  AND (container_name=%s OR container_name IS NULL)",
                (site_id, svname))
        else:
            cur.execute(
                "UPDATE whp.ai_monitor_issues "
                "SET status = 'fixed', "
                "    auto_resolved_at = COALESCE(auto_resolved_at, NOW()), "
                "    status_changed_by = 'system', "
                "    status_changed_at = NOW() "
                "WHERE category='site_down' AND status='open' "
                "  AND container_name=%s",
                (svname,))
    mysql_conn.commit()


# ---------- Site auto-recovery (Layer 3) ----------
#
# Safety net beneath the cac-litespeed daemon supervisor and the unless-stopped
# restart policy: detect a customer site that is DOWN (HAProxy backend DOWN) or
# UP-but-not-serving-PHP (active probe returns 5xx) and restart it. Bounded by a
# per-container rate cap and gated on the site being active (not suspended), so
# we never fight an operator's suspend/teardown or hammer a flapping site.

SITE_AUTORESTART_ENABLED = True   # master switch for restart actions
SITE_PHP_PROBE_ENABLED   = True   # active in-container PHP probe pass
MAX_RESTARTS_PER_WINDOW  = 3      # per container...
RESTART_WINDOW_SECONDS   = 3600   # ...within this rolling window
PHP_PROBE_TIMEOUT        = 8      # seconds per in-container curl (covers slow cold renders)
PHP_PROBE_FAIL_THRESHOLD = 2      # consecutive 5xx/timeout before a restart


def _docker_state(container: str):
    """Return (exists: bool, running: bool) for a container."""
    try:
        out = subprocess.run(
            ['docker', 'inspect', '-f', '{{.State.Running}}', container],
            capture_output=True, timeout=10, text=True)
        if out.returncode != 0:
            return (False, False)
        return (True, out.stdout.strip() == 'true')
    except Exception:
        return (False, False)


def _docker_health(container: str) -> str:
    """Container's own HEALTHCHECK status: 'healthy'|'unhealthy'|'starting', or
    'none' when the image defines no healthcheck. Used as the PHP-health signal
    for cac-fpm (its healthcheck pings php-fpm on :9000 via cgi-fcgi)."""
    try:
        out = subprocess.run(
            ['docker', 'inspect', '-f',
             '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}',
             container],
            capture_output=True, timeout=10, text=True)
        if out.returncode != 0:
            return 'none'
        return out.stdout.strip() or 'none'
    except Exception:
        return 'none'


def is_site_suspended(mysql_conn, site_id) -> bool:
    """True if the site exists and is marked inactive (suspended/disabled).
    Unknown/unresolved site_id → not suspended (caller decides whether to act).
    On any DB read error, fail safe as suspended so we don't fight an operator
    action we couldn't read."""
    if not site_id:
        return False
    try:
        with mysql_conn.cursor() as cur:
            cur.execute("SELECT active FROM whp.sites WHERE id=%s", (site_id,))
            row = cur.fetchone()
        if not row:
            return False
        return int(row['active']) == 0
    except Exception:
        return True


def _recovery_allowed(sqlite_conn, container: str) -> bool:
    now = int(time.time())
    row = sqlite_conn.execute(
        "SELECT attempts, window_started_at FROM site_recovery WHERE container_name=?",
        (container,)).fetchone()
    if not row:
        return True
    if now - row['window_started_at'] >= RESTART_WINDOW_SECONDS:
        return True  # window expired; _record_recovery resets it
    return row['attempts'] < MAX_RESTARTS_PER_WINDOW


def _record_recovery(sqlite_conn, container: str, result: str):
    now = int(time.time())
    row = sqlite_conn.execute(
        "SELECT window_started_at FROM site_recovery WHERE container_name=?",
        (container,)).fetchone()
    if not row or (now - row['window_started_at'] >= RESTART_WINDOW_SECONDS):
        sqlite_conn.execute(
            "INSERT OR REPLACE INTO site_recovery "
            "(container_name, attempts, window_started_at, last_attempt_at, last_result) "
            "VALUES (?, 1, ?, ?, ?)",
            (container, now, now, result))
    else:
        sqlite_conn.execute(
            "UPDATE site_recovery SET attempts=attempts+1, last_attempt_at=?, last_result=? "
            "WHERE container_name=?",
            (now, result, container))
    sqlite_conn.commit()


def attempt_site_recovery(sqlite_conn, mysql_conn, svname: str,
                          site_id, domain, reason: str) -> str:
    """Restart a customer site container, gated on: feature enabled, name
    resolves to a customer site, site not suspended, container exists, and the
    per-container rate cap. Returns a human-readable outcome for the Issue
    trail (and never raises)."""
    if not SITE_AUTORESTART_ENABLED:
        return "auto-restart disabled by config"
    # Only act on containers we can tie to a customer site. An unresolved name
    # might be infra or a torn-down site — leave it alone.
    if not site_id:
        return f"skipped: '{svname}' did not resolve to a customer site"
    if is_site_suspended(mysql_conn, site_id):
        return f"skipped: site {domain} is suspended (active=0)"
    exists, running = _docker_state(svname)
    if not exists:
        return f"skipped: container {svname} does not exist (removed?)"
    if not _recovery_allowed(sqlite_conn, svname):
        return (f"skipped: rate cap hit "
                f"({MAX_RESTARTS_PER_WINDOW}/{RESTART_WINDOW_SECONDS}s) for {svname}")

    action = 'restart' if running else 'start'
    try:
        out = subprocess.run(['docker', action, svname],
                             capture_output=True, timeout=60, text=True)
        result = 'ok' if out.returncode == 0 \
            else f"fail rc={out.returncode}: {out.stderr.strip()[:200]}"
    except Exception as e:
        result = f"error: {e}"
    _record_recovery(sqlite_conn, svname, result)
    msg = f"auto-{action} {svname} ({reason}): {result}"
    print(f"[site-recovery] {msg}", flush=True)
    return msg


def _list_customer_containers():
    """Running customer containers as (name, image) tuples, identified by the
    whp.tier=customer label that docker-api.php stamps at create time. The image
    is needed to pick the right probe scheme; the caller further filters to
    containers that resolve to a real site (skips valkey-cust-* and other
    non-web customer containers that also carry the label)."""
    try:
        out = subprocess.run(
            ['docker', 'ps', '--filter', 'label=whp.tier=customer',
             '--filter', 'status=running', '--format', '{{.Names}}\t{{.Image}}'],
            capture_output=True, timeout=10, text=True)
        if out.returncode != 0:
            return []
        result = []
        for line in out.stdout.splitlines():
            parts = line.split('\t')
            if len(parts) >= 2 and parts[0]:
                result.append((parts[0], parts[1]))
        return result
    except Exception:
        return []


def poll_site_php_health(sqlite_conn, mysql_conn):
    """Active per-site PHP/app health probe with per-family signal:
      - cac-litespeed: in-container HTTPS GET '/' with the site Host header.
        Only a 5xx is treated as wedged (a genuine pool wedge makes OLS 503); a
        timeout means slow-not-dead and is NOT a restart trigger.
      - cac-fpm: the container's own docker healthcheck (pings php-fpm :9000).
    Restart only after PHP_PROBE_FAIL_THRESHOLD consecutive failures —
    suspension-aware and rate-capped via attempt_site_recovery — so a transient
    blip never triggers a restart."""
    if not SITE_PHP_PROBE_ENABLED:
        return
    now = int(time.time())
    for container, image in _list_customer_containers():
        site_id, domain, _sub, _user = resolve_backend_to_site(mysql_conn, container)
        # Only probe real customer SITE containers. Names that don't resolve
        # (valkey-cust-*, other infra) aren't web servers — skip them.
        if not site_id:
            continue
        host = domain
        # The two image families have different architectures, so the PHP-health
        # signal differs:
        #
        #   cac-litespeed — a full web server in the container (TLS on :443). Its
        #     OWN docker healthcheck only hits the static /healthz, so it can't
        #     see a wedged lsphp. We actively probe https://127.0.0.1/ (with the
        #     site Host header) to exercise lsphp; 5xx/timeout == wedged.
        #
        #   cac-fpm — php-fpm ONLY, listening on :9000 (no HTTP server inside;
        #     the shared-httpd containers proxy to it). An HTTP probe is
        #     impossible, BUT its docker healthcheck already pings php-fpm via
        #     cgi-fcgi, so the container's own health status IS the PHP signal.
        #     This also covers the HAProxy blind spot: a cac-fpm site's HAProxy
        #     backend is the shared-httpd container, so handle_backend_down never
        #     sees a single site's php-fpm die — docker health does.
        if 'litespeed' in image:
            try:
                out = subprocess.run(
                    ['docker', 'exec', container, 'curl', '-sS', '-o', '/dev/null',
                     '-w', '%{http_code}', '-m', str(PHP_PROBE_TIMEOUT), '-k',
                     '-H', f'Host: {host}', 'https://127.0.0.1/'],
                    capture_output=True, timeout=PHP_PROBE_TIMEOUT + 5, text=True)
                code = out.stdout.strip() or '000'
            except Exception:
                code = '000'  # exec/timeout failure
            # ONLY a 5xx counts as wedged. A timeout (000) means the page is slow,
            # not that lsphp is dead — a genuine pool wedge makes OLS return 503,
            # whereas a heavy/cold uncached render can legitimately take >8s. We
            # must NOT restart a slow-but-working site (it doesn't help and hurts).
            # A truly hung container is caught by the HAProxy backend-DOWN path.
            wedged = code.startswith('5')
        else:
            # cac-fpm (and any non-litespeed web image): trust the container's
            # own healthcheck. 'starting'/'none'/'healthy' are not wedged.
            code = _docker_health(container)
            wedged = code == 'unhealthy'

        prev = sqlite_conn.execute(
            "SELECT consecutive_failures FROM site_php_probe WHERE container_name=?",
            (container,)).fetchone()
        streak = (prev['consecutive_failures'] if prev else 0) + 1 if wedged else 0

        sqlite_conn.execute(
            "INSERT OR REPLACE INTO site_php_probe "
            "(container_name, consecutive_failures, last_code, last_checked_at) "
            "VALUES (?, ?, ?, ?)",
            (container, streak, code, now))
        sqlite_conn.commit()

        if wedged and streak >= PHP_PROBE_FAIL_THRESHOLD:
            outcome = attempt_site_recovery(
                sqlite_conn, mysql_conn, container, site_id, domain,
                reason=f"php_wedged code={code} x{streak}")
            bucket = ten_minute_bucket(now)
            signature = compute_signature(
                kind='site', site_id=site_id if site_id else 0,
                category='site_php_wedged', bucket=bucket)
            try:
                insert_or_increment_issue(
                    mysql_conn, signature=signature, site_id=site_id,
                    subscription_id=None, username=None, container_name=container,
                    severity='critical', category='site_php_wedged', subsystem=None,
                    source='php_probe',
                    title=f"PHP not responding: {host} ({container})",
                    diagnosis=(f"In-container probe returned {code} {streak}x "
                               f"consecutively. Recovery: {outcome}"),
                    recommendation='', raw_log_excerpt='',
                    context_data={'container': container, 'http_code': code,
                                  'streak': streak, 'recovery': outcome})
            except Exception as e:
                print(f"[php-probe] issue insert failed: {e}", flush=True)
            # Reset the streak after acting so we don't re-fire every cycle while
            # the restart settles; the rate cap is the real backstop.
            sqlite_conn.execute(
                "UPDATE site_php_probe SET consecutive_failures=0 WHERE container_name=?",
                (container,))
            sqlite_conn.commit()


# ---------- Crash-event drain ----------

# Cause classifications produced by haproxy-crash-monitor.sh + watchdog hook.
# Anything outside this list still gets surfaced as 'warning' by default.
_CRASH_CAUSE_SEVERITY = {
    'OOM_DOCKER':        'critical',
    'OOM_KERNEL':        'critical',
    'SEGFAULT':          'critical',
    'SIGKILL_EXTERNAL':  'critical',
    'PROCESS_ERROR':     'critical',
    'ABORT':             'critical',
    'WATCHDOG_FAILED':   'critical',  # both restart and recreate failed; needs a human
    'WATCHDOG_RECREATE': 'warning',   # docker restart didn't work; container was recreated
    'WATCHDOG_RESTART':  'warning',   # health checks failed; auto-recovery succeeded
    'WATCHDOG_START':    'warning',   # container wasn't running; we started it
    'CLEAN_SHUTDOWN':    'info',      # docker stop / restart by an operator
}


def drain_crash_events(sqlite_conn, mysql_conn):
    """Pull any new rows from whp.container_crash_events and surface them
    as Issues so auto-recovery actions become visible in the unified
    Issues tracker. High-water mark stored in SQLite poll_cursor table.
    """
    row = sqlite_conn.execute(
        "SELECT value FROM poll_cursor WHERE key='last_crash_event_id'"
    ).fetchone()
    last_seen = row['value'] if row else 0

    with mysql_conn.cursor() as cur:
        cur.execute("""
            SELECT id, container_name, event_type, exit_code, oom_killed,
                   cause, restart_count, downtime_seconds,
                   memory_usage, host_memory_free, dmesg_excerpt,
                   container_logs, created_at
            FROM whp.container_crash_events
            WHERE id > %s
            ORDER BY id ASC
            LIMIT 50
        """, (last_seen,))
        events = cur.fetchall()

    if not events:
        return 0

    for ev in events:
        try:
            _emit_issue_for_crash(mysql_conn, ev)
        except Exception as e:
            print(f"[drain-crash-events] event id={ev['id']} failed: {e}",
                  flush=True)
        last_seen = max(last_seen, ev['id'])

    sqlite_conn.execute(
        "INSERT OR REPLACE INTO poll_cursor (key, value) VALUES ('last_crash_event_id', ?)",
        (last_seen,))
    sqlite_conn.commit()
    print(f"[drain-crash-events] processed {len(events)} new event(s); "
          f"last_seen_id={last_seen}", flush=True)
    return len(events)


def _emit_issue_for_crash(mysql_conn, ev):
    container = ev['container_name'] or 'unknown'
    cause     = (ev['cause'] or 'UNKNOWN').upper()
    severity  = _CRASH_CAUSE_SEVERITY.get(cause, 'warning')

    # haproxy-manager is the panel-shared HAProxy — server-wide. Site
    # containers (domain.tld-NN) resolve to a site_id via existing helper.
    site_id = subscription_id = username = None
    domain  = container
    if container != 'haproxy-manager':
        try:
            from whp_monitor_core import connect_mysql as _ignore  # only to force-import
        except Exception:
            pass
        site_id, domain, subscription_id, username = resolve_backend_to_site(
            mysql_conn, container)

    title_cause = cause.replace('_', ' ').title()
    if cause == 'WATCHDOG_RESTART':
        title = f"Auto-restart performed: {container} (watchdog)"
    elif cause == 'WATCHDOG_RECREATE':
        title = f"Container recreated: {container} (watchdog fallback)"
    elif cause == 'WATCHDOG_START':
        title = f"Container started: {container} (was not running)"
    elif cause == 'WATCHDOG_FAILED':
        title = f"Auto-recovery FAILED: {container} (manual intervention needed)"
    elif cause == 'CLEAN_SHUTDOWN':
        title = f"Container restarted cleanly: {container}"
    else:
        title = f"Container crash detected: {container} ({title_cause})"

    # Evidence: a compact one-liner the customer / root can read.
    parts = [f"cause={cause}", f"event={ev['event_type'] or '?'}"]
    if ev.get('exit_code') is not None: parts.append(f"exit={ev['exit_code']}")
    if ev.get('oom_killed'):            parts.append("oom_killed=true")
    if ev.get('restart_count') is not None: parts.append(f"docker_restart_count={ev['restart_count']}")
    if ev.get('downtime_seconds') is not None: parts.append(f"downtime={ev['downtime_seconds']}s")
    if ev.get('memory_usage'):     parts.append(f"mem={ev['memory_usage']}")
    if ev.get('host_memory_free'): parts.append(f"host_free={ev['host_memory_free']}")
    evidence = " | ".join(parts)

    diagnosis_lines = [f"{title} at {ev['created_at']}.", evidence]
    if ev.get('dmesg_excerpt'):
        diagnosis_lines.append("--- dmesg excerpt ---\n" + str(ev['dmesg_excerpt'])[:2000])
    if ev.get('container_logs'):
        diagnosis_lines.append("--- last container logs before exit ---\n"
                               + str(ev['container_logs'])[:3000])
    diagnosis = "\n\n".join(diagnosis_lines)

    if cause == 'WATCHDOG_RESTART':
        recommendation = ("The watchdog auto-restarted the container after health checks "
                          "failed. Review the container logs for the underlying cause; "
                          "if it recurs within an hour, escalate to a human.")
    elif cause == 'WATCHDOG_RECREATE':
        recommendation = ("docker restart failed, so the watchdog recreated the "
                          "container from scratch. Investigate why a plain restart "
                          "wasn't enough — likely image / config issue.")
    elif cause == 'WATCHDOG_START':
        recommendation = ("The watchdog found the container stopped and started it. "
                          "Check why it stopped (haproxy-crash-monitor.sh or dmesg "
                          "should have a separate event with the cause).")
    elif cause == 'WATCHDOG_FAILED':
        recommendation = ("Both 'docker restart' and recreate failed. The platform's "
                          "auto-recovery is exhausted — manual intervention is required. "
                          "Inspect the watchdog log at /var/log/haproxy-watchdog.log.")
    elif severity == 'critical':
        recommendation = ("Investigate the underlying cause (OOM, segfault, signal). "
                          "If memory-related, raise the container memory limit or fix "
                          "the leak; if SIGKILL_EXTERNAL, check who's killing it.")
    else:
        recommendation = ("Recorded for visibility. No action needed unless the "
                          "container is now unstable.")

    # Dedup per (container, cause) so repeated auto-restarts / crashes of the
    # same container collapse into one counted issue (was keyed on site_id,
    # which is 0 for server-wide containers and time-bucketed, spawning a new
    # row each event).
    signature = compute_signature(kind='crash', container=container, cause=cause.lower())

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=site_id,
        subscription_id=subscription_id,
        username=username,
        container_name=container,
        severity=severity,
        category='container_crash',
        subsystem=cause.lower(),
        source='crash_monitor',
        title=title,
        diagnosis=diagnosis,
        recommendation=recommendation,
        raw_log_excerpt=ev.get('container_logs'),
        context_data={
            'crash_event_id':   ev['id'],
            'cause':            cause,
            'event_type':       ev['event_type'],
            'exit_code':        ev.get('exit_code'),
            'oom_killed':       bool(ev.get('oom_killed')),
            'restart_count':    ev.get('restart_count'),
            'downtime_seconds': ev.get('downtime_seconds'),
            'occurred_at':      str(ev['created_at']),
        },
    )


# ---------- Main ----------

def main():
    sqlite_conn = connect_sqlite()
    ensure_sqlite_schema(sqlite_conn)
    try:
        mysql_conn = connect_mysql()
    except Exception as e:
        print(f"[fatal] MySQL connect failed: {e}", flush=True)
        write_heartbeat(sqlite_conn, 'health_api', 'fail', f"mysql connect: {e}")
        write_heartbeat(sqlite_conn, 'haproxy_stats', 'fail', f"mysql connect: {e}")
        sqlite_conn.close()
        return 1

    health_err = haproxy_err = crash_err = php_err = None
    try:
        try:
            poll_health(sqlite_conn, mysql_conn)
        except Exception as e:
            health_err = str(e)
            print(f"[health] poll cycle failed: {e}", flush=True)
        try:
            drain_crash_events(sqlite_conn, mysql_conn)
        except Exception as e:
            crash_err = str(e)
            print(f"[drain-crash-events] cycle failed: {e}", flush=True)
        try:
            poll_haproxy_stats(sqlite_conn, mysql_conn)
        except Exception as e:
            haproxy_err = str(e)
            print(f"[haproxy-stats] poll cycle failed: {e}", flush=True)
        try:
            poll_site_php_health(sqlite_conn, mysql_conn)
        except Exception as e:
            php_err = str(e)
            print(f"[php-probe] poll cycle failed: {e}", flush=True)

        write_heartbeat(sqlite_conn, 'health_api',
                        'ok' if not health_err else 'fail', health_err)
        write_heartbeat(sqlite_conn, 'haproxy_stats',
                        'ok' if not haproxy_err else 'fail', haproxy_err)
        write_heartbeat(sqlite_conn, 'crash_events',
                        'ok' if not crash_err else 'fail', crash_err)
        write_heartbeat(sqlite_conn, 'php_probe',
                        'ok' if not php_err else 'fail', php_err)
    finally:
        mysql_conn.close()
        sqlite_conn.close()
    return 0


if __name__ == '__main__':
    sys.exit(main())
