#!/usr/bin/env python3
"""
Minute-cadence poll for AI-monitor health and HAProxy stats.

Runs non-interactively from cron. Never long-running — one poll cycle per
invocation to stay in line with WHP's cron-driven pattern (see also
haproxy-watchdog.sh).

Sources:
  - https://localhost:8443/api/health.php?check=all&detailed=1
  - http://localhost:8404/stats;csv

On a healthy→warning/critical transition for any subsystem or backend, fetches
relevant context, calls AI for diagnosis, and inserts (or increments) an Issue
via whp_monitor_core.
"""
import csv
import io
import json
import os
import re
import shlex
import subprocess
import sys
import time
import sqlite3
import urllib3

# Suppress localhost self-signed cert warning for the health.php poll.
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from whp_monitor_core import (
    connect_mysql, connect_sqlite, compute_signature, ten_minute_bucket,
    insert_or_increment_issue, mark_auto_resolved, call_ai,
)

import requests

HEALTH_URL = 'https://localhost:8443/api/health.php?check=all&detailed=1'
# HAProxy stats bind to 127.0.0.1:8404 INSIDE the haproxy-manager container
# and are not exposed to the host. Fetch via `docker exec` instead.
HAPROXY_STATS_CMD = ['docker', 'exec', 'haproxy-manager',
                     'curl', '-s', 'http://127.0.0.1:8404/stats;csv']


def ensure_sqlite_schema(conn):
    """Create health_state, backend_state, poll_heartbeat, poll_cursor,
    health_transitions tables. Also runs an idempotent ALTER for the
    severity_level column added later for tiered disk alerting."""
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS health_state (
            subsystem TEXT PRIMARY KEY,
            status TEXT NOT NULL,
            detail_json TEXT,
            transitioned_at INTEGER NOT NULL
        );
        CREATE TABLE IF NOT EXISTS backend_state (
            backend_name TEXT PRIMARY KEY,
            status TEXT NOT NULL,
            transitioned_at INTEGER NOT NULL
        );
        CREATE TABLE IF NOT EXISTS poll_heartbeat (
            source TEXT PRIMARY KEY,
            last_run_at INTEGER NOT NULL,
            last_status TEXT NOT NULL,
            last_error  TEXT
        );
        CREATE TABLE IF NOT EXISTS poll_cursor (
            key   TEXT PRIMARY KEY,
            value INTEGER NOT NULL
        );
        -- Row-per-event log of subsystem state changes. Powers the AI Monitor
        -- dashboard timeline so we can correlate with external alerts (e.g.
        -- Uptime Kuma) and see what the panel did about each event.
        CREATE TABLE IF NOT EXISTS health_transitions (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            subsystem TEXT NOT NULL,           -- e.g. 'cpu', 'disk:root', 'haproxy'
            prev_status TEXT NOT NULL,
            new_status TEXT NOT NULL,
            prev_severity INTEGER DEFAULT 0,   -- disk tier 0-6; 0 for non-tiered subsystems
            new_severity INTEGER DEFAULT 0,
            detail_json TEXT,                  -- raw subsystem snapshot at transition
            ai_diagnosis TEXT,                 -- AI-extracted diagnosis (warning/critical only)
            mysql_signature VARCHAR(64),       -- joins to whp.ai_monitor_issues.signature
            transitioned_at INTEGER NOT NULL
        );
        CREATE INDEX IF NOT EXISTS idx_ht_subsystem_time ON health_transitions(subsystem, transitioned_at);
        CREATE INDEX IF NOT EXISTS idx_ht_time ON health_transitions(transitioned_at);
    """)
    # Idempotent column add (SQLite has no IF NOT EXISTS for columns; check
    # PRAGMA first).
    cols = {row[1] for row in conn.execute("PRAGMA table_info(health_state)").fetchall()}
    if 'severity_level' not in cols:
        try:
            conn.execute("ALTER TABLE health_state ADD COLUMN severity_level INTEGER DEFAULT 0")
        except sqlite3.OperationalError:
            pass  # raced with another invocation
    conn.commit()


def write_heartbeat(conn, source: str, status: str, error: str = None):
    """Stamp a heartbeat row so the AI Monitor admin page can show
    'last health poll: Ns ago' without having to invent it from log
    transitions. Called at the end of every poll cycle, success or not."""
    conn.execute(
        "INSERT OR REPLACE INTO poll_heartbeat (source, last_run_at, last_status, last_error) "
        "VALUES (?, ?, ?, ?)",
        (source, int(time.time()), status, error))
    conn.commit()


# ---------- Health API polling ----------

HEALTH_SUBSYSTEMS = ['cpu', 'connections', 'disk', 'swap', 'docker', 'haproxy']

CATEGORY_BY_SUBSYSTEM = {
    'cpu': 'cpu_pressure',
    'swap': 'swap_pressure',
    'disk': 'disk_full',
    'connections': 'connection_storm',
    'docker': 'docker_degraded',
    'haproxy': 'haproxy_unhealthy',
}


def disk_tier(usage_pct: float) -> int:
    """Map a partition's usage_percent to a 0-6 severity tier.

    0  = healthy (<80)        -- no alert
    1  = warning  (>=80)
    2  = warning  (>=85)
    3  = warning  (>=90)
    4  = critical (>=95)
    5  = critical (>=99)
    6  = critical (==100)     -- system at risk of collapse
    """
    if usage_pct >= 100: return 6
    if usage_pct >= 99:  return 5
    if usage_pct >= 95:  return 4
    if usage_pct >= 90:  return 3
    if usage_pct >= 85:  return 2
    if usage_pct >= 80:  return 1
    return 0


def status_from_tier(tier: int) -> str:
    if tier >= 4: return 'critical'
    if tier >= 1: return 'warning'
    return 'healthy'


def record_transition(sqlite_conn, subsystem: str,
                      prev_status: str, new_status: str,
                      prev_tier: int, new_tier: int,
                      detail: dict, ai_diagnosis: str = None,
                      signature: str = None):
    """Persist a row-per-transition log entry. Caller is responsible for
    the surrounding INSERT OR REPLACE on health_state."""
    sqlite_conn.execute(
        "INSERT INTO health_transitions "
        "(subsystem, prev_status, new_status, prev_severity, new_severity, "
        " detail_json, ai_diagnosis, mysql_signature, transitioned_at) "
        "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
        (subsystem, prev_status, new_status, prev_tier, new_tier,
         json.dumps(detail), ai_diagnosis, signature, int(time.time())))
    sqlite_conn.commit()


def poll_health(sqlite_conn, mysql_conn):
    try:
        r = requests.get(HEALTH_URL, verify=False, timeout=15)
        r.raise_for_status()
        data = r.json()
    except Exception as e:
        # Don't error-loop on transient failures. Log once and return.
        print(f"[health] poll failed: {e}", flush=True)
        return

    checks = data.get('data', {}).get('checks', {})
    for sub in HEALTH_SUBSYSTEMS:
        sub_data = checks.get(sub) or {}
        status = sub_data.get('status', 'unknown')
        if status not in ('healthy', 'warning', 'critical', 'error'):
            continue

        # Disk: evaluate each partition independently with 6-tier severity.
        # A 1->2 tier change inside 'warning' is still a transition we want
        # logged (and notified on) -- the existing status comparison would
        # miss it.
        if sub == 'disk':
            filesystems = sub_data.get('filesystems') or {}
            for partition, fs in filesystems.items():
                pct = float(fs.get('usage_percent') or 0)
                evaluate_disk_partition(sqlite_conn, mysql_conn, partition, fs, pct)
            continue

        # Non-disk subsystems use the original status-only model.
        evaluate_simple_subsystem(sqlite_conn, mysql_conn, sub, status, sub_data)


def evaluate_simple_subsystem(sqlite_conn, mysql_conn, sub, new_status, sub_data):
    last = sqlite_conn.execute(
        "SELECT status FROM health_state WHERE subsystem=?", (sub,)).fetchone()
    last_status = last['status'] if last else 'healthy'
    if new_status == last_status:
        return

    sig = (compute_signature(kind='health', subsystem=sub, severity=new_status)
           if new_status in ('warning', 'critical') else None)

    sqlite_conn.execute(
        "INSERT OR REPLACE INTO health_state (subsystem, status, detail_json, transitioned_at, severity_level) "
        "VALUES (?, ?, ?, ?, 0)",
        (sub, new_status, json.dumps(sub_data), int(time.time())))
    sqlite_conn.commit()

    if new_status == 'healthy':
        for sev in ('critical', 'warning'):
            recovered_sig = compute_signature(kind='health', subsystem=sub, severity=sev)
            mark_auto_resolved(mysql_conn, recovered_sig)
        record_transition(sqlite_conn, sub, last_status, new_status, 0, 0, sub_data)
        return

    if new_status in ('warning', 'critical'):
        ai_text = handle_health_transition(sqlite_conn, mysql_conn, sub, new_status, sub_data)
        record_transition(sqlite_conn, sub, last_status, new_status,
                          0, 0, sub_data, ai_text, sig)


def evaluate_disk_partition(sqlite_conn, mysql_conn, partition, fs, pct):
    """Per-partition tracking with 6-tier severity. Each 5% crossing on the
    way up is a transition; downward movement still updates state but only
    counts as a recovery once we cross back below 80% (tier 0).

    Signature is partition-scoped (not severity-scoped) so the issue row
    in MySQL is reused across tier bumps -- 'one issue per partition,
    severity climbs as the partition fills.'"""
    sub_key = f'disk:{partition}'
    new_tier = disk_tier(pct)
    new_status = status_from_tier(new_tier)
    signature = compute_signature(kind='health', subsystem=sub_key, severity='all')  # tier-agnostic

    last = sqlite_conn.execute(
        "SELECT status, severity_level FROM health_state WHERE subsystem=?",
        (sub_key,)).fetchone()
    last_status = last['status'] if last else 'healthy'
    last_tier = (last['severity_level'] if last else 0) or 0

    if new_tier == last_tier and new_status == last_status:
        return  # genuinely unchanged

    sqlite_conn.execute(
        "INSERT OR REPLACE INTO health_state "
        "(subsystem, status, detail_json, transitioned_at, severity_level) "
        "VALUES (?, ?, ?, ?, ?)",
        (sub_key, new_status, json.dumps(fs), int(time.time()), new_tier))
    sqlite_conn.commit()

    # Recovery: dropped to tier 0. Mark the open issue auto-resolved.
    if new_tier == 0:
        mark_auto_resolved(mysql_conn, signature)
        record_transition(sqlite_conn, sub_key, last_status, new_status,
                          last_tier, new_tier, fs, signature=signature)
        return

    # Downward but still elevated (e.g. 5->4): just log the transition;
    # don't re-call AI, don't re-notify.
    if new_tier < last_tier:
        record_transition(sqlite_conn, sub_key, last_status, new_status,
                          last_tier, new_tier, fs, signature=signature)
        return

    # Upward: enrich, ask AI for diagnosis, write/update the issue.
    detail = dict(fs)
    detail['partition'] = partition
    detail['severity_level'] = new_tier
    ai_text = handle_health_transition(sqlite_conn, mysql_conn, sub_key,
                                       new_status, detail)
    record_transition(sqlite_conn, sub_key, last_status, new_status,
                      last_tier, new_tier, fs, ai_text, signature)


def handle_health_transition(sqlite_conn, mysql_conn, subsystem, severity, detail):
    """Run the AI diagnosis + persist (or update) the issue row. Returns the
    AI diagnosis text so callers can also stamp it on the timeline row."""
    # Strip 'disk:partition' suffix when looking up category/context — both
    # are keyed by base subsystem.
    base = subsystem.split(':', 1)[0]
    category = CATEGORY_BY_SUBSYSTEM.get(base, 'general')

    # Disk uses a partition-scoped signature so all six tier bumps share one
    # MySQL issue row that simply has its severity bumped.
    if base == 'disk':
        signature = compute_signature(kind='health', subsystem=subsystem, severity='all')
    else:
        signature = compute_signature(kind='health', subsystem=subsystem, severity=severity)

    context = gather_health_context(base)
    snapshot = {'health_detail': detail, 'context': context}

    prompt = (
        f"Subsystem '{subsystem}' transitioned to '{severity}' on a WHP hosting server.\n\n"
        f"Health snapshot:\n{json.dumps(detail, indent=2)}\n\n"
        f"Context:\n{context[:3000]}\n\n"
        f"Diagnose the most likely root cause and suggest one concrete remediation. "
        f"Be concise (under 300 words)."
    )
    try:
        ai = call_ai(prompt, tier='basic', max_tokens=600)
        diagnosis = ai['content']
    except Exception as e:
        diagnosis = f"AI call failed: {e}"

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=None,
        subscription_id=None,
        username=None,
        container_name=None,
        severity=severity,
        category=category,
        subsystem=subsystem,
        source='health_api',
        title=f"{subsystem} {severity}",
        diagnosis=diagnosis,
        recommendation='',
        raw_log_excerpt=None,
        context_data=snapshot,
    )
    return diagnosis


def gather_health_context(subsystem: str) -> str:
    """Return up to 3KB of subsystem-specific diagnostic context."""
    commands = {
        'cpu':         "top -bn1 | head -20",
        'swap':        "ps axo pid,user,rss,comm --sort=-rss | head -10",
        # Disk: per-partition usage, top space hogs, and cleanup candidates.
        # The probe lists rotated logs (>30d), large Docker container json
        # logs, and the journal size so the AI can see exactly which paths
        # are eating space and recommend specific cleanups (the user reviews
        # and clicks-to-execute -- we don't auto-delete).
        'disk':        ("df -h; echo ---; "
                        "echo 'TOP SPACE HOGS:'; "
                        "du -hx --max-depth=2 /docker /var 2>/dev/null | sort -rh | head -20; "
                        "echo ---; "
                        "echo 'ROTATED LOGS >30d (>10MB):'; "
                        "find /var/log /docker/users -type f \\( -name '*.gz' -o -name '*.[0-9]' -o -name '*.[0-9].gz' \\) "
                        "    -mtime +30 -size +10M -printf '%s %p\\n' 2>/dev/null | sort -rn | head -20; "
                        "echo ---; "
                        "echo 'DOCKER CONTAINER JSON LOGS (>100MB):'; "
                        "find /var/lib/docker/containers -name '*-json.log' -size +100M -printf '%s %p\\n' 2>/dev/null | sort -rn | head -10; "
                        "echo ---; "
                        "echo 'JOURNAL SIZE:'; "
                        "journalctl --disk-usage 2>/dev/null"),
        'connections': "ss -s",
        'docker':      "docker ps --format 'table {{.Names}}\\t{{.Status}}' | head -20",
        'haproxy':     "docker logs haproxy-manager --tail 50 2>&1",
    }
    cmd = commands.get(subsystem)
    if not cmd:
        return ''
    try:
        out = subprocess.run(cmd, shell=True, capture_output=True, timeout=10, text=True)
        return (out.stdout + out.stderr)[:3000]
    except Exception as e:
        return f"(context collection failed: {e})"


# ---------- HAProxy stats polling ----------

def poll_haproxy_stats(sqlite_conn, mysql_conn):
    try:
        out = subprocess.run(HAPROXY_STATS_CMD, capture_output=True, timeout=10,
                             text=True, check=True)
        text = out.stdout
        if not text:
            print("[haproxy-stats] empty response from docker exec", flush=True)
            return
    except Exception as e:
        print(f"[haproxy-stats] poll failed: {e}", flush=True)
        return

    reader = csv.reader(io.StringIO(text.lstrip('# ')))
    header = next(reader, None)
    if not header:
        return
    try:
        pxname_i = header.index('pxname')
        svname_i = header.index('svname')
        status_i = header.index('status')
    except ValueError:
        print("[haproxy-stats] unexpected CSV header", flush=True)
        return

    for row in reader:
        if len(row) <= max(pxname_i, svname_i, status_i):
            continue
        pxname = row[pxname_i]
        svname = row[svname_i]
        status = row[status_i]
        # svname 'BACKEND' and 'FRONTEND' are aggregates, skip.
        if svname in ('BACKEND', 'FRONTEND'):
            continue
        if not status:
            continue

        backend_key = f"{pxname}/{svname}"
        last = sqlite_conn.execute(
            "SELECT status FROM backend_state WHERE backend_name=?",
            (backend_key,)).fetchone()
        last_status = last['status'] if last else 'UP'

        if status == last_status:
            continue

        sqlite_conn.execute(
            "INSERT OR REPLACE INTO backend_state (backend_name, status, transitioned_at) "
            "VALUES (?, ?, ?)",
            (backend_key, status, int(time.time())))
        sqlite_conn.commit()

        # Normalize HAProxy statuses.
        # HAProxy returns compound statuses like 'MAINT (resolution)' or
        # 'DOWN 1/3' — match by prefix rather than exact string.
        def _is_down(s):
            return s.startswith(('DOWN', 'NOLB', 'MAINT'))
        is_down_now = _is_down(status)
        was_down = _is_down(last_status)

        if is_down_now and not was_down:
            handle_backend_down(mysql_conn, pxname, svname, status)
        elif not is_down_now and was_down:
            handle_backend_up(mysql_conn, pxname, svname)


def resolve_backend_to_site(mysql_conn, svname: str):
    """
    Backend container name follows 'domain.tld-NN' convention.
    Returns (site_id, primary_domain, subscription_id, username)
    or (None, svname, None, None).

    Subdomain sites (e.g. transcribe.shadowdao.com) share the apex row
    in whp.domains with their parent site; the specific FQDN lives in
    whp.site_domains. Try site_domains first, fall back to the apex
    linkage for legacy sites that predate site_domains.
    """
    m = re.match(r'^(.+)-\d{1,2}$', svname)
    if not m:
        return (None, svname, None, None)
    domain = m.group(1)
    # Container names use wildcard.x.com; DB stores *.x.com. Reverse the
    # translation so the lookup hits the subscribed domain.
    if domain.startswith('wildcard.'):
        domain = '*.' + domain[len('wildcard.'):]
    with mysql_conn.cursor() as cur:
        # Preferred: exact FQDN match in site_domains.
        cur.execute("""
            SELECT s.id AS site_id, s.username,
                   COALESCE(sdp.fqdn, d.domain_name) AS primary_domain,
                   sub.id AS subscription_id
            FROM whp.site_domains sd
            JOIN whp.sites s ON s.id = sd.site_id
            JOIN whp.domains d ON d.id = s.primary_domain_id
            LEFT JOIN whp.site_domains sdp
                   ON sdp.site_id = s.id AND sdp.is_primary = 1
            LEFT JOIN whp.ai_monitor_subscriptions sub
                   ON sub.site_id = s.id AND sub.status = 'active'
            WHERE sd.fqdn = %s
            LIMIT 1
        """, (domain,))
        row = cur.fetchone()
        if not row:
            # Fallback: apex registered in whp.domains (legacy sites).
            cur.execute("""
                SELECT s.id AS site_id, s.username,
                       d.domain_name AS primary_domain,
                       sub.id AS subscription_id
                FROM whp.domains d
                JOIN whp.sites s ON s.primary_domain_id = d.id
                LEFT JOIN whp.ai_monitor_subscriptions sub
                       ON sub.site_id = s.id AND sub.status = 'active'
                WHERE d.domain_name = %s
                LIMIT 1
            """, (domain,))
            row = cur.fetchone()
    if not row:
        return (None, domain, None, None)
    return (row['site_id'], row['primary_domain'],
            row['subscription_id'], row['username'])


def handle_backend_down(mysql_conn, pxname, svname, raw_status):
    site_id, domain, subscription_id, username = resolve_backend_to_site(mysql_conn, svname)
    bucket = ten_minute_bucket(int(time.time()))
    signature = compute_signature(
        kind='site',
        site_id=site_id if site_id else 0,
        category='site_down',
        bucket=bucket)

    # Pull container logs.
    container = svname
    try:
        out = subprocess.run(
            f"docker logs {shlex.quote(container)} --since 5m 2>&1 | tail -100",
            shell=True, capture_output=True, timeout=15, text=True)
        logs = out.stdout[:5000]
    except Exception as e:
        logs = f"(log fetch failed: {e})"

    prompt = (
        f"HAProxy backend '{pxname}/{svname}' transitioned to {raw_status} on a WHP hosting server.\n\n"
        f"Last 100 log lines from container '{container}':\n{logs}\n\n"
        f"Diagnose the most likely cause of the backend going down and suggest a remediation. "
        f"Be concise (under 300 words)."
    )
    try:
        ai = call_ai(prompt, tier='basic', max_tokens=600)
        diagnosis = ai['content']
    except Exception as e:
        diagnosis = f"AI call failed: {e}"

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=site_id,
        subscription_id=subscription_id,
        username=username,
        container_name=svname,
        severity='critical',
        category='site_down',
        subsystem=None,
        source='haproxy_stats',
        title=f"Backend DOWN: {domain} ({svname})",
        diagnosis=diagnosis,
        recommendation='',
        raw_log_excerpt=logs,
        context_data={'pxname': pxname, 'svname': svname, 'haproxy_status': raw_status},
    )


def handle_backend_up(mysql_conn, pxname, svname):
    # On UP transition, close any open site_down Issue for this site/container.
    # Lookup by (site_id, container_name) — not by signature, because the original
    # Down event's 10-minute bucket may have rolled over by the time recovery
    # is observed.
    site_id, _domain, _sub, _username = resolve_backend_to_site(mysql_conn, svname)
    with mysql_conn.cursor() as cur:
        if site_id:
            cur.execute(
                "UPDATE whp.ai_monitor_issues "
                "SET status = 'fixed', "
                "    auto_resolved_at = COALESCE(auto_resolved_at, NOW()), "
                "    status_changed_by = 'system', "
                "    status_changed_at = NOW() "
                "WHERE site_id=%s AND category='site_down' AND status='open' "
                "  AND (container_name=%s OR container_name IS NULL)",
                (site_id, svname))
        else:
            cur.execute(
                "UPDATE whp.ai_monitor_issues "
                "SET status = 'fixed', "
                "    auto_resolved_at = COALESCE(auto_resolved_at, NOW()), "
                "    status_changed_by = 'system', "
                "    status_changed_at = NOW() "
                "WHERE category='site_down' AND status='open' "
                "  AND container_name=%s",
                (svname,))
    mysql_conn.commit()


# ---------- Crash-event drain ----------

# Cause classifications produced by haproxy-crash-monitor.sh + watchdog hook.
# Anything outside this list still gets surfaced as 'warning' by default.
_CRASH_CAUSE_SEVERITY = {
    'OOM_DOCKER':        'critical',
    'OOM_KERNEL':        'critical',
    'SEGFAULT':          'critical',
    'SIGKILL_EXTERNAL':  'critical',
    'PROCESS_ERROR':     'critical',
    'ABORT':             'critical',
    'WATCHDOG_FAILED':   'critical',  # both restart and recreate failed; needs a human
    'WATCHDOG_RECREATE': 'warning',   # docker restart didn't work; container was recreated
    'WATCHDOG_RESTART':  'warning',   # health checks failed; auto-recovery succeeded
    'WATCHDOG_START':    'warning',   # container wasn't running; we started it
    'CLEAN_SHUTDOWN':    'info',      # docker stop / restart by an operator
}


def drain_crash_events(sqlite_conn, mysql_conn):
    """Pull any new rows from whp.container_crash_events and surface them
    as Issues so auto-recovery actions become visible in the unified
    Issues tracker. High-water mark stored in SQLite poll_cursor table.
    """
    row = sqlite_conn.execute(
        "SELECT value FROM poll_cursor WHERE key='last_crash_event_id'"
    ).fetchone()
    last_seen = row['value'] if row else 0

    with mysql_conn.cursor() as cur:
        cur.execute("""
            SELECT id, container_name, event_type, exit_code, oom_killed,
                   cause, restart_count, downtime_seconds,
                   memory_usage, host_memory_free, dmesg_excerpt,
                   container_logs, created_at
            FROM whp.container_crash_events
            WHERE id > %s
            ORDER BY id ASC
            LIMIT 50
        """, (last_seen,))
        events = cur.fetchall()

    if not events:
        return 0

    for ev in events:
        try:
            _emit_issue_for_crash(mysql_conn, ev)
        except Exception as e:
            print(f"[drain-crash-events] event id={ev['id']} failed: {e}",
                  flush=True)
        last_seen = max(last_seen, ev['id'])

    sqlite_conn.execute(
        "INSERT OR REPLACE INTO poll_cursor (key, value) VALUES ('last_crash_event_id', ?)",
        (last_seen,))
    sqlite_conn.commit()
    print(f"[drain-crash-events] processed {len(events)} new event(s); "
          f"last_seen_id={last_seen}", flush=True)
    return len(events)


def _emit_issue_for_crash(mysql_conn, ev):
    container = ev['container_name'] or 'unknown'
    cause     = (ev['cause'] or 'UNKNOWN').upper()
    severity  = _CRASH_CAUSE_SEVERITY.get(cause, 'warning')
    bucket    = ten_minute_bucket(int(time.time()))

    # haproxy-manager is the panel-shared HAProxy — server-wide. Site
    # containers (domain.tld-NN) resolve to a site_id via existing helper.
    site_id = subscription_id = username = None
    domain  = container
    if container != 'haproxy-manager':
        try:
            from whp_monitor_core import connect_mysql as _ignore  # only to force-import
        except Exception:
            pass
        site_id, domain, subscription_id, username = resolve_backend_to_site(
            mysql_conn, container)

    title_cause = cause.replace('_', ' ').title()
    if cause == 'WATCHDOG_RESTART':
        title = f"Auto-restart performed: {container} (watchdog)"
    elif cause == 'WATCHDOG_RECREATE':
        title = f"Container recreated: {container} (watchdog fallback)"
    elif cause == 'WATCHDOG_START':
        title = f"Container started: {container} (was not running)"
    elif cause == 'WATCHDOG_FAILED':
        title = f"Auto-recovery FAILED: {container} (manual intervention needed)"
    elif cause == 'CLEAN_SHUTDOWN':
        title = f"Container restarted cleanly: {container}"
    else:
        title = f"Container crash detected: {container} ({title_cause})"

    # Evidence: a compact one-liner the customer / root can read.
    parts = [f"cause={cause}", f"event={ev['event_type'] or '?'}"]
    if ev.get('exit_code') is not None: parts.append(f"exit={ev['exit_code']}")
    if ev.get('oom_killed'):            parts.append("oom_killed=true")
    if ev.get('restart_count') is not None: parts.append(f"docker_restart_count={ev['restart_count']}")
    if ev.get('downtime_seconds') is not None: parts.append(f"downtime={ev['downtime_seconds']}s")
    if ev.get('memory_usage'):     parts.append(f"mem={ev['memory_usage']}")
    if ev.get('host_memory_free'): parts.append(f"host_free={ev['host_memory_free']}")
    evidence = " | ".join(parts)

    diagnosis_lines = [f"{title} at {ev['created_at']}.", evidence]
    if ev.get('dmesg_excerpt'):
        diagnosis_lines.append("--- dmesg excerpt ---\n" + str(ev['dmesg_excerpt'])[:2000])
    if ev.get('container_logs'):
        diagnosis_lines.append("--- last container logs before exit ---\n"
                               + str(ev['container_logs'])[:3000])
    diagnosis = "\n\n".join(diagnosis_lines)

    if cause == 'WATCHDOG_RESTART':
        recommendation = ("The watchdog auto-restarted the container after health checks "
                          "failed. Review the container logs for the underlying cause; "
                          "if it recurs within an hour, escalate to a human.")
    elif cause == 'WATCHDOG_RECREATE':
        recommendation = ("docker restart failed, so the watchdog recreated the "
                          "container from scratch. Investigate why a plain restart "
                          "wasn't enough — likely image / config issue.")
    elif cause == 'WATCHDOG_START':
        recommendation = ("The watchdog found the container stopped and started it. "
                          "Check why it stopped (haproxy-crash-monitor.sh or dmesg "
                          "should have a separate event with the cause).")
    elif cause == 'WATCHDOG_FAILED':
        recommendation = ("Both 'docker restart' and recreate failed. The platform's "
                          "auto-recovery is exhausted — manual intervention is required. "
                          "Inspect the watchdog log at /var/log/haproxy-watchdog.log.")
    elif severity == 'critical':
        recommendation = ("Investigate the underlying cause (OOM, segfault, signal). "
                          "If memory-related, raise the container memory limit or fix "
                          "the leak; if SIGKILL_EXTERNAL, check who's killing it.")
    else:
        recommendation = ("Recorded for visibility. No action needed unless the "
                          "container is now unstable.")

    signature = compute_signature(
        kind='site',
        site_id=site_id if site_id else 0,
        category=f"crash:{cause.lower()}",
        bucket=bucket)

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=site_id,
        subscription_id=subscription_id,
        username=username,
        container_name=container,
        severity=severity,
        category='container_crash',
        subsystem=cause.lower(),
        source='crash_monitor',
        title=title,
        diagnosis=diagnosis,
        recommendation=recommendation,
        raw_log_excerpt=ev.get('container_logs'),
        context_data={
            'crash_event_id':   ev['id'],
            'cause':            cause,
            'event_type':       ev['event_type'],
            'exit_code':        ev.get('exit_code'),
            'oom_killed':       bool(ev.get('oom_killed')),
            'restart_count':    ev.get('restart_count'),
            'downtime_seconds': ev.get('downtime_seconds'),
            'occurred_at':      str(ev['created_at']),
        },
    )


# ---------- Main ----------

def main():
    sqlite_conn = connect_sqlite()
    ensure_sqlite_schema(sqlite_conn)
    try:
        mysql_conn = connect_mysql()
    except Exception as e:
        print(f"[fatal] MySQL connect failed: {e}", flush=True)
        write_heartbeat(sqlite_conn, 'health_api', 'fail', f"mysql connect: {e}")
        write_heartbeat(sqlite_conn, 'haproxy_stats', 'fail', f"mysql connect: {e}")
        sqlite_conn.close()
        return 1

    health_err = haproxy_err = crash_err = None
    try:
        try:
            poll_health(sqlite_conn, mysql_conn)
        except Exception as e:
            health_err = str(e)
            print(f"[health] poll cycle failed: {e}", flush=True)
        try:
            drain_crash_events(sqlite_conn, mysql_conn)
        except Exception as e:
            crash_err = str(e)
            print(f"[drain-crash-events] cycle failed: {e}", flush=True)
        try:
            poll_haproxy_stats(sqlite_conn, mysql_conn)
        except Exception as e:
            haproxy_err = str(e)
            print(f"[haproxy-stats] poll cycle failed: {e}", flush=True)

        write_heartbeat(sqlite_conn, 'health_api',
                        'ok' if not health_err else 'fail', health_err)
        write_heartbeat(sqlite_conn, 'haproxy_stats',
                        'ok' if not haproxy_err else 'fail', haproxy_err)
        write_heartbeat(sqlite_conn, 'crash_events',
                        'ok' if not crash_err else 'fail', crash_err)
    finally:
        mysql_conn.close()
        sqlite_conn.close()
    return 0


if __name__ == '__main__':
    sys.exit(main())
