#!/usr/bin/env python3
"""
Minute-cadence poll for AI-monitor health and HAProxy stats.

Runs non-interactively from cron. Never long-running — one poll cycle per
invocation to stay in line with WHP's cron-driven pattern (see also
haproxy-watchdog.sh).

Sources:
  - https://localhost:8443/api/health.php?check=all&detailed=1
  - http://localhost:8404/stats;csv

On a healthy→warning/critical transition for any subsystem or backend, fetches
relevant context, calls AI for diagnosis, and inserts (or increments) an Issue
via whp_monitor_core.
"""
import csv
import io
import json
import os
import re
import shlex
import subprocess
import sys
import time
import sqlite3
import urllib3

# Suppress localhost self-signed cert warning for the health.php poll.
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from whp_monitor_core import (
    connect_mysql, connect_sqlite, compute_signature, ten_minute_bucket,
    insert_or_increment_issue, mark_auto_resolved, call_ai,
)

import requests

HEALTH_URL = 'https://localhost:8443/api/health.php?check=all&detailed=1'
# HAProxy stats bind to 127.0.0.1:8404 INSIDE the haproxy-manager container
# and are not exposed to the host. Fetch via `docker exec` instead.
HAPROXY_STATS_CMD = ['docker', 'exec', 'haproxy-manager',
                     'curl', '-s', 'http://127.0.0.1:8404/stats;csv']


def ensure_sqlite_schema(conn):
    """Create health_state and backend_state tables if missing."""
    conn.executescript("""
        CREATE TABLE IF NOT EXISTS health_state (
            subsystem TEXT PRIMARY KEY,
            status TEXT NOT NULL,
            detail_json TEXT,
            transitioned_at INTEGER NOT NULL
        );
        CREATE TABLE IF NOT EXISTS backend_state (
            backend_name TEXT PRIMARY KEY,
            status TEXT NOT NULL,
            transitioned_at INTEGER NOT NULL
        );
    """)
    conn.commit()


# ---------- Health API polling ----------

HEALTH_SUBSYSTEMS = ['cpu', 'connections', 'disk', 'swap', 'docker', 'haproxy']

CATEGORY_BY_SUBSYSTEM = {
    'cpu': 'cpu_pressure',
    'swap': 'swap_pressure',
    'disk': 'disk_full',
    'connections': 'connection_storm',
    'docker': 'docker_degraded',
    'haproxy': 'haproxy_unhealthy',
}


def poll_health(sqlite_conn, mysql_conn):
    try:
        r = requests.get(HEALTH_URL, verify=False, timeout=15)
        r.raise_for_status()
        data = r.json()
    except Exception as e:
        # Don't error-loop on transient failures. Log once and return.
        print(f"[health] poll failed: {e}", flush=True)
        return

    checks = data.get('data', {}).get('checks', {})
    for sub in HEALTH_SUBSYSTEMS:
        sub_data = checks.get(sub) or {}
        status = sub_data.get('status', 'unknown')
        if status not in ('healthy', 'warning', 'critical', 'error'):
            continue

        last = sqlite_conn.execute(
            "SELECT status FROM health_state WHERE subsystem=?", (sub,)).fetchone()
        last_status = last['status'] if last else 'healthy'

        if status == last_status:
            continue  # no transition

        # Persist new state
        sqlite_conn.execute(
            "INSERT OR REPLACE INTO health_state (subsystem, status, detail_json, transitioned_at) "
            "VALUES (?, ?, ?, ?)",
            (sub, status, json.dumps(sub_data), int(time.time())))
        sqlite_conn.commit()

        if status == 'healthy':
            # Recovery: stamp auto_resolved_at on any matching open issue.
            # Try both severities — whichever was open will match on signature.
            for sev in ('critical', 'warning'):
                sig = compute_signature(kind='health', subsystem=sub, severity=sev)
                mark_auto_resolved(mysql_conn, sig)
            continue

        if status in ('warning', 'critical'):
            handle_health_transition(sqlite_conn, mysql_conn, sub, status, sub_data)


def handle_health_transition(sqlite_conn, mysql_conn, subsystem, severity, detail):
    category = CATEGORY_BY_SUBSYSTEM[subsystem]
    signature = compute_signature(kind='health', subsystem=subsystem, severity=severity)

    # Gather subsystem-specific context.
    context = gather_health_context(subsystem)
    snapshot = {'health_detail': detail, 'context': context}

    # AI diagnosis.
    prompt = (
        f"Subsystem '{subsystem}' transitioned to '{severity}' on a WHP hosting server.\n\n"
        f"Health snapshot:\n{json.dumps(detail, indent=2)}\n\n"
        f"Context:\n{context[:3000]}\n\n"
        f"Diagnose the most likely root cause and suggest one concrete remediation. "
        f"Be concise (under 300 words)."
    )
    try:
        ai = call_ai(prompt, tier='basic', max_tokens=600)
        diagnosis = ai['content']
    except Exception as e:
        diagnosis = f"AI call failed: {e}"

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=None,
        subscription_id=None,
        username=None,
        container_name=None,
        severity=severity,
        category=category,
        subsystem=subsystem,
        source='health_api',
        title=f"{subsystem} {severity}",
        diagnosis=diagnosis,
        recommendation='',
        raw_log_excerpt=None,
        context_data=snapshot,
    )


def gather_health_context(subsystem: str) -> str:
    """Return up to 3KB of subsystem-specific diagnostic context."""
    commands = {
        'cpu':         "top -bn1 | head -20",
        'swap':        "ps axo pid,user,rss,comm --sort=-rss | head -10",
        'disk':        "df -h; echo ---; du -sh /docker/users/*/userfiles 2>/dev/null | sort -rh | head -10",
        'connections': "ss -s",
        'docker':      "docker ps --format 'table {{.Names}}\\t{{.Status}}' | head -20",
        'haproxy':     "docker logs haproxy-manager --tail 50 2>&1",
    }
    cmd = commands.get(subsystem)
    if not cmd:
        return ''
    try:
        out = subprocess.run(cmd, shell=True, capture_output=True, timeout=10, text=True)
        return (out.stdout + out.stderr)[:3000]
    except Exception as e:
        return f"(context collection failed: {e})"


# ---------- HAProxy stats polling ----------

def poll_haproxy_stats(sqlite_conn, mysql_conn):
    try:
        out = subprocess.run(HAPROXY_STATS_CMD, capture_output=True, timeout=10,
                             text=True, check=True)
        text = out.stdout
        if not text:
            print("[haproxy-stats] empty response from docker exec", flush=True)
            return
    except Exception as e:
        print(f"[haproxy-stats] poll failed: {e}", flush=True)
        return

    reader = csv.reader(io.StringIO(text.lstrip('# ')))
    header = next(reader, None)
    if not header:
        return
    try:
        pxname_i = header.index('pxname')
        svname_i = header.index('svname')
        status_i = header.index('status')
    except ValueError:
        print("[haproxy-stats] unexpected CSV header", flush=True)
        return

    for row in reader:
        if len(row) <= max(pxname_i, svname_i, status_i):
            continue
        pxname = row[pxname_i]
        svname = row[svname_i]
        status = row[status_i]
        # svname 'BACKEND' and 'FRONTEND' are aggregates, skip.
        if svname in ('BACKEND', 'FRONTEND'):
            continue
        if not status:
            continue

        backend_key = f"{pxname}/{svname}"
        last = sqlite_conn.execute(
            "SELECT status FROM backend_state WHERE backend_name=?",
            (backend_key,)).fetchone()
        last_status = last['status'] if last else 'UP'

        if status == last_status:
            continue

        sqlite_conn.execute(
            "INSERT OR REPLACE INTO backend_state (backend_name, status, transitioned_at) "
            "VALUES (?, ?, ?)",
            (backend_key, status, int(time.time())))
        sqlite_conn.commit()

        # Normalize HAProxy statuses.
        # HAProxy returns compound statuses like 'MAINT (resolution)' or
        # 'DOWN 1/3' — match by prefix rather than exact string.
        def _is_down(s):
            return s.startswith(('DOWN', 'NOLB', 'MAINT'))
        is_down_now = _is_down(status)
        was_down = _is_down(last_status)

        if is_down_now and not was_down:
            handle_backend_down(mysql_conn, pxname, svname, status)
        elif not is_down_now and was_down:
            handle_backend_up(mysql_conn, pxname, svname)


def resolve_backend_to_site(mysql_conn, svname: str):
    """
    Backend container name follows 'domain.tld-NN' convention.
    Returns (site_id, primary_domain, subscription_id, username)
    or (None, svname, None, None).
    """
    m = re.match(r'^(.+)-\d{1,2}$', svname)
    if not m:
        return (None, svname, None, None)
    domain = m.group(1)
    # Container names use wildcard.x.com; DB stores *.x.com. Reverse the
    # translation so the lookup hits the subscribed domain.
    if domain.startswith('wildcard.'):
        domain = '*.' + domain[len('wildcard.'):]
    with mysql_conn.cursor() as cur:
        cur.execute("""
            SELECT s.id AS site_id, s.username, d.domain_name AS primary_domain,
                   sub.id AS subscription_id
            FROM whp.domains d
            JOIN whp.sites s ON s.primary_domain_id = d.id
            LEFT JOIN whp.ai_monitor_subscriptions sub
                   ON sub.site_id = s.id AND sub.status = 'active'
            WHERE d.domain_name = %s
            LIMIT 1
        """, (domain,))
        row = cur.fetchone()
    if not row:
        return (None, domain, None, None)
    return (row['site_id'], row['primary_domain'],
            row['subscription_id'], row['username'])


def handle_backend_down(mysql_conn, pxname, svname, raw_status):
    site_id, domain, subscription_id, username = resolve_backend_to_site(mysql_conn, svname)
    bucket = ten_minute_bucket(int(time.time()))
    signature = compute_signature(
        kind='site',
        site_id=site_id if site_id else 0,
        category='site_down',
        bucket=bucket)

    # Pull container logs.
    container = svname
    try:
        out = subprocess.run(
            f"docker logs {shlex.quote(container)} --since 5m 2>&1 | tail -100",
            shell=True, capture_output=True, timeout=15, text=True)
        logs = out.stdout[:5000]
    except Exception as e:
        logs = f"(log fetch failed: {e})"

    prompt = (
        f"HAProxy backend '{pxname}/{svname}' transitioned to {raw_status} on a WHP hosting server.\n\n"
        f"Last 100 log lines from container '{container}':\n{logs}\n\n"
        f"Diagnose the most likely cause of the backend going down and suggest a remediation. "
        f"Be concise (under 300 words)."
    )
    try:
        ai = call_ai(prompt, tier='basic', max_tokens=600)
        diagnosis = ai['content']
    except Exception as e:
        diagnosis = f"AI call failed: {e}"

    insert_or_increment_issue(
        mysql_conn,
        signature=signature,
        site_id=site_id,
        subscription_id=subscription_id,
        username=username,
        container_name=svname,
        severity='critical',
        category='site_down',
        subsystem=None,
        source='haproxy_stats',
        title=f"Backend DOWN: {domain} ({svname})",
        diagnosis=diagnosis,
        recommendation='',
        raw_log_excerpt=logs,
        context_data={'pxname': pxname, 'svname': svname, 'haproxy_status': raw_status},
    )


def handle_backend_up(mysql_conn, pxname, svname):
    # On UP transition, close any open site_down Issue for this site/container.
    # Lookup by (site_id, container_name) — not by signature, because the original
    # Down event's 10-minute bucket may have rolled over by the time recovery
    # is observed.
    site_id, _domain, _sub, _username = resolve_backend_to_site(mysql_conn, svname)
    with mysql_conn.cursor() as cur:
        if site_id:
            cur.execute(
                "UPDATE whp.ai_monitor_issues "
                "SET status = 'fixed', "
                "    auto_resolved_at = COALESCE(auto_resolved_at, NOW()), "
                "    status_changed_by = 'system', "
                "    status_changed_at = NOW() "
                "WHERE site_id=%s AND category='site_down' AND status='open' "
                "  AND (container_name=%s OR container_name IS NULL)",
                (site_id, svname))
        else:
            cur.execute(
                "UPDATE whp.ai_monitor_issues "
                "SET status = 'fixed', "
                "    auto_resolved_at = COALESCE(auto_resolved_at, NOW()), "
                "    status_changed_by = 'system', "
                "    status_changed_at = NOW() "
                "WHERE category='site_down' AND status='open' "
                "  AND container_name=%s",
                (svname,))
    mysql_conn.commit()


# ---------- Main ----------

def main():
    sqlite_conn = connect_sqlite()
    ensure_sqlite_schema(sqlite_conn)
    try:
        mysql_conn = connect_mysql()
    except Exception as e:
        print(f"[fatal] MySQL connect failed: {e}", flush=True)
        return 1
    try:
        poll_health(sqlite_conn, mysql_conn)
        poll_haproxy_stats(sqlite_conn, mysql_conn)
    finally:
        mysql_conn.close()
        sqlite_conn.close()
    return 0


if __name__ == '__main__':
    sys.exit(main())
