#!/usr/bin/env python3
"""
WHP AI-Powered Log Monitor and Auto-Remediation Engine
======================================================
Monitors system, application, and container logs for errors,
uses AI to diagnose issues, and performs safe auto-remediation.

Usage:
    whp-log-monitor-ai.py --mode critical   # Real-time critical error handling
    whp-log-monitor-ai.py --mode daily       # Daily batch analysis of non-critical errors
    whp-log-monitor-ai.py --dry-run          # Preview actions without executing
    whp-log-monitor-ai.py --reset-positions  # Reset all log position tracking

Configuration: /etc/whp/ai-monitor.conf
Database:      /var/lib/whp-ai-monitor/monitor.db
"""

import os
import sys
import re
import json
import time
import hashlib
import sqlite3
import subprocess
import configparser
import argparse
import logging
import urllib.request
import urllib.error
from abc import ABC, abstractmethod

try:
    import openai
except ImportError:
    openai = None
    _OPENAI_IMPORT_ERROR = (
        "The 'openai' package is required for AI analysis.\n"
        "Install it with: pip install openai\n"
        "This package is also compatible with LiteLLM proxy endpoints."
    )

try:
    import pymysql
    import pymysql.cursors
except ImportError:
    pymysql = None

logger = logging.getLogger("whp-ai-monitor")

# ---------------------------------------------------------------------------
# Default configuration values
# ---------------------------------------------------------------------------
DEFAULTS = {
    "general": {
        "db_path": "/var/lib/whp-ai-monitor/monitor.db",
        "log_level": "INFO",
        "dry_run": "false",
        "backup_dir": "/var/lib/whp-ai-monitor/backups",
    },
    "ai": {
        "api_key_file": "/etc/whp/ai-api-key",
        "base_url": "https://api.openai.com/v1",
        "basic_model": "global.anthropic.claude-haiku-4-5-20251001-v1:0",
        "advanced_model": "global.anthropic.claude-sonnet-4-6",
        "max_calls_per_run": "20",
        "max_calls_per_day": "200",
        "timeout": "60",
    },
    "sources": {
        "apache_error": "/var/log/httpd/error_log",
        "apache_suexec": "/var/log/httpd/suexec.log",
        "php_fpm_error": "/var/log/php-fpm/error.log",
        "php_fpm_slow": "/var/log/php-fpm/www-slow.log",
        "mysql_error": "",
        "haproxy_log": "",
        "proftpd_log": "/var/log/proftpd/proftpd.log",
        "whp_app_log": "/docker/whp/web/storage/logs/laravel.log",
        "docker_containers": "mysql,haproxy-manager,postgresql,memcache",
        "journal_services": "httpd,php-fpm,proftpd,whp-terminal,whp-backup-upload",
        "enable_dmesg": "true",
    },
    "notifications": {
        "discord_webhook_url": "",
        "discord_role_critical": "",
        "discord_role_warning": "",
        "discord_role_info": "",
        "slack_webhook_url": "",
        "telegram_bot_token": "",
        "telegram_chat_id": "",
    },
    "remediation": {
        "enabled": "true",
        "cooldown_minutes": "5",
        "max_remediations_per_hour": "10",
        "circuit_breaker_threshold": "3",
        "allowed_services": "httpd,php-fpm,proftpd,whp-terminal,whp-backup-upload",
        "allowed_containers": "mysql,postgresql,haproxy-manager,memcache",
        "allowed_config_paths": "/etc/httpd,/etc/php-fpm.d,/etc/proftpd.conf,"
                                "/docker/whp/web/.env,/docker/haproxy",
    },
    # [customer_monitoring] INI section retired. Global toggle lives in
    # /docker/whp/settings.json; site list in whp.ai_monitor_subscriptions.
    "daily": {
        "report_dir": "/var/lib/whp-ai-monitor/reports",
        "batch_size": "50",
    },
}


# ===================================================================
# Config
# ===================================================================
class Config:
    """Loads and provides access to the INI configuration file."""

    CONFIG_PATH = "/etc/whp/ai-monitor.conf"

    def __init__(self, path=None):
        self._parser = configparser.ConfigParser()
        # Seed with defaults
        for section, values in DEFAULTS.items():
            if not self._parser.has_section(section):
                self._parser.add_section(section)
            for key, val in values.items():
                self._parser.set(section, key, val)

        config_path = path or self.CONFIG_PATH
        if os.path.isfile(config_path):
            self._parser.read(config_path)
            logger.info("Loaded config from %s", config_path)
        else:
            logger.warning("Config file %s not found; using defaults", config_path)

        self._api_key = self._load_api_key()

    def _load_api_key(self):
        key_file = self._get("ai", "api_key_file")
        if key_file and os.path.isfile(key_file):
            try:
                with open(key_file, "r") as fh:
                    return fh.read().strip()
            except OSError as exc:
                logger.error("Cannot read API key file %s: %s", key_file, exc)
        # Fallback to environment
        return os.environ.get("OPENAI_API_KEY", "")

    def _get(self, section, key):
        try:
            return self._parser.get(section, key)
        except (configparser.NoSectionError, configparser.NoOptionError):
            return DEFAULTS.get(section, {}).get(key, "")

    def _getint(self, section, key):
        try:
            return self._parser.getint(section, key)
        except (configparser.NoSectionError, configparser.NoOptionError, ValueError):
            return int(DEFAULTS.get(section, {}).get(key, "0"))

    def _getbool(self, section, key):
        try:
            return self._parser.getboolean(section, key)
        except (configparser.NoSectionError, configparser.NoOptionError, ValueError):
            return DEFAULTS.get(section, {}).get(key, "false").lower() == "true"

    # ---- Properties ----
    @property
    def db_path(self):
        return self._get("general", "db_path")

    @property
    def log_level(self):
        return self._get("general", "log_level")

    @property
    def dry_run(self):
        return self._getbool("general", "dry_run")

    @property
    def backup_dir(self):
        return self._get("general", "backup_dir")

    @property
    def ai_api_key(self):
        return self._api_key

    @property
    def ai_base_url(self):
        return self._get("ai", "base_url")

    @property
    def basic_model(self):
        return self._get("ai", "basic_model")

    @property
    def advanced_model(self):
        return self._get("ai", "advanced_model")

    @property
    def max_calls_per_run(self):
        return self._getint("ai", "max_calls_per_run")

    @property
    def max_calls_per_day(self):
        return self._getint("ai", "max_calls_per_day")

    @property
    def ai_timeout(self):
        return self._getint("ai", "timeout")

    @property
    def source_files(self):
        """Return dict of source_id -> file path for file-based log sources."""
        result = {}
        for key in ("apache_error", "apache_suexec", "php_fpm_error",
                     "php_fpm_slow", "mysql_error", "haproxy_log",
                     "proftpd_log", "whp_app_log"):
            val = self._get("sources", key)
            if val and val.strip():
                result[key] = val.strip()
        return result

    @property
    def docker_containers(self):
        raw = self._get("sources", "docker_containers")
        return [c.strip() for c in raw.split(",") if c.strip()]

    @property
    def journal_services(self):
        raw = self._get("sources", "journal_services")
        return [s.strip() for s in raw.split(",") if s.strip()]

    @property
    def enable_dmesg(self):
        return self._getbool("sources", "enable_dmesg")

    @property
    def discord_webhook(self):
        return self._get("notifications", "discord_webhook_url")

    @property
    def slack_webhook(self):
        return self._get("notifications", "slack_webhook_url")

    @property
    def telegram_token(self):
        return self._get("notifications", "telegram_bot_token")

    @property
    def telegram_chat_id(self):
        return self._get("notifications", "telegram_chat_id")

    @property
    def remediation_enabled(self):
        return self._getbool("remediation", "enabled")

    @property
    def cooldown_minutes(self):
        return self._getint("remediation", "cooldown_minutes")

    @property
    def max_remediations_per_hour(self):
        return self._getint("remediation", "max_remediations_per_hour")

    @property
    def circuit_breaker_threshold(self):
        return self._getint("remediation", "circuit_breaker_threshold")

    @property
    def allowed_services(self):
        raw = self._get("remediation", "allowed_services")
        return [s.strip() for s in raw.split(",") if s.strip()]

    @property
    def allowed_containers(self):
        raw = self._get("remediation", "allowed_containers")
        return [c.strip() for c in raw.split(",") if c.strip()]

    @property
    def allowed_config_paths(self):
        raw = self._get("remediation", "allowed_config_paths")
        return [p.strip() for p in raw.split(",") if p.strip()]

    @property
    def report_dir(self):
        return self._get("daily", "report_dir")

    @property
    def daily_batch_size(self):
        return self._getint("daily", "batch_size")

    # INI [customer_monitoring] section is retired. Site list comes from
    # MySQL whp.ai_monitor_subscriptions; global toggle from settings.json.
    # See _load_customer_subscriptions() in WHPLogMonitor for the read path.


# ===================================================================
# Database
# ===================================================================
class Database:
    """SQLite persistence layer with auto-schema creation."""

    SCHEMA_SQL = [
        """CREATE TABLE IF NOT EXISTS log_positions (
            source_id TEXT PRIMARY KEY,
            last_position INTEGER DEFAULT 0,
            last_inode INTEGER DEFAULT 0,
            last_timestamp TEXT DEFAULT '',
            last_check INTEGER DEFAULT 0
        )""",
        """CREATE TABLE IF NOT EXISTS errors (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            signature TEXT UNIQUE NOT NULL,
            first_seen INTEGER NOT NULL,
            last_seen INTEGER NOT NULL,
            occurrence_count INTEGER DEFAULT 1,
            severity TEXT NOT NULL,
            source TEXT NOT NULL,
            raw_message TEXT NOT NULL,
            category TEXT DEFAULT '',
            ai_diagnosis TEXT DEFAULT '',
            ai_recommendation TEXT DEFAULT '',
            fix_type TEXT DEFAULT '',
            fix_applied INTEGER DEFAULT 0,
            fix_successful INTEGER DEFAULT 0,
            processed INTEGER DEFAULT 0,
            created_at INTEGER NOT NULL
        )""",
        """CREATE TABLE IF NOT EXISTS remediation_log (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            error_id INTEGER NOT NULL,
            action_type TEXT NOT NULL,
            target TEXT NOT NULL,
            backup_path TEXT DEFAULT '',
            command_run TEXT DEFAULT '',
            stdout TEXT DEFAULT '',
            stderr TEXT DEFAULT '',
            exit_code INTEGER DEFAULT -1,
            success INTEGER DEFAULT 0,
            timestamp INTEGER NOT NULL
        )""",
        """CREATE TABLE IF NOT EXISTS daily_api_calls (
            date TEXT PRIMARY KEY,
            count INTEGER DEFAULT 0
        )""",
    ]

    def __init__(self, db_path):
        db_dir = os.path.dirname(db_path)
        if db_dir and not os.path.isdir(db_dir):
            os.makedirs(db_dir, mode=0o750, exist_ok=True)
        self._path = db_path
        self._conn = sqlite3.connect(db_path, timeout=30)
        self._conn.row_factory = sqlite3.Row
        self._conn.execute("PRAGMA journal_mode=WAL")
        self._conn.execute("PRAGMA busy_timeout=5000")
        self._create_tables()

    def _create_tables(self):
        cursor = self._conn.cursor()
        for sql in self.SCHEMA_SQL:
            cursor.execute(sql)
        self._conn.commit()

    def close(self):
        if self._conn:
            self._conn.close()

    # ---- Position tracking ----
    def get_position(self, source_id):
        row = self._conn.execute(
            "SELECT last_position, last_inode, last_timestamp, last_check "
            "FROM log_positions WHERE source_id = ?",
            (source_id,),
        ).fetchone()
        if row:
            return dict(row)
        return {
            "last_position": 0,
            "last_inode": 0,
            "last_timestamp": "",
            "last_check": 0,
        }

    def set_position(self, source_id, position=0, inode=0, timestamp=""):
        now = int(time.time())
        self._conn.execute(
            "INSERT INTO log_positions (source_id, last_position, last_inode, "
            "last_timestamp, last_check) VALUES (?, ?, ?, ?, ?) "
            "ON CONFLICT(source_id) DO UPDATE SET "
            "last_position=excluded.last_position, "
            "last_inode=excluded.last_inode, "
            "last_timestamp=excluded.last_timestamp, "
            "last_check=excluded.last_check",
            (source_id, position, inode, timestamp, now),
        )
        self._conn.commit()

    def reset_all_positions(self):
        self._conn.execute("DELETE FROM log_positions")
        self._conn.commit()
        logger.info("All log positions have been reset")

    # ---- Error tracking ----
    def error_exists(self, signature):
        row = self._conn.execute(
            "SELECT id, occurrence_count FROM errors WHERE signature = ?",
            (signature,),
        ).fetchone()
        if row:
            now = int(time.time())
            self._conn.execute(
                "UPDATE errors SET last_seen = ?, occurrence_count = occurrence_count + 1 "
                "WHERE signature = ?",
                (now, signature),
            )
            self._conn.commit()
            return dict(row)
        return None

    def add_error(self, signature, severity, source, raw_message, category=""):
        now = int(time.time())
        try:
            self._conn.execute(
                "INSERT INTO errors "
                "(signature, first_seen, last_seen, severity, source, "
                "raw_message, category, created_at) "
                "VALUES (?, ?, ?, ?, ?, ?, ?, ?)",
                (signature, now, now, severity, source, raw_message, category, now),
            )
            self._conn.commit()
            return self._conn.execute(
                "SELECT last_insert_rowid()"
            ).fetchone()[0]
        except sqlite3.IntegrityError:
            # Race condition: another process inserted it
            existing = self.error_exists(signature)
            return existing["id"] if existing else None

    def update_error_with_ai(self, error_id, diagnosis, recommendation,
                             fix_type="", fix_applied=0, fix_successful=0):
        self._conn.execute(
            "UPDATE errors SET ai_diagnosis = ?, ai_recommendation = ?, "
            "fix_type = ?, fix_applied = ?, fix_successful = ?, processed = 1 "
            "WHERE id = ?",
            (diagnosis, recommendation, fix_type, fix_applied, fix_successful,
             error_id),
        )
        self._conn.commit()

    def add_remediation(self, error_id, action_type, target, backup_path="",
                        command_run="", stdout="", stderr="", exit_code=-1,
                        success=0):
        now = int(time.time())
        self._conn.execute(
            "INSERT INTO remediation_log "
            "(error_id, action_type, target, backup_path, command_run, "
            "stdout, stderr, exit_code, success, timestamp) "
            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
            (error_id, action_type, target, backup_path, command_run,
             stdout, stderr, exit_code, success, now),
        )
        self._conn.commit()

    def get_unprocessed_noncritical(self, limit=50):
        rows = self._conn.execute(
            "SELECT id, signature, severity, source, raw_message, category, "
            "occurrence_count FROM errors "
            "WHERE processed = 0 AND severity = 'non_critical' "
            "ORDER BY last_seen DESC LIMIT ?",
            (limit,),
        ).fetchall()
        return [dict(r) for r in rows]

    def mark_processed(self, error_ids):
        if not error_ids:
            return
        placeholders = ",".join("?" for _ in error_ids)
        self._conn.execute(
            f"UPDATE errors SET processed = 1 WHERE id IN ({placeholders})",
            error_ids,
        )
        self._conn.commit()

    # ---- API call tracking ----
    def increment_api_calls(self):
        today = time.strftime("%Y-%m-%d")
        self._conn.execute(
            "INSERT INTO daily_api_calls (date, count) VALUES (?, 1) "
            "ON CONFLICT(date) DO UPDATE SET count = count + 1",
            (today,),
        )
        self._conn.commit()

    def get_api_calls_today(self):
        today = time.strftime("%Y-%m-%d")
        row = self._conn.execute(
            "SELECT count FROM daily_api_calls WHERE date = ?",
            (today,),
        ).fetchone()
        return row["count"] if row else 0

    # ---- Remediation helpers ----
    def get_recent_remediations(self, target, hours=1):
        cutoff = int(time.time()) - (hours * 3600)
        rows = self._conn.execute(
            "SELECT * FROM remediation_log WHERE target = ? AND timestamp > ? "
            "ORDER BY timestamp DESC",
            (target, cutoff),
        ).fetchall()
        return [dict(r) for r in rows]

    def get_consecutive_failures(self, target, limit=10):
        rows = self._conn.execute(
            "SELECT success FROM remediation_log WHERE target = ? "
            "ORDER BY timestamp DESC LIMIT ?",
            (target, limit),
        ).fetchall()
        count = 0
        for row in rows:
            if row["success"] == 0:
                count += 1
            else:
                break
        return count


class CustomerDatabase:
    """MySQL persistence layer for customer-facing AI monitoring data."""

    def __init__(self):
        self._conn = None
        if pymysql is None:
            logger.warning("pymysql not installed — customer monitoring disabled")
            return
        try:
            # Read MySQL credentials from /root/.my.cnf
            import configparser as cp
            mycnf = cp.ConfigParser()
            mycnf.read('/root/.my.cnf')
            password = mycnf.get('client', 'password', fallback='').strip()
            self._conn = pymysql.connect(
                host='127.0.0.1',
                port=3306,
                user='root',
                password=password,
                database='whp',
                charset='utf8mb4',
                cursorclass=pymysql.cursors.DictCursor,
                autocommit=True,
                connect_timeout=5,
            )
            logger.info("Customer MySQL database connected")
        except Exception as exc:
            logger.error("Failed to connect to MySQL for customer monitoring: %s", exc)
            self._conn = None

    @property
    def available(self):
        return self._conn is not None

    def close(self):
        if self._conn:
            try:
                self._conn.close()
            except Exception:
                pass

    def get_active_subscriptions(self):
        """Load active monitoring subscriptions with site and domain info."""
        if not self.available:
            return []
        try:
            with self._conn.cursor() as cur:
                cur.execute("""
                    SELECT sub.id AS subscription_id, sub.site_id, sub.username,
                           sub.notification_email, sub.daily_review_limit,
                           d.domain_name AS primary_domain,
                           s.site_name, s.container_count
                    FROM ai_monitor_subscriptions sub
                    JOIN sites s ON sub.site_id = s.id
                    JOIN domains d ON s.primary_domain_id = d.id
                    WHERE sub.status = 'active'
                """)
                return cur.fetchall()
        except Exception as exc:
            logger.error("Failed to load subscriptions from MySQL: %s", exc)
            return []

    def upsert_issue(self, subscription_id, site_id, username, container_name,
                     severity, category, title, diagnosis, recommendation,
                     raw_log_excerpt, signature):
        """Insert or update a customer-visible issue. Returns issue_id or None."""
        if not self.available:
            return None
        try:
            # Sanitize log excerpt
            raw_log_excerpt = self._sanitize_log(raw_log_excerpt) if raw_log_excerpt else None

            with self._conn.cursor() as cur:
                # Check if issue with this signature already exists for this subscription
                cur.execute(
                    "SELECT id, occurrence_count FROM ai_monitor_issues "
                    "WHERE subscription_id = %s AND signature = %s AND status = 'open'",
                    (subscription_id, signature)
                )
                existing = cur.fetchone()

                if existing:
                    cur.execute(
                        "UPDATE ai_monitor_issues SET occurrence_count = occurrence_count + 1, "
                        "last_seen = NOW() WHERE id = %s",
                        (existing['id'],)
                    )
                    return existing['id']
                else:
                    cur.execute(
                        "INSERT INTO ai_monitor_issues "
                        "(subscription_id, site_id, username, container_name, severity, "
                        "category, title, diagnosis, recommendation, raw_log_excerpt, "
                        "signature, source, occurrence_count, first_seen, last_seen) "
                        "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, 'logs', 1, NOW(), NOW())",
                        (subscription_id, site_id, username, container_name, severity,
                         category, title, diagnosis, recommendation, raw_log_excerpt,
                         signature)
                    )
                    return cur.lastrowid
        except Exception as exc:
            logger.error("Failed to upsert customer issue: %s", exc)
            return None

    def queue_email(self, subscription_id, issue_id, recipient_email, subject,
                    body_html, body_text):
        """Queue an email notification for a customer issue."""
        if not self.available or not recipient_email:
            return
        try:
            with self._conn.cursor() as cur:
                cur.execute(
                    "INSERT INTO ai_monitor_email_queue "
                    "(subscription_id, issue_id, recipient_email, subject, body_html, body_text) "
                    "VALUES (%s, %s, %s, %s, %s, %s)",
                    (subscription_id, issue_id, recipient_email, subject, body_html, body_text)
                )
        except Exception as exc:
            logger.error("Failed to queue email notification: %s", exc)

    def mark_issue_email_sent(self, issue_id):
        """Mark that an email notification was queued for this issue."""
        if not self.available:
            return
        try:
            with self._conn.cursor() as cur:
                cur.execute(
                    "UPDATE ai_monitor_issues SET email_sent = 1, email_sent_at = NOW() WHERE id = %s",
                    (issue_id,)
                )
        except Exception as exc:
            logger.error("Failed to mark issue email sent: %s", exc)

    @staticmethod
    def _sanitize_log(text):
        """Remove server-internal paths and IPs from log text for customer display."""
        if not text:
            return text
        # Strip server-internal paths
        text = re.sub(r'/docker/whp/[^\s]*', '[server-path]', text)
        text = re.sub(r'/etc/(httpd|php-fpm\.d|haproxy|proftpd)[^\s]*', '[server-config]', text)
        text = re.sub(r'/root/[^\s]*', '[server-path]', text)
        text = re.sub(r'/var/log/[^\s]*', '[server-log]', text)
        # Strip internal IPs (keep public-facing domains)
        text = re.sub(r'\b(10\.\d+\.\d+\.\d+|172\.(1[6-9]|2\d|3[01])\.\d+\.\d+|192\.168\.\d+\.\d+)\b',
                       '[internal-ip]', text)
        return text[:2000]  # Limit excerpt length


# ===================================================================
# Log Sources
# ===================================================================
class LogSource(ABC):
    """Base class for all log sources with position tracking."""

    def __init__(self, source_id, db):
        self.source_id = source_id
        self.db = db

    @abstractmethod
    def read_new_entries(self):
        """Return a list of new log entry strings since last read."""

    def _group_multiline(self, lines):
        """Group multi-line log entries (stack traces etc.) into single entries."""
        if not lines:
            return []

        grouped = []
        current = []
        multiline_pattern = re.compile(
            r"^(\s+|Traceback|#\d+|at\s+|Caused by|\.\.\.|\}\s*$|\]\s*$)"
        )
        timestamp_pattern = re.compile(
            r"^(\[\d{4}|[A-Z][a-z]{2}\s+\d|"
            r"\d{4}[-/]\d{2}[-/]\d{2}|"
            r"\d{2}:\d{2}:\d{2})"
        )

        for line in lines:
            stripped = line.rstrip()
            if not stripped:
                continue
            # A new entry starts with a timestamp or doesn't look like a continuation
            if current and (timestamp_pattern.match(stripped)
                           and not multiline_pattern.match(stripped)):
                grouped.append("\n".join(current))
                current = [stripped]
            else:
                current.append(stripped)

        if current:
            grouped.append("\n".join(current))

        return grouped


class FileLogSource(LogSource):
    """Reads from a file-based log with inode tracking for rotation detection."""

    def __init__(self, source_id, db, filepath):
        super().__init__(source_id, db)
        self.filepath = filepath

    def read_new_entries(self):
        if not os.path.isfile(self.filepath):
            logger.debug("Log file %s does not exist, skipping", self.filepath)
            return []

        try:
            stat = os.stat(self.filepath)
        except OSError as exc:
            logger.warning("Cannot stat %s: %s", self.filepath, exc)
            return []

        current_inode = stat.st_ino
        pos_data = self.db.get_position(self.source_id)
        last_pos = pos_data["last_position"]
        last_inode = pos_data["last_inode"]

        # Detect log rotation: inode changed or file is smaller than last position
        if last_inode != 0 and (current_inode != last_inode or stat.st_size < last_pos):
            logger.info("Log rotation detected for %s (inode %d -> %d)",
                        self.filepath, last_inode, current_inode)
            last_pos = 0

        if stat.st_size <= last_pos:
            # No new data
            self.db.set_position(self.source_id, last_pos, current_inode, "")
            return []

        lines = []
        try:
            with open(self.filepath, "r", errors="replace") as fh:
                fh.seek(last_pos)
                raw = fh.read()
                new_pos = fh.tell()
                lines = raw.splitlines()
        except OSError as exc:
            logger.error("Cannot read %s: %s", self.filepath, exc)
            return []

        self.db.set_position(self.source_id, new_pos, current_inode, "")
        return self._group_multiline(lines)


class DockerLogSource(LogSource):
    """Reads logs from a Docker container via docker logs --since."""

    def __init__(self, source_id, db, container_name):
        super().__init__(source_id, db)
        self.container_name = container_name

    def read_new_entries(self):
        pos_data = self.db.get_position(self.source_id)
        last_ts = pos_data["last_timestamp"]

        cmd = ["docker", "logs", self.container_name, "--tail", "500"]
        if last_ts:
            cmd.extend(["--since", last_ts])
        else:
            # First run: only get last 5 minutes
            cmd.extend(["--since", "5m"])

        try:
            result = subprocess.run(
                cmd, capture_output=True, text=True, timeout=30
            )
            # Docker logs writes to both stdout and stderr
            output = result.stdout + result.stderr
        except subprocess.TimeoutExpired:
            logger.warning("Timeout reading docker logs for %s",
                           self.container_name)
            return []
        except FileNotFoundError:
            logger.warning("docker command not found")
            return []

        new_ts = time.strftime("%Y-%m-%dT%H:%M:%S")
        self.db.set_position(self.source_id, 0, 0, new_ts)

        lines = output.splitlines()
        return self._group_multiline(lines)


class DmesgSource(LogSource):
    """Reads kernel messages via dmesg -T."""

    def __init__(self, db):
        super().__init__("dmesg", db)

    def read_new_entries(self):
        pos_data = self.db.get_position(self.source_id)
        last_ts_str = pos_data["last_timestamp"]

        try:
            result = subprocess.run(
                ["dmesg", "-T"], capture_output=True, text=True, timeout=30
            )
            output = result.stdout
        except subprocess.TimeoutExpired:
            logger.warning("Timeout running dmesg")
            return []
        except FileNotFoundError:
            logger.warning("dmesg command not found")
            return []

        lines = output.splitlines()
        if not lines:
            return []

        # Only return lines newer than last check
        new_lines = []
        if last_ts_str:
            found_new = False
            for line in lines:
                if found_new:
                    new_lines.append(line)
                elif last_ts_str in line:
                    found_new = True
                    # Skip the matching line itself (already processed)
        else:
            # First run: take only last 50 lines
            new_lines = lines[-50:]

        # Record last line as timestamp marker
        if lines:
            # Extract the timestamp portion from the last dmesg line
            # Format: [Tue Mar 31 10:00:00 2026] message
            last_line = lines[-1]
            ts_match = re.match(r"^\[([^\]]+)\]", last_line)
            if ts_match:
                self.db.set_position(self.source_id, 0, 0, ts_match.group(0))
            else:
                self.db.set_position(self.source_id, 0, 0, last_line[:80])

        return self._group_multiline(new_lines)


class JournalSource(LogSource):
    """Reads from systemd journal for a specific unit."""

    def __init__(self, source_id, db, service_name):
        super().__init__(source_id, db)
        self.service_name = service_name

    def read_new_entries(self):
        pos_data = self.db.get_position(self.source_id)
        last_ts = pos_data["last_timestamp"]

        cmd = ["journalctl", "-u", self.service_name, "--no-pager", "-q"]
        if last_ts:
            cmd.extend(["--since", last_ts])
        else:
            # First run: last 5 minutes
            since = time.strftime(
                "%Y-%m-%d %H:%M:%S",
                time.localtime(time.time() - 300)
            )
            cmd.extend(["--since", since])

        try:
            result = subprocess.run(
                cmd, capture_output=True, text=True, timeout=30
            )
            output = result.stdout
        except subprocess.TimeoutExpired:
            logger.warning("Timeout reading journal for %s", self.service_name)
            return []
        except FileNotFoundError:
            logger.warning("journalctl command not found")
            return []

        new_ts = time.strftime("%Y-%m-%d %H:%M:%S")
        self.db.set_position(self.source_id, 0, 0, new_ts)

        lines = output.splitlines()
        return self._group_multiline(lines)


# ===================================================================
# Error Classifier
# ===================================================================
class ErrorClassifier:
    """Classifies log messages into severity levels and categories."""

    _monitored_sites = set()

    CRITICAL_PATTERNS = [
        (re.compile(r"oom-kill|Out of memory|Killed process", re.IGNORECASE),
         "oom"),
        (re.compile(r"(exited|died|stopped|unhealthy)"),
         "container_failure"),
        (re.compile(r"(failed|dead|inactive)"),
         "service_failure"),
        (re.compile(r"Segmentation fault|segfault|core dumped", re.IGNORECASE),
         "crash"),
        (re.compile(r"Fatal error.*Allowed memory size", re.IGNORECASE),
         "php_oom"),
        (re.compile(r"server reached.*max_children", re.IGNORECASE),
         "php_fpm_exhaustion"),
        (re.compile(
            r"Syntax error|failed to parse|configuration file.*error",
            re.IGNORECASE),
         "config_error"),
        (re.compile(r"No space left on device", re.IGNORECASE),
         "disk_full"),
        (re.compile(r"FATAL|EMERGENCY|PANIC"),  # case sensitive
         "fatal_error"),
    ]

    # Source-specific critical patterns (only match against certain sources)
    SOURCE_CRITICAL = {
        "container_failure": re.compile(r"docker|container", re.IGNORECASE),
        "service_failure": re.compile(r"journal|systemd|service", re.IGNORECASE),
    }

    NOISE_PATTERNS = [
        re.compile(r"File does not exist|script not found", re.IGNORECASE),
        re.compile(r"^\s*$"),
    ]

    # PHP/PHP-FPM errors in customer sites — these are the customer's problem
    # unless they pay for container monitoring. Matches paths like /docker/users/
    CUSTOMER_PHP_PATTERN = re.compile(
        r"(Fatal error.*Allowed memory size|PHP Fatal error|PHP Warning|PHP Notice)"
        r".*(/docker/users/|/home/)",
        re.IGNORECASE,
    )

    # OOM events from customer containers in dmesg — these are customer resource
    # issues, not system-level emergencies. Customer container names follow the
    # pattern domain.tld-NN (e.g., peptidesaver.net-01, example.com-01).
    # System containers (mysql, haproxy-manager, memcache, postgresql, shared-httpd-*)
    # should still trigger critical alerts.
    SYSTEM_CONTAINERS = re.compile(
        r"(mysql|haproxy-manager|memcache|postgresql|shared-httpd)",
        re.IGNORECASE,
    )
    # Matches dmesg OOM messages from Docker containers. Kernel OOM dumps include
    # the cgroup path (e.g., /docker/<container_id>) or the container name in
    # various positions. We check for:
    # 1. Domain-like container names anywhere in the multiline dump
    # 2. Docker cgroup paths (/docker/<hex_id> or /system.slice/docker-<hex_id>)
    CUSTOMER_CONTAINER_OOM_PATTERN = re.compile(
        r"(oom-kill|Out of memory|Killed process|oom_kill_process|memory cgroup)",
        re.IGNORECASE,
    )
    # Extract container name from anywhere in the OOM dump text
    DOCKER_CONTAINER_NAME_PATTERN = re.compile(
        r"(?:docker[/-])([0-9a-f]{12,64})"  # Docker cgroup with container ID
        r"|"
        r"(?:task_css_set|oom-kill:.*?task=|Killed process \d+ \()"
        r"([a-z0-9][-a-z0-9]*\.[a-z]{2,})",  # Container name in task info
        re.IGNORECASE,
    )
    # Broader pattern: find domain-like container names (domain.tld-NN) anywhere
    DOMAIN_CONTAINER_PATTERN = re.compile(
        r"\b([a-z0-9][-a-z0-9]*\.[a-z]{2,})-\d{1,2}\b",
        re.IGNORECASE,
    )

    NOISE_404_PATTERN = re.compile(r"404", re.IGNORECASE)
    APACHE_ACCESS_CONTEXT = re.compile(
        r"(access_log|access\.log|customer|apache.*access)", re.IGNORECASE
    )

    NON_CRITICAL_PATTERN = re.compile(
        r"error|warning|fail|exception", re.IGNORECASE
    )

    INFO_PATTERN = re.compile(
        r"^\[?(INFO|NOTICE|DEBUG)\]?\s|"
        r"(successfully|completed|started|loaded|enabled|ready|listening)",
        re.IGNORECASE,
    )

    def classify(self, message, source):
        """Classify a log message. Returns (severity, category) or None to skip."""
        if not message or not message.strip():
            return None

        # Noise filter
        for pattern in self.NOISE_PATTERNS:
            if pattern.search(message):
                return None

        # PHP errors in customer sites — skip unless paid monitoring
        if self.CUSTOMER_PHP_PATTERN.search(message):
            # Check if this is from a monitored site
            domain_match = self.DOMAIN_CONTAINER_PATTERN.search(message)
            if domain_match and self._is_monitored_site(domain_match.group(0)):
                return ("customer_php_error", domain_match.group(0))
            # Also check the path for username-based matching
            path_match = re.search(r'/docker/users/([^/]+)/', message)
            if path_match:
                username = path_match.group(1)
                # Check if any monitored site belongs to this user
                site_domain = self._get_monitored_domain_for_user(username)
                if site_domain:
                    return ("customer_php_error", site_domain)
            return None

        # OOM events from customer containers — skip unless it's a system
        # container or a customer container with paid monitoring.
        # Health checks handle customer container restarts automatically.
        if self.CUSTOMER_CONTAINER_OOM_PATTERN.search(message):
            container_name = self._identify_oom_container(message)
            if container_name:
                # System container OOM → stay critical (handled by normal path)
                if self.SYSTEM_CONTAINERS.search(container_name):
                    pass  # fall through to critical patterns below
                # Monitored customer site → send advisory notice
                # Match by site domain prefix (e.g., "peptidesaver.net" matches
                # "peptidesaver.net-01", "peptidesaver.net-02", etc.)
                elif self._is_monitored_site(container_name):
                    return ("customer_oom", container_name)
                # Unmonitored customer container → skip entirely
                else:
                    logger.debug(
                        "Ignoring OOM from unmonitored container: %s",
                        container_name,
                    )
                    return None

    def set_monitored_sites(self, sites):
        """Set the list of customer site domains with paid monitoring.

        Sites are matched as prefixes against container names, so
        'example.com' matches 'example.com-01', 'example.com-02', etc.
        """
        self._monitored_sites = set(s.lower() for s in sites)

    def _is_monitored_site(self, container_name):
        """Check if a container belongs to a monitored customer site."""
        name_lower = container_name.lower()
        for site in self._monitored_sites:
            if name_lower.startswith(site):
                return True
        return False

    _monitored_user_domains = {}  # username -> primary_domain mapping

    def set_monitored_user_domains(self, user_domains):
        """Set mapping of usernames to their monitored site domains."""
        self._monitored_user_domains = {u.lower(): d for u, d in user_domains.items()}

    def _get_monitored_domain_for_user(self, username):
        """Get the monitored domain for a username, if any."""
        return self._monitored_user_domains.get(username.lower())

    def _identify_oom_container(self, message):
        """Extract container name from a dmesg OOM message.

        Returns the container name, or None if unidentifiable.
        """
        # Check for domain-like container names (e.g., peptidesaver.net-01)
        domain_match = self.DOMAIN_CONTAINER_PATTERN.search(message)
        if domain_match:
            return domain_match.group(0)

        # Check for Docker container IDs in cgroup paths and resolve them
        docker_match = self.DOCKER_CONTAINER_NAME_PATTERN.search(message)
        if docker_match:
            cid = docker_match.group(1) or docker_match.group(2) or ""
            if re.match(r"^[0-9a-f]{12,}$", cid):
                resolved = resolve_container_ids(cid)
                if resolved != cid:
                    return resolved

        return None

        # Customer 404s in apache access context
        if (self.NOISE_404_PATTERN.search(message)
                and self.APACHE_ACCESS_CONTEXT.search(source)):
            return None

        # Skip purely informational messages
        if self.INFO_PATTERN.search(message):
            # Unless they also contain critical patterns
            has_critical = False
            for pattern, _ in self.CRITICAL_PATTERNS:
                if pattern.search(message):
                    has_critical = True
                    break
            if not has_critical:
                return None

        # Check critical patterns
        for pattern, category in self.CRITICAL_PATTERNS:
            if pattern.search(message):
                # Some categories are source-specific
                if category in self.SOURCE_CRITICAL:
                    source_pattern = self.SOURCE_CRITICAL[category]
                    if not source_pattern.search(source):
                        continue
                return ("critical", category)

        # Non-critical: anything that looks like an error
        if self.NON_CRITICAL_PATTERN.search(message):
            return ("non_critical", "general_error")

        return None


# ===================================================================
# Error Signature
# ===================================================================
class ErrorSignature:
    """Generates deduplication signatures for error messages."""

    # Patterns to normalize out of messages before hashing
    NORMALIZE_PATTERNS = [
        # Timestamps: various formats
        (re.compile(
            r"\d{4}[-/]\d{2}[-/]\d{2}[T\s]\d{2}:\d{2}:\d{2}(\.\d+)?"
            r"(\s*[+-]\d{2}:?\d{2})?"),
         "<TS>"),
        (re.compile(
            r"[A-Z][a-z]{2}\s+\d{1,2}\s+\d{2}:\d{2}:\d{2}"),
         "<TS>"),
        (re.compile(
            r"\[\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2}\s*[^\]]*\]"),
         "<TS>"),
        # PIDs
        (re.compile(r"\[\d{2,7}\]"), "[PID]"),
        (re.compile(r"pid\s*[:=]\s*\d+", re.IGNORECASE), "pid=PID"),
        # IP addresses (v4)
        (re.compile(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}(:\d+)?"), "<IP>"),
        # Memory addresses
        (re.compile(r"0x[0-9a-fA-F]+"), "0xADDR"),
        # Line numbers in code references
        (re.compile(r"(line\s*)\d+", re.IGNORECASE), r"\1NUM"),
        (re.compile(r"(:\d+)(?=\s|$|\))"), ":NUM"),
        # Large numeric IDs / sequence numbers
        (re.compile(r"(?<=[^.\d])\d{6,}(?=[^.\d])"), "NUMID"),
        # Dmesg OOM specific: normalize memory sizes (e.g., "anon:869MB", "1024 pages")
        (re.compile(r"\d+\s*(MB|KB|GB|pages|bytes)", re.IGNORECASE), "MEMVAL"),
        # Dmesg OOM: normalize "Killed process NNNN (name)"
        (re.compile(r"Killed process \d+", re.IGNORECASE), "Killed process PID"),
        # Dmesg: normalize failcnt values
        (re.compile(r"failcnt\s*[:=]?\s*\d+", re.IGNORECASE), "failcnt=N"),
    ]

    @staticmethod
    def generate(message):
        """Generate a SHA-256 signature for an error message."""
        normalized = message.strip()
        for pattern, replacement in ErrorSignature.NORMALIZE_PATTERNS:
            normalized = pattern.sub(replacement, normalized)
        # Collapse whitespace
        normalized = re.sub(r"\s+", " ", normalized).strip()
        return hashlib.sha256(normalized.encode("utf-8")).hexdigest()


# ===================================================================
# AI Analyzer
# ===================================================================
AI_SYSTEM_PROMPT = """You are a Linux systems administration expert analyzing error logs from a WHP (Web Hosting Panel) server running CentOS/RHEL with:
- Apache httpd (port 8080, behind HAProxy) with PHP-FPM 8.2
- HAProxy in Docker container "haproxy-manager" (ports 80/443/8000)
- MySQL/MariaDB 10.11 (Docker "mysql"), PostgreSQL (Docker "postgresql")
- Memcache (Docker "memcache"), ProFTPD (systemd)
- Web app at /docker/whp/web/, scripts at /root/whp/scripts/

Respond with ONLY valid JSON matching the required schema. No markdown, no explanation outside JSON.

For fix_type:
- restart_service: target must be one of: httpd, php-fpm, proftpd, whp-terminal, whp-backup-upload
- restart_container: target must be one of: mysql, postgresql, haproxy-manager, memcache
- edit_config: target must be an allowed config path
- manual_only: for anything risky or uncertain

If risk_level is "high", always use fix_type "manual_only".
Never suggest destructive actions, firewall changes, or package installations."""

CRITICAL_USER_PROMPT = """Analyze this critical error from a WHP server.

Source: {source}
Category: {category}
Error message:
```
{message}
```

System context:
{context}

Respond with ONLY this JSON schema:
{{
  "diagnosis": "string - what is happening and why",
  "needs_escalation": false,
  "severity_confirmed": "critical or non_critical",
  "fix_type": "restart_service|restart_container|edit_config|manual_only",
  "fix_details": {{
    "target": "service or container name or config path",
    "command": "specific command to run if applicable",
    "config_changes": "what config changes if edit_config",
    "verification_command": "command to verify fix worked"
  }},
  "risk_level": "low|medium|high",
  "explanation": "human-readable explanation for notification"
}}"""

ESCALATION_USER_PROMPT = """A basic AI analysis flagged this issue for escalation. Please provide a more thorough analysis.

Original error:
Source: {source}
Category: {category}
Message:
```
{message}
```

Basic analysis:
```json
{basic_analysis}
```

System context:
{context}

Provide an improved JSON response with the same schema, with more detailed diagnosis and safer fix recommendations."""

DAILY_BATCH_PROMPT = """Analyze this batch of non-critical errors from the past day on a WHP server.
Group them by pattern and provide recommendations.

Errors by category:
{errors_json}

Respond with ONLY this JSON:
{{
  "summary": "brief overall assessment",
  "patterns": [
    {{
      "category": "string",
      "count": 0,
      "description": "what these errors mean",
      "recommendation": "what to do",
      "priority": "low|medium|high"
    }}
  ],
  "immediate_actions": ["list of things to do now if any"],
  "monitoring_suggestions": ["list of things to watch"]
}}"""


class AIAnalyzer:
    """Sends error data to an AI model for diagnosis and fix suggestions."""

    def __init__(self, config):
        self.config = config
        self._calls_this_run = 0
        self._client = None

        if openai is None:
            logger.error(_OPENAI_IMPORT_ERROR)
            return

        if not config.ai_api_key:
            logger.error("No AI API key configured. AI analysis will be skipped.")
            return

        self._client = openai.OpenAI(
            base_url=config.ai_base_url,
            api_key=config.ai_api_key,
        )

    def _can_call(self, db):
        """Check per-run and per-day API call limits."""
        if self._client is None:
            return False
        if self._calls_this_run >= self.config.max_calls_per_run:
            logger.warning("Reached max AI calls per run (%d)",
                           self.config.max_calls_per_run)
            return False
        if db.get_api_calls_today() >= self.config.max_calls_per_day:
            logger.warning("Reached max AI calls per day (%d)",
                           self.config.max_calls_per_day)
            return False
        return True

    def _call_model(self, model, system_prompt, user_prompt, db):
        """Make a single API call and return parsed JSON response."""
        if not self._can_call(db):
            return None

        try:
            response = self._client.chat.completions.create(
                model=model,
                messages=[
                    {"role": "system", "content": system_prompt},
                    {"role": "user", "content": user_prompt},
                ],
                temperature=0.1,
                max_tokens=2000,
                timeout=self.config.ai_timeout,
            )
            self._calls_this_run += 1
            db.increment_api_calls()

            content = response.choices[0].message.content.strip()

            # Strip markdown code fences if present
            if content.startswith("```"):
                lines = content.split("\n")
                # Remove first and last fence lines
                if lines[0].startswith("```"):
                    lines = lines[1:]
                if lines and lines[-1].strip() == "```":
                    lines = lines[:-1]
                content = "\n".join(lines)

            return json.loads(content)

        except json.JSONDecodeError as exc:
            logger.error("AI returned invalid JSON: %s", exc)
            logger.debug("Raw response: %s", content if 'content' in dir() else "N/A")
            return None
        except openai.APITimeoutError:
            logger.error("AI API call timed out after %ds", self.config.ai_timeout)
            return None
        except openai.RateLimitError:
            logger.error("AI API rate limit reached")
            return None
        except openai.APIError as exc:
            logger.error("AI API error: %s", exc)
            return None
        except Exception as exc:
            logger.error("Unexpected error calling AI: %s", exc)
            return None

    def analyze_critical(self, error_message, category, context, source, db):
        """Analyze a critical error, with optional escalation to advanced model."""
        user_prompt = CRITICAL_USER_PROMPT.format(
            source=source,
            category=category,
            message=error_message[:3000],  # Limit message size
            context=context,
        )

        result = self._call_model(
            self.config.basic_model, AI_SYSTEM_PROMPT, user_prompt, db
        )

        if result is None:
            return self._fallback_response(error_message, category)

        # Escalate to advanced model if needed
        if result.get("needs_escalation", False):
            logger.info("Escalating to advanced model for deeper analysis")
            escalation_prompt = ESCALATION_USER_PROMPT.format(
                source=source,
                category=category,
                message=error_message[:3000],
                basic_analysis=json.dumps(result, indent=2),
                context=context,
            )
            advanced_result = self._call_model(
                self.config.advanced_model, AI_SYSTEM_PROMPT,
                escalation_prompt, db
            )
            if advanced_result is not None:
                result = advanced_result

        return result

    def analyze_daily_batch(self, errors_by_category, db):
        """Analyze a batch of non-critical errors grouped by category."""
        # Truncate to avoid huge prompts
        summary = {}
        for cat, errors in errors_by_category.items():
            summary[cat] = {
                "count": len(errors),
                "samples": [e["raw_message"][:500] for e in errors[:5]],
                "sources": list(set(e["source"] for e in errors)),
            }

        user_prompt = DAILY_BATCH_PROMPT.format(
            errors_json=json.dumps(summary, indent=2)
        )

        result = self._call_model(
            self.config.basic_model, AI_SYSTEM_PROMPT, user_prompt, db
        )

        if result is None:
            return {
                "summary": "AI analysis unavailable",
                "patterns": [],
                "immediate_actions": [],
                "monitoring_suggestions": [],
            }

        return result

    @staticmethod
    def _fallback_response(message, category):
        """Return a safe fallback when AI is unavailable."""
        return {
            "diagnosis": f"AI analysis unavailable. Category: {category}",
            "needs_escalation": False,
            "severity_confirmed": "critical",
            "fix_type": "manual_only",
            "fix_details": {
                "target": "",
                "command": "",
                "config_changes": "",
                "verification_command": "",
            },
            "risk_level": "high",
            "explanation": (
                f"Error detected in category '{category}' but AI analysis "
                f"is not available. Manual investigation required."
            ),
        }


# ===================================================================
# Container ID resolver
# ===================================================================
_container_name_cache = {}

def resolve_container_ids(text):
    """Replace Docker container ID hex strings with human-readable names."""
    def _lookup(match):
        cid = match.group(0)
        short = cid[:12]
        if short in _container_name_cache:
            return _container_name_cache[short]
        try:
            result = subprocess.run(
                ["docker", "inspect", "--format", "{{.Name}}", short],
                capture_output=True, text=True, timeout=5,
            )
            if result.returncode == 0:
                name = result.stdout.strip().lstrip("/")
                _container_name_cache[short] = name
                return name
        except Exception:
            pass
        _container_name_cache[short] = short
        return short

    # Match 12+ hex char sequences typical of container IDs
    return re.sub(r'[0-9a-f]{12,64}', _lookup, text)


# ===================================================================
# Notifier
# ===================================================================
class Notifier:
    """Sends notifications to configured platforms (Discord, Slack, Telegram)."""

    SEVERITY_PREFIXES = {
        "critical": "[CRITICAL]",
        "warning": "[WARNING]",
        "info": "[INFO]",
        "success": "[OK]",
    }

    def __init__(self, config):
        self.config = config

    def send(self, message, severity="info"):
        """Send notification to all configured platforms."""
        prefix = self.SEVERITY_PREFIXES.get(severity, "[INFO]")
        # Resolve any Docker container IDs to human-readable names
        message = resolve_container_ids(message)
        full_message = f"{prefix} WHP Monitor: {message}"

        if self.config.discord_webhook:
            self._send_discord(full_message, severity)
        if self.config.slack_webhook:
            self._send_slack(full_message)
        if self.config.telegram_token and self.config.telegram_chat_id:
            self._send_telegram(full_message)

        if not any([
            self.config.discord_webhook,
            self.config.slack_webhook,
            self.config.telegram_token,
        ]):
            logger.debug("No notification channels configured")

    def _send_discord(self, message, severity="info"):
        """Send notification to Discord webhook with optional role mention."""
        try:
            # Get role ID for this severity level
            role_id = self.config._get("notifications", f"discord_role_{severity}")
            mention = f"<@&{role_id}> " if role_id else ""
            # Discord has a 2000 character limit
            truncated = f"{mention}{message}"[:1990]
            payload = json.dumps({"content": truncated}).encode("utf-8")
            req = urllib.request.Request(
                self.config.discord_webhook,
                data=payload,
                headers={
                    "Content-Type": "application/json",
                    "User-Agent": "WHP-AI-Monitor/1.0",
                },
                method="POST",
            )
            urllib.request.urlopen(req, timeout=10)
            logger.debug("Discord notification sent")
        except (urllib.error.URLError, OSError) as exc:
            logger.warning("Failed to send Discord notification: %s", exc)

    def _send_slack(self, message):
        """Send notification to Slack webhook."""
        try:
            truncated = message[:3000]
            payload = json.dumps({"text": truncated}).encode("utf-8")
            req = urllib.request.Request(
                self.config.slack_webhook,
                data=payload,
                headers={
                    "Content-Type": "application/json",
                    "User-Agent": "WHP-AI-Monitor/1.0",
                },
                method="POST",
            )
            urllib.request.urlopen(req, timeout=10)
            logger.debug("Slack notification sent")
        except (urllib.error.URLError, OSError) as exc:
            logger.warning("Failed to send Slack notification: %s", exc)

    def _send_telegram(self, message):
        """Send notification to Telegram bot."""
        try:
            truncated = message[:4000]
            url = (
                f"https://api.telegram.org/"
                f"bot{self.config.telegram_token}/sendMessage"
            )
            payload = json.dumps({
                "chat_id": self.config.telegram_chat_id,
                "text": truncated,
                "parse_mode": "Markdown",
            }).encode("utf-8")
            req = urllib.request.Request(
                url,
                data=payload,
                headers={
                    "Content-Type": "application/json",
                    "User-Agent": "WHP-AI-Monitor/1.0",
                },
                method="POST",
            )
            urllib.request.urlopen(req, timeout=10)
            logger.debug("Telegram notification sent")
        except (urllib.error.URLError, OSError) as exc:
            logger.warning("Failed to send Telegram notification: %s", exc)


# ===================================================================
# Remediation Engine
# ===================================================================
class RemediationEngine:
    """Executes safe auto-remediation actions based on AI recommendations."""

    def __init__(self, config, db, notifier):
        self.config = config
        self.db = db
        self.notifier = notifier
        self._consecutive_failures = 0

    def can_remediate(self):
        """Check if remediation is allowed (circuit breaker + hourly limits)."""
        if not self.config.remediation_enabled:
            logger.info("Remediation is disabled in config")
            return False

        if self._consecutive_failures >= self.config.circuit_breaker_threshold:
            logger.warning(
                "Circuit breaker open: %d consecutive failures (threshold %d)",
                self._consecutive_failures,
                self.config.circuit_breaker_threshold,
            )
            self.notifier.send(
                "Circuit breaker OPEN: too many consecutive remediation failures. "
                "Manual intervention required.",
                severity="critical",
            )
            return False

        # Check hourly limit across all targets
        cutoff = int(time.time()) - 3600
        try:
            conn = self.db._conn
            row = conn.execute(
                "SELECT COUNT(*) as cnt FROM remediation_log WHERE timestamp > ?",
                (cutoff,),
            ).fetchone()
            if row and row["cnt"] >= self.config.max_remediations_per_hour:
                logger.warning(
                    "Hourly remediation limit reached (%d/%d)",
                    row["cnt"], self.config.max_remediations_per_hour,
                )
                return False
        except sqlite3.Error:
            pass

        return True

    def _check_cooldown(self, target):
        """Check if target was recently remediated (cooldown period)."""
        recent = self.db.get_recent_remediations(
            target, hours=self.config.cooldown_minutes / 60.0 or 1
        )
        if recent:
            # Check if any remediation happened within cooldown period
            cooldown_seconds = self.config.cooldown_minutes * 60
            now = int(time.time())
            for r in recent:
                if now - r["timestamp"] < cooldown_seconds:
                    logger.info(
                        "Target '%s' is in cooldown (last remediation %ds ago)",
                        target, now - r["timestamp"],
                    )
                    return False
        return True

    def execute(self, error_id, ai_response, dry_run=False):
        """Execute remediation based on AI response. Returns True on success."""
        fix_type = ai_response.get("fix_type", "manual_only")
        risk_level = ai_response.get("risk_level", "high")
        fix_details = ai_response.get("fix_details", {})
        target = fix_details.get("target", "")
        explanation = ai_response.get("explanation", "")

        # Safety checks
        if risk_level == "high":
            logger.info("High risk level - logging recommendation only")
            self.notifier.send(
                f"High-risk issue detected. Manual action needed:\n"
                f"{explanation}\nTarget: {target}",
                severity="critical",
            )
            self.db.add_remediation(
                error_id, "manual_only", target or "unknown",
                command_run="N/A (high risk)", success=0,
            )
            return False

        if fix_type == "manual_only":
            logger.info("Fix type is manual_only - logging recommendation")
            self.notifier.send(
                f"Manual action recommended:\n{explanation}\nTarget: {target}",
                severity="warning",
            )
            self.db.add_remediation(
                error_id, "manual_only", target or "unknown",
                command_run="N/A (manual)", success=0,
            )
            return False

        if not self.can_remediate():
            return False

        if not self._check_cooldown(target):
            self.notifier.send(
                f"Remediation skipped for '{target}' (cooldown active).",
                severity="info",
            )
            return False

        if dry_run:
            logger.info("[DRY RUN] Would execute %s on %s", fix_type, target)
            self.notifier.send(
                f"[DRY RUN] Would {fix_type}: {target}\n{explanation}",
                severity="info",
            )
            return True

        # Dispatch to appropriate handler
        success = False
        if fix_type == "restart_service":
            success = self._restart_service(error_id, target)
        elif fix_type == "restart_container":
            success = self._restart_container(error_id, target)
        elif fix_type == "edit_config":
            success = self._edit_config(
                error_id, target, fix_details, ai_response
            )
        else:
            logger.warning("Unknown fix_type: %s", fix_type)
            self.db.add_remediation(
                error_id, fix_type, target or "unknown",
                command_run=f"Unknown fix_type: {fix_type}", success=0,
            )
            return False

        # Update circuit breaker state
        if success:
            self._consecutive_failures = 0
            self.notifier.send(
                f"Auto-remediation SUCCESS: {fix_type} on {target}\n"
                f"{explanation}",
                severity="success",
            )
        else:
            self._consecutive_failures += 1
            self.notifier.send(
                f"Auto-remediation FAILED: {fix_type} on {target}\n"
                f"{explanation}\n"
                f"Consecutive failures: {self._consecutive_failures}/"
                f"{self.config.circuit_breaker_threshold}",
                severity="critical",
            )

        return success

    def _restart_service(self, error_id, service):
        """Restart a systemd service with pre/post verification."""
        if service not in self.config.allowed_services:
            logger.error("Service '%s' not in allowed list", service)
            self.db.add_remediation(
                error_id, "restart_service", service,
                command_run="BLOCKED: not in allowed_services", success=0,
            )
            return False

        # Check if service is actually down
        pre_status = self._run_cmd(
            ["systemctl", "is-active", service], timeout=10
        )
        logger.info(
            "Pre-restart status for %s: %s",
            service, pre_status.get("stdout", "").strip(),
        )

        # Execute restart
        cmd = ["systemctl", "restart", service]
        result = self._run_cmd(cmd, timeout=60)

        # Verify service came up (wait briefly)
        time.sleep(2)
        post_status = self._run_cmd(
            ["systemctl", "is-active", service], timeout=10
        )
        is_active = post_status.get("stdout", "").strip() == "active"

        self.db.add_remediation(
            error_id,
            action_type="restart_service",
            target=service,
            command_run=" ".join(cmd),
            stdout=result.get("stdout", ""),
            stderr=result.get("stderr", ""),
            exit_code=result.get("exit_code", -1),
            success=1 if is_active else 0,
        )

        if is_active:
            logger.info("Service %s restarted successfully", service)
        else:
            logger.error("Service %s failed to start after restart", service)

        return is_active

    def _restart_container(self, error_id, container):
        """Restart a Docker container with fallback to container-manager.sh."""
        if container not in self.config.allowed_containers:
            logger.error("Container '%s' not in allowed list", container)
            self.db.add_remediation(
                error_id, "restart_container", container,
                command_run="BLOCKED: not in allowed_containers", success=0,
            )
            return False

        # Check current container status
        pre_status = self._get_container_health(container)
        logger.info("Pre-restart container %s status: %s", container, pre_status)

        # Attempt docker restart first
        cmd = ["docker", "restart", container]
        result = self._run_cmd(cmd, timeout=120)

        # Verify container is running
        time.sleep(3)
        post_status = self._get_container_health(container)
        is_running = post_status in ("running", "healthy")

        if not is_running:
            # Fallback: use container-manager.sh recreate
            logger.warning(
                "docker restart failed for %s, trying container-manager.sh",
                container,
            )
            fallback_cmd = [
                "/root/whp/scripts/container-manager.sh", "recreate", container
            ]
            fallback_result = self._run_cmd(fallback_cmd, timeout=180)

            time.sleep(5)
            post_status = self._get_container_health(container)
            is_running = post_status in ("running", "healthy")

            # Record the fallback attempt
            self.db.add_remediation(
                error_id,
                action_type="restart_container",
                target=container,
                command_run=" ".join(fallback_cmd),
                stdout=fallback_result.get("stdout", ""),
                stderr=fallback_result.get("stderr", ""),
                exit_code=fallback_result.get("exit_code", -1),
                success=1 if is_running else 0,
            )
        else:
            self.db.add_remediation(
                error_id,
                action_type="restart_container",
                target=container,
                command_run=" ".join(cmd),
                stdout=result.get("stdout", ""),
                stderr=result.get("stderr", ""),
                exit_code=result.get("exit_code", -1),
                success=1 if is_running else 0,
            )

        if is_running:
            logger.info("Container %s restarted successfully", container)
        else:
            logger.error("Container %s failed to restart", container)

        return is_running

    def _edit_config(self, error_id, target, fix_details, ai_response):
        """Edit a configuration file with backup and rollback support."""
        config_changes = fix_details.get("config_changes", "")
        verification_cmd = fix_details.get("verification_command", "")

        if not target or not config_changes:
            logger.error("Missing target or config_changes for edit_config")
            self.db.add_remediation(
                error_id, "edit_config", target or "unknown",
                command_run="BLOCKED: missing details", success=0,
            )
            return False

        # Verify target is in allowed paths
        allowed = False
        for allowed_path in self.config.allowed_config_paths:
            if target.startswith(allowed_path):
                allowed = True
                break

        if not allowed:
            logger.error("Config path '%s' not in allowed paths", target)
            self.db.add_remediation(
                error_id, "edit_config", target,
                command_run="BLOCKED: path not allowed", success=0,
            )
            return False

        if not os.path.isfile(target):
            logger.error("Config file '%s' does not exist", target)
            self.db.add_remediation(
                error_id, "edit_config", target,
                command_run="BLOCKED: file not found", success=0,
            )
            return False

        # Create backup
        backup_dir = self.config.backup_dir
        os.makedirs(backup_dir, mode=0o750, exist_ok=True)
        timestamp = time.strftime("%Y%m%d_%H%M%S")
        backup_name = os.path.basename(target) + f".{timestamp}.bak"
        backup_path = os.path.join(backup_dir, backup_name)

        try:
            with open(target, "r") as src:
                original_content = src.read()
            with open(backup_path, "w") as dst:
                dst.write(original_content)
            logger.info("Backed up %s to %s", target, backup_path)
        except OSError as exc:
            logger.error("Failed to backup %s: %s", target, exc)
            self.db.add_remediation(
                error_id, "edit_config", target,
                command_run=f"BLOCKED: backup failed: {exc}", success=0,
            )
            return False

        # Apply the change via the AI-suggested command
        command = fix_details.get("command", "")
        if not command:
            logger.error("No command provided for config edit")
            self.db.add_remediation(
                error_id, "edit_config", target,
                backup_path=backup_path,
                command_run="BLOCKED: no command provided", success=0,
            )
            return False

        result = self._run_cmd(["bash", "-c", command], timeout=30)

        if result.get("exit_code", -1) != 0:
            logger.error(
                "Config edit command failed (exit %d): %s",
                result["exit_code"], result.get("stderr", ""),
            )
            # Rollback
            self._rollback_config(target, backup_path)
            self.db.add_remediation(
                error_id, "edit_config", target,
                backup_path=backup_path,
                command_run=command,
                stdout=result.get("stdout", ""),
                stderr=result.get("stderr", ""),
                exit_code=result.get("exit_code", -1),
                success=0,
            )
            return False

        # Run syntax check / verification if provided
        if verification_cmd:
            verify_result = self._run_cmd(
                ["bash", "-c", verification_cmd], timeout=30
            )
            if verify_result.get("exit_code", -1) != 0:
                logger.error(
                    "Config verification failed, rolling back: %s",
                    verify_result.get("stderr", ""),
                )
                self._rollback_config(target, backup_path)
                self.db.add_remediation(
                    error_id, "edit_config", target,
                    backup_path=backup_path,
                    command_run=f"{command} && {verification_cmd}",
                    stdout=verify_result.get("stdout", ""),
                    stderr=verify_result.get("stderr", ""),
                    exit_code=verify_result.get("exit_code", -1),
                    success=0,
                )
                return False

        self.db.add_remediation(
            error_id, "edit_config", target,
            backup_path=backup_path,
            command_run=command,
            stdout=result.get("stdout", ""),
            stderr=result.get("stderr", ""),
            exit_code=result.get("exit_code", -1),
            success=1,
        )
        logger.info("Config edit applied successfully to %s", target)
        return True

    def _rollback_config(self, target, backup_path):
        """Restore a config file from backup."""
        try:
            with open(backup_path, "r") as src:
                content = src.read()
            with open(target, "w") as dst:
                dst.write(content)
            logger.info("Rolled back %s from %s", target, backup_path)
        except OSError as exc:
            logger.error("ROLLBACK FAILED for %s: %s", target, exc)

    def _get_container_health(self, container):
        """Get the running status of a Docker container."""
        result = self._run_cmd(
            ["docker", "inspect", "--format", "{{.State.Status}}", container],
            timeout=10,
        )
        return result.get("stdout", "").strip()

    @staticmethod
    def _run_cmd(cmd, timeout=30):
        """Run a subprocess command safely with timeout."""
        try:
            result = subprocess.run(
                cmd, capture_output=True, text=True, timeout=timeout
            )
            return {
                "stdout": result.stdout,
                "stderr": result.stderr,
                "exit_code": result.returncode,
            }
        except subprocess.TimeoutExpired:
            logger.error("Command timed out after %ds: %s", timeout, " ".join(cmd))
            return {"stdout": "", "stderr": "TIMEOUT", "exit_code": -1}
        except FileNotFoundError:
            logger.error("Command not found: %s", cmd[0])
            return {"stdout": "", "stderr": "COMMAND_NOT_FOUND", "exit_code": -1}
        except OSError as exc:
            logger.error("Command execution error: %s", exc)
            return {"stdout": "", "stderr": str(exc), "exit_code": -1}


# ===================================================================
# Main Orchestrator
# ===================================================================
class WHPLogMonitor:
    """Main orchestrator that ties all components together."""

    def __init__(self, config, dry_run=False, verbose=False):
        self.config = config
        self.dry_run = dry_run or config.dry_run
        self.verbose = verbose
        self.db = Database(config.db_path)
        self.classifier = ErrorClassifier()
        self.ai = AIAnalyzer(config)
        self.notifier = Notifier(config)
        self.remediator = RemediationEngine(config, self.db, self.notifier)
        # Customer monitoring MySQL connection
        self.customer_db = CustomerDatabase()
        self._subscriptions = {}  # domain -> subscription dict
        self._load_customer_subscriptions()
        self._sources = []
        self._init_sources()

    def _load_customer_subscriptions(self):
        """Load active customer monitoring subscriptions from MySQL."""
        if not self.customer_db.available:
            logger.info("Customer MySQL not available — customer monitoring disabled")
            return

        # Check global toggle in settings.json
        try:
            with open('/docker/whp/settings.json', 'r') as f:
                settings = json.load(f)
            if not settings.get('customer_monitoring', {}).get('enabled', True):
                logger.info("Customer monitoring globally disabled in settings")
                return
        except (FileNotFoundError, json.JSONDecodeError):
            pass  # If settings.json missing, allow customer monitoring

        subs = self.customer_db.get_active_subscriptions()
        if not subs:
            logger.info("No active customer monitoring subscriptions")
            return

        monitored_domains = []
        user_domain_map = {}
        for sub in subs:
            domain = sub['primary_domain'].lower()
            self._subscriptions[domain] = sub
            monitored_domains.append(domain)
            user_domain_map[sub['username']] = domain

        # Update the classifier with monitored sites from MySQL
        # This replaces the static INI config
        self.classifier.set_monitored_sites(monitored_domains)
        self.classifier.set_monitored_user_domains(user_domain_map)
        logger.info("Loaded %d customer monitoring subscriptions from MySQL", len(subs))

    def _init_sources(self):
        """Initialize all configured log sources."""
        # File-based sources
        for source_id, filepath in self.config.source_files.items():
            if os.path.exists(filepath) or os.path.exists(os.path.dirname(filepath)):
                self._sources.append(
                    FileLogSource(source_id, self.db, filepath)
                )
                logger.debug("Added file source: %s -> %s", source_id, filepath)
            else:
                logger.debug(
                    "Skipping file source %s: path %s not accessible",
                    source_id, filepath,
                )

        # Docker container sources
        for container in self.config.docker_containers:
            source_id = f"docker_{container}"
            self._sources.append(
                DockerLogSource(source_id, self.db, container)
            )
            logger.debug("Added docker source: %s", container)

        # Systemd journal sources
        for service in self.config.journal_services:
            source_id = f"journal_{service}"
            self._sources.append(
                JournalSource(source_id, self.db, service)
            )
            logger.debug("Added journal source: %s", service)

        # Kernel messages
        if self.config.enable_dmesg:
            self._sources.append(DmesgSource(self.db))
            logger.debug("Added dmesg source")

        logger.info("Initialized %d log sources", len(self._sources))

    def run_critical(self):
        """Process logs for critical errors with immediate AI analysis."""
        logger.info("=== Starting critical error scan ===")
        total_entries = 0
        critical_count = 0
        noncritical_count = 0
        skipped_count = 0

        for source in self._sources:
            try:
                entries = source.read_new_entries()
            except Exception as exc:
                logger.error(
                    "Error reading source %s: %s", source.source_id, exc
                )
                continue

            total_entries += len(entries)
            logger.debug(
                "Source %s: %d new entries", source.source_id, len(entries)
            )

            for entry in entries:
                result = self.classifier.classify(entry, source.source_id)
                if result is None:
                    skipped_count += 1
                    continue

                severity, category = result

                # Customer container OOM — auto-restart without AI analysis
                if severity == "customer_oom":
                    container_name = category  # category holds the container name
                    self._handle_customer_oom(container_name, entry, source)
                    continue

                # Customer PHP/application error — write to MySQL for customer dashboard
                if severity == "customer_php_error":
                    container_or_domain = category
                    self._handle_customer_php_error(container_or_domain, entry, source)
                    continue

                signature = ErrorSignature.generate(entry)

                # Dedup check
                existing = self.db.error_exists(signature)
                if existing:
                    logger.debug(
                        "Duplicate error (id=%d, count incremented)",
                        existing["id"],
                    )
                    continue

                # Store the error
                error_id = self.db.add_error(
                    signature=signature,
                    severity=severity,
                    source=source.source_id,
                    raw_message=entry[:5000],  # Limit stored message size
                    category=category,
                )

                if error_id is None:
                    continue

                if severity == "critical":
                    critical_count += 1
                    self._handle_critical(error_id, entry, category, source)
                else:
                    noncritical_count += 1

        # Scan on-disk application logs for monitored customer sites
        self._scan_site_logs()

        logger.info(
            "Scan complete: %d entries, %d critical, %d non-critical, %d skipped",
            total_entries, critical_count, noncritical_count, skipped_count,
        )

        if critical_count == 0:
            logger.info("No new critical errors found")

    def _handle_customer_oom(self, container_name, message, source):
        """Handle a monitored customer container OOM: log and send advisory.

        No restart is performed — Docker health checks handle container recovery.
        This only fires for customer containers with paid AI monitoring enabled.
        """
        # Dedup: use a simplified signature based on container name
        signature = ErrorSignature.generate(f"customer_oom:{container_name}")
        existing = self.db.error_exists(signature)
        if existing:
            logger.debug(
                "Duplicate customer OOM for %s (id=%d, count incremented)",
                container_name, existing["id"],
            )
            return

        logger.info(
            "Monitored customer container OOM: %s — sending advisory",
            container_name,
        )

        # Store the error
        error_id = self.db.add_error(
            signature=signature,
            severity="warning",
            source=source.source_id,
            raw_message=message[:5000],
            category="customer_oom",
        )

        # Update with static diagnosis (no AI call needed)
        self.db.update_error_with_ai(
            error_id,
            diagnosis=(
                f"Customer container {container_name} hit its memory limit "
                f"and was killed by the kernel OOM killer."
            ),
            recommendation=(
                f"The container will be restarted automatically by health checks. "
                f"Consider optimizing application code to reduce memory usage or "
                f"increasing the memory allocation for this site."
            ),
            fix_type="manual_only",
        )

        # Send advisory notification
        self.notifier.send(
            f"Customer container memory limit exceeded:\n"
            f"Container: {container_name}\n"
            f"The container was killed by the kernel OOM killer due to exceeding "
            f"its memory allocation. Health checks will restart it automatically.\n"
            f"Recommendation: Optimize application code or scale up RAM allocation.",
            severity="warning",
        )

        # Write to MySQL for customer dashboard
        domain = self._resolve_domain(container_name)
        sub = self._subscriptions.get(domain)
        if sub:
            self._write_customer_issue(
                sub, "critical", "oom",
                "Container Memory Limit Exceeded",
                f"Container {container_name} hit its memory limit and was killed by the kernel OOM killer.",
                "The container will restart automatically. Consider optimizing your application's memory usage or increasing the memory allocation for this site.",
                message[:2000], signature,
                container_name=container_name,
            )

    def _handle_customer_php_error(self, container_or_domain, message, source):
        """Handle a PHP error from a monitored customer site."""
        domain = self._resolve_domain(container_or_domain)
        sub = self._subscriptions.get(domain)
        if not sub:
            return

        signature = hashlib.sha256(
            f"php_error:{domain}:{message[:200]}".encode()
        ).hexdigest()

        # Determine specific PHP error type for title
        if re.search(r'Fatal error.*Allowed memory size', message, re.IGNORECASE):
            title = "PHP Memory Limit Exceeded"
            severity = "critical"
        elif re.search(r'PHP Fatal error', message, re.IGNORECASE):
            title = "PHP Fatal Error"
            severity = "critical"
        elif re.search(r'PHP Warning', message, re.IGNORECASE):
            title = "PHP Warning"
            severity = "warning"
        elif re.search(r'Uncaught Exception|Uncaught Error', message, re.IGNORECASE):
            title = "Uncaught PHP Exception"
            severity = "critical"
        else:
            title = "PHP Application Error"
            severity = "warning"

        diagnosis = f"A {title.lower()} was detected in your site's application code."
        recommendation = (
            "Review your application's error logs for the full stack trace. "
            "If this is a memory limit issue, consider optimizing your code or "
            "increasing your site's memory allocation."
        )

        self._write_customer_issue(
            sub, severity, "php_error", title, diagnosis,
            recommendation, message[:2000], signature,
            container_name=container_or_domain if '-' in container_or_domain else None,
        )

    def _write_customer_issue(self, subscription, severity, category, title,
                              diagnosis, recommendation, raw_log_excerpt,
                              signature, container_name=None):
        """Write a customer-visible issue to MySQL and queue email notification."""
        if not self.customer_db.available:
            return

        issue_id = self.customer_db.upsert_issue(
            subscription_id=subscription['subscription_id'],
            site_id=subscription['site_id'],
            username=subscription['username'],
            container_name=container_name,
            severity=severity,
            category=category,
            title=title,
            diagnosis=diagnosis,
            recommendation=recommendation,
            raw_log_excerpt=raw_log_excerpt,
            signature=signature,
        )

        if issue_id and subscription.get('notification_email'):
            hostname = subprocess.getoutput("hostname").strip()
            subject = f"[WHP] {severity.upper()}: {title} on {subscription['site_name']}"
            body_text = (
                f"AI Monitor Alert for {subscription['site_name']}\n\n"
                f"Severity: {severity.upper()}\n"
                f"Issue: {title}\n\n"
                f"Diagnosis:\n{diagnosis}\n\n"
                f"Recommendation:\n{recommendation}\n\n"
                f"View details in your WHP control panel."
            )
            body_html = (
                f"<h2>AI Monitor Alert for {subscription['site_name']}</h2>"
                f"<p><strong>Severity:</strong> {severity.upper()}</p>"
                f"<p><strong>Issue:</strong> {title}</p>"
                f"<h3>Diagnosis</h3><p>{diagnosis}</p>"
                f"<h3>Recommendation</h3><p>{recommendation}</p>"
                f"<p><a href='https://{hostname}/?page=ai-monitor-customer'>View in Dashboard</a></p>"
            )
            self.customer_db.queue_email(
                subscription['subscription_id'], issue_id,
                subscription['notification_email'], subject, body_html, body_text,
            )
            self.customer_db.mark_issue_email_sent(issue_id)

    def _scan_site_logs(self):
        """Scan on-disk application log files for monitored customer sites."""
        if not self._subscriptions:
            return

        for domain, sub in self._subscriptions.items():
            username = sub['username']
            # Build the site directory path (handle wildcard domains)
            safe_domain = domain.replace('*.', 'wildcard.')
            logs_dir = f"/docker/users/{username}/{safe_domain}/logs"

            if not os.path.isdir(logs_dir):
                continue

            try:
                self._scan_log_directory(logs_dir, sub)
            except Exception as exc:
                logger.error(
                    "Error scanning site logs for %s: %s", domain, exc
                )

    def _scan_log_directory(self, logs_dir, subscription):
        """Scan all .log files in a site's logs directory for errors."""
        try:
            log_files = [
                f for f in os.listdir(logs_dir)
                if f.endswith('.log') and os.path.isfile(os.path.join(logs_dir, f))
            ]
        except PermissionError:
            logger.debug("Permission denied reading %s", logs_dir)
            return

        for log_file in log_files:
            filepath = os.path.join(logs_dir, log_file)
            source_id = f"site_log:{subscription['primary_domain']}:{log_file}"

            # Use the existing position tracking via our SQLite DB
            try:
                stat = os.stat(filepath)
            except OSError:
                continue

            # Get last read position from SQLite
            pos_data = self.db.get_position(source_id)
            last_pos = pos_data.get('last_position', 0) if pos_data else 0
            last_inode = pos_data.get('last_inode', 0) if pos_data else 0

            # Check for file rotation (inode changed or file shrunk)
            current_inode = stat.st_ino
            current_size = stat.st_size
            if current_inode != last_inode or current_size < last_pos:
                last_pos = 0

            if current_size <= last_pos:
                continue  # No new data

            # Read new content (limit to 64KB per file per run to avoid memory issues)
            max_read = 65536
            try:
                with open(filepath, 'r', errors='replace') as f:
                    f.seek(last_pos)
                    content = f.read(max_read)
                    new_pos = f.tell()
            except (PermissionError, OSError) as exc:
                logger.debug("Cannot read %s: %s", filepath, exc)
                continue

            if not content.strip():
                continue

            # Process line by line
            lines = content.split('\n')
            error_lines = []
            for line in lines:
                line = line.strip()
                if not line:
                    continue
                # Look for error indicators in application logs
                if re.search(
                    r'(?:error|ERR|Error|ERROR|FATAL|fatal|WARN|warn|Warning|'
                    r'uncaught|unhandled|exception|TypeError|ReferenceError|'
                    r'ECONNREFUSED|ENOTFOUND|ENOMEM|SIGKILL|SIGTERM|OOMKilled|'
                    r'heap out of memory|allocation failure|JavaScript heap)',
                    line
                ):
                    error_lines.append(line)

            # Save position
            self.db.set_position(source_id, new_pos, current_inode)

            if not error_lines:
                continue

            # Group similar errors and write customer issues
            # Use the first error line as representative, count total
            representative = error_lines[0][:500]
            error_count = len(error_lines)

            # Determine severity based on content
            if re.search(
                r'FATAL|fatal|heap out of memory|OOMKilled|SIGKILL|'
                r'ENOMEM|allocation failure',
                representative, re.IGNORECASE
            ):
                severity = "critical"
                title = "Application Fatal Error"
            elif re.search(
                r'uncaught|unhandled|TypeError|ReferenceError|exception',
                representative, re.IGNORECASE
            ):
                severity = "warning"
                title = "Uncaught Application Exception"
            elif re.search(r'ECONNREFUSED|ENOTFOUND', representative):
                severity = "warning"
                title = "Application Connection Error"
            else:
                severity = "warning"
                title = "Application Error"

            # Add log file context to the title
            title = f"{title} in {log_file}"

            signature = hashlib.sha256(
                f"site_log:{subscription['primary_domain']}:{log_file}:{representative[:200]}".encode()
            ).hexdigest()

            diagnosis = (
                f"Found {error_count} error(s) in {log_file}. "
                f"Most recent: {representative[:300]}"
            )
            recommendation = (
                "Review your application logs for the full error details. "
                "Common causes include unhandled exceptions, memory limits, "
                "and connection issues to external services."
            )

            # Combine the last few error lines as the log excerpt
            excerpt = "\n".join(error_lines[-10:])

            self._write_customer_issue(
                subscription, severity, "app_error", title, diagnosis,
                recommendation, excerpt, signature,
            )

            logger.info(
                "Found %d errors in %s for site %s",
                error_count, log_file, subscription['primary_domain'],
            )

    def _resolve_domain(self, container_or_domain):
        """Resolve a container name or domain reference to the monitored domain.

        Container names like 'example.com-01' resolve to 'example.com'.
        Domain names return as-is if they're in the subscription map.
        """
        name = container_or_domain.lower()
        # Direct match
        if name in self._subscriptions:
            return name
        # Strip container suffix (-01, -02, etc.)
        match = re.match(r'^(.+)-\d{1,2}$', name)
        if match:
            domain = match.group(1)
            if domain in self._subscriptions:
                return domain
        # Prefix match (container name starts with a monitored domain)
        for domain in self._subscriptions:
            if name.startswith(domain):
                return domain
        return name

    def _handle_critical(self, error_id, message, category, source):
        """Handle a single critical error: analyze, remediate, notify."""
        logger.warning(
            "CRITICAL [%s] from %s: %s",
            category, source.source_id, message[:200],
        )

        # Gather context for AI
        context = self._get_system_context()

        # Add source-specific context
        if "service" in category or "journal" in source.source_id:
            service_name = source.source_id.replace("journal_", "")
            context += f"\nService status ({service_name}):\n"
            context += self._get_service_status(service_name)
        elif "container" in category or "docker" in source.source_id:
            container_name = source.source_id.replace("docker_", "")
            context += f"\nContainer status ({container_name}):\n"
            context += self._get_container_status(container_name)

        # AI analysis
        ai_result = self.ai.analyze_critical(
            message, category, context, source.source_id, self.db
        )

        # Update database with AI findings
        diagnosis = ai_result.get("diagnosis", "")
        explanation = ai_result.get("explanation", "")
        fix_type = ai_result.get("fix_type", "manual_only")

        self.db.update_error_with_ai(
            error_id,
            diagnosis=diagnosis,
            recommendation=explanation,
            fix_type=fix_type,
        )

        # Attempt remediation
        if fix_type != "manual_only" and self.config.remediation_enabled:
            success = self.remediator.execute(
                error_id, ai_result, dry_run=self.dry_run
            )
            self.db.update_error_with_ai(
                error_id,
                diagnosis=diagnosis,
                recommendation=explanation,
                fix_type=fix_type,
                fix_applied=1,
                fix_successful=1 if success else 0,
            )
        else:
            # Send notification for manual-only items
            self.notifier.send(
                f"Critical error detected [{category}]:\n"
                f"Source: {source.source_id}\n"
                f"Diagnosis: {diagnosis}\n"
                f"Recommendation: {explanation}",
                severity="critical",
            )

    def run_daily(self):
        """Daily batch analysis of non-critical errors."""
        logger.info("=== Starting daily batch analysis ===")

        errors = self.db.get_unprocessed_noncritical(
            limit=self.config.daily_batch_size
        )

        if not errors:
            logger.info("No unprocessed non-critical errors")
            return

        logger.info("Processing %d non-critical errors", len(errors))

        # Group by category
        errors_by_category = {}
        for err in errors:
            cat = err.get("category", "general_error")
            if cat not in errors_by_category:
                errors_by_category[cat] = []
            errors_by_category[cat].append(err)

        # AI batch analysis
        analysis = self.ai.analyze_daily_batch(errors_by_category, self.db)

        # Write daily report
        report = self._format_daily_report(analysis, errors_by_category)
        report_path = self._write_daily_report(report)

        # Send summary notification
        summary = analysis.get("summary", "Daily analysis complete")
        pattern_count = len(analysis.get("patterns", []))
        actions = analysis.get("immediate_actions", [])

        notification = (
            f"Daily Log Analysis Report\n"
            f"Errors analyzed: {len(errors)}\n"
            f"Categories: {len(errors_by_category)}\n"
            f"Patterns found: {pattern_count}\n"
            f"Summary: {summary}\n"
        )
        if actions:
            notification += "Immediate actions:\n"
            for action in actions[:5]:
                notification += f"  - {action}\n"
        if report_path:
            notification += f"Full report: {report_path}"

        self.notifier.send(notification, severity="info")

        # Mark all as processed
        error_ids = [e["id"] for e in errors]
        self.db.mark_processed(error_ids)

        logger.info(
            "Daily analysis complete: %d errors in %d categories",
            len(errors), len(errors_by_category),
        )

    def _format_daily_report(self, analysis, errors_by_category):
        """Format the daily analysis into a human-readable report."""
        lines = [
            "=" * 72,
            f"WHP Log Monitor - Daily Report - {time.strftime('%Y-%m-%d %H:%M')}",
            "=" * 72,
            "",
            f"Summary: {analysis.get('summary', 'N/A')}",
            "",
        ]

        patterns = analysis.get("patterns", [])
        if patterns:
            lines.append("PATTERNS DETECTED:")
            lines.append("-" * 40)
            for p in patterns:
                lines.append(
                    f"  [{p.get('priority', 'low').upper()}] "
                    f"{p.get('category', 'unknown')} "
                    f"(count: {p.get('count', 0)})"
                )
                lines.append(f"    Description: {p.get('description', 'N/A')}")
                lines.append(
                    f"    Recommendation: {p.get('recommendation', 'N/A')}"
                )
                lines.append("")

        actions = analysis.get("immediate_actions", [])
        if actions:
            lines.append("IMMEDIATE ACTIONS:")
            lines.append("-" * 40)
            for a in actions:
                lines.append(f"  * {a}")
            lines.append("")

        suggestions = analysis.get("monitoring_suggestions", [])
        if suggestions:
            lines.append("MONITORING SUGGESTIONS:")
            lines.append("-" * 40)
            for s in suggestions:
                lines.append(f"  - {s}")
            lines.append("")

        lines.append("ERROR COUNTS BY CATEGORY:")
        lines.append("-" * 40)
        for cat, errs in sorted(
            errors_by_category.items(), key=lambda x: -len(x[1])
        ):
            lines.append(f"  {cat}: {len(errs)}")
        lines.append("")
        lines.append("=" * 72)

        return "\n".join(lines)

    def _write_daily_report(self, report):
        """Write the daily report to a file."""
        report_dir = self.config.report_dir
        try:
            os.makedirs(report_dir, mode=0o750, exist_ok=True)
            filename = f"daily-report-{time.strftime('%Y-%m-%d')}.txt"
            filepath = os.path.join(report_dir, filename)
            with open(filepath, "w") as fh:
                fh.write(report)
            logger.info("Daily report written to %s", filepath)
            return filepath
        except OSError as exc:
            logger.error("Failed to write daily report: %s", exc)
            return None

    def _get_system_context(self):
        """Gather system context for AI analysis."""
        parts = []

        # Uptime
        result = self._safe_cmd(["uptime"], timeout=5)
        if result:
            parts.append(f"Uptime: {result.strip()}")

        # Disk usage for key partitions
        result = self._safe_cmd(
            ["df", "-h", "/", "/docker"], timeout=5
        )
        if result:
            parts.append(f"Disk usage:\n{result.strip()}")

        # Memory
        result = self._safe_cmd(["free", "-h"], timeout=5)
        if result:
            parts.append(f"Memory:\n{result.strip()}")

        return "\n\n".join(parts) if parts else "System context unavailable"

    def _get_service_status(self, service):
        """Get the status of a systemd service."""
        result = self._safe_cmd(
            ["systemctl", "status", service, "--no-pager", "-l"],
            timeout=10,
        )
        return result or "Status unavailable"

    def _get_container_status(self, container):
        """Get the status of a Docker container."""
        result = self._safe_cmd(
            ["docker", "inspect", "--format",
             "Status: {{.State.Status}}, "
             "Health: {{if .State.Health}}{{.State.Health.Status}}{{else}}N/A{{end}}, "
             "RestartCount: {{.RestartCount}}, "
             "StartedAt: {{.State.StartedAt}}",
             container],
            timeout=10,
        )
        return result or "Status unavailable"

    @staticmethod
    def _safe_cmd(cmd, timeout=10):
        """Run a command safely and return stdout or empty string."""
        try:
            result = subprocess.run(
                cmd, capture_output=True, text=True, timeout=timeout
            )
            return result.stdout
        except (subprocess.TimeoutExpired, FileNotFoundError, OSError):
            return ""

    def close(self):
        """Clean up resources."""
        if self.db:
            self.db.close()
        self.customer_db.close()


# ===================================================================
# Main entry point
# ===================================================================
def main():
    parser = argparse.ArgumentParser(
        description="WHP AI-Powered Log Monitor and Auto-Remediation",
        formatter_class=argparse.RawDescriptionHelpFormatter,
        epilog=(
            "Examples:\n"
            "  %(prog)s --mode critical          # Scan for critical errors\n"
            "  %(prog)s --mode daily              # Daily batch analysis\n"
            "  %(prog)s --mode critical --dry-run # Preview without acting\n"
            "  %(prog)s --reset-positions         # Reset all log tracking\n"
        ),
    )
    parser.add_argument(
        "--mode",
        choices=["critical", "daily"],
        help="Run mode: 'critical' for real-time error handling, "
             "'daily' for batch analysis",
    )
    parser.add_argument(
        "--dry-run",
        action="store_true",
        default=False,
        help="Preview actions without executing remediation",
    )
    parser.add_argument(
        "--verbose",
        action="store_true",
        default=False,
        help="Enable verbose/debug logging output",
    )
    parser.add_argument(
        "--reset-positions",
        action="store_true",
        default=False,
        help="Reset all log position tracking and exit",
    )
    parser.add_argument(
        "--config",
        default=None,
        help="Path to config file (default: /etc/whp/ai-monitor.conf)",
    )
    parser.add_argument(
        "--test-ai",
        action="store_true",
        default=False,
        help="Send a synthetic test error to the AI and display the response",
    )

    args = parser.parse_args()

    # Setup logging
    log_level = logging.DEBUG if args.verbose else logging.INFO
    logging.basicConfig(
        level=log_level,
        format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
        stream=sys.stdout,
    )

    # Load configuration
    try:
        config = Config(path=args.config)
    except Exception as exc:
        logger.critical("Failed to load configuration: %s", exc)
        return 1

    # Override log level from config if not in verbose mode
    if not args.verbose:
        level = getattr(logging, config.log_level.upper(), logging.INFO)
        logging.getLogger().setLevel(level)

    # Handle position reset
    if args.reset_positions:
        db = Database(config.db_path)
        db.reset_all_positions()
        db.close()
        logger.info("All log positions reset successfully")
        return 0

    # Handle AI test
    if args.test_ai:
        logger.info("=== AI Connection Test ===")
        analyzer = AIAnalyzer(config)
        test_error = (
            "[Tue Mar 31 20:01:04 2026] PHP Fatal error: Allowed memory size of "
            "134217728 bytes exhausted (tried to allocate 65536 bytes) in "
            "/docker/users/testuser/public_html/wp-includes/class-wpdb.php on line 2056"
        )
        logger.info("Sending test error to basic model (%s)...", config.basic_model)
        db = Database(config.db_path)
        context = {
            "uptime": subprocess.getoutput("uptime -p"),
            "disk": subprocess.getoutput("df -h / /docker 2>/dev/null | tail -2"),
            "memory": subprocess.getoutput("free -h | head -2"),
        }
        result = analyzer.analyze_critical(test_error, "php_oom", context, "test", db)
        if result:
            logger.info("AI Response received successfully!")
            logger.info("Diagnosis: %s", result.get("diagnosis", "N/A"))
            logger.info("Fix type: %s", result.get("fix_type", "N/A"))
            logger.info("Risk level: %s", result.get("risk_level", "N/A"))
            logger.info("Explanation: %s", result.get("explanation", "N/A"))
            logger.info("Full response:\n%s", json.dumps(result, indent=2))
            # Test notifications too
            notifier = Notifier(config)
            notifier.send(
                "🧪 WHP AI Monitor — Test successful!\n"
                f"Model: {config.basic_model}\n"
                f"Diagnosis: {result.get('diagnosis', 'N/A')}",
                severity="info",
            )
            logger.info("Test notification sent to configured channels")
        else:
            logger.error("AI returned no response. Check your base_url, API key, and model name.")
        return 0

    # Require --mode for normal operation
    if not args.mode:
        parser.error("--mode is required (use 'critical' or 'daily')")

    # Run the monitor
    monitor = None
    try:
        monitor = WHPLogMonitor(
            config,
            dry_run=args.dry_run,
            verbose=args.verbose,
        )

        if args.dry_run:
            logger.info("*** DRY RUN MODE - no changes will be made ***")

        if args.mode == "critical":
            monitor.run_critical()
        elif args.mode == "daily":
            monitor.run_daily()

        return 0

    except KeyboardInterrupt:
        logger.info("Interrupted by user")
        return 0
    except Exception as exc:
        logger.critical("Fatal error: %s", exc, exc_info=True)
        # Try to send a notification about the crash
        try:
            notifier = Notifier(config)
            notifier.send(
                f"WHP Log Monitor CRASHED: {exc}",
                severity="critical",
            )
        except Exception:
            pass
        return 1
    finally:
        if monitor:
            monitor.close()


if __name__ == "__main__":
    sys.exit(main())
