#!/usr/bin/env bash
set -euo pipefail

###############################################################################
# haproxy-crash-monitor.sh
#
# Real-time Docker events listener daemon for the haproxy-manager container.
# Reacts to container die/oom/kill/start events within seconds and captures
# forensic data (exit code, OOM status, memory, kernel logs, container logs)
# before Docker's auto-restart erases the evidence.
#
# Runs as a systemd service (haproxy-crash-monitor.service).
# Writes structured JSONL to /var/log/haproxy-crash-events.json and inserts
# rows into the whp.container_crash_events database table.
#
# Notifications are sent via Discord/Slack/Telegram using the same webhook
# configuration as the AI log monitor (/etc/whp/ai-monitor.conf).
###############################################################################

CONTAINER_NAME="haproxy-manager"
CRASH_LOG="/var/log/haproxy-crash-events.json"
DAEMON_LOG="/var/log/haproxy-crash-monitor.log"
AI_MONITOR_CONF="/etc/whp/ai-monitor.conf"
MYSQL_DEFAULTS="/root/.my.cnf"

# Notification rate limiting
LAST_NOTIFY_FILE="/tmp/haproxy_crash_last_notify"
NOTIFY_COOLDOWN=60  # seconds between notifications

# Reconnection backoff
MAX_BACKOFF=60
BACKOFF=5

# Track last die timestamp for downtime calculation
LAST_DIE_TS=""

log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') [haproxy-crash-monitor] $*"
}

# ---------------------------------------------------------------------------
# INI config parser — reads a value from /etc/whp/ai-monitor.conf
# Usage: ini_get "section" "key"
# ---------------------------------------------------------------------------
ini_get() {
    local section="$1" key="$2"
    local in_section=0
    local value=""

    [[ ! -f "$AI_MONITOR_CONF" ]] && return

    while IFS= read -r line; do
        # Strip leading/trailing whitespace
        line="${line#"${line%%[![:space:]]*}"}"
        line="${line%"${line##*[![:space:]]}"}"

        # Skip comments and empty lines
        [[ -z "$line" || "$line" == \#* ]] && continue

        # Section header
        if [[ "$line" =~ ^\[([^]]+)\]$ ]]; then
            if [[ "${BASH_REMATCH[1]}" == "$section" ]]; then
                in_section=1
            else
                in_section=0
            fi
            continue
        fi

        # Key = value
        if (( in_section )) && [[ "$line" =~ ^${key}[[:space:]]*=[[:space:]]*(.*) ]]; then
            value="${BASH_REMATCH[1]}"
            # Strip trailing whitespace
            value="${value%"${value##*[![:space:]]}"}"
            break
        fi
    done < "$AI_MONITOR_CONF"

    echo "$value"
}

# ---------------------------------------------------------------------------
# Notification senders
# ---------------------------------------------------------------------------
send_discord() {
    local message="$1"
    local webhook
    webhook=$(ini_get "notifications" "discord_webhook_url")
    [[ -z "$webhook" ]] && return

    local role_id
    role_id=$(ini_get "notifications" "discord_role_critical")
    local mention=""
    [[ -n "$role_id" ]] && mention="<@&${role_id}> "

    local payload
    payload=$(python3 -c "import json,sys; print(json.dumps({'content': sys.argv[1]+sys.argv[2]}))" "$mention" "$message" 2>/dev/null) || return
    curl -s -o /dev/null -m 10 -H "Content-Type: application/json" -d "$payload" "$webhook" 2>/dev/null || true
}

send_slack() {
    local message="$1"
    local webhook
    webhook=$(ini_get "notifications" "slack_webhook_url")
    [[ -z "$webhook" ]] && return

    local payload
    payload=$(python3 -c "import json,sys; print(json.dumps({'text': sys.argv[1]}))" "$message" 2>/dev/null) || return
    curl -s -o /dev/null -m 10 -H "Content-Type: application/json" -d "$payload" "$webhook" 2>/dev/null || true
}

send_telegram() {
    local message="$1"
    local bot_token chat_id
    bot_token=$(ini_get "notifications" "telegram_bot_token")
    chat_id=$(ini_get "notifications" "telegram_chat_id")
    [[ -z "$bot_token" || -z "$chat_id" ]] && return

    curl -s -o /dev/null -m 10 \
        "https://api.telegram.org/bot${bot_token}/sendMessage" \
        -d "chat_id=${chat_id}" \
        -d "text=${message}" \
        -d "parse_mode=HTML" 2>/dev/null || true
}

send_notification() {
    local message="$1"

    # Check if notifications are enabled
    local enabled
    enabled=$(ini_get "notifications" "enabled")
    [[ "$enabled" != "true" ]] && return

    # Rate limit: max 1 notification per NOTIFY_COOLDOWN seconds
    if [[ -f "$LAST_NOTIFY_FILE" ]]; then
        local last_notify now elapsed
        last_notify=$(cat "$LAST_NOTIFY_FILE" 2>/dev/null || echo 0)
        now=$(date +%s)
        elapsed=$(( now - last_notify ))
        if (( elapsed < NOTIFY_COOLDOWN )); then
            log "Notification rate-limited (${elapsed}s since last, cooldown ${NOTIFY_COOLDOWN}s)"
            return
        fi
    fi

    date +%s > "$LAST_NOTIFY_FILE"

    send_discord "$message"
    send_slack "$message"
    send_telegram "$message"
    log "Notification sent"
}

# ---------------------------------------------------------------------------
# Classify crash cause from exit code and OOM status
# ---------------------------------------------------------------------------
classify_cause() {
    local exit_code="$1" oom_killed="$2" dmesg_oom="$3"

    if [[ "$oom_killed" == "true" ]]; then
        echo "OOM_DOCKER"
    elif [[ -n "$dmesg_oom" ]]; then
        echo "OOM_KERNEL"
    elif (( exit_code == 137 )); then
        echo "SIGKILL_EXTERNAL"
    elif (( exit_code == 139 )); then
        echo "SEGFAULT"
    elif (( exit_code == 134 )); then
        echo "ABORT"
    elif (( exit_code == 0 )); then
        echo "CLEAN_SHUTDOWN"
    elif (( exit_code == 1 )); then
        echo "PROCESS_ERROR"
    else
        echo "UNKNOWN_EXIT_${exit_code}"
    fi
}

# ---------------------------------------------------------------------------
# Capture forensic data and log the crash event
# ---------------------------------------------------------------------------
handle_crash_event() {
    local event_type="$1" exit_code="${2:-0}"

    log "=== CRASH EVENT DETECTED: ${event_type} (exit code: ${exit_code}) ==="

    # Docker inspect for OOM and state details
    local oom_killed="false"
    local inspect_exit_code="$exit_code"
    local finished_at=""
    local restart_count=""
    local inspect_json=""

    inspect_json=$(docker inspect "$CONTAINER_NAME" --format='{{json .State}}' 2>/dev/null || echo "{}")
    oom_killed=$(docker inspect "$CONTAINER_NAME" --format='{{.State.OOMKilled}}' 2>/dev/null || echo "false")
    inspect_exit_code=$(docker inspect "$CONTAINER_NAME" --format='{{.State.ExitCode}}' 2>/dev/null || echo "$exit_code")
    finished_at=$(docker inspect "$CONTAINER_NAME" --format='{{.State.FinishedAt}}' 2>/dev/null || echo "")
    restart_count=$(docker inspect "$CONTAINER_NAME" --format='{{.RestartCount}}' 2>/dev/null || echo "")

    # Use inspect exit code if we didn't get one from the event
    if (( exit_code == 0 )) && (( inspect_exit_code != 0 )); then
        exit_code="$inspect_exit_code"
    fi

    log "OOMKilled: $oom_killed | ExitCode: $exit_code | RestartCount: $restart_count"

    # Kernel OOM check
    local dmesg_oom=""
    dmesg_oom=$(dmesg -T 2>/dev/null | grep -i -E "oom|kill" | grep -i -E "haproxy|${CONTAINER_NAME}" | tail -5 || true)
    if [[ -z "$dmesg_oom" ]]; then
        # Broader check — recent OOM events in the last 60 seconds
        dmesg_oom=$(dmesg -T 2>/dev/null | grep -i -E "oom-kill|Out of memory|Killed process" | tail -3 || true)
    fi
    [[ -n "$dmesg_oom" ]] && log "Kernel OOM detected: $dmesg_oom"

    # Container logs (last 50 lines before crash)
    local container_logs=""
    container_logs=$(docker logs "$CONTAINER_NAME" --tail 50 --timestamps 2>&1 || echo "logs unavailable")

    # Host memory
    local host_memory=""
    host_memory=$(free -h 2>/dev/null | grep -E "^Mem:" | awk '{print $3 " used / " $2 " total (" $7 " available)"}' || echo "unavailable")

    # Container memory at crash (may not be available after restart)
    local container_memory=""
    container_memory=$(timeout 5 docker stats "$CONTAINER_NAME" --no-stream --format "{{.MemUsage}} ({{.MemPerc}})" 2>/dev/null || echo "unavailable")

    # Classify the cause
    local cause
    cause=$(classify_cause "$exit_code" "$oom_killed" "$dmesg_oom")

    log "Cause classification: $cause"
    log "Host memory: $host_memory"

    # Calculate downtime if we have a previous die timestamp
    local downtime_seconds=""
    if [[ -n "$LAST_DIE_TS" ]]; then
        local now_ts
        now_ts=$(date +%s)
        downtime_seconds=$(( now_ts - LAST_DIE_TS ))
    fi

    # Record the die timestamp
    LAST_DIE_TS=$(date +%s)

    # --- Write JSONL entry ---
    local timestamp
    timestamp=$(date -u '+%Y-%m-%dT%H:%M:%SZ')

    # Escape strings for JSON
    local escaped_logs escaped_dmesg escaped_inspect
    escaped_logs=$(echo "$container_logs" | head -50 | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')
    escaped_dmesg=$(echo "$dmesg_oom" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')
    escaped_inspect=$(echo "$inspect_json" | python3 -c 'import sys,json; print(json.dumps(sys.stdin.read()))' 2>/dev/null || echo '""')

    local json_entry
    json_entry=$(cat <<JSONEOF
{"timestamp":"${timestamp}","event":"${event_type}","container":"${CONTAINER_NAME}","exit_code":${exit_code},"oom_killed":${oom_killed},"cause":"${cause}","restart_count":${restart_count:-0},"downtime_seconds":${downtime_seconds:-null},"memory_at_crash":"${container_memory}","host_memory":"${host_memory}","dmesg_oom":${escaped_dmesg},"container_logs":${escaped_logs},"docker_inspect_state":${escaped_inspect}}
JSONEOF
)

    echo "$json_entry" >> "$CRASH_LOG"
    log "Crash event written to $CRASH_LOG"

    # --- Insert into database ---
    if [[ -f "$MYSQL_DEFAULTS" ]]; then
        # Escape backslashes then single quotes for safe SQL insertion
        local sql_logs sql_dmesg sql_inspect
        sql_logs=$(echo "$container_logs" | head -50 | sed 's/\\/\\\\/g' | sed "s/'/''/g")
        sql_dmesg=$(echo "$dmesg_oom" | sed 's/\\/\\\\/g' | sed "s/'/''/g")
        sql_inspect=$(echo "$inspect_json" | sed 's/\\/\\\\/g' | sed "s/'/''/g")

        mysql --defaults-file="$MYSQL_DEFAULTS" -e "
            INSERT INTO whp.container_crash_events
                (container_name, event_type, exit_code, oom_killed, cause,
                 restart_count, downtime_seconds, memory_usage, host_memory_free,
                 dmesg_excerpt, container_logs, docker_inspect_json)
            VALUES
                ('${CONTAINER_NAME}', '${event_type}', ${exit_code}, $([ "$oom_killed" = "true" ] && echo 1 || echo 0), '${cause}',
                 ${restart_count:-0}, ${downtime_seconds:-"NULL"}, '${container_memory}', '${host_memory}',
                 '${sql_dmesg}', '${sql_logs}', '${sql_inspect}')
        " 2>/dev/null && log "Crash event inserted into database" || log "WARNING: Failed to insert crash event into database"
    else
        log "WARNING: MySQL defaults file not found, skipping database insert"
    fi

    # --- Send notification ---
    if [[ "$cause" != "CLEAN_SHUTDOWN" ]]; then
        local notify_msg
        notify_msg="HAPROXY CRASH DETECTED
Cause: ${cause}
Exit Code: ${exit_code} | OOM: ${oom_killed} | Restart #${restart_count:-?}
Host Memory: ${host_memory}
Container Memory: ${container_memory}
Time: ${timestamp}"

        [[ -n "$downtime_seconds" ]] && notify_msg="${notify_msg}
Downtime: ~${downtime_seconds}s"

        send_notification "$notify_msg"
    fi

    log "=== END CRASH EVENT ==="
}

# ---------------------------------------------------------------------------
# Handle container start event
# ---------------------------------------------------------------------------
handle_start_event() {
    local downtime_msg=""
    if [[ -n "$LAST_DIE_TS" ]]; then
        local now_ts
        now_ts=$(date +%s)
        local downtime=$(( now_ts - LAST_DIE_TS ))
        downtime_msg=" (downtime: ~${downtime}s)"
    fi
    log "Container '${CONTAINER_NAME}' started${downtime_msg}"
}

# ---------------------------------------------------------------------------
# Main event loop
# ---------------------------------------------------------------------------
main() {
    log "Starting HAProxy crash monitor daemon"
    log "Watching container: $CONTAINER_NAME"
    log "Crash log: $CRASH_LOG"

    # Ensure log file exists
    touch "$CRASH_LOG"

    while true; do
        log "Listening for docker events..."

        # Process substitution keeps the while loop in the current shell,
        # preserving LAST_DIE_TS and BACKOFF state across events.
        while IFS= read -r event_json; do
            # Successfully receiving events means connection is healthy — reset backoff
            BACKOFF=5

            # Parse event type and attributes
            local event_status=""
            local event_exit_code="0"

            event_status=$(echo "$event_json" | python3 -c 'import sys,json; e=json.load(sys.stdin); print(e.get("status",""))' 2>/dev/null || echo "")
            event_exit_code=$(echo "$event_json" | python3 -c 'import sys,json; e=json.load(sys.stdin); print(e.get("Actor",{}).get("Attributes",{}).get("exitCode","0"))' 2>/dev/null || echo "0")

            case "$event_status" in
                die)
                    handle_crash_event "die" "$event_exit_code"
                    ;;
                oom)
                    handle_crash_event "oom" "$event_exit_code"
                    ;;
                kill)
                    # Log kills but let the die event handle the full forensics
                    local signal=""
                    signal=$(echo "$event_json" | python3 -c 'import sys,json; e=json.load(sys.stdin); print(e.get("Actor",{}).get("Attributes",{}).get("signal",""))' 2>/dev/null || echo "")
                    log "Container killed with signal: $signal"
                    ;;
                start)
                    handle_start_event
                    ;;
            esac
        done < <(docker events \
            --filter "container=${CONTAINER_NAME}" \
            --filter "event=die" \
            --filter "event=oom" \
            --filter "event=kill" \
            --filter "event=start" \
            --format '{{json .}}' 2>/dev/null || true)

        # If we get here, the docker events stream disconnected
        log "Docker events stream disconnected. Reconnecting in ${BACKOFF}s..."
        sleep "$BACKOFF"
        BACKOFF=$(( BACKOFF * 2 ))
        (( BACKOFF > MAX_BACKOFF )) && BACKOFF=$MAX_BACKOFF
    done
}

main "$@"
