#!/usr/bin/env bash
#
# haproxy-watchdog.sh
#
# Cron-driven watchdog script that runs every minute to monitor the
# HAProxy container (haproxy-manager). If HAProxy becomes unresponsive
# for 3 consecutive checks (~3 minutes), the script will attempt a
# docker restart, falling back to a full container recreate if that fails.
# A 5-minute cooldown prevents restart storms.
#
# Intended crontab entry:
#   * * * * * /root/whp/scripts/haproxy-watchdog.sh
#

LOGFILE="/var/log/haproxy-watchdog.log"
FAILURE_FILE="/tmp/haproxy_watchdog_failures"
LAST_RESTART_FILE="/tmp/haproxy_watchdog_last_restart"
CONTAINER_NAME="haproxy-manager"
MAX_FAILURES=3
COOLDOWN_SECONDS=300  # 5 minutes

log() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') [haproxy-watchdog] $*" >> "$LOGFILE"
}

get_failure_count() {
    if [[ -f "$FAILURE_FILE" ]]; then
        cat "$FAILURE_FILE" 2>/dev/null || echo 0
    else
        echo 0
    fi
}

set_failure_count() {
    echo "$1" > "$FAILURE_FILE"
}

is_in_cooldown() {
    if [[ ! -f "$LAST_RESTART_FILE" ]]; then
        return 1  # No cooldown active
    fi
    local last_restart
    last_restart=$(cat "$LAST_RESTART_FILE" 2>/dev/null || echo 0)
    local now
    now=$(date +%s)
    local elapsed=$(( now - last_restart ))
    if (( elapsed < COOLDOWN_SECONDS )); then
        return 0  # Still in cooldown
    fi
    return 1  # Cooldown expired
}

record_restart_time() {
    date +%s > "$LAST_RESTART_FILE"
}

# Insert a row into whp.container_crash_events so the AI Monitor's
# crash-event drain (whp-monitor-poll.py) surfaces this watchdog action
# as an Issue. The drain runs every minute, so the Issue appears within
# ~60s of the restart.
#
# Args:
#   $1 = cause (WATCHDOG_RESTART | WATCHDOG_RECREATE | WATCHDOG_FAILED)
#   $2 = outcome ('ok' or 'fail')
#   $3 = consecutive_failures that triggered the action
record_watchdog_event() {
    local cause="$1"
    local outcome="$2"
    local failures="$3"

    if ! command -v mysql >/dev/null 2>&1; then
        log "record_watchdog_event: mysql client not present, skipping DB insert"
        return 0
    fi

    # Capture a small forensics blob (free memory + last 20 log lines) so
    # the Issue carries useful evidence even if the crash-monitor daemon
    # didn't catch the underlying cause.
    local mem_free
    mem_free=$(free -h 2>/dev/null | awk '/^Mem:/{print $4 " free / " $2 " total"}')
    local logs_tail
    logs_tail=$(docker logs "$CONTAINER_NAME" --tail 20 --timestamps 2>&1 | head -c 4000 || true)

    # Use stdin heredoc + a python literal escape via mysql --batch's
    # built-in input parsing. We rely on the mysql client's prepared-style
    # variables to avoid quoting problems with newlines in $logs_tail.
    mysql -u root whp <<SQL || log "record_watchdog_event: DB insert failed for cause=${cause}"
SET @cn = '$CONTAINER_NAME';
SET @ev = 'watchdog';
SET @cause = '${cause}';
SET @notes = $(printf '%s' "outcome=${outcome} | consecutive_failures=${failures} | mem=${mem_free}" | sed "s/'/''/g; s/^/'/; s/\$/'/");
SET @logs = $(printf '%s' "$logs_tail" | sed "s/'/''/g; s/^/'/; s/\$/'/");
INSERT INTO whp.container_crash_events
    (container_name, event_type, oom_killed, cause, container_logs)
VALUES (@cn, @ev, 0, @cause, CONCAT(@notes, '\n\n--- container logs ---\n', @logs));
SQL

    log "record_watchdog_event: cause=${cause} outcome=${outcome} (Issue will appear in AI Monitor within ~60s)"
}

# --- Main logic ---

# Check if the container is running
if ! docker inspect --format='{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null | grep -q "true"; then
    log "Container '$CONTAINER_NAME' is not running. Starting it..."
    if docker start "$CONTAINER_NAME" >> "$LOGFILE" 2>&1; then
        record_watchdog_event "WATCHDOG_START" "ok" "0"
    else
        record_watchdog_event "WATCHDOG_START" "fail" "0"
    fi
    log "Start command issued. Exit."
    exit 0
fi

# --- Detect Docker auto-restarts we didn't initiate ---
RESTART_COUNT_FILE="/tmp/haproxy_watchdog_restart_count"
CURRENT_RESTART_COUNT=$(docker inspect --format='{{.RestartCount}}' "$CONTAINER_NAME" 2>/dev/null || echo 0)
PREV_RESTART_COUNT=$(cat "$RESTART_COUNT_FILE" 2>/dev/null || echo 0)
echo "$CURRENT_RESTART_COUNT" > "$RESTART_COUNT_FILE"

if (( CURRENT_RESTART_COUNT > PREV_RESTART_COUNT )); then
    RESTART_DELTA=$(( CURRENT_RESTART_COUNT - PREV_RESTART_COUNT ))
    log "ALERT: Docker auto-restarted '$CONTAINER_NAME' $RESTART_DELTA time(s) since last check (RestartCount: $PREV_RESTART_COUNT -> $CURRENT_RESTART_COUNT)"
    CONTAINER_STARTED=$(docker inspect --format='{{.State.StartedAt}}' "$CONTAINER_NAME" 2>/dev/null || echo "unknown")
    log "Container started at: $CONTAINER_STARTED"

    # Capture basic forensics as a fallback if crash monitor missed it
    if [[ ! -f /var/log/haproxy-crash-events.json ]] || ! tail -1 /var/log/haproxy-crash-events.json 2>/dev/null | grep -q "$(date +%Y-%m-%d)"; then
        log "No recent crash monitor data — capturing fallback diagnostics"
        docker inspect "$CONTAINER_NAME" --format='OOMKilled: {{.State.OOMKilled}} | ExitCode: {{.State.ExitCode}} | FinishedAt: {{.State.FinishedAt}}' >> "$LOGFILE" 2>&1
        dmesg -T 2>/dev/null | grep -i -E "oom|kill" | tail -5 >> "$LOGFILE" 2>&1
        free -h >> "$LOGFILE" 2>&1
    fi
fi

# Container is running — check if HAProxy is actually responding
if curl -s -o /dev/null --max-time 10 http://localhost/; then
    # Success — reset failure counter
    failures=$(get_failure_count)
    if (( failures > 0 )); then
        log "Health check passed. Resetting failure counter (was $failures)."
    fi
    set_failure_count 0
    exit 0
fi

# Health check failed
failures=$(get_failure_count)
failures=$(( failures + 1 ))
set_failure_count "$failures"
log "Health check failed. Consecutive failures: $failures/$MAX_FAILURES."

if (( failures < MAX_FAILURES )); then
    exit 0
fi

# Reached failure threshold — attempt recovery
if is_in_cooldown; then
    log "Restart needed but still in cooldown period. Skipping."
    exit 0
fi

# --- Capture diagnostics before restarting ---
log "=== PRE-RESTART DIAGNOSTICS ==="

# Container resource usage
log "--- Container Stats ---"
docker stats "$CONTAINER_NAME" --no-stream --format "CPU: {{.CPUPerc}} | MEM: {{.MemUsage}} ({{.MemPerc}}) | NET: {{.NetIO}} | PIDS: {{.PIDs}}" >> "$LOGFILE" 2>&1

# Process list inside container
log "--- Container Processes ---"
docker exec "$CONTAINER_NAME" ps aux >> "$LOGFILE" 2>&1

# HAProxy connection state
log "--- HAProxy Connections (netstat summary) ---"
docker exec "$CONTAINER_NAME" bash -c 'netstat -ant 2>/dev/null | awk "/^tcp/ {state[\$6]++} END {for (s in state) print s, state[s]}" | sort -k2 -rn' >> "$LOGFILE" 2>&1

# Total connection count on ports 80/443
log "--- Connection Counts (ports 80/443) ---"
docker exec "$CONTAINER_NAME" bash -c 'echo "Port 80: $(netstat -ant 2>/dev/null | grep ":80 " | wc -l) | Port 443: $(netstat -ant 2>/dev/null | grep ":443 " | wc -l)"' >> "$LOGFILE" 2>&1

# Check if HAProxy process is alive vs hung
log "--- HAProxy Process State ---"
docker exec "$CONTAINER_NAME" bash -c 'for pid in $(pidof haproxy); do echo "PID $pid: $(cat /proc/$pid/status 2>/dev/null | grep -E "^(State|Threads|VmRSS|VmSize|FDSize)" | tr "\n" " ")"; done' >> "$LOGFILE" 2>&1

# Open file descriptors
log "--- Open File Descriptors ---"
docker exec "$CONTAINER_NAME" bash -c 'for pid in $(pidof haproxy); do echo "PID $pid FDs: $(ls /proc/$pid/fd 2>/dev/null | wc -l)"; done' >> "$LOGFILE" 2>&1

# Flask API check (port 8000) — is just HAProxy hung, or the whole container?
log "--- Flask API (port 8000) ---"
if curl -s --max-time 5 http://localhost:8000/health >> "$LOGFILE" 2>&1; then
    log "Flask API responding (HAProxy is hung but management API is alive)"
else
    log "Flask API also unresponsive (entire container may be hung)"
fi

# Check host memory/swap
log "--- Host Memory ---"
free -h >> "$LOGFILE" 2>&1

# Check for OOM kills
log "--- Recent OOM Events ---"
dmesg -T 2>/dev/null | grep -i -E "oom|kill" | tail -5 >> "$LOGFILE" 2>&1

# Last 20 lines of container logs
log "--- Container Logs (last 20 lines) ---"
docker logs "$CONTAINER_NAME" --tail 20 --timestamps >> "$LOGFILE" 2>&1

log "=== END DIAGNOSTICS ==="

log "Threshold reached ($MAX_FAILURES consecutive failures). Attempting docker restart..."
if docker restart "$CONTAINER_NAME" >> "$LOGFILE" 2>&1; then
    log "Docker restart succeeded."
    record_watchdog_event "WATCHDOG_RESTART" "ok" "$failures"
    record_restart_time
    set_failure_count 0
else
    log "Docker restart failed. Falling back to container-manager.sh recreate..."
    if /root/whp/scripts/container-manager.sh recreate "$CONTAINER_NAME" >> "$LOGFILE" 2>&1; then
        log "Container recreate succeeded."
        record_watchdog_event "WATCHDOG_RECREATE" "ok" "$failures"
        record_restart_time
        set_failure_count 0
    else
        log "Container recreate also failed. Manual intervention required."
        record_watchdog_event "WATCHDOG_FAILED" "fail" "$failures"
        record_restart_time
    fi
fi
