#!/usr/bin/env bash
# Wrap the whp-monitor-poll cron entry with flock + timeout so a hung
# instance can never pile up or run forever.
#
# Background: 2026-04-28 — a whp-monitor-poll.py instance hung for 56
# minutes on a MySQL socket after a SQLite multi-statement DDL error
# (since fixed). With cron firing every minute and no lock, additional
# instances would have piled up. flock -n makes new firings exit
# immediately if a previous instance is still running, and timeout 90
# kills any instance running longer than 90 seconds.
#
# This script REPLACES the existing cron entry (installed by
# setup-whp-monitor-poll.sh) with the watchdog-wrapped version.
# Idempotent — safe to run multiple times.

set -euo pipefail

SCRIPT_PATH='/root/whp/scripts/whp-monitor-poll.py'
LOG_PATH='/var/log/whp-monitor-poll.log'
LOCK_PATH='/run/whp-monitor-poll.lock'
CRON_TAG='# WHP AI Monitor - Minute-cadence poll'

NEW_CRON_LINE="* * * * * /usr/bin/flock -n ${LOCK_PATH} /usr/bin/timeout 90 /usr/bin/python3 ${SCRIPT_PATH} >> ${LOG_PATH} 2>&1"

# Build a fresh crontab: drop any existing line that mentions the script
# (with or without the watchdog wrapper), then re-add the wrapped one.
TMP_CRON="$(mktemp)"
trap 'rm -f "${TMP_CRON}"' EXIT

crontab -l 2>/dev/null \
    | grep -vF "${SCRIPT_PATH}" \
    | grep -vF "${CRON_TAG}" \
    > "${TMP_CRON}" || true

{
    cat "${TMP_CRON}"
    echo "${CRON_TAG}"
    echo "${NEW_CRON_LINE}"
} | crontab -

echo "Updated cron entry to:"
echo "  ${NEW_CRON_LINE}"

# Reap any in-flight instance that's been running > 90 seconds. Cron
# would normally let it keep going indefinitely; we kill anything that
# already exceeded the new ceiling so the next scheduled run starts
# clean.
for pid in $(pgrep -f 'whp-monitor-poll.py' || true); do
    if [[ -d "/proc/${pid}" ]]; then
        # etimes is seconds — portable on procps-ng.
        elapsed=$(ps -p "${pid}" -o etimes= 2>/dev/null | tr -d ' ' || echo 0)
        if [[ -n "${elapsed}" && "${elapsed}" -gt 90 ]]; then
            echo "Killing stale monitor-poll PID ${pid} (running ${elapsed}s)"
            kill -TERM "${pid}" 2>/dev/null || true
            sleep 2
            kill -KILL "${pid}" 2>/dev/null || true
        fi
    fi
done

# Ensure the lock directory exists. /run is a tmpfs that's recreated
# each boot, so the lock file itself doesn't need pre-creation — flock
# will create it on first use.
mkdir -p "$(dirname "${LOCK_PATH}")"

echo "update-whp-monitor-poll-cron-watchdog.sh: OK"
