#!/usr/bin/env bash
# Wrap every per-minute (and a couple of per-2-min / per-5-min) WHP cron
# entry with `flock -n` + `timeout` to prevent the hang/pile-up pattern
# that took out monitor-poll on 2026-04-28.
#
# - flock -n means new firings exit immediately if a previous instance
#   is still running. No pile-up.
# - timeout SIGTERMs anything that exceeds the cap. No silent forever-
#   hangs.
#
# Per-script timeouts are picked to (a) be larger than typical run time,
# (b) be small enough that a true hang gets cleaned up within minutes.
# The flock takes the brunt of the protection; the timeout is the
# safety net.
#
# Idempotent: scans for entries already wrapped in flock and skips them.

set -euo pipefail

# script_path|timeout_seconds|lock_basename
ENTRIES=(
    "/docker/whp/web/scripts/backup-worker.php|240|whp-backup-worker"
    "/docker/whp/web/scripts/backup-upload-worker.php|240|whp-backup-upload"
    "/root/whp/scripts/verify-dns-challenges.php|50|whp-verify-dns"
    "/root/whp/scripts/haproxy-watchdog.sh|50|whp-haproxy-watchdog"
    "/root/whp/scripts/process-email-queue.php|90|whp-process-email-queue"
    "/root/whp/scripts/process-log-review.php|240|whp-process-log-review"
    "/root/whp/scripts/backup-scheduler.php|240|whp-backup-scheduler"
    "/root/whp/scripts/log_monitor_cron.sh|120|whp-log-monitor-cron"
    "/root/whp/scripts/whp-log-monitor-ai.py|240|whp-log-monitor-ai-critical"
)

CURRENT="$(crontab -l 2>/dev/null || true)"
NEW="${CURRENT}"

wrap_one() {
    local path="$1"
    local secs="$2"
    local lockbase="$3"
    local lockfile="/run/${lockbase}.lock"

    # Already wrapped? Skip.
    if echo "${NEW}" | grep -qF "/usr/bin/flock -n ${lockfile}"; then
        echo "  ${path}: already wrapped, skipping"
        return 0
    fi

    # Find the existing line that runs this script. We anchor on the
    # script path so we don't accidentally rewrite an unrelated entry.
    local matched
    matched="$(echo "${NEW}" | grep -F "${path}" || true)"
    if [[ -z "${matched}" ]]; then
        echo "  ${path}: not in crontab, skipping"
        return 0
    fi

    # Iterate in case there's more than one entry referencing the same
    # script (e.g. different modes). Wrap each.
    while IFS= read -r line; do
        [[ -z "${line}" ]] && continue
        # Skip comment lines.
        [[ "${line}" =~ ^[[:space:]]*# ]] && continue
        # Skip if already has flock for THIS lockfile.
        if [[ "${line}" == *"flock -n ${lockfile}"* ]]; then
            continue
        fi

        # Split: keep the schedule (first 5 fields), wrap the command.
        local schedule cmd
        schedule="$(echo "${line}" | awk '{print $1, $2, $3, $4, $5}')"
        cmd="$(echo "${line}" | cut -d' ' -f6-)"

        # New command: flock -n LOCK timeout SECS <original cmd>
        local new_cmd="/usr/bin/flock -n ${lockfile} /usr/bin/timeout ${secs} ${cmd}"
        local new_line="${schedule} ${new_cmd}"

        # Replace in NEW. Use awk to avoid sed's regex pitfalls with
        # paths and special chars.
        NEW="$(echo "${NEW}" | awk -v old="${line}" -v new="${new_line}" '{
            if ($0 == old) { print new } else { print $0 }
        }')"
        echo "  ${path}: wrapped (timeout ${secs}s, lock ${lockfile})"
    done <<< "${matched}"
}

echo "Wrapping per-minute WHP crons with flock + timeout..."
for e in "${ENTRIES[@]}"; do
    IFS='|' read -r path secs lockbase <<< "${e}"
    wrap_one "${path}" "${secs}" "${lockbase}"
done

# Reap any in-flight instance currently running past its new timeout.
for e in "${ENTRIES[@]}"; do
    IFS='|' read -r path secs _lockbase <<< "${e}"
    for pid in $(pgrep -f "${path}" || true); do
        [[ -d "/proc/${pid}" ]] || continue
        elapsed="$(ps -p "${pid}" -o etimes= 2>/dev/null | tr -d ' ' || echo 0)"
        if [[ -n "${elapsed}" && "${elapsed}" -gt "${secs}" ]]; then
            echo "  reaping stale PID ${pid} for ${path} (running ${elapsed}s, cap ${secs}s)"
            kill -TERM "${pid}" 2>/dev/null || true
            sleep 1
            kill -KILL "${pid}" 2>/dev/null || true
        fi
    done
done

# Install the rewritten crontab if it actually changed.
if [[ "${NEW}" == "${CURRENT}" ]]; then
    echo "No changes — crontab already wrapped."
else
    echo "${NEW}" | crontab -
    echo "Updated crontab installed."
fi

mkdir -p /run

echo "wrap-per-minute-crons.sh: OK"
