#!/usr/bin/env bash
# Per-minute customer container watchdog. Restarts stopped autostart=true
# customer containers with exponential backoff [1,2,5,15,30] min.

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib-container-tiers.sh
source "${SCRIPT_DIR}/lib-container-tiers.sh"
# shellcheck source=lib-customer-state.sh
source "${SCRIPT_DIR}/lib-customer-state.sh"

STATE=/var/lib/whp/watchdog-state.json
LOG=/var/log/whp/customer-watchdog.log
mkdir -p "$(dirname "${STATE}")" "$(dirname "${LOG}")"
[ -f "${STATE}" ] || echo '{}' > "${STATE}"
chmod 0600 "${STATE}"
# Guard against a previously truncated or corrupt state file. A bad JSON file
# causes every subsequent jq call to emit nothing, leading to silent logic errors.
if ! jq -e . "${STATE}" >/dev/null 2>&1; then
    log_line ERROR watchdog action=state_corrupt resetting=true
    echo '{}' > "${STATE}"
    chmod 0600 "${STATE}"
fi

# Initialise customer state file (creates it with correct perms if absent)
whp_state_init

exec >> "${LOG}" 2>&1

ENABLED="$(whp_setting_bool customer_watchdog.enabled true)"
[ "${ENABLED}" = "true" ] || { log_line DEBUG watchdog action=skip reason=disabled; exit 0; }

MAX_PER_TICK="$(whp_setting_num customer_watchdog.max_restarts_per_tick 3 1 20)"
SETTLE_SEC="$(whp_setting_num customer_watchdog.settle_window_sec 30 5 120)"
VERIFY_SEC="$(whp_setting_num customer_watchdog.post_start_verify_sec 5 1 60)"
ALERT_AFTER="$(whp_setting_num customer_watchdog.alert_after_consecutive_failures 5 2 20)"
BACKOFF_MINUTES=(1 2 5 15 30)

mapfile -t CANDIDATES < <(docker ps -aq \
    --filter "label=whp.tier=customer" \
    --filter "status=exited" \
    --filter "status=dead" \
    --filter "status=created" 2>/dev/null \
    | while IFS= read -r id; do
        name="$(docker inspect -f '{{.Name}}' "$id" 2>/dev/null | sed 's|^/||')"
        [ -n "$name" ] && [ "$(whp_state_autostart "$name")" = "true" ] && echo "$id"
    done)

[ "${#CANDIDATES[@]}" -eq 0 ] && exit 0

log_line INFO watchdog action=tick candidates="${#CANDIDATES[@]}"

now_epoch=$(date -u +%s)
restarts_this_tick=0

read_state() {
    local id="$1" key="$2"
    jq -r --arg id "$id" --arg k "$key" '(.[$id][$k] // "")' "${STATE}"
}

write_state() {
    local id="$1" fails="$2" next_after="$3" alerted="$4"
    local tmp; tmp="$(mktemp)" || { log_line ERROR watchdog action=state_mktemp_failed; return 1; }
    if ! jq --arg id "$id" --argjson fails "$fails" --arg next "$next_after" --arg alerted "$alerted" \
        '.[$id] = {
            "last_attempt_at": (now | strftime("%Y-%m-%dT%H:%M:%SZ")),
            "consecutive_failures": $fails,
            "next_attempt_after": $next,
            "alerted_at": $alerted
        }' "${STATE}" > "${tmp}" 2>/dev/null; then
        log_line ERROR watchdog action=state_write_failed id="${id}"
        rm -f "${tmp}"
        return 1
    fi
    if ! jq -e . "${tmp}" >/dev/null 2>&1; then
        log_line ERROR watchdog action=state_invalid_json id="${id}"
        rm -f "${tmp}"
        return 1
    fi
    mv "${tmp}" "${STATE}"
    chmod 0600 "${STATE}"
}

clear_state() {
    local id="$1"
    local tmp; tmp="$(mktemp)" || return 1
    if ! jq --arg id "$id" 'del(.[$id])' "${STATE}" > "${tmp}" 2>/dev/null; then
        log_line ERROR watchdog action=state_clear_failed id="${id}"
        rm -f "${tmp}"
        return 1
    fi
    if ! jq -e . "${tmp}" >/dev/null 2>&1; then
        log_line ERROR watchdog action=state_clear_invalid_json id="${id}"
        rm -f "${tmp}"
        return 1
    fi
    mv "${tmp}" "${STATE}"
    chmod 0600 "${STATE}"
}

iso_to_epoch() { date -u -d "$1" +%s 2>/dev/null || echo 0; }

backoff_for_failures() {
    local n="$1"
    local idx=$((n - 1))
    [ "$idx" -lt 0 ] && idx=0
    [ "$idx" -ge "${#BACKOFF_MINUTES[@]}" ] && idx=$((${#BACKOFF_MINUTES[@]} - 1))
    echo "${BACKOFF_MINUTES[$idx]}"
}

started_at_epoch() {
    local id="$1"
    local s; s="$(docker inspect -f '{{.State.StartedAt}}' "${id}" 2>/dev/null)"
    iso_to_epoch "$s"
}

for id in "${CANDIDATES[@]}"; do
    if [ "${restarts_this_tick}" -ge "${MAX_PER_TICK}" ]; then break; fi

    next_after="$(read_state "$id" next_attempt_after)"
    if [ -n "${next_after}" ]; then
        next_epoch="$(iso_to_epoch "${next_after}")"
        if [ "${now_epoch}" -lt "${next_epoch}" ]; then
            log_line DEBUG watchdog container="${id}" action=skip reason=in_backoff next="${next_after}"
            continue
        fi
    fi

    s_epoch=$(started_at_epoch "$id")
    if [ "$s_epoch" -gt 0 ] && [ $((now_epoch - s_epoch)) -lt "${SETTLE_SEC}" ]; then
        log_line DEBUG watchdog container="${id}" action=skip reason=settle_window
        continue
    fi

    # Re-check state inside loop — state file may have changed since enumeration
    # (disable_site() may have written autostart=false during the ~50 s window).
    cur_name="$(docker inspect -f '{{.Name}}' "${id}" 2>/dev/null | sed 's|^/||')"
    if [ -n "${cur_name}" ] && [ "$(whp_state_autostart "${cur_name}")" != "true" ]; then
        log_line DEBUG watchdog container="${id}" action=skip reason=autostart_disabled_during_tick
        continue
    fi

    fails_str="$(read_state "$id" consecutive_failures)"
    fails="${fails_str:-0}"

    log_line INFO watchdog container="${id}" action=restart attempt="$((fails + 1))"
    if docker start "${id}" >/dev/null 2>&1; then
        sleep "${VERIFY_SEC}"
        if [ "$(docker inspect -f '{{.State.Running}}' "${id}" 2>/dev/null)" = "true" ]; then
            log_line INFO watchdog container="${id}" action=restart result=success
            clear_state "${id}"
            restarts_this_tick=$((restarts_this_tick + 1))
            continue
        fi
    fi

    fails=$((fails + 1))
    mins=$(backoff_for_failures "${fails}")
    next_epoch=$((now_epoch + mins * 60))
    next_iso="$(date -u -d "@${next_epoch}" +%Y-%m-%dT%H:%M:%SZ)"
    log_line WARN watchdog container="${id}" action=restart result=fail consecutive="${fails}" backoff_min="${mins}"

    alerted_at="$(read_state "$id" alerted_at)"
    alerted_epoch="$(iso_to_epoch "${alerted_at}")"
    if [ "${fails}" -ge "${ALERT_AFTER}" ] && [ $((now_epoch - alerted_epoch)) -gt 21600 ]; then
        if [ -x /root/whp/scripts/send-support-alert.sh ]; then
            tail_log="$(docker logs --tail 100 "${id}" 2>&1 | tail -20)"
            /root/whp/scripts/send-support-alert.sh \
                "WHP watchdog: container ${id} failing on $(hostname) (consec=${fails})" \
                "${tail_log}" || true
        fi
        alerted_at_iso="$(date -u -d "@${now_epoch}" +%Y-%m-%dT%H:%M:%SZ)"
        write_state "${id}" "${fails}" "${next_iso}" "${alerted_at_iso}"
    else
        write_state "${id}" "${fails}" "${next_iso}" "${alerted_at}"
    fi

    restarts_this_tick=$((restarts_this_tick + 1))
done

log_line INFO watchdog action=tick_done restarts="${restarts_this_tick}"
exit 0
