#!/usr/bin/env bash
# WHP tiered container boot orchestrator. Systemd oneshot After=docker.service.
# Tier 1 (db) / Tier 2 (dns) fatal on healthcheck timeout.
# Tier 3 (edge) soft-fatal.
# Tier 4 (customer) per-container best-effort; watchdog handles recovery.

set -uo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=lib-container-tiers.sh
source "${SCRIPT_DIR}/lib-container-tiers.sh"
# shellcheck source=lib-customer-state.sh
source "${SCRIPT_DIR}/lib-customer-state.sh"

LOG=/var/log/whp/boot-orchestrator.log
SUMMARY=/var/lib/whp/boot-orchestrator-last.json
mkdir -p "$(dirname "${SUMMARY}")" "$(dirname "${LOG}")"
chmod 0755 "$(dirname "${SUMMARY}")" "$(dirname "${LOG}")"

# Redirect stdout/stderr to the log file. Note: previous version used
# `tee -a` via process substitution, which spawns a background tee process
# that `wait` later joined unintentionally — causing the customer-batch
# `wait` (joining backgrounded `docker start` calls) to block forever.
# Plain redirection has no subprocess and works correctly. The systemd unit
# has StandardOutput=journal / StandardError=journal so the journal also
# captures output when invoked by systemd.
exec >> "${LOG}" 2>&1

BOOT_START=$(date -u +%s)
BOOT_START_ISO="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

ENABLED="$(whp_setting_bool boot_orchestrator.enabled true)"
BATCH_SIZE="$(whp_setting_num boot_orchestrator.customer_batch_size 5 1 50)"
BATCH_DELAY="$(whp_setting_num boot_orchestrator.customer_batch_delay_sec 10 1 120)"
TIER_TIMEOUT="$(whp_setting_num boot_orchestrator.tier_healthcheck_timeout_sec 60 10 300)"

if [ "${ENABLED}" != "true" ]; then
    log_line INFO boot action=skip reason=disabled_via_settings
    exit 0
fi

log_line INFO boot action=start batch_size="${BATCH_SIZE}" batch_delay="${BATCH_DELAY}" tier_timeout="${TIER_TIMEOUT}"

# Initialise customer state file (creates it with correct perms if absent)
whp_state_init

# Preflight: docker daemon responsive
preflight_attempts=0
while ! docker info >/dev/null 2>&1; do
    preflight_attempts=$((preflight_attempts + 1))
    if [ "${preflight_attempts}" -ge 5 ]; then
        log_line ERROR boot action=preflight result=fail attempts=5
        exit 1
    fi
    log_line WARN boot action=preflight attempt="${preflight_attempts}" result=retry
    sleep 2
done

TIER_RESULTS=()

start_tier() {
    local tier="$1" healthcheck_cmd="$2" fatal_on_fail="$3"
    log_line INFO boot tier="${tier}" action=enumerate
    local ids; ids="$(tier_containers "${tier}")"
    local count=0
    if [ -n "${ids}" ]; then
        count="$(printf '%s\n' "${ids}" | wc -l | tr -d ' ')"
    fi
    log_line INFO boot tier="${tier}" action=found count="${count}"
    local started=0
    # shellcheck disable=SC2086
    for id in ${ids}; do
        if docker start "${id}" >/dev/null 2>&1; then started=$((started + 1))
        else log_line ERROR boot tier="${tier}" container="${id}" action=start result=fail; fi
    done
    log_line INFO boot tier="${tier}" action=started count="${started}/${count}"
    # Always run health check — dockerd may have started these via restart policy
    # before the orchestrator queried. An empty enumeration does NOT mean the
    # tier is absent; it means containers are already running.
    local hc_start; hc_start=$(date -u +%s)
    if wait_for_health "${tier}" "${TIER_TIMEOUT}" "${healthcheck_cmd}"; then
        local hc_end; hc_end=$(date -u +%s)
        log_line INFO boot tier="${tier}" action=healthy elapsed=$((hc_end - hc_start))
        TIER_RESULTS+=("${tier}:healthy:${started}/${count}")
        return 0
    fi
    log_line ERROR boot tier="${tier}" action=healthcheck result=timeout timeout="${TIER_TIMEOUT}"
    TIER_RESULTS+=("${tier}:unhealthy:${started}/${count}")
    if [ "${fatal_on_fail}" = "1" ]; then
        send_alert_fatal "${tier}"
        write_summary
        exit 2
    fi
    return 0
}

start_tier_customer() {
    log_line INFO boot tier=customer action=enumerate
    # Enumerate by tier=customer OR legacy whp.user label, then filter
    # disabled containers via state file (keyed by container name, not ID).
    # (docker update --label-add/rm is unsupported on plain containers, so we
    # cannot rely on whp.autostart labels for dynamic state — v109 pivot.)
    local ids
    ids="$(customer_container_ids | while IFS= read -r id; do
        [ -z "$id" ] && continue
        name="$(docker inspect -f '{{.Name}}' "$id" 2>/dev/null | sed 's|^/||')"
        if [ -n "$name" ] && [ "$(whp_state_autostart "$name")" = "true" ]; then
            echo "$id"
        fi
    done)"
    if [ -z "${ids}" ]; then
        log_line INFO boot tier=customer action=enumerate result=empty
        TIER_RESULTS+=("customer:empty:0")
        return 0
    fi
    local total; total="$(echo "${ids}" | wc -l | tr -d ' ')"
    log_line INFO boot tier=customer action=start total="${total}" batch_size="${BATCH_SIZE}" delay="${BATCH_DELAY}"
    local started=0 failed=()
    local batch=()
    local i=0
    # shellcheck disable=SC2086
    for id in ${ids}; do
        batch+=("${id}")
        i=$((i + 1))
        if [ "${#batch[@]}" -ge "${BATCH_SIZE}" ] || [ "${i}" -eq "${total}" ]; then
            for c in "${batch[@]}"; do docker start "${c}" >/dev/null 2>&1 & done
            wait
            for c in "${batch[@]}"; do
                if [ "$(docker inspect -f '{{.State.Running}}' "${c}" 2>/dev/null)" = "true" ]; then
                    started=$((started + 1))
                else
                    failed+=("${c}")
                    log_line WARN boot tier=customer container="${c}" action=start result=not_running
                fi
            done
            if [ "${i}" -lt "${total}" ]; then sleep "${BATCH_DELAY}"; fi
            batch=()
        fi
    done
    log_line INFO boot tier=customer action=done started="${started}/${total}" failed_count="${#failed[@]}"
    TIER_RESULTS+=("customer:done:${started}/${total}")
}

write_summary() {
    local boot_end; boot_end=$(date -u +%s)
    local duration=$((boot_end - BOOT_START))
    local tier_lines=""
    for r in "${TIER_RESULTS[@]}"; do
        if [ -n "${tier_lines}" ]; then tier_lines+=","; fi
        tier_lines+="\"${r%%:*}\":\"${r#*:}\""
    done
    cat > "${SUMMARY}" <<EOF
{ "booted_at": "${BOOT_START_ISO}", "duration_sec": ${duration}, "tiers": { ${tier_lines} } }
EOF
    chmod 0644 "${SUMMARY}"
}

send_alert_fatal() {
    local tier="$1"
    if [ -x /root/whp/scripts/send-support-alert.sh ]; then
        /root/whp/scripts/send-support-alert.sh \
            "WHP boot orchestrator: tier ${tier} healthcheck timeout on $(hostname)" \
            "$(tail -50 "${LOG}")" \
            || true
    fi
}

# All healthchecks use docker exec or docker inspect (no host-side curl/dig
# required, since haproxy-manager/coraza-spoa don't expose ports to host
# and dig isn't installed on whp01 / staging).
HC_DB="docker exec mysql mysqladmin ping 2>/dev/null | grep -q 'mysqld is alive' && docker exec postgresql pg_isready -U postgres >/dev/null 2>&1"
# DNS: powerdns image is debian-slim (no pgrep, no curl). Running state is
# the most portable check.
HC_DNS="[ \"\$(docker inspect -f '{{.State.Running}}' powerdns 2>/dev/null)\" = 'true' ]"
# Edge: haproxy-manager has a built-in HEALTHCHECK directive (container-manager.sh:113);
# read its result. coraza-spoa has no HEALTHCHECK so verify Running state.
# whp-suspended runs with --restart unless-stopped so dockerd manages it;
# not gating boot on it (also still being mirrored in some envs).
HC_EDGE="[ \"\$(docker inspect -f '{{.State.Health.Status}}' haproxy-manager 2>/dev/null)\" = 'healthy' ] && [ \"\$(docker inspect -f '{{.State.Running}}' coraza-spoa 2>/dev/null)\" = 'true' ]"

start_tier db   "${HC_DB}"   1
start_tier dns  "${HC_DNS}"  1
start_tier edge "${HC_EDGE}" 0
start_tier_customer

write_summary
BOOT_END=$(date -u +%s)
log_line INFO boot action=done duration_sec=$((BOOT_END - BOOT_START))
exit 0
