#!/usr/bin/env bash
##
## whp-stack — flip a host between the PHP panel (httpd + php-fpm) and the Go
## panel (whp-panel) over the same state. Spec: docs/superpowers/specs/
## 2026-10-23-go-panel-design.md section 10. Installed to /usr/local/sbin by
## scripts/ensure-whp-panel.sh.
##
##   whp-stack status
##   whp-stack go  [--force]
##   whp-stack php [--force]
##
## Exit codes: status 0 = exactly one stack active, matching /etc/whp/active-stack
## and the cron set, and (go mode) no host PHP binary exec'd since the switch
## (audit watch, see AUDIT_KEY; without auditd: a warning, not checked) and
## the legacy whp account has no NOPASSWD sudo (ruling P39);
## 1 = not (reason on stderr) — on a LUKS host whose /docker is locked the
## summary line carries luks=locked and the reason names the lock (CR-2).
## Switch: 3 = refused, a job is in
## flight (use --force), or (php) the copy-back of the Go panel's SQLite files was
## refused (whp-stack-converge-php.sh); 2 = usage; otherwise the exit code of the
## final status. `go` also exits 3 when its preflight refuses (a Go piece missing,
## getfacl/auditctl/ausearch/psql not installed (go_packages), or no /etc/whp/go-port-allowed
## opt-in; see go_preflight) or when the cutover gate refuses (a host that is
## not ^whp-go-test without /etc/whp/go-cutover-approved and a clean
## `scripts/whp-stack-preflight.sh --phase pre`; see cutover_gate). When the
## gate ran, `go` ends with `whp-stack-preflight.sh --phase post` (the checks
## only the switch can make true) and exits 4 if any FAILs: the switch
## completed, the Go stack is left RUNNING, nothing is rolled back (fix forward).
##
## Cutover-gate TEST SEAM (VMs only): a ^whp-go-test host skips the gate unless
## WHP_FORCE_CUTOVER_GATE=1 is in the environment or /etc/whp/force-cutover-gate
## exists; then it runs the same gate as production (approval file, pre phase,
## post phase). The seam is consulted ONLY for ^whp-go-test names: it can turn
## the gate on, never off.
set -Eeuo pipefail

# Test seams (every default is the production path).
STACK_FILE="${WHP_STACK_FILE:-/etc/whp/active-stack}"
CRONS_SCRIPT="${WHP_STACK_CRONS:-/root/whp/scripts/ensure-whp-crons.sh}"
# D6 (canary .148): /etc/whp/logrotate-coraza.conf embeds the prerotate rollup
# command of the ACTIVE stack (PHP line vs `whp waf rollup`). It is re-rendered on
# every switch, right after the stack file is written, so a go host never keeps the
# PHP line (every Coraza rotation then exec'd php and tripped the no-PHP watch).
CORAZA_LOGROTATE_SCRIPT="${WHP_STACK_CORAZA_LOGROTATE:-/root/whp/scripts/ensure-coraza-logrotate.sh}"
FM_CONF="${WHP_FM_CONF:-/etc/whp/filemanagerd.conf}"
# The file-manager daemon binary: probed for peer_groups support (fm_peer_capable).
FM_BIN="${WHP_FM_BIN:-/usr/libexec/whp/whp-filemanagerd}"
# D2 (VM 410 r16): status' fm_peer also asks the DAEMON (unit active + a
# connect to its socket), not only the conf. WHP_STACK_FM_PROBE seams the
# connect (a command given the socket path; unset = python3 AF_UNIX connect).
FM_UNIT="${WHP_STACK_FM_UNIT:-whp-filemanagerd.service}"
FM_PROBE="${WHP_STACK_FM_PROBE:-}"
LOCK_DIR="${WHP_LOCK_DIR:-/run}"
CROND_DIR="${WHP_CROND_DIR:-/etc/cron.d}"
PANEL_SCRIPT="${WHP_STACK_PANEL:-/root/whp/scripts/ensure-whp-panel.sh}"
CONVERGE_SCRIPT="${WHP_STACK_CONVERGE:-/root/whp/scripts/whp-stack-converge-go.sh}"
# Task 13: LUKS hosts keep Go state on /docker (bind mounts); installs the mount
# units + drop-ins and relocates boot-drive state BEFORE the converger writes any.
GO_LUKS_STATE_SCRIPT="${WHP_STACK_GO_LUKS_STATE:-/root/whp/scripts/ensure-whp-go-luks-state.sh}"
# PFIX-10: privd.json's per-area defaults (configs/privd.d) are merged by
# ensure-whp-privd.sh (`whp privd-config merge-areas`: adds missing keys only).
# Every switch to go runs it, so a host that reaches go by ANY path has them.
ENSURE_PRIVD_SCRIPT="${WHP_STACK_ENSURE_PRIVD:-/root/whp/scripts/ensure-whp-privd.sh}"
CONVERGE_PHP_SCRIPT="${WHP_STACK_CONVERGE_PHP:-/root/whp/scripts/whp-stack-converge-php.sh}"
# K1 (prep Task 6 fix round 1): the switch window. While a converger runs,
# nothing may start as whp-jobs: this marker exists (every `whp jobs` entry
# point refuses while it does: jobs.CheckNotSwitching) and crond is stopped.
SWITCHING="${WHP_STACK_SWITCHING:-/run/whp-stack.switching}"
CROND_UNIT="${WHP_STACK_CROND_UNIT:-crond.service}"
# ops r3 (M1): the Node terminal is retired in go mode; status asserts it.
TERMINAL_UNIT="${WHP_STACK_TERMINAL_UNIT:-whp-terminal.service}"
TERMINAL_UNIT_FILE="${WHP_STACK_TERMINAL_UNIT_FILE:-/etc/systemd/system/whp-terminal.service}"
PROC_DIR="${WHP_STACK_PROC:-/proc}"
# PFIX-13 fix round 1 (M3): the non-daemon whp-jobs processes are waited out BEFORE
# anything is stopped (same pgrep/user seams as the converger).
PGREP="${WHP_CONVERGE_PGREP:-/usr/bin/pgrep}"
JOBS_USER="${WHP_CONVERGE_JOBS_USER:-whp-jobs}"
PGREP_POLL="${WHP_STACK_PGREP_POLL:-1}"
# Same directory (and seam) the converger runs hooks from.
TERMINAL_HOOK_D="${WHP_CONVERGE_D:-/root/whp/scripts/whp-stack-converge.d}"
# terminal_retire_hook: a converge hook that masks whp-terminal (same test as
# scripts/whp-stack-preflight.sh terminal_hook).
terminal_retire_hook() {
  local f
  for f in "$TERMINAL_HOOK_D"/*; do
    [ -f "$f" ] || continue
    grep -qE '^[[:space:]]*systemctl[[:space:]]+(--[a-z-]+[[:space:]]+)*mask[[:space:]].*whp-terminal' "$f" 2>/dev/null && return 0
  done
  return 1
}
# ... and atd (prep security review S1): no at job starts in the window either.
ATD_UNIT="${WHP_STACK_ATD_UNIT:-atd.service}"
# The Go binary: `whp jobs inflight|list` (prep Tasks 7-8).
WHP_BIN="${WHP_BIN:-/usr/libexec/whp/whp}"
# Go preflight (final review A1/A2): what `whp-stack go` needs before it
# touches anything, and the explicit per-host opt-in.
PRIVD_BIN="${WHP_STACK_PRIVD_BIN:-/usr/libexec/whp/whp-privd}"
PAM_CHECK="${WHP_STACK_PAM_CHECK:-/usr/libexec/whp/whp-pam-check}"
PRIVD_UNIT="${WHP_STACK_PRIVD_UNIT:-/etc/systemd/system/whp-privd.service}"
PRIVD_JSON="${WHP_STACK_PRIVD_JSON:-/etc/whp/privd.json}"
GO_ALLOWED="${WHP_STACK_GO_ALLOWED:-/etc/whp/go-port-allowed}"
LUKS_CONF="${WHP_STACK_LUKS_CONF:-/etc/whp/luks.conf}"
# CR-2 (ops-install-luks): the LUKS state helpers (the same lib setup.sh and
# the unlock helper source) answer "is /docker locked?" for `status`.
LUKS_STATE_LIB="${WHP_STACK_LUKS_STATE_LIB:-/root/whp/scripts/lib-luks-state.sh}"
# CR-3 (ops-install-luks): the cutover gate for `go` on a host that is not a
# ^whp-go-test VM: /etc/whp/go-cutover-approved (validated like GO_ALLOWED)
# plus a clean run of the cutover preflight script. WHP_STACK_HOSTNAME seams
# the host name (unset = the real `hostname`).
CUTOVER_APPROVED="${WHP_STACK_CUTOVER_APPROVED:-/etc/whp/go-cutover-approved}"
PREFLIGHT_SCRIPT="${WHP_STACK_PREFLIGHT:-/root/whp/scripts/whp-stack-preflight.sh}"
HOSTNAME_SEAM="${WHP_STACK_HOSTNAME:-}"
# The cutover-gate test seam (header): marker path, and the bounds of the
# preflight runs and of the wait for privd's socket before the post phase
# (P15 renders through privd).
FORCE_GATE_MARKER="${WHP_STACK_FORCE_GATE_MARKER:-/etc/whp/force-cutover-gate}"
PREFLIGHT_TIMEOUT="${WHP_STACK_PREFLIGHT_TIMEOUT:-900}"
PRIVD_SOCK="${WHP_STACK_PRIVD_SOCK:-/run/whp-privd/privd.sock}"
POST_WAIT="${WHP_STACK_POST_WAIT:-60}"
CUTOVER_GATED=0; CUTOVER_FORCED=0
# Go-mode "no PHP process" watch (prep Task 14): kernel audit rules on every
# host PHP binary's execve, keyed AUDIT_KEY, loaded on `go`, removed on `php`;
# `status` in go mode fails on any exec since GO_SINCE. Container PHP is not
# matched: an audit path rule follows the host inode, not a container's copy.
AUDITCTL="${WHP_STACK_AUDITCTL:-/usr/sbin/auditctl}"
AUSEARCH="${WHP_STACK_AUSEARCH:-/usr/sbin/ausearch}"
# The go-secrets converger's ACL check needs getfacl (ruling P36: required, never skipped).
GETFACL="${WHP_STACK_GETFACL:-/usr/bin/getfacl}"
# The privd database converger (40-databases.sh) runs the PostgreSQL client: psql
# (PFIX-7, ruling P36: AL10 minimal lacks it; the VM checks installed it by hand).
PSQL="${WHP_STACK_PSQL:-/usr/bin/psql}"
# P01 of the preflight inspects whp-pam-check with readelf (binutils), which a minimal
# AlmaLinux install lacks (AL10 aarch64 measured).
READELF="${WHP_STACK_READELF:-/usr/bin/readelf}"
GO_SINCE="${WHP_STACK_GO_SINCE:-/var/lib/whp/go-mode-since}"
# Sticky no_php=VIOLATED, cleared legitimately: `whp-stack ack-php-exec <time> <why>`
# appends a line "<baseline-epoch> <who> <iso> <why>" here. status then counts only
# PHP execs at or after the newest baseline (= the acknowledged exec's second + 1),
# so a NEW exec still violates. Root-owned, not group/other-writable, or ignored
# with a failure (fail closed). Removed with GO_SINCE on a switch to php.
PHP_ACK_FILE="${WHP_STACK_PHP_ACK_FILE:-/var/lib/whp/go-php-exec-ack}"
PHP_ACK_UID="${WHP_STACK_PHP_ACK_UID:-0}"
# Ruling P39: in go mode the legacy whp account has no NOPASSWD sudo (the go
# converger holds its sudoers file aside); status checks it.
SUDOERS_SCRIPT="${WHP_STACK_SUDOERS_SCRIPT:-/root/whp/scripts/whp-stack-sudoers.sh}"
# auditctl rules do not survive a reboot. This file does: augenrules --load
# (audit-rules.service on AL10, auditd.service on AL9) compiles rules.d/ at boot.
AUDIT_RULES_FILE="${WHP_STACK_AUDIT_RULES_FILE:-/etc/audit/rules.d/whp-gomode.rules}"
PHP_BIN_DIR="${WHP_STACK_PHP_BIN_DIR:-/usr/bin}"
PHP_FPM_BIN="${WHP_STACK_PHP_FPM:-/usr/sbin/php-fpm}"
AUDIT_KEY=whp-gomode-php
# Both exec syscalls (systemd starts services with execveat(AT_EMPTY_PATH)) and
# both x86-64 syscall ABIs (a 32-bit execve via int 0x80 bypasses arch=b64).
AUDIT_ARCHES=(b64 b32)
# b32 is DROPPED (never b64) only when this kernel rejects an arch=b32 rule, e.g. an
# AArch32-less aarch64 kernel (docs/arm64/FIRST-INSTALL-PLAN.md B4). A kernel that
# accepts both (every x86-64 kernel with ia32 emulation, the Pi 4 Alma kernel
# measured 2026-11-11) keeps the full set, so nothing changes there. The decision
# is recorded in the persisted rules file by this marker line, which `status`
# reads: no marker = both arches are required, exactly as before.
AUDIT_PROBE_KEY=whp-gomode-probe
AUDIT_B64_ONLY_MARK="## whp-gomode-arches: b64 (this kernel rejects arch=b32 rules)"
AUDIT_ACTIVE_ARCHES=("${AUDIT_ARCHES[@]}")
# Where auditd keeps audit.log (+ rotated .1 .. .N): the retention check.
AUDIT_LOG_DIR="${WHP_STACK_AUDIT_LOG_DIR:-/var/log/audit}"

PHP_UNITS=(httpd.service php-fpm.service whp-backup-upload.service)
GO_UNITS=(whp-privd.service whp-panel.service)
# Plus every ENABLED whp-jobs@<name>.service daemon (prep Task 6): an instance
# is enabled when `systemctl enable` linked it into whp-stack-go.target.wants/
# (its [Install] WantedBy=), so it is exactly a whp-jobs@ entry of the go
# target's Wants=. Those links are the record of "enabled": a php switch
# stops the daemons but never disables them.
# Units the whp-stack before prep Task 14 masked in go mode (the shipped
# whp-filemanagerd.service then had Wants=httpd.service). Nothing pulls them in
# any more, so nothing is masked; a switch only lifts such a leftover mask.
LEGACY_MASKED_UNITS=(httpd.service php-fpm.service)

# Job template units whose active instances mean work is in flight.
# whp-job-* = privd systemd.run_job transient units (prep Task 9)
JOB_TEMPLATES=('whp-cpanel-import@*' 'whp-delete-user@*' 'whp-bulk-sites@*' 'whp-patch-campaign@*' 'whp-job-*')
# flock files of the backup workers (ensure-whp-crons.sh: whp-backup-cleanup,
# -scheduler, -upload, -worker). A held lock = a worker is running right now.
BACKUP_LOCK_GLOB="whp-backup-*.lock"
# Every WHP cron's flock file (ensure-whp-crons.sh: /run/whp-<job>.lock).
# whp-stack go waits up to LOCK_WAIT seconds for each held one once crond is
# stopped (final review M2), so no PHP cron that started just before the
# switch window is still writing /docker/whp/sql when the go converger
# snapshots it.
CRON_LOCK_GLOB="whp-*.lock"
LOCK_WAIT="${WHP_STACK_LOCK_WAIT:-300}"
# Quiesce-then-drain wait for in-flight one-shot jobs before a switch refuses.
DRAIN_WAIT="${WHP_STACK_DRAIN_WAIT:-660}"

die() { echo "whp-stack: $*" >&2; exit "${2:-1}"; }

# stack_lib NAME: locate scripts/lib/NAME. The installed /usr/local/sbin/whp-stack
# is a COPY with no lib/ beside it (PFIX-12), so look beside the script first (a
# repo run), then in the repo checkout (/root/whp). Sets STACK_LIB_FOUND; fails
# loudly, naming both places, when neither has it.
# The lib is SOURCED, so as root its location is never taken from the
# environment: WHP_REPO is honoured only when the REAL uid is not 0 (a dev box,
# or the test suite under fakeroot, whose id -u lies but whose /proc/self/status
# does not). Real root always uses /root/whp.
real_uid() { awk '/^Uid:/ { print $2; exit }' /proc/self/status 2>/dev/null; }
if [ "$(real_uid)" != 0 ] && [ -n "${WHP_REPO:-}" ]; then
  REPO_SCRIPTS="$WHP_REPO/scripts"
else
  REPO_SCRIPTS="/root/whp/scripts"
fi
stack_lib() {
  local name="$1" self cand
  self="$(dirname -- "$(readlink -f -- "${BASH_SOURCE[0]}")")"
  for cand in "$self/lib/$name" "$REPO_SCRIPTS/lib/$name"; do
    if [ -f "$cand" ]; then STACK_LIB_FOUND="$cand"; return 0; fi
  done
  echo "whp-stack: lib/$name not found in $self/lib or $REPO_SCRIPTS/lib" >&2
  return 1
}

# aw_load: source scripts/lib/atomic-write.sh once (durable temp + fsync +
# rename + dir fsync; VM 410 r22: a plain temp + mv left active-stack and the
# ack file ZERO-LENGTH after a power-off). Every writer below needs it, so a
# missing lib is fatal there rather than a silent non-durable write.
aw_load() {
  declare -F atomic_install >/dev/null 2>&1 && return 0
  if ! stack_lib atomic-write.sh; then
    die "cannot load scripts/lib/atomic-write.sh (looked in the script's own lib/ and $REPO_SCRIPTS/lib); refusing a non-durable write"
  fi
  # shellcheck source=scripts/lib/atomic-write.sh
  . "$STACK_LIB_FOUND"
}

jobs_daemons() { # the enabled whp-jobs@<name>.service daemons, sorted
  local w
  for w in $(systemctl show -p Wants --value whp-stack-go.target 2>/dev/null || true); do
    case "$w" in whp-jobs@?*.service) echo "$w" ;; esac
  done | LC_ALL=C sort -u
}

units_of() { # $1 = php|go
  if [ "$1" = php ]; then printf '%s\n' "${PHP_UNITS[@]}"; else printf '%s\n' "${GO_UNITS[@]}"; jobs_daemons; fi
}

file_stack() {
  local s=php
  if [ -r "$STACK_FILE" ]; then
    s="$(tr -d '[:space:]' < "$STACK_FILE")"
  fi
  printf '%s' "$s"
}

# unit_states STACK: sets ACTIVE_N / TOTAL_N and prints "unit:state,..." in UNIT_DESC.
unit_states() {
  local u st
  ACTIVE_N=0; TOTAL_N=0; UNIT_DESC=""
  while read -r u; do
    st="$(systemctl is-active "$u" 2>/dev/null || true)"
    [ -n "$st" ] || st=inactive
    TOTAL_N=$((TOTAL_N + 1))
    [ "$st" = active ] && ACTIVE_N=$((ACTIVE_N + 1))
    UNIT_DESC="${UNIT_DESC:+$UNIT_DESC,}${u%.service}:$st"
  done < <(units_of "$1")
}

# cron_state: prints php|go|mixed from the cron set this host should have.
#   php    = every PHP key present and no $CROND_DIR/whp-jobs (the go file
#            must be gone).
#   go     = no live PHP line at all — no known key present and no unknown
#            PHP line ($CRONS_SCRIPT --list-unknown-php; the go marker
#            disables those, so a marked line is not live) — AND
#            $CROND_DIR/whp-jobs byte-equal to `whp jobs cron-table` (only
#            the registry renders that file; a stale or hand-edited one is
#            not the go set).
#   mixed  = anything else, including a registry that cannot be read: fail
#            closed, never guess (ops CR-4 / plan Task 7).
cron_state() {
  local keys total=0 present=0 kind key cur unknown
  keys="$(bash "$CRONS_SCRIPT" --list-php-keys)" || die "cannot list PHP crons via $CRONS_SCRIPT" 1
  [ -n "$keys" ] || die "cannot judge the cron set: $CRONS_SCRIPT --list-php-keys returned no keys" 1
  cur="$(crontab -l 2>/dev/null || true)"
  while IFS=: read -r kind key; do
    [ -n "$kind" ] || continue
    total=$((total + 1))
    case "$kind" in
      crontab) grep -qF -- "$key" <<<"$cur" && present=$((present + 1)) ;;
      crond)   [ -e "$CROND_DIR/$key" ] && present=$((present + 1)) ;;
    esac
  done <<<"$keys"
  if [ "$present" -eq "$total" ] && [ ! -e "$CROND_DIR/whp-jobs" ] && [ ! -L "$CROND_DIR/whp-jobs" ]; then
    echo php; return 0
  fi
  if [ "$present" -eq 0 ]; then
    unknown="$(bash "$CRONS_SCRIPT" --list-unknown-php)" || die "cannot list unknown PHP crons via $CRONS_SCRIPT" 1
    if [ -z "$unknown" ] && [ -x "$WHP_BIN" ] && [ -f "$CROND_DIR/whp-jobs" ] && [ ! -L "$CROND_DIR/whp-jobs" ] \
       && cmp -s "$CROND_DIR/whp-jobs" <("$WHP_BIN" jobs cron-table 2>/dev/null); then
      echo go; return 0
    fi
  fi
  echo mixed
}

# go_jobs_summary: "go_jobs=<registered Go jobs> php_without_go=<php job keys no
# registered Go cron replaces, comma-separated>". Informational: "?" when the
# registry cannot be read (no binary yet on a php host), never a status failure.
go_jobs_summary() {
  local n replaced keys k out="" first=1
  if [ -x "$WHP_BIN" ] && n="$("$WHP_BIN" jobs list --json 2>/dev/null | jq 'length' 2>/dev/null)" \
     && replaced="$(bash "$CRONS_SCRIPT" --list-go-replaced 2>/dev/null)" \
     && keys="$(bash "$CRONS_SCRIPT" --list-php-keys 2>/dev/null)"; then
    while IFS= read -r k; do
      [ -n "$k" ] || continue
      grep -qxF -- "$k" <<<"$replaced" && continue
      if [ "$first" = 1 ]; then out="$k"; first=0; else out="$out,$k"; fi
    done <<<"$keys"
    echo "go_jobs=$n php_without_go=$out"
  else
    echo "go_jobs=? php_without_go=?"
  fi
}

# luks_state: sets LUKS_STATE (disabled|unlocked|locked|unknown) and
# LUKS_STATUS_REASON (the status failure text for locked and unknown; empty
# otherwise). No luks.conf = not a LUKS host (disabled, no field printed).
# A LUKS host whose state cannot be judged — the lib is missing, so luks_
# is_unlocked cannot run — is unknown, and status fails closed instead of
# guessing (CR-2).
luks_state() {
  LUKS_STATE=disabled; LUKS_STATUS_REASON=""
  if [ ! -e "$LUKS_CONF" ] && [ ! -L "$LUKS_CONF" ]; then return 0; fi
  if [ ! -r "$LUKS_STATE_LIB" ]; then
    LUKS_STATE=unknown
    LUKS_STATUS_REASON="/docker LUKS state unknown: $LUKS_STATE_LIB is missing or unreadable (a luks.conf is present, so this is a LUKS host; install the repo at /root/whp or set WHP_STACK_LUKS_STATE_LIB)"
    return 0
  fi
  LUKS_CONF_PATH="$LUKS_CONF"
  . "$LUKS_STATE_LIB"
  luks_is_enabled || return 0
  if luks_is_unlocked; then LUKS_STATE=unlocked; return 0; fi
  LUKS_STATE=locked
  LUKS_STATUS_REASON="/docker is locked (unlock it; the active stack starts after unlock)"
}

cmd_status() {
  local want php_active php_total php_desc go_active go_total go_desc crons reason="" nophp="" luks=""
  want="$(file_stack)"
  case "$want" in php|go) ;; *) reason="active-stack file holds '$want' (want php or go)" ;; esac
  # CR-2: a locked /docker is the root cause of every dead-stack reading, so
  # its reason wins over the unit-count/cron/audit reasons below (an invalid
  # active-stack file still wins: it is the more actionable defect). An
  # unknown LUKS state fails closed the same way.
  luks_state
  reason="${reason:-$LUKS_STATUS_REASON}"
  case "$LUKS_STATE" in locked|unknown) luks=" luks=$LUKS_STATE" ;; esac
  unit_states php; php_active=$ACTIVE_N; php_total=$TOTAL_N; php_desc=$UNIT_DESC
  unit_states go;  go_active=$ACTIVE_N;  go_total=$TOTAL_N;  go_desc=$UNIT_DESC
  crons="$(cron_state)"
  local sudo_reason="" sudo_out
  if [ "$want" = go ]; then
    no_php_check; nophp=" no_php=$NOPHP_STATE"
    if sudo_out="$(bash "$SUDOERS_SCRIPT" check 2>&1)"; then nophp="$nophp whp_sudo=ok"
    else nophp="$nophp whp_sudo=VIOLATED"; sudo_reason="the whp account has passwordless sudo in go mode (ruling P39): $sudo_out"; fi
  fi
  # crond is the engine of every cron job (php-mode crontab and go-mode
  # whp-jobs entries alike). VM pass 3 found it `failed (start-limit-hit)` for
  # ~6.7 h with status still green: report it, in both stacks.
  local crond_reason="" crond_state
  if ! systemctl is-active --quiet "$CROND_UNIT" >/dev/null 2>&1; then
    crond_state="$(systemctl is-active "$CROND_UNIT" 2>/dev/null || true)"
    crond_reason="$CROND_UNIT is not active (${crond_state:-unknown}): no cron job runs; fix with: systemctl reset-failed $CROND_UNIT && systemctl start $CROND_UNIT"
    echo "whp-stack: VIOLATION: $crond_reason" >&2
  fi
  # Go mode with terminal_backend=go only: the Node terminal (docker exec as any uid, root MySQL) must not
  # run as root. Authoritative post-switch check for the retire hook (files-import-
  # terminal Task 16; preflight P10 only trusts the hook's text). Root = User= empty
  # or root in the unit file, or a main PID owned by uid 0. Never fires on php.
  local term_reason="" term_backend=node
  if [ "$want" = go ]; then
    # Owner decision 2026-10-26: the Node terminal stays the live terminal under go;
    # the Go terminal is off unless settings.json terminal_backend=go
    # (scripts/lib/terminal-backend.sh). Only backend=go retires Node.
    # shellcheck source=scripts/lib/terminal-backend.sh
    term_backend="$(stack_lib terminal-backend.sh && . "$STACK_LIB_FOUND" && whp_terminal_backend)" \
      || die "cannot load scripts/lib/terminal-backend.sh (looked in the script's own lib/ and $REPO_SCRIPTS/lib); refusing to guess the terminal backend" 1
    if [ "$term_backend" = node ]; then
      echo "whp-stack: note: terminal_backend=node -- the Node terminal ($TERMINAL_UNIT) stays the live terminal (owner decision 2026-10-26); the Go terminal is available and off (settings.json terminal_backend=go enables it after its VM checks)" >&2
    fi
  fi
  if [ "$want" = go ] && [ "$term_backend" = go ] && systemctl is-active --quiet "$TERMINAL_UNIT" >/dev/null 2>&1; then
    local t_user t_pid t_uid=""
    t_user="$(grep -E '^User=' -- "$TERMINAL_UNIT_FILE" 2>/dev/null | head -1 | cut -d= -f2 || true)"
    t_pid="$(systemctl show -p MainPID --value "$TERMINAL_UNIT" 2>/dev/null || true)"
    case "$t_pid" in ''|*[!0-9]*|0) ;; *) t_uid="$(stat -c %u -- "$PROC_DIR/$t_pid" 2>/dev/null || true)" ;; esac
    if [ -z "$t_user" ] || [ "$t_user" = root ] || [ "$t_uid" = 0 ]; then
      if terminal_retire_hook; then
        # The retire hook is installed and the unit still runs as root: it did not do its job.
        term_reason="$TERMINAL_UNIT is active as root (User='${t_user:-}'${t_uid:+, pid $t_pid uid $t_uid}) in go mode although a converge hook that masks it is installed; the Node terminal must be stopped and masked: systemctl stop $TERMINAL_UNIT && systemctl mask $TERMINAL_UNIT"
        echo "whp-stack: VIOLATION: $term_reason" >&2
      else
        # The retire hook (scripts/whp-stack-converge.d/50-server-admin.sh, files-import-terminal Task 16) is missing from this release tree, so nothing masks the Node terminal: a cutover blocker, not a failed switch.
        echo "whp-stack: WARN: $TERMINAL_UNIT is active as root in go mode — cutover blocker: no converge hook retires it (scripts/whp-stack-converge.d/50-server-admin.sh from files-import-terminal Task 16 is missing; ledger unit:whp-terminal.service, server-admin)" >&2
      fi
    fi
  fi
  local fm=""
  case "$want" in php|go) fm_peer_state "$want"; fm=" fm_peer=$FM_PEER_STATE" ;; esac
  echo "active=$want php_units=$php_desc go_units=$go_desc crons=$crons $(go_jobs_summary)$nophp$fm$luks"
  if [ "$want" = go ]; then
    echo "whp-stack: note: go mode -- /docker/whp/sql is PHP's stale copy; root CLIs that write it (rotate-encryption-key.sh, regenerate-coraza-overrides.sh, whp-waf-agent.php, coraza-blocking-set-cli.php, security-cli.php) act on that copy, and whp-stack php then refuses to overwrite it" >&2
  fi

  local live=none
  if [ "$php_active" -gt 0 ] && [ "$go_active" -gt 0 ]; then
    reason="${reason:-both stacks have active units (mixed state)}"
  elif [ "$php_active" -eq "$php_total" ]; then live=php
  elif [ "$go_active" -eq "$go_total" ]; then live=go
  else
    reason="${reason:-no stack is fully active}"
  fi
  if [ -z "$reason" ] && [ "$live" != "$want" ]; then
    reason="active-stack file says $want but the running stack is $live"
  fi
  if [ -z "$reason" ] && [ "$crons" != "$want" ]; then
    reason="active-stack is $want but the cron set is $crons"
  fi
  if [ -z "$reason" ] && [ -n "$NOPHP_REASON" ]; then reason="$NOPHP_REASON"; fi
  if [ -z "$reason" ] && [ -n "$sudo_reason" ]; then reason="$sudo_reason"; fi
  if [ -z "$reason" ] && [ -n "$FM_PEER_REASON" ]; then reason="$FM_PEER_REASON"; fi
  if [ -z "$reason" ] && [ -n "$crond_reason" ]; then reason="$crond_reason"; fi
  if [ -z "$reason" ] && [ -n "$term_reason" ]; then reason="$term_reason"; fi
  if [ -n "$reason" ]; then
    echo "whp-stack: status FAILED: $reason" >&2
    return 1
  fi
  return 0
}

# --- go-mode "no PHP process" watch (prep Task 14) ---------------------------
# php_watch_paths: the host PHP binaries to watch, one per line: php and
# php-fpm always, plus every php[0-9]* and php-cgi that exists.
php_watch_paths() {
  local f
  printf '%s\n' "$PHP_BIN_DIR/php" "$PHP_FPM_BIN"
  for f in "$PHP_BIN_DIR"/php[0-9]* "$PHP_BIN_DIR/php-cgi"; do
    if [ -f "$f" ]; then printf '%s\n' "$f"; fi
  done
}

# watch_available: auditctl + ausearch installed and auditd running. On AL10
# auditctl ships in the separate audit-rules package; without auditd the
# kernel's records never reach /var/log/audit, so a clean search means nothing.
# `auditctl -s` "enabled 0" = kernel auditing off: no records at all.
watch_available() {
  local st pid en
  [ -x "$AUDITCTL" ] && [ -x "$AUSEARCH" ] || return 1
  st="$("$AUDITCTL" -s 2>/dev/null)" || return 1
  pid="$(awk '$1 == "pid" { print $2; exit }' <<<"$st")"
  en="$(awk '$1 == "enabled" { print $2; exit }' <<<"$st")"
  [ -n "$pid" ] && [ "$pid" != 0 ] && [ -n "$en" ] && [ "$en" != 0 ]
}
WATCH_MISSING_MSG="no-php watch unavailable (auditd missing)"

# audit_arches_from_file: the arch set `status` must find loaded: both, unless
# install_php_watch recorded that this kernel rejected b32.
audit_arches_from_file() {
  AUDIT_ACTIVE_ARCHES=("${AUDIT_ARCHES[@]}")
  if [ -r "$AUDIT_RULES_FILE" ] && grep -qxF -- "$AUDIT_B64_ONLY_MARK" "$AUDIT_RULES_FILE"; then AUDIT_ACTIVE_ARCHES=(b64); fi
}

# audit_host_arch: uname -m (seam WHP_STACK_UNAME_M for tests).
audit_host_arch() { echo "${WHP_STACK_UNAME_M:-$(uname -m 2>/dev/null)}"; }

# audit_probe_b32 PATH: 0 = keep arch=b32 (accepted, or the probe proved nothing),
# 1 = the kernel REJECTS b32 (EINVAL / EOPNOTSUPP: "Invalid argument" / "Operation
# not supported" from auditctl, e.g. an aarch64 kernel without AArch32). Any other
# failure (ENOMEM, "Rule exists", a rule limit ...) is NOT a verdict: b32 is kept and
# the real add fails loudly. x86_64 never probes: every x86-64 kernel with audit
# supports the ia32 ABI, and the x86 path must stay exactly what it was. A stale
# probe rule (same key, leaked by an earlier interrupted run) is deleted first.
audit_probe_b32() {
  local err rc=0
  [ "$(audit_host_arch)" != x86_64 ] || return 0
  "$AUDITCTL" -D -k "$AUDIT_PROBE_KEY" >/dev/null 2>&1 || true
  err="$("$AUDITCTL" -a always,exit -F arch=b32 -S execve -S execveat -F path="$1" -k "$AUDIT_PROBE_KEY" 2>&1 >/dev/null)" || rc=1
  "$AUDITCTL" -D -k "$AUDIT_PROBE_KEY" >/dev/null 2>&1 || true
  [ "$rc" -eq 1 ] || return 0
  case "$err" in
    *"Invalid argument"*|*"Operation not supported"*|*"not supported"*) return 1 ;;
  esac
  return 0
}

# install_php_watch: (re)load the rules (fixed argv; -D -k first, so a re-run
# never duplicates and other keys stay), then persist them for the next boot.
# Returns 1 on an auditctl failure, after removing what it loaded.
install_php_watch() {
  local p rules=""
  if ! watch_available; then
    echo "whp-stack: WARNING: $WATCH_MISSING_MSG: go mode is not checked for PHP execs (needs auditd running and $AUDITCTL; AL10: dnf install audit-rules, then re-run whp-stack go)" >&2
    return 0
  fi
  "$AUDITCTL" -D -k "$AUDIT_KEY" >/dev/null || return 1
  local a probe
  AUDIT_ACTIVE_ARCHES=("${AUDIT_ARCHES[@]}")
  # first path, SIGPIPE-safe under set -eo pipefail (no `| head`): the whole list, then its first line
  probe="$(php_watch_paths)"; probe="${probe%%$'\n'*}"
  if [ -n "$probe" ] && ! audit_probe_b32 "$probe"; then
    AUDIT_ACTIVE_ARCHES=(b64)
    echo "whp-stack: WARNING: this kernel rejects audit rules for arch=b32: watching arch=b64 only (a 32-bit exec of php cannot happen here without AArch32/ia32 support)" >&2
  fi
  while read -r p; do
    for a in "${AUDIT_ACTIVE_ARCHES[@]}"; do
      if ! "$AUDITCTL" -a always,exit -F arch="$a" -S execve -S execveat -F path="$p" -k "$AUDIT_KEY" >/dev/null; then
        "$AUDITCTL" -D -k "$AUDIT_KEY" >/dev/null 2>&1 || true
        return 1
      fi
      rules="$rules-a always,exit -F arch=$a -S execve -S execveat -F path=$p -k $AUDIT_KEY"$'\n'
    done
  done < <(php_watch_paths)
  local dir tmp
  dir="$(dirname "$AUDIT_RULES_FILE")"
  if [ ! -d "$dir" ]; then
    echo "whp-stack: WARNING: $dir missing: the no-php audit rules are loaded but will not survive a reboot (re-run whp-stack go after one)" >&2
    return 0
  fi
  aw_load
  tmp="$(mktemp "$dir/.whp-gomode.XXXXXX")"
  { echo "## Written by whp-stack go (go-mode no-PHP watch); removed by whp-stack php. Do not edit."
    [ "${#AUDIT_ACTIVE_ARCHES[@]}" -eq "${#AUDIT_ARCHES[@]}" ] || echo "$AUDIT_B64_ONLY_MARK"
    printf '%s' "$rules"; } > "$tmp"
  atomic_install "$tmp" "$AUDIT_RULES_FILE" 0640 || die "could not durably write $AUDIT_RULES_FILE"
}

# remove_php_watch: php mode. Unload the rules, drop the boot file and the epoch.
remove_php_watch() {
  if [ -x "$AUDITCTL" ]; then
    "$AUDITCTL" -D -k "$AUDIT_KEY" >/dev/null 2>&1 \
      || echo "whp-stack: WARNING: could not unload the $AUDIT_KEY audit rules ($AUDITCTL -D -k $AUDIT_KEY)" >&2
  fi
  rm -f -- "$AUDIT_RULES_FILE" "$GO_SINCE" "$PHP_ACK_FILE" "$COLLECTOR_CROND" "$PHP_EXEC_HB" "$PHP_EXEC_LOG" "$PHP_EXEC_LOG".1 "$PHP_EXEC_LOG".2 "$PHP_EXEC_LOG".3
}

# record_go_since KEEP: write the epoch from which status counts PHP execs.
# KEEP=1 (a re-run of `go` on a go host) keeps a valid existing one: moving it
# would hide an exec that already happened in this go-mode period.
record_go_since() {
  if [ "$1" = 1 ] && [ -r "$GO_SINCE" ] && [[ "$(cat "$GO_SINCE")" =~ ^[0-9]+$ ]]; then return 0; fi
  aw_load
  mkdir -p "$(dirname "$GO_SINCE")"
  date +%s | atomic_write_stdin "$GO_SINCE" || die "could not durably write $GO_SINCE"
}

# audit_stamp_iso LINE: the msg=audit(<stamp>:<serial>) time of an ausearch
# record as UTC ISO-8601 (2026-11-08T10:00:00Z); the raw stamp if unparseable;
# "?" if the line carries none.
audit_stamp_iso() {
  local st="${1#*msg=audit(}" iso="" ep
  st="${st%%)*}"; st="${st%:*}"          # drop ":serial"
  [ -n "$st" ] || { echo "?"; return 0; }
  if [[ "$st" =~ ^([0-9]+)(\.[0-9]+)?$ ]]; then
    iso="$(LC_ALL=C date -u -d "@${BASH_REMATCH[1]}" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || iso=""
  elif [[ "$st" =~ ^[0-9]{1,2}/[0-9]{1,2}/[0-9]{2,4}\ [0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?$ ]]; then
    # -i prints local time: resolve it to an epoch first (no -u on the parse)
    ep="$(LC_ALL=C date -d "${st%.*}" +%s 2>/dev/null)" && iso="$(LC_ALL=C date -u -d "@$ep" +%Y-%m-%dT%H:%M:%SZ 2>/dev/null)" || iso=""
  fi
  echo "${iso:-$st}"
}


# parse_exec_record LINE: classify ONE line of RAW (non -i) `ausearch -m SYSCALL
# -k KEY` output. Structure is taken only from fixed positions and anchors,
# never from substrings of attacker-influenced fields (comm / exe / argv are
# quoted or hex-encoded in raw records, and sit AFTER the fixed fields):
#   head   "type=SYSCALL msg=audit(<stamp>:<serial>): arch=A syscall=S ..."
#   tail   the key is the FINAL field: the text after the LAST " key=".
# Sets PR_KIND = ignore (not a SYSCALL, or key=(null)) | exec | unparsed (keyed
# but not understood: counted as an exec by the callers, fail closed), and
# PR_ID PR_AT(UTC ISO or "") PR_PID PR_PPID PR_AUID PR_EXE PR_COMM PR_WHY.
PR_KIND=ignore; PR_ID=""; PR_AT=""; PR_PID="?"; PR_PPID="?"; PR_AUID="?"; PR_EXE="?"; PR_COMM="?"; PR_WHY=""
parse_exec_record() {
  local line="$1" key sc stamp h rest tok
  PR_KIND=ignore; PR_ID=""; PR_AT=""; PR_PID="?"; PR_PPID="?"; PR_AUID="?"; PR_EXE="?"; PR_COMM="?"; PR_WHY=""
  line="${line#node=* }"                      # optional ENRICHED/remote node prefix (first token only)
  case "$line" in type=SYSCALL\ *) ;; *) return 0 ;; esac
  case "$line" in *" key="*) key="${line##* key=}" ;; *) key="" ;; esac
  case "$key" in "(null)"|'"(null)"') return 0 ;; esac
  PR_KIND=unparsed
  stamp=""
  if [[ "$line" =~ ^type=SYSCALL\ msg=audit\(([^\)]*)\)\ ?:\ arch=[^\ ]+\ syscall=([^\ ]+)\  ]]; then
    stamp="${BASH_REMATCH[1]}"; sc="${BASH_REMATCH[2]}"
    PR_ID="${stamp// /_}"
    PR_AT="$(audit_stamp_iso "$line")"; [ "$PR_AT" != "?" ] || PR_AT=""
    [[ "$PR_AT" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$ ]] || PR_AT=""
  else
    PR_WHY="SYSCALL record head not understood"
  fi
  if [ -z "$key" ] || [[ "$key" =~ [[:space:]] ]] || ! [[ "$key" =~ ^(\"[^\"]*\"|[^\"[:space:]]+)$ ]]; then
    PR_WHY="${PR_WHY:-no unambiguous final key= field}"
  elif [ -n "$stamp" ]; then
    case "$sc" in
      execve|execveat|59|322|11|358|221|281) PR_KIND="exec" ;;
      *) PR_WHY="keyed SYSCALL is not an exec (syscall=$sc)" ;;
    esac
  fi
  if [ -n "$stamp" ]; then
    h="${line%% comm=*}"                       # fixed fields precede the first " comm="
    [[ "$h" =~ \ ppid=([0-9]+)\ pid=([0-9]+)\ auid=([^\ ]+) ]] && { PR_PPID="${BASH_REMATCH[1]}"; PR_PID="${BASH_REMATCH[2]}"; PR_AUID="${BASH_REMATCH[3]}"; }
    case "$line" in
      *" comm="*) rest="${line#* comm=}"; tok="${rest%% *}"; PR_COMM="${tok:0:64}"
                  case "$rest" in *" exe="*) rest="${rest#* exe=}"; tok="${rest%% *}"; PR_EXE="${tok:0:128}" ;; esac ;;
    esac
    PR_COMM="$(LC_ALL=C tr -cd '[:graph:]' <<<"$PR_COMM")"; PR_EXE="$(LC_ALL=C tr -cd '[:graph:]' <<<"$PR_EXE")"
  fi
  if [ "$PR_KIND" = unparsed ]; then
    [ -n "$PR_ID" ] || PR_ID="unparsed-$(printf '%s' "$line" | cksum | cut -d' ' -f1)"
    PR_COMM=UNPARSED; PR_EXE="UNPARSED($PR_WHY)"; PR_EXE="${PR_EXE// /_}"
  fi
  return 0
}

# no_php_check (go mode): sets NOPHP_STATE (ok|VIOLATED|unavailable|unknown)
# and NOPHP_REASON (empty = no failure). Every PHP exec found is named on stderr.
NOPHP_STATE=""; NOPHP_REASON=""; NOPHP_MAX_EPOCH=0
no_php_check() {
  NOPHP_STATE=unknown; NOPHP_REASON=""; NOPHP_MAX_EPOCH=0
  if ! watch_available; then
    NOPHP_STATE=unavailable
    echo "whp-stack: WARNING: $WATCH_MISSING_MSG: the go-mode no-PHP invariant is NOT checked" >&2
    return 0
  fi
  audit_arches_from_file
  local loaded p a missing="" since out rc=0 n=0 keyed=0 line exe comm ppid pcomm ep ackb
  loaded="$("$AUDITCTL" -l -k "$AUDIT_KEY" 2>&1)" || { NOPHP_REASON="cannot list the audit rules ($AUDITCTL -l): $loaded"; return 0; }
  # the kernel lists repeated -S as one comma list (measured, audit 4.0.3)
  while read -r p; do
    for a in "${AUDIT_ACTIVE_ARCHES[@]}"; do
      grep -qxF -- "-a always,exit -F arch=$a -S execve,execveat -F path=$p -F key=$AUDIT_KEY" <<<"$loaded" || missing="$missing $p ($a)"
    done
  done < <(php_watch_paths)
  if [ -n "$missing" ]; then
    NOPHP_REASON="no-php audit rule(s) not loaded for:$missing (re-run whp-stack go)"
    return 0
  fi
  since="$(cat "$GO_SINCE" 2>/dev/null || true)"
  if ! [[ "$since" =~ ^[0-9]+$ ]]; then
    NOPHP_REASON="$GO_SINCE missing or not an epoch: cannot tell when go mode began (re-run whp-stack go)"
    return 0
  fi
  # a retention gap (or no readable/parseable log) is not a clean pass
  RETENTION_GAP="$(audit_retention_check "$since")"
  # acknowledged execs (ack-php-exec): count only what happened after the newest baseline
  ackb=""
  if ! ackb="$(php_ack_baseline)"; then
    NOPHP_REASON="$PHP_ACK_FILE is not trustworthy (not a regular root-owned file without group/other write, or a malformed line): fix or remove it"
    return 0
  fi
  if [ -n "$ackb" ] && [ "$ackb" -gt "$since" ]; then since="$ackb"; fi
  # the collector (independent of audit retention): decides alone when its coverage is continuous
  collector_assess "$since" "$ackb"
  if [ "$COLL_COVERED" = 1 ]; then
    if [ "$COLL_N" -gt 0 ]; then
      NOPHP_STATE=VIOLATED; NOPHP_MAX_EPOCH="$COLL_MAX"
      NOPHP_REASON="$COLL_N PHP exec(s) on the host since go mode began ($(date -d "@$since" '+%F %T %Z')); see above and $PHP_EXEC_LOG. After the CAUSE is fixed, clear this legitimately with: whp-stack ack-php-exec <time of the newest exec above, UTC ISO> \"<what was fixed>\" (records who/why in $PHP_ACK_FILE; a later exec still violates)"
    else
      RETENTION_GAP=""; nophp_clean
    fi
    return 0
  fi
  # ausearch (measured, audit 4.0.3): --input-logs, or it reads stdin whenever
  # stdin is not a tty; --start takes the locale's date (%x) and a time, not an
  # epoch: C locale = mm/dd/yy. -k also matches auditctl's own add/remove
  # events (a CONFIG_CHANGE carrying the key plus auditctl's sendto SYSCALL,
  # whose own key is "(null)"); -sc takes ONE syscall ("Syscall
  # execve,execveat not found"), so there is no -sc and the loop below counts
  # only execve/execveat SYSCALL lines that carry the key. A keyed SYSCALL
  # line it cannot parse is never read as clean.
  out="$(LC_ALL=C "$AUSEARCH" --input-logs -m SYSCALL -k "$AUDIT_KEY" \
           --start "$(LC_ALL=C date -d "@$since" +%m/%d/%y)" "$(LC_ALL=C date -d "@$since" +%H:%M:%S)" 2>&1)" || rc=$?
  if [ "$rc" -ne 0 ]; then
    if grep -qxF '<no matches>' <<<"$out" && [ "$COLL_N" -eq 0 ]; then nophp_clean; return 0; fi
    if grep -qxF '<no matches>' <<<"$out"; then out=""
    else
      NOPHP_REASON="cannot search the audit log (ausearch exit $rc): $(head -1 <<<"$out")"
      return 0
    fi
  fi
  local at="" pid="" auid=""
  while IFS= read -r line; do
    # WHEN: `ausearch -m SYSCALL` (raw or -i) never prints "time->" lines (VM 408
    # r17 measured at=? for every record); the time lives in each record's
    # msg=audit(<stamp>:<serial>). Raw stamp = epoch.ms; -i in the C locale =
    # "MM/DD/YY[YY] HH:MM:SS.ms" in local time. Normalised to UTC ISO below.
    # A "time->" header (ausearch --format text / other builds) is still honoured.
    case "$line" in time-\>*) at="${line#time->}"; continue ;; esac
    parse_exec_record "$line"
    [ "$PR_KIND" != ignore ] || continue
    keyed=$((keyed + 1))
    [ -z "$PR_AT" ] || at="$PR_AT"
    # an unparsed keyed record stays in `keyed` only: reported as "unparsed ausearch output"
    exe="$PR_EXE"; comm="$PR_COMM"; ppid="$PR_PPID"; pid="$PR_PID"; auid="$PR_AUID"
    pcomm="?"
    if [ -n "$ppid" ] && [ -r "/proc/$ppid/comm" ]; then pcomm="$(cat "/proc/$ppid/comm" 2>/dev/null || echo '?')"; fi
    ep=""
    if [[ "$at" =~ ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9:]{8}Z$ ]]; then ep="$(LC_ALL=C date -u -d "$at" +%s 2>/dev/null || true)"; fi
    # an exec before the acknowledged baseline (ack-php-exec) is not counted
    # again; ausearch --start only has whole-second, locale-dependent precision
    if [ -n "$ackb" ] && [[ "$ep" =~ ^[0-9]+$ ]] && [ "$ep" -lt "$ackb" ]; then keyed=$((keyed - 1)); continue; fi
    if [ "$PR_KIND" = unparsed ]; then continue; fi
    n=$((n + 1))
    if [[ "$ep" =~ ^[0-9]+$ ]] && [ "$ep" -gt "$NOPHP_MAX_EPOCH" ]; then NOPHP_MAX_EPOCH="$ep"; fi
    if [ "$n" -le 50 ]; then
      # the parent's comm NOW (the pid may since have been reused)
      echo "whp-stack: PHP exec in go mode: exe=${exe:-?} comm=${comm:-?} ppid=${ppid:-?} (now: ${pcomm:-?}) pid=${pid:-?} auid=${auid:-?} at=${at:-?} [argv: ausearch --input-logs -m SYSCALL -k $AUDIT_KEY]" >&2
    fi
  done <<<"$out"
  local unparsed=$((keyed - n)) also=""
  [ "$unparsed" -le 0 ] || also="; plus $unparsed unparsed keyed SYSCALL record(s)"
  if [ "$n" -eq 0 ] && [ "$unparsed" -le 0 ] && [ "$COLL_N" -gt 0 ]; then
    NOPHP_STATE=VIOLATED; NOPHP_MAX_EPOCH="$COLL_MAX"
    NOPHP_REASON="$COLL_N PHP exec(s) collected in $PHP_EXEC_LOG since go mode began; see above. Clear with: whp-stack ack-php-exec <time of the newest exec above, UTC ISO> \"<what was fixed>\""
    return 0
  fi
  if [ "$n" -eq 0 ] && [ "$unparsed" -le 0 ]; then nophp_clean; return 0; fi
  if [ "$n" -eq 0 ]; then
    NOPHP_REASON="unparsed ausearch output: $unparsed SYSCALL record(s) carry key $AUDIT_KEY but were not understood; inspect: ausearch --input-logs -m SYSCALL -k $AUDIT_KEY -i"
    return 0
  fi
  [ "$n" -le 50 ] || echo "whp-stack: ... and $((n - 50)) more (ausearch -k $AUDIT_KEY -i)" >&2
  NOPHP_STATE=VIOLATED
  NOPHP_REASON="$n PHP exec(s) on the host since go mode began ($(date -d "@$since" '+%F %T %Z'))$also; see above and ausearch --input-logs -k $AUDIT_KEY -i. After the CAUSE is fixed, clear this legitimately with: whp-stack ack-php-exec <time of the newest exec above, UTC ISO> \"<what was fixed>\" (records who/why in $PHP_ACK_FILE; a later exec still violates)"
}

# php_ack_baseline: prints the newest acknowledged baseline epoch ("" = none).
# Returns 1 when the ack file exists but cannot be trusted (fail closed).
php_ack_baseline() {
  local best="" line e own mode
  if [ ! -e "$PHP_ACK_FILE" ] && [ ! -L "$PHP_ACK_FILE" ]; then echo ""; return 0; fi
  [ -f "$PHP_ACK_FILE" ] && [ ! -L "$PHP_ACK_FILE" ] || return 1
  read -r own mode < <(stat -c '%u %a' -- "$PHP_ACK_FILE") || return 1
  [ "$own" = "$PHP_ACK_UID" ] && ! (( 8#$mode & 8#022 )) || return 1
  while IFS= read -r line; do
    [ -n "$line" ] || continue
    e="${line%% *}"
    [[ "$e" =~ ^[0-9]+$ ]] || return 1
    if [ -z "$best" ] || [ "$e" -gt "$best" ]; then best="$e"; fi
  done < "$PHP_ACK_FILE"
  echo "$best"
}

# cmd_ack_php_exec <time> <why...>: acknowledge the PHP execs currently reported.
# <time> must equal the newest exec's time as status prints it (at=...), which
# proves the operator read the list; the baseline is that second + 1, so an exec
# at a later second is still caught. Needs root (the ack file is root-owned).
cmd_ack_php_exec() {
  local want_at="${1:-}" why="" who base iso
  shift 1 2>/dev/null || true
  why="$*"
  if [ -z "$want_at" ] || [ -z "${why// /}" ]; then
    echo "usage: whp-stack ack-php-exec <time of the newest PHP exec, UTC ISO as status prints it> <why>" >&2; exit 2
  fi
  [ "$(id -u)" = "$PHP_ACK_UID" ] || { echo "whp-stack: ack-php-exec must run as root" >&2; exit 3; }
  [ "$(file_stack)" = go ] || { echo "whp-stack: ack-php-exec applies to go mode only" >&2; exit 3; }
  no_php_check 2>/dev/null || true
  if [ "$NOPHP_STATE" != VIOLATED ]; then
    echo "whp-stack: nothing to acknowledge (no_php=$NOPHP_STATE${NOPHP_REASON:+: $NOPHP_REASON})" >&2
    exit 3
  fi
  if [ "$NOPHP_MAX_EPOCH" -le 0 ]; then
    echo "whp-stack: refusing: the newest exec has no parseable time, so a baseline cannot be set safely" >&2
    exit 3
  fi
  iso="$(LC_ALL=C date -u -d "@$NOPHP_MAX_EPOCH" +%Y-%m-%dT%H:%M:%SZ)"
  if [ "$want_at" != "$iso" ]; then
    echo "whp-stack: refusing: '$want_at' is not the newest PHP exec ($iso); run whp-stack status and read the list first" >&2
    exit 3
  fi
  who="${SUDO_USER:-$(id -un)}"
  base=$((NOPHP_MAX_EPOCH + 1))
  aw_load
  mkdir -p "$(dirname "$PHP_ACK_FILE")"
  [ ! -L "$PHP_ACK_FILE" ] || die "$PHP_ACK_FILE is a symlink; refusing to write through it" 3
  # Rewrite (old lines + the new one) durably instead of appending in place: an
  # append without fsync read back ZERO-LENGTH after a power-off (VM 410 r22),
  # which un-acknowledges every exec; a torn append would read as malformed.
  { if [ -f "$PHP_ACK_FILE" ]; then cat -- "$PHP_ACK_FILE"; fi
    printf '%s %s %s %s\n' "$base" "$who" "$iso" "$(tr -d '\n\r' <<<"$why")"; } \
    | atomic_write_stdin "$PHP_ACK_FILE" 0600 || die "could not durably write $PHP_ACK_FILE" 1
  echo "whp-stack: acknowledged PHP exec(s) up to $iso (by $who: $why); baseline recorded in $PHP_ACK_FILE. Any later exec still violates."
}

# nophp_clean: nothing found -- ok, unless the search could not cover the
# whole go-mode period (RETENTION_GAP): then unverified, a failure.
nophp_clean() {
  if [ -n "$RETENTION_GAP" ]; then
    NOPHP_STATE=unverified
    NOPHP_REASON="no-php watch unverified: $RETENTION_GAP; and ${COLL_WHY:-the exec collector does not cover go-mode start}. Fix: $COLLECTOR_FIX"
  else
    NOPHP_STATE=ok
  fi
}

# audit_retention_check SINCE: prints why the audit log cannot cover the
# go-mode period since SINCE, or nothing. Uncovered = the oldest record still
# in audit.log* is newer than SINCE (auditd rotated the gap away), or no
# audit.log* is readable, or none has a parseable first record.
RETENTION_GAP=""
audit_retention_check() {
  local since="$1" f t oldest="" readable=0
  for f in "$AUDIT_LOG_DIR"/audit.log "$AUDIT_LOG_DIR"/audit.log.*; do
    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
    readable=1
    t="$(head -c 512 -- "$f" | sed -nE '1s/.*msg=audit\(([0-9]+)\..*/\1/p')"
    [[ "$t" =~ ^[0-9]+$ ]] || continue
    if [ -z "$oldest" ] || [ "$t" -lt "$oldest" ]; then oldest="$t"; fi
  done
  if [ "$readable" = 0 ]; then
    echo "no readable audit.log* in $AUDIT_LOG_DIR, so the search proves nothing"
  elif [ -z "$oldest" ]; then
    echo "no parseable record (msg=audit(<epoch>.) at the start of any audit.log* in $AUDIT_LOG_DIR"
  elif [ "$oldest" -gt "$since" ]; then
    echo "the oldest retained audit record ($(date -d "@$oldest" '+%F %T %Z')) is newer than the go-mode start ($(date -d "@$since" '+%F %T %Z')): auditd rotated the gap away, so PHP execs in it cannot be checked (raise num_logs/max_log_file in /etc/audit/auditd.conf)"
  fi
}

# ---- PHP-exec collector (independent of audit-log retention) -------------------
# auditd rotates audit.log every few hours on a small VM, so "ausearch since
# go-mode start" can never be proven on a long-lived host. A cron job
# (COLLECTOR_CROND, every 5 minutes) therefore runs `whp-stack collect-php-exec`,
# which copies NEW keyed PHP exec records into PHP_EXEC_LOG (append-only, size
# capped) and writes a heartbeat. status trusts the collector when its coverage
# is continuous from go-mode start; otherwise it falls back to the audit log and
# reports `unverified` with the fix. Heartbeat (key=value, root-owned):
#   coverage_start  epoch from which every exec is in the log or still in audit
#   last_run        epoch the last successful collection STARTED
#   last_exec       epoch of the newest exec ever collected (0 = none)
PHP_EXEC_LOG="${WHP_STACK_PHP_EXEC_LOG:-/var/lib/whp/go-php-exec.log}"
PHP_EXEC_HB="${WHP_STACK_PHP_EXEC_HB:-/var/lib/whp/go-php-exec-collector}"
PHP_EXEC_LOG_MAX="${WHP_STACK_PHP_EXEC_LOG_MAX:-4194304}"      # bytes; rotates to .1 .. .3
COLLECTOR_CROND="$CROND_DIR/whp-php-exec-collector"
COLLECTOR_LOCK="${WHP_STACK_COLLECTOR_LOCK:-$LOCK_DIR/whp-php-exec-collector.lock}"
COLLECTOR_INTERVAL_MIN=5
COLLECTOR_STALE="${WHP_STACK_COLLECTOR_STALE:-900}"           # seconds: > 2 intervals + slack
COLLECTOR_BIN="${WHP_STACK_COLLECTOR_BIN:-$(dirname "$CRONS_SCRIPT")/whp-stack}"
COLLECTOR_FIX="systemctl enable --now crond; whp-stack go (re-installs $COLLECTOR_CROND); optionally raise num_logs=10 / max_log_file=50 (MB) in /etc/audit/auditd.conf"

# hb_get KEY: value from the heartbeat ("" if absent/untrusted).
hb_get() {
  [ -f "$PHP_EXEC_HB" ] && [ ! -L "$PHP_EXEC_HB" ] || return 0
  local own
  own="$(stat -c %u -- "$PHP_EXEC_HB")" || return 0
  [ "$own" = "$PHP_ACK_UID" ] || return 0
  sed -nE "s/^$1=([0-9]+)\$/\\1/p" "$PHP_EXEC_HB" | head -1
}

# audit_oldest_epoch: epoch of the oldest parseable record retained in audit.log*, "" if none.
audit_oldest_epoch() {
  local f t oldest=""
  for f in "$AUDIT_LOG_DIR"/audit.log "$AUDIT_LOG_DIR"/audit.log.*; do
    [ -f "$f" ] && [ -r "$f" ] || continue
    t="$(head -c 512 -- "$f" | sed -nE '1s/.*msg=audit\(([0-9]+)\..*/\1/p')"
    [[ "$t" =~ ^[0-9]+$ ]] || continue
    if [ -z "$oldest" ] || [ "$t" -lt "$oldest" ]; then oldest="$t"; fi
  done
  echo "$oldest"
}

# collect_php_exec: one collection pass. Exit 0 = heartbeat advanced; 1 = could
# not collect (heartbeat untouched, so status sees the gap).
collect_php_exec() {
  local now last cov lastx oldest args out rc=0 line at ep id exe comm ppid pid auid tmp n=0 i
  mkdir -p "$(dirname "$PHP_EXEC_LOG")"
  # status calls this in-shell, so a missing lib must return 1 (heartbeat
  # untouched => unverified), not aw_load's die
  if ! declare -F atomic_write_stdin >/dev/null 2>&1; then
    # shellcheck source=scripts/lib/atomic-write.sh
    { stack_lib atomic-write.sh && . "$STACK_LIB_FOUND"; } || { echo "whp-stack: collect-php-exec: cannot load lib/atomic-write.sh" >&2; return 1; }
  fi
  exec 8>"$COLLECTOR_LOCK" && flock -w 60 8 || { echo "whp-stack: collector lock busy" >&2; return 1; }
  watch_available || { echo "whp-stack: collect-php-exec: $WATCH_MISSING_MSG" >&2; return 1; }
  now="$(date +%s)"
  last="$(hb_get last_run)"; cov="$(hb_get coverage_start)"; lastx="$(hb_get last_exec)"; lastx="${lastx:-0}"
  oldest="$(audit_oldest_epoch)"
  # continuity: the previous run's start must still be inside the retained audit log
  if [ -n "$last" ] && [ -n "$cov" ] && [ -n "$oldest" ] && [ "$oldest" -le "$last" ]; then :
  elif [ -n "$last" ] && [ -n "$cov" ] && [ -z "$oldest" ]; then cov="$now"
  else cov="${oldest:-$now}"; fi
  args=(--input-logs -m SYSCALL -k "$AUDIT_KEY")
  if [ -n "$last" ] && [ -n "$oldest" ] && [ "$oldest" -le "$last" ]; then
    local s=$((last - 2)); [ "$s" -ge 0 ] || s=0
    args+=(--start "$(LC_ALL=C date -d "@$s" +%m/%d/%y)" "$(LC_ALL=C date -d "@$s" +%H:%M:%S)")
  fi
  out="$(LC_ALL=C "$AUSEARCH" "${args[@]}" 2>&1)" || rc=$?
  if [ "$rc" -ne 0 ] && ! grep -qxF '<no matches>' <<<"$out"; then
    echo "whp-stack: collect-php-exec: ausearch exit $rc: $(head -1 <<<"$out")" >&2; return 1
  fi
  [ "$rc" -eq 0 ] || out=""
  tmp="$(mktemp "$(dirname "$PHP_EXEC_LOG")/.go-php-exec.XXXXXX")" || return 1
  while IFS= read -r line; do
    parse_exec_record "$line"
    [ "$PR_KIND" != ignore ] || continue
    id="$PR_ID"; at="$PR_AT"; ep=""
    if [ -n "$at" ]; then ep="$(LC_ALL=C date -u -d "$at" +%s 2>/dev/null || true)"; fi
    # unparseable time: count it as happening now (never silently dropped)
    if ! [[ "$ep" =~ ^[0-9]+$ ]]; then ep="$now"; at="$(LC_ALL=C date -u -d "@$now" +%Y-%m-%dT%H:%M:%SZ)"; fi
    exe="$PR_EXE"; comm="$PR_COMM"; ppid="$PR_PPID"; pid="$PR_PID"; auid="$PR_AUID"
    # dedupe by event id against this run and the recent log
    if grep -qF -- "	$id	" "$tmp" 2>/dev/null; then continue; fi
    if { tail -n 2000 -- "$PHP_EXEC_LOG" 2>/dev/null || true; } | grep -qF -- "	$id	"; then continue; fi
    printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$ep" "$at" "$id" "${pid:-?}" "${auid:-?}" "${ppid:-?}" "${exe:-?}" "${comm:-?}" >> "$tmp"
    n=$((n + 1))
    if [ "$ep" -gt "$lastx" ]; then lastx="$ep"; fi
  done <<<"$out"
  if [ "$n" -gt 0 ]; then
    ( umask 077; cat -- "$tmp" >> "$PHP_EXEC_LOG" ) || { rm -f -- "$tmp"; return 1; }
    sync -- "$PHP_EXEC_LOG" 2>/dev/null || sync
  fi
  rm -f -- "$tmp"
  chmod 0600 -- "$PHP_EXEC_LOG" 2>/dev/null || true
  # size cap: .2 -> .3, .1 -> .2, log -> .1 (evidence beyond the cap survives as last_exec)
  if [ -f "$PHP_EXEC_LOG" ] && [ "$(stat -c %s -- "$PHP_EXEC_LOG")" -gt "$PHP_EXEC_LOG_MAX" ]; then
    for i in 2 1; do [ ! -f "$PHP_EXEC_LOG.$i" ] || mv -f -- "$PHP_EXEC_LOG.$i" "$PHP_EXEC_LOG.$((i + 1))"; done
    mv -f -- "$PHP_EXEC_LOG" "$PHP_EXEC_LOG.1"
    ( umask 077; : > "$PHP_EXEC_LOG" )
  fi
  # whole-file rewrite: durable (fsync temp, rename, fsync dir) like active-stack
  printf 'coverage_start=%s\nlast_run=%s\nlast_exec=%s\n' "$cov" "$now" "$lastx" \
    | atomic_write_stdin "$PHP_EXEC_HB" 0600 || { echo "whp-stack: collect-php-exec: could not durably write $PHP_EXEC_HB" >&2; return 1; }
  return 0
}

cmd_collect_php_exec() {
  [ "$(id -u)" = "$PHP_ACK_UID" ] || { echo "whp-stack: collect-php-exec must run as root" >&2; exit 3; }
  [ "$(file_stack)" = go ] || exit 0       # nothing to watch outside go mode
  collect_php_exec
}

install_collector() {
  local tmp
  aw_load
  mkdir -p "$CROND_DIR"
  tmp="$(mktemp "$CROND_DIR/.whp-php-exec-collector.XXXXXX")" || return 1
  { echo "# Written by whp-stack go (go-mode no-PHP watch collector); removed by whp-stack php. Do not edit."
    echo "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
    echo "*/$COLLECTOR_INTERVAL_MIN * * * * root flock -n $LOCK_DIR/whp-php-exec-collector-cron.lock timeout 120 $COLLECTOR_BIN collect-php-exec >/dev/null 2>&1"; } > "$tmp"
  atomic_install "$tmp" "$COLLECTOR_CROND" 0644
}

# collector_assess SINCE ACKB: sets COLL_COVERED (1 = continuous since SINCE),
# COLL_WHY, COLL_N / COLL_MAX (un-acked execs in the log at or after the floor);
# prints each of those to stderr like the ausearch path does.
COLL_COVERED=0; COLL_WHY=""; COLL_N=0; COLL_MAX=0
collector_assess() {
  local since="$1" ackb="$2" floor f ep at id pid auid ppid exe comm cov last lastx now
  COLL_COVERED=0; COLL_WHY=""; COLL_N=0; COLL_MAX=0
  floor="$since"; if [ -n "$ackb" ] && [ "$ackb" -gt "$floor" ]; then floor="$ackb"; fi
  now="$(date +%s)"
  local collect_failed=0
  if [ -e "$COLLECTOR_CROND" ]; then collect_php_exec >/dev/null 2>&1 || collect_failed=1; fi
  cov="$(hb_get coverage_start)"; last="$(hb_get last_run)"; lastx="$(hb_get last_exec)"; lastx="${lastx:-0}"
  if [ "$collect_failed" = 1 ]; then
    COLL_WHY="the exec collector could not run just now (ausearch/auditd error; run whp-stack collect-php-exec)"
  elif [ -z "$cov" ] || [ -z "$last" ]; then
    COLL_WHY="the exec collector has never run (no trusted heartbeat $PHP_EXEC_HB)"
  elif [ "$cov" -gt "$since" ]; then
    COLL_WHY="the exec collector's coverage starts at $(date -d "@$cov" '+%F %T %Z'), after the go-mode start ($(date -d "@$since" '+%F %T %Z'))"
  elif [ $((now - last)) -gt "$COLLECTOR_STALE" ]; then
    COLL_WHY="the exec collector last ran $(date -d "@$last" '+%F %T %Z') ($((now - last))s ago, limit ${COLLECTOR_STALE}s): coverage has a gap"
  else
    COLL_COVERED=1
  fi
  for f in "$PHP_EXEC_LOG.3" "$PHP_EXEC_LOG.2" "$PHP_EXEC_LOG.1" "$PHP_EXEC_LOG"; do
    [ -f "$f" ] && [ ! -L "$f" ] || continue
    while IFS=$'\t' read -r ep at id pid auid ppid exe comm; do
      [[ "$ep" =~ ^[0-9]+$ ]] && [ "$ep" -ge "$floor" ] || continue
      COLL_N=$((COLL_N + 1))
      [ "$ep" -le "$COLL_MAX" ] || COLL_MAX="$ep"
      if [ "$COLL_N" -le 50 ]; then
        echo "whp-stack: PHP exec in go mode: exe=${exe:-?} comm=${comm:-?} ppid=${ppid:-?} pid=${pid:-?} auid=${auid:-?} at=${at:-?} [collected: $PHP_EXEC_LOG]" >&2
      fi
    done < "$f"
  done
  # evidence rotated out of the capped log still counts (heartbeat remembers the newest)
  if [ "$COLL_N" -eq 0 ] && [ "$lastx" -ge "$floor" ] && [ "$lastx" -gt 0 ]; then
    COLL_N=1; COLL_MAX="$lastx"
    echo "whp-stack: PHP exec in go mode: at=$(LC_ALL=C date -u -d "@$lastx" +%Y-%m-%dT%H:%M:%SZ) (details rotated out of $PHP_EXEC_LOG; ausearch -k $AUDIT_KEY)" >&2
  fi
}

# lift_legacy_masks: a host switched by the old whp-stack still has httpd /
# php-fpm masked; lift that (only a mask: is-enabled says exactly "masked").
lift_legacy_masks() {
  local u
  for u in "${LEGACY_MASKED_UNITS[@]}"; do
    if [ "$(systemctl is-enabled "$u" 2>/dev/null || true)" = masked ]; then
      systemctl unmask "$u" >/dev/null
    fi
  done
}

# lock_held_only_by_daemons FILE: true only when the kernel's lock table
# ($PROC_DIR/locks) names holders for FILE's inode and EVERY holder runs in a
# whp-jobs@<name>.service cgroup. A long-lived daemon (backup-upload) is not an
# in-flight one-shot job: the switch stops it (SIGTERM lets uploads finish) and
# waits (wait_daemons_stopped). Fail closed: no holder found, an unreadable
# table or cgroup, or any holder outside a daemon unit = a real job (false).
# The WHOLE cgroup path of a daemon instance (cgroup v2 `0::` or the v1 systemd
# hierarchy): /system.slice/[system-whp\x2djobs.slice/]whp-jobs@<name>.service.
# Anchored at both ends, so a user unit, a prefix/suffix spoof or a sub-cgroup
# never counts.
DAEMON_CGROUP_RE='^(0::|[0-9]+:name=systemd:)/system\.slice/(system-whp\\x2djobs\.slice/)?whp-jobs@[A-Za-z0-9_-]+\.service$'
lock_held_only_by_daemons() {
  local ino pids pid n=0
  ino="$(stat -c %i -- "$1" 2>/dev/null)" || return 1
  [ -n "$ino" ] && [ -r "$PROC_DIR/locks" ] || return 1
  pids="$(awk -v ino="$ino" '{ split($6, a, ":"); if (a[3] == ino) print $5 }' "$PROC_DIR/locks" 2>/dev/null | sort -u)"
  [ -n "$pids" ] || return 1
  for pid in $pids; do
    grep -qE "$DAEMON_CGROUP_RE" "$PROC_DIR/$pid/cgroup" 2>/dev/null || return 1
    n=$((n + 1))
  done
  [ "$n" -gt 0 ]
}

in_flight() { # prints one line per in-flight job
  local pat u f
  for pat in "${JOB_TEMPLATES[@]}"; do
    while read -r u _; do
      [ -n "$u" ] && echo "unit $u"
    done < <(systemctl list-units --type=service --state=active --plain --no-legend --no-pager "$pat" 2>/dev/null || true)
  done
  for f in "$LOCK_DIR"/$BACKUP_LOCK_GLOB; do
    [ -e "$f" ] || continue
    if ! flock -n "$f" true 2>/dev/null; then
      lock_held_only_by_daemons "$f" || echo "lock $f (held)"
    fi
  done
  # Go jobs: held /run/whp-jobs locks and active whp-job-* units. A Go run
  # holds no legacy /run/<x>.lock, so the lock glob above cannot see it; this
  # is what makes `whp-stack php` refuse while a Go job runs. No binary = no
  # Go jobs. A failing query is reported as in flight: "could not ask" is not "idle".
  if [ ! -x "$WHP_BIN" ]; then
    # Fail closed: on a go host (or one that ever ran Go jobs) a missing binary
    # means "cannot ask", not "idle". A genuine php host stays silent.
    if [ "$(file_stack)" = go ] || [ -e "$LOCK_DIR/whp-jobs" ]; then
      echo "go jobs: $WHP_BIN missing; cannot rule out a running job"
    fi
  else
    local out rc=0
    out="$("$WHP_BIN" jobs inflight 2>/dev/null)" || rc=$?
    # A held-lock line is dropped when only a whp-jobs daemon holds that lock.
    local line lf
    while IFS= read -r line; do
      [ -n "$line" ] || continue
      case "$line" in
        "lock "*" (held)")
          lf="${line#lock }"; lf="${lf% (held)}"
          if [ -f "$lf" ] && lock_held_only_by_daemons "$lf"; then continue; fi ;;
      esac
      printf '%s\n' "$line"
    done <<<"$out"
    [ "$rc" -eq 0 ] || echo "go jobs: '$WHP_BIN jobs inflight' failed (exit $rc); cannot rule out a running job"
  fi
}

# held_cron_locks: the held $LOCK_DIR/whp-*.lock files, space-separated.
held_cron_locks() {
  local f out=""
  for f in "$LOCK_DIR"/$CRON_LOCK_GLOB; do
    if [ -L "$f" ] || [ ! -f "$f" ]; then continue; fi
    flock -n "$f" true 2>/dev/null || out="$out $f"
  done
  printf '%s' "${out# }"
}

# wait_cron_locks FORCE: wait up to LOCK_WAIT s for every held cron lock;
# still held then: refuse (exit 3), or with --force warn and go on.
# Covers only flock-wrapped crons: a bare legacy line holds no lock.
wait_cron_locks() {
  local force="$1" held deadline=$((SECONDS + LOCK_WAIT)) said=0
  while held="$(held_cron_locks)"; [ -n "$held" ]; do
    if [ "$SECONDS" -ge "$deadline" ]; then
      if [ "$force" = 1 ]; then
        echo "whp-stack: WARNING: cron locks still held after ${LOCK_WAIT}s: $held; --force given, switching anyway" >&2
        return 0
      fi
      die "cron locks still held after ${LOCK_WAIT}s: $held (a cron job is still running); nothing was changed, retry or use --force" 3
    fi
    [ "$said" = 1 ] || { echo "whp-stack: crond stopped; waiting for running cron job(s) (up to ${LOCK_WAIT}s): $held" >&2; said=1; }
    sleep 1
  done
}

# cutover_gate (CR-3): `whp-stack go` on a host whose name does not match
# ^whp-go-test needs the per-host cutover approval (/etc/whp/go-cutover-
# approved, validated like GO_ALLOWED: root's regular file, no symlink, not
# group/other-writable) AND a clean run of scripts/whp-stack-preflight.sh.
# Fail closed: a missing or failing preflight script refuses (exit 3,
# nothing was changed) — the preflight cannot be skipped by --force because
# this runs at step 0, before the --force handling.
#
# Phases (canary .148, 2026-11-08): the gate runs only `--phase pre` (what
# must hold BEFORE the switch; P03/P05/P11 (LUKS)/P13/P15 there are capability checks,
# since only this script makes them true); `--phase post` runs at the end of
# the switch (post_switch_checks).
#
# Test seam: a ^whp-go-test host runs the gate too when WHP_FORCE_CUTOVER_GATE=1
# or $FORCE_GATE_MARKER exists (CUTOVER_FORCED=1, passed on to the preflight so
# P16 drops its test-host exemption). The seam sits INSIDE the test-host branch:
# a non-test host never reads it, so it cannot switch the gate off.
cutover_gate() {
  local h
  # an unreadable host name fails closed: it cannot be proven to be a test VM
  if [ -n "$HOSTNAME_SEAM" ]; then h="$HOSTNAME_SEAM"; else h="$(hostname 2>/dev/null || true)"; fi
  case "$h" in
    whp-go-test*)
      if [ "${WHP_FORCE_CUTOVER_GATE:-}" = 1 ] || [ -e "$FORCE_GATE_MARKER" ] || [ -L "$FORCE_GATE_MARKER" ]; then
        CUTOVER_FORCED=1
        echo "whp-stack: cutover gate: test host $h with the force-cutover-gate seam: running the production gate" >&2
      else
        return 0
      fi ;;
  esac
  CUTOVER_GATED=1
  if [ -L "$CUTOVER_APPROVED" ] || [ ! -f "$CUTOVER_APPROVED" ] || [ "$(stat -c %u -- "$CUTOVER_APPROVED")" != 0 ] \
     || [ $(( 0$(stat -c %a -- "$CUTOVER_APPROVED") & 022 )) -ne 0 ]; then
    die "cutover gate: $h is not a ^whp-go-test host (or the gate is forced) and $CUTOVER_APPROVED is missing or invalid (a root-owned regular file, not a symlink, not group/other-writable, approves this host's cutover); nothing was changed" 3
  fi
  if [ ! -f "$PREFLIGHT_SCRIPT" ]; then
    die "cutover gate: $PREFLIGHT_SCRIPT is missing, so the cutover preflight cannot run on $h; nothing was changed" 3
  fi
  if ! run_preflight pre; then
    die "cutover gate: $PREFLIGHT_SCRIPT failed on $h (--phase pre: every pre-switch check must pass before this host may switch to go; the FAIL lines above say what to fix); nothing was changed" 3
  fi
}

# run_preflight PHASE: the preflight, bounded, with the seam passed on.
run_preflight() {
  local forced=()
  [ "$CUTOVER_FORCED" != 1 ] || forced=(WHP_FORCE_CUTOVER_GATE=1)
  env "${forced[@]}" timeout "$PREFLIGHT_TIMEOUT" bash "$PREFLIGHT_SCRIPT" --phase "$1"
}

# post_switch_checks: after the Go stack started (only when the cutover gate
# ran). PASS: a loud PASS banner, return 0. FAIL: the table, a loud FAIL banner
# naming the failing checks and the fix-forward line, exit 4 -- the stack is
# NOT stopped or rolled back (the host is on go and serving; a rollback would
# be a second switch the operator should choose).
post_switch_checks() {
  local out rc=0 fails waited=0
  # P15 renders through privd: give its socket a bounded moment to appear
  while [ ! -S "$PRIVD_SOCK" ] && [ "$waited" -lt "$POST_WAIT" ]; do sleep 1; waited=$((waited + 1)); done
  echo "whp-stack: ==================== POST-SWITCH CHECKS ($PREFLIGHT_SCRIPT --phase post) ===================="
  if [ ! -f "$PREFLIGHT_SCRIPT" ]; then
    out="FAIL P-- post-switch: $PREFLIGHT_SCRIPT is missing; nothing verified the switch"; rc=1
  else
    out="$(run_preflight post 2>&1)" || rc=$?
  fi
  printf '%s\n' "$out" | sed 's/^/  /'
  if [ "$rc" -eq 0 ]; then
    echo "whp-stack: ==================== POST-SWITCH: PASS ===================="
    return 0
  fi
  fails="$(awk '$1 == "FAIL" { printf "%s%s", sep, $2; sep = " " }' <<<"$out")"
  {
    echo "whp-stack: ==================== POST-SWITCH: FAIL (${fails:-preflight exit $rc}) ===================="
    echo "whp-stack: the switch to go COMPLETED and the Go stack is RUNNING; nothing was rolled back."
    echo "whp-stack: fix forward: fix each FAIL above, then re-check with: bash $PREFLIGHT_SCRIPT --phase post"
    echo "whp-stack: (re-running 'whp-stack go' is safe and re-checks too; 'whp-stack php' only if it cannot be fixed now)"
  } >&2
  exit 4
}

# go_preflight (final review A1/A2): refuse `whp-stack go` (exit 3, nothing
# changed) unless the Go stack's pieces are installed -- the whp binary (cron
# table, in-flight query), whp-privd + whp-pam-check, the whp-privd unit and
# /etc/whp/privd.json -- AND the host opted in: /etc/whp/go-port-allowed, a
# root-owned regular file (not a symlink, not group/other-writable). The marker's presence is the opt-in;
# its content matters only on a LUKS host (any /etc/whp/luks.conf present,
# whatever LUKS_ENABLED says): there the marker must also hold the exact line
# `luks-ok`, because go mode's LUKS blockers (docs/go-port/README.md "Before
# any production host runs go mode") are still open. VM 410 is the one LUKS
# host allowed to opt in. No production host has the marker.
# os_major: the EL major (VERSION_ID of an AlmaLinux os-release) or empty.
# Test seam: WHP_STACK_OS_RELEASE.
os_major() {
  local f="${WHP_STACK_OS_RELEASE:-/etc/os-release}" v
  [ -r "$f" ] || return 0
  v="$(sed -n 's/^VERSION_ID=//p' "$f" | head -n 1 | tr -d '"'"'"' \r')"
  v="${v%%.*}"
  case "$v" in 9|10) printf '%s\n' "$v" ;; esac
  return 0
}

go_preflight() {
  local p
  for p in "$WHP_BIN" "$PRIVD_BIN" "$PAM_CHECK"; do
    if [ ! -f "$p" ] || [ ! -x "$p" ]; then die "go preflight: $p is missing or not executable; nothing was changed" 3; fi
  done
  for p in "$PRIVD_UNIT" "$PRIVD_JSON"; do
    [ -f "$p" ] || die "go preflight: $p is missing; nothing was changed" 3
  done
  if [ -L "$GO_ALLOWED" ] || [ ! -f "$GO_ALLOWED" ] || [ "$(stat -c %u -- "$GO_ALLOWED")" != 0 ] \
     || [ $(( 0$(stat -c %a -- "$GO_ALLOWED") & 022 )) -ne 0 ]; then
    die "go preflight: $GO_ALLOWED is missing, not a regular file, a symlink, not root's or group/other-writable: this host has not opted in to go mode (test VMs only: touch it as root); nothing was changed" 3
  fi
  if [ -e "$LUKS_CONF" ] || [ -L "$LUKS_CONF" ]; then
    grep -qx 'luks-ok' -- "$GO_ALLOWED" \
      || die "go preflight: $LUKS_CONF is present (LUKS host) and $GO_ALLOWED has no 'luks-ok' line: go mode's LUKS blockers are open (docs/go-port/README.md); nothing was changed" 3
  fi
}

# go_packages: the OS tools `whp-stack go` needs, checked FIRST (cheap, before
# the cutover gate), refusing with ONE dnf line naming every missing package.
# whp-stack never installs packages itself. Preflight P00 is the same check by
# package name (rpm); this one checks the binaries the code actually execs.
go_packages() {
  local p missing="" paths=""
  # Ruling P36 (batch 3): the tools go mode's checks rely on. AL10 minimal has
  # none of them (measured on VMs 409/410): without getfacl the go-secrets ACL
  # check cannot run, without auditctl/ausearch the no-PHP watch cannot, without
  # psql the privd database converger cannot (PFIX-7).
  # Package names differ by OS (VM 412, AL9): auditctl ships in `audit` on el9
  # and in the separate `audit-rules` on el10; psql is `postgresql` on both.
  local spec major auditctl_pkg=audit-rules needs="AL10 needs acl, audit, audit-rules, binutils and postgresql"
  major="$(os_major)"
  case "$major" in
    9)  auditctl_pkg=audit; needs="this el9 host needs acl, audit, binutils and postgresql" ;;
    10) needs="this el10 host needs acl, audit, audit-rules, binutils and postgresql" ;;
  esac
  for spec in "$GETFACL:acl" "$AUDITCTL:$auditctl_pkg" "$AUSEARCH:audit" "$PSQL:postgresql" "$READELF:binutils"; do
    p="${spec%:*}"
    if [ ! -f "$p" ] || [ ! -x "$p" ]; then
      paths="$paths${paths:+, }$p"
      case " $missing " in *" ${spec##*:} "*) ;; *) missing="$missing${missing:+ }${spec##*:}" ;; esac
    fi
  done
  if [ -n "$missing" ]; then
    die "go preflight: $paths not installed; install first (whp-stack never installs packages): dnf install $missing; $needs; nothing was changed" 3
  fi
}

# --- file-manager daemon peer policy (PFIX-1 item 8, R-FM-PEER) -------------
# The daemon has two gates. The FILESYSTEM gate is socket_group: the runtime
# dir and socket are root:<socket_group> 0750/0660, so only its members can
# connect. The PEER gate admits a connection by the peer's PRIMARY gid: the
# peer_groups list, or {socket_group} when the key is absent.
#   php mode: socket_group apache, no peer_groups (php-fpm runs with primary
#             group apache: both gates are apache, the PHP-era conf).
#   go mode:  socket_group whp-fm-clients (whp-panel and whp-jobs are its
#             supplementary members, made by the go converger) and peer_groups
#             ["whp-panel","whp-jobs"] (their primary groups). socket_group
#             whp-fm-clients alone would refuse both: no daemon has it as
#             its primary group.
# A daemon older than the files-import lane's Task 3 knows neither
# peer_groups (it refuses unknown keys: it would not start) nor a peer set
# other than socket_group, so for it go mode keeps the interim conf
# (socket_group whp-panel, no peer_groups) and says that whp-jobs is locked
# out until the daemon is updated. The merge order of the two changes then
# does not matter. Asked of the program: the binary carries the
# json:"peer_groups" tag exactly when its config reader decodes the key.
FM_GO_GROUP=whp-fm-clients
FM_GO_PEERS='"whp-panel","whp-jobs"'
FM_INTERIM_GROUP=whp-panel
FM_PHP_GROUP=apache

fm_peer_capable() { [ -f "$FM_BIN" ] && grep -qaF 'json:"peer_groups"' -- "$FM_BIN"; }

# fm_want MODE: "<socket_group> <peer_groups JSON list or ->" for MODE.
fm_want() {
  if [ "$1" = go ]; then
    if fm_peer_capable; then echo "$FM_GO_GROUP [$FM_GO_PEERS]"; else echo "$FM_INTERIM_GROUP -"; fi
  else
    echo "$FM_PHP_GROUP -"
  fi
}

# fm_have: "<socket_group> <peer_groups list or ->" as the conf holds them
# (the conf is the single-line JSON the installers write; peer_groups is
# written on one line right after socket_group).
fm_have() {
  local g p
  g="$(grep -oE '"socket_group"[[:space:]]*:[[:space:]]*"[^"]*"' -- "$FM_CONF" | head -n1 | sed -E 's/.*"([^"]*)"$/\1/')" || true
  p="$(grep -oE '"peer_groups"[[:space:]]*:[[:space:]]*\[[^]]*\]' -- "$FM_CONF" | head -n1 | sed -E 's/^"peer_groups"[[:space:]]*:[[:space:]]*//; s/[[:space:]]//g')" || true
  echo "${g:-?} ${p:--}"
}

# set_fm_peers MODE: rewrite the conf for MODE (atomic rename, mode and owner
# kept, untouched when already right), then restart the daemon.
set_fm_peers() {
  if [ ! -f "$FM_CONF" ]; then
    echo "whp-stack: $FM_CONF absent; file-manager peer policy not changed" >&2
    return 0
  fi
  grep -qE '"socket_group"[[:space:]]*:' "$FM_CONF" \
    || die "no socket_group key in $FM_CONF" 4
  local want group peers tmp
  want="$(fm_want "$1")"; group="${want%% *}"; peers="${want#* }"
  if [ "$1" = go ] && ! fm_peer_capable; then
    echo "whp-stack: WARNING: $FM_BIN predates peer_groups (files-import lane Task 3): file-manager socket group stays $FM_INTERIM_GROUP, so whp-jobs cannot reach filemanagerd until the daemon is updated and whp-stack go is re-run" >&2
  fi
  tmp="$(mktemp "$FM_CONF.XXXXXX")"   # same dir: mv below is an atomic rename
  # drop any peer_groups (where we write it, or as a leading key), then set
  # socket_group and, for go, append peer_groups right after it
  sed -E -e 's/,[[:space:]]*"peer_groups"[[:space:]]*:[[:space:]]*\[[^]]*\]//g' \
         -e 's/"peer_groups"[[:space:]]*:[[:space:]]*\[[^]]*\][[:space:]]*,[[:space:]]*//g' \
         -e 's/("socket_group"[[:space:]]*:[[:space:]]*")[^"]*(")/\1'"$group"'\2/' "$FM_CONF" > "$tmp"
  if [ "$peers" != - ]; then
    sed -i -E 's/("socket_group"[[:space:]]*:[[:space:]]*"[^"]*")/\1,"peer_groups":'"$peers"'/' "$tmp"
  fi
  if ! FM_CONF="$tmp" fm_have_is "$1"; then
    rm -f "$tmp"
    die "could not rewrite $FM_CONF for $1 mode (want $(fm_want "$1"), the rewrite reads $(FM_CONF="$tmp" fm_have 2>/dev/null)); nothing was changed" 4
  fi
  if cmp -s "$tmp" "$FM_CONF"; then
    rm -f "$tmp"
  else
    chown --reference="$FM_CONF" "$tmp" 2>/dev/null || true
    chmod --reference="$FM_CONF" "$tmp"
    aw_load
    atomic_install "$tmp" "$FM_CONF" || die "could not durably write $FM_CONF; nothing was restarted" 4
  fi
  systemctl restart whp-filemanagerd.service
}

fm_have_is() { [ "$(fm_have)" = "$(fm_want "$1")" ]; }

# fm_sock: the conf's socket_path (the shipped default when absent).
fm_sock() {
  local p
  p="$(grep -oE '"socket_path"[[:space:]]*:[[:space:]]*"[^"]*"' -- "$FM_CONF" 2>/dev/null | head -n1 | sed -E 's/.*"([^"]*)"$/\1/')" || true
  echo "${p:-/run/whp/filemanagerd.sock}"
}
# fm_probe SOCK: 0 when a listener accepts a connection on SOCK (bounded).
# Root is an allowed peer (filemanagerd PeerPolicy AllowUIDs 0); the probe
# connects and closes without a request. Without python3 only the socket's
# presence is checked (FM_PROBE_NOTE says so).
FM_PROBE_NOTE=""
fm_probe() {
  if [ -n "$FM_PROBE" ]; then timeout 5 "$FM_PROBE" "$1"; return; fi
  [ -S "$1" ] || return 1
  if command -v python3 >/dev/null 2>&1; then
    timeout 5 python3 -I -c 'import socket,sys; s=socket.socket(socket.AF_UNIX); s.settimeout(3); s.connect(sys.argv[1]); s.close()' "$1" 2>/dev/null
  else
    FM_PROBE_NOTE=" (python3 absent: socket present, connect not probed)"; return 0
  fi
}

# fm_peer_state MODE: sets FM_PEER_STATE (ok, absent, VIOLATED, DOWN) and
# FM_PEER_REASON for cmd_status. DOWN (D2, VM 410 r16): the conf is right but
# the daemon is not serving -- after a LUKS unlock filemanagerd stayed
# inactive, every File Manager call failed FM_E_INTERNAL, and status still
# said fm_peer=ok because only the conf was read.
FM_PEER_STATE=""; FM_PEER_REASON=""
fm_peer_state() {
  FM_PEER_STATE=ok; FM_PEER_REASON=""
  if [ ! -f "$FM_CONF" ]; then FM_PEER_STATE=absent; return 0; fi
  if ! fm_have_is "$1"; then
    FM_PEER_STATE=VIOLATED
    FM_PEER_REASON="$FM_CONF has socket_group/peer_groups $(fm_have), $1 mode wants $(fm_want "$1")"
    return 0
  fi
  local st sock
  if ! systemctl is-active --quiet "$FM_UNIT" >/dev/null 2>&1; then
    st="$(systemctl is-active "$FM_UNIT" 2>/dev/null || true)"
    FM_PEER_STATE=DOWN
    FM_PEER_REASON="$FM_UNIT is ${st:-unknown}: every File Manager call fails (FM_E_INTERNAL); start it: systemctl reset-failed $FM_UNIT && systemctl start $FM_UNIT"
    return 0
  fi
  sock="$(fm_sock)"
  if ! fm_probe "$sock"; then
    FM_PEER_STATE=DOWN
    FM_PEER_REASON="$FM_UNIT is active but nothing accepts on $sock: every File Manager call fails; see journalctl -u $FM_UNIT"
    return 0
  fi
  [ -z "$FM_PROBE_NOTE" ] || echo "whp-stack: note: fm_peer$FM_PROBE_NOTE" >&2
}

# stop_unit UNIT: stop, treating "not loaded" (systemctl exit 5) as already stopped.
stop_unit() {
  local rc=0
  systemctl stop "$1" || rc=$?
  [ "$rc" -eq 0 ] || [ "$rc" -eq 5 ]
}

# jobs_user_holders: one line per process running as whp-jobs that is NOT inside a
# whp-jobs@<name>.service daemon cgroup (the daemons are stopped, and waited for,
# later, while the panel still serves): "pid N unit U cmd C". Fails closed: a
# process whose cgroup cannot be read counts as a holder.
jobs_user_holders() {
  local pid unit cmd
  getent passwd "$JOBS_USER" >/dev/null 2>&1 || return 0
  while read -r pid; do
    [[ "$pid" =~ ^[0-9]+$ ]] || continue
    if grep -qE "$DAEMON_CGROUP_RE" "$PROC_DIR/$pid/cgroup" 2>/dev/null; then continue; fi
    unit="$({ sed -nE 's#^.*/([^/]+\.(service|scope))$#\1#p' "$PROC_DIR/$pid/cgroup" | tail -1; } 2>/dev/null || true)"
    cmd="$({ tr '\0' ' ' < "$PROC_DIR/$pid/cmdline" | cut -c1-120 | sed 's/ *$//'; } 2>/dev/null || true)"
    echo "pid $pid unit ${unit:-unknown} cmd ${cmd:-unknown}"
  done < <("$PGREP" -u "$JOBS_USER" 2>/dev/null || true)
}
# wait_jobs_user_idle FORCE: with crond/atd stopped and the switching marker set
# nothing new starts as whp-jobs, so this only falls. Bounded by DRAIN_WAIT (the
# one-shot bound, no job outlives 600 s); progress every 60 s; refuses (exit 3,
# nothing stopped) naming the holders, or with --force warns and goes on.
wait_jobs_user_idle() {
  local force="$1" holders deadline=$((SECONDS + DRAIN_WAIT)) next=$((SECONDS + 60)) said=0
  while holders="$(jobs_user_holders)"; [ -n "$holders" ]; do
    if [ "$force" = 1 ]; then
      echo "whp-stack: WARNING: process(es) still run as $JOBS_USER; --force given, switching anyway:" >&2
      printf '  %s\n' "$holders" >&2
      return 0
    fi
    if [ "$SECONDS" -ge "$deadline" ]; then
      echo "whp-stack: process(es) still run as $JOBS_USER after ${DRAIN_WAIT}s:" >&2
      printf '  %s\n' "$holders" >&2
      echo "whp-stack: REFUSED: nothing was stopped; wait for them (or stop them) and retry, or use --force" >&2
      exit 3
    fi
    if [ "$said" = 0 ]; then
      echo "whp-stack: waiting up to ${DRAIN_WAIT}s for process(es) running as $JOBS_USER (the panel keeps serving meanwhile):" >&2
      printf '  %s\n' "$holders" >&2
      said=1
    elif [ "$SECONDS" -ge "$next" ]; then
      echo "whp-stack: still waiting for $JOBS_USER process(es) ($((deadline - SECONDS))s left)" >&2
      next=$((SECONDS + 60))
    fi
    sleep "$PGREP_POLL"
  done
}

# Interrupts (INT/TERM): say exactly what state the host is in and what to run.
# SWITCH_PHASE: pre = nothing stopped; stopping = units stopped / being waited for,
# the converger has not started (safe to roll back); converge = a converger runs or
# ran (rolled back only when it reports it changed nothing); post = past the converger.
SWITCH_PHASE=pre
CONVERGE_STATUS_FILE=""
on_signal() {
  local sig="$1" st=""
  trap '' INT TERM
  trap - ERR
  echo "whp-stack: INTERRUPTED ($sig) during the switch to ${SWITCH_TO:-?}." >&2
  case "$SWITCH_PHASE" in
    pre)
      echo "whp-stack: nothing was stopped; the host is as it was (crond/atd are restarted). Re-run: whp-stack ${SWITCH_TO:-<go|php>}" >&2 ;;
    stopping)
      if rollback_stack; then
        echo "whp-stack: rolled back to the ${ROLLED_BACK_TO} stack; the host is as it was. Re-run: whp-stack ${SWITCH_TO}" >&2
      else
        echo "whp-stack: ERROR: the rollback could not restart every unit of the ${ROLLED_BACK_TO} stack; run 'whp-stack status' and start what is down by hand, then 'whp-stack ${ROLLED_BACK_TO}'" >&2
      fi ;;
    converge)
      [ -z "$CONVERGE_STATUS_FILE" ] || st="$(cat "$CONVERGE_STATUS_FILE" 2>/dev/null || true)"
      if [ "$SWITCH_TO" = php ] && [ "$st" = unchanged ]; then
        if rollback_stack; then
          echo "whp-stack: the copy-back changed nothing; rolled back to the ${ROLLED_BACK_TO} stack. Re-run: whp-stack php" >&2
        else
          echo "whp-stack: ERROR: the rollback could not restart every unit of the ${ROLLED_BACK_TO} stack; run 'whp-stack status' and start what is down by hand" >&2
        fi
      elif [ "$SWITCH_TO" = php ]; then
        echo "whp-stack: interrupted during the copy-back; it may have changed state (the Go units are stopped, PHP was NOT started). Run 'whp-stack status', then finish with: whp-stack php (or resume go mode with: whp-stack go)" >&2
      else
        echo "whp-stack: interrupted during the go converge; the host is half-switched (PHP is stopped). Finish with: whp-stack go (or go back with: whp-stack php)" >&2
      fi ;;
    *)
      echo "whp-stack: the converge finished; the host is half-switched. Finish with: whp-stack ${SWITCH_TO}" >&2 ;;
  esac
  [ "$sig" = INT ] && exit 130
  exit 143
}

# PFIX-13 item 3: a REFUSAL after the switch started stopping units must leave
# the host exactly as it was (the VM left the Go stack down after a refused
# `whp-stack php`). rollback_stack restarts the stack that was running before
# the switch (go when the host was on go, else php): its units, and -- via the
# EXIT trap (resume_jobs) -- crond and atd. It also undoes what the refused
# switch had already set up for its target (the go-mode watch rules when the
# target was go). Only refusals that happen BEFORE the converger changed
# anything may roll back; see the copy-back's COMMITTED point in
# whp-stack-converge-php.sh. A failure (ERR trap) is not a refusal and keeps
# its recovery advice (on_switch_error).
rollback_stack() {
  local prev=php u rc=0
  [ "${SWITCH_WAS_GO:-0}" = 1 ] && prev=go
  ROLLED_BACK_TO="$prev"
  echo "whp-stack: rolling back: restarting the $prev stack" >&2
  if [ "${SWITCH_TO:-}" = go ] && [ "$prev" = php ]; then remove_php_watch; fi
  systemctl reset-failed "whp-stack-$prev.target" >/dev/null 2>&1 || true
  systemctl start "whp-stack-$prev.target" || rc=1
  while read -r u; do
    [ -n "$u" ] || continue
    systemctl start "$u" || rc=1
  done < <(units_of "$prev")
  systemctl is-active --quiet "whp-stack-$prev.target" >/dev/null 2>&1 || true
  return $rc
}
# refuse_and_roll_back MESSAGE...: print the refusal, roll back, exit 3 (1 when
# the rollback itself failed: the host then needs a hand, and says so).
refuse_and_roll_back() {
  trap - ERR
  echo "whp-stack: REFUSED: $*" >&2
  if rollback_stack; then
    echo "whp-stack: rolled back to the ${ROLLED_BACK_TO:-previous} stack; the host is as it was (no Go or PHP state was changed; a planted whp-jobs/whp-panel crontab or at job, if there was one, was removed)" >&2
    exit 3
  fi
  echo "whp-stack: ERROR: the rollback could not restart every unit of the previous stack; check 'whp-stack status' and start what is down by hand" >&2
  exit 1
}

# wait_daemons_stopped FORCE: after the daemons were stopped, wait up to
# DAEMON_WAIT s for each whp-jobs@ daemon to be really inactive (an upload in
# flight finishes on SIGTERM; the unit stays "deactivating" meanwhile). Still
# active then: refuse (exit 3), or with --force warn and go on.
# Default 660 s: the unit's TimeoutStopSec is 600 (configs/systemd/whp-jobs@backup-upload.service.d),
# after which systemd kills it, so a daemon is gone within ~600 s; +60 s margin.
DAEMON_WAIT="${WHP_STACK_DAEMON_WAIT:-660}"
wait_daemons_stopped() {
  local force="$1" d st live deadline=$((SECONDS + DAEMON_WAIT)) said=0
  while :; do
    live=""
    for d in $(jobs_daemons); do
      # ActiveState, not is-active: a daemon still draining an upload is
      # "deactivating", for which is-active exits non-zero. Only inactive and
      # failed mean gone; anything else (also an unreadable state) is live.
      st="$(systemctl show -p ActiveState --value "$d" 2>/dev/null || echo unknown)"
      case "$st" in inactive|failed) ;; *) live="$live $d($st)" ;; esac
    done
    [ -n "$live" ] || return 0
    if [ "$SECONDS" -ge "$deadline" ]; then
      if [ "$force" = 1 ]; then
        echo "whp-stack: WARNING: daemon(s) still running after ${DAEMON_WAIT}s:$live; --force given, switching anyway" >&2
        return 0
      fi
      refuse_and_roll_back "daemon(s) still running after ${DAEMON_WAIT}s:$live (an upload is still finishing); retry the switch later, or use --force"
    fi
    [ "$said" = 1 ] || { echo "whp-stack: waiting for daemon(s) to stop (up to ${DAEMON_WAIT}s):$live" >&2; said=1; }
    sleep 1
  done
}

cmd_switch() { # $1 = go|php, rest = flags
  local to="$1" from force=0 busy was_go=0
  shift
  for a in "$@"; do
    case "$a" in --force) force=1 ;; *) usage ;; esac
  done
  if [ "$to" = go ]; then from=php; else from=go; fi
  if [ "$(file_stack)" = go ]; then was_go=1; fi

  # 0. going to go: the OS packages first (cheap; one dnf line), then the
  # cutover gate (CR-3: non-test hosts need approval and a clean PRE-switch
  # preflight), then every piece the Go stack needs and this host's opt-in,
  # before anything is stopped (all exit 3 on the first gap).
  if [ "$to" = go ]; then go_packages; cutover_gate; go_preflight; fi

  # The switch window opens first: crond/atd stopped, the switching marker set (nothing new
  # can start as whp-jobs; the old stack keeps SERVING). The EXIT trap restarts crond/atd
  # on any refusal or failure; the INT/TERM trap says what state the host is in.
  SWITCH_FROM="$from"; SWITCH_TO="$to"; SWITCH_WAS_GO="$was_go"; SWITCH_PHASE=pre
  trap resume_jobs EXIT
  trap 'on_signal INT' INT
  trap 'on_signal TERM' TERM
  quiesce_jobs

  # 1. in-flight jobs, drained with the old stack still serving. They are bounded: no cron
  # job's Timeout exceeds 600 s (areas/*/jobs.go; the backup crons are 600 s) so a job
  # started before the quiesce is gone by then; DRAIN_WAIT = 600 + 60 s margin.
  busy="$(in_flight)"
  if [ -n "$busy" ] && [ "$force" -ne 1 ]; then
    echo "whp-stack: crond stopped; waiting for running job(s) to finish (up to ${DRAIN_WAIT}s):" >&2
    printf '  %s\n' "$busy" >&2
    local drain_deadline=$((SECONDS + DRAIN_WAIT))
    while [ -n "$busy" ] && [ "$SECONDS" -lt "$drain_deadline" ]; do
      sleep 1
      busy="$(in_flight)"
    done
    [ -z "$busy" ] || echo "whp-stack: still running after ${DRAIN_WAIT}s:" >&2
  fi
  if [ -n "$busy" ]; then
    echo "whp-stack: jobs in flight:" >&2
    printf '  %s\n' "$busy" >&2
    if [ "$force" -ne 1 ]; then
      echo "whp-stack: refusing to switch; wait for them or use --force" >&2
      exit 3
    fi
    echo "whp-stack: --force given; switching anyway" >&2
  fi

  # 1a. any non-daemon whp-jobs process (the converger refuses while one runs): waited out
  # now, before anything is stopped.
  wait_jobs_user_idle "$force"

  # 1b. going to go: make whp-panel's TLS copy now (ensure-whp-panel.sh makes it
  # only on go hosts, so a host that ran ensure in php mode has none). Before
  # anything is stopped, so a failure leaves the host untouched.
  if [ "$to" = go ]; then
    bash "$PANEL_SCRIPT" --tls-only || die "TLS copy for whp-panel failed ($PANEL_SCRIPT --tls-only); nothing was changed" 1
  fi

  # (The switch window, K1, opened above and stays open until the new cron set is in place, step 4.)

  # 1c. going to go: crond is stopped, so no new cron job starts; wait for the
  # ones already running (held /run/whp-*.lock) before PHP is stopped and the
  # converger snapshots its SQLite files. A refusal here changed nothing (the
  # EXIT trap restarts crond and removes the marker).
  if [ "$to" = go ]; then wait_cron_locks "$force"; fi

  # 1d. going to go: load the no-PHP watch before anything is stopped, so an
  # auditctl failure leaves the stack untouched. The epoch it counts from is
  # recorded only in step 6, after the last step that may still run PHP.
  if [ "$to" = go ] && ! install_php_watch; then
    rm -f -- "$AUDIT_RULES_FILE"
    die "auditctl could not load the no-php watch rules ($AUDIT_KEY); nothing was changed (the watch's own rules were unloaded)" 1
  fi

  # From here on a failure leaves the host half-switched: say how to recover.
  trap 'on_switch_error $LINENO' ERR

  # 2. stop the stack we are leaving (the target does not stop its Wants= units).
  # Leaving go: the whp-jobs daemons first (SIGTERM: an upload in flight finishes) and
  # WAIT for them while the panel and privd still serve; only then the serving units.
  # The daemons are not in-flight one-shots, but they must be gone before we go on.
  SWITCH_PHASE=stopping
  local u
  if [ "$from" = go ]; then
    while read -r u; do [ -z "$u" ] || stop_unit "$u"; done < <(jobs_daemons)
    wait_daemons_stopped "$force"
  fi
  stop_unit "whp-stack-$from.target"
  while read -r u; do stop_unit "$u"; done < <(units_of "$from")

  # 2a. going to php: copy the panel's working copies of the shared SQLite
  # files back to /docker/whp/sql (whp-stack-converge-php.sh validates every
  # entry first and refuses to overwrite a PHP file that changed in go mode).
  # Before anything else changes, so a refusal leaves PHP stopped and the Go
  # copies intact: inspect, then `whp-stack go` to resume go mode.
  if [ "$to" = php ]; then
    local crc=0 cstatus
    cstatus="$(mktemp)"
    CONVERGE_STATUS_FILE="$cstatus"; SWITCH_PHASE=converge
    WHP_CONVERGE_STATUS_FILE="$cstatus" bash "$CONVERGE_PHP_SCRIPT" || crc=$?
    if [ "$crc" -ne 0 ]; then
      trap - ERR
      echo "whp-stack: copy-back of the Go panel's SQLite files REFUSED (exit $crc, reason above);" >&2
      if [ "$(cat "$cstatus" 2>/dev/null)" = unchanged ]; then
        # refused before the converger changed anything the Go stack depends on
        rm -f -- "$cstatus"
        refuse_and_roll_back "the copy-back changed nothing; fix the reason above, then retry: whp-stack php"
      fi
      rm -f -- "$cstatus"
      echo "whp-stack: the converger had already started changing state, so the Go stack was NOT restarted: the Go units are stopped and PHP was NOT started. Inspect, then resume go mode with: whp-stack go" >&2
      exit 3
    fi
    rm -f -- "$cstatus"; CONVERGE_STATUS_FILE=""
  fi

  # A mask left by the old whp-stack go (see LEGACY_MASKED_UNITS) would stop
  # php mode starting; in go mode is-enabled must not read "masked" either.
  lift_legacy_masks
  # going to php: the go-mode watch goes before PHP starts.
  if [ "$to" = php ]; then remove_php_watch; fi

  # 2b. going to go: prepare the panel's state (whp-stack-converge-go.sh: the
  # panel's root-made copies of the shared SQLite files under
  # /var/lib/whp-panel, /docker/whp/sql kept root's, suspension dirs,
  # settings.json group, the panel's MySQL user). whp-panel is stopped first
  # so nothing running as its uid can race root (a re-run of `whp-stack go`
  # on a go host); the target restarts it in step 6. Never run by
  # post-update.sh: a host that never switches is never touched.
  if [ "$to" = go ]; then
    # the go -> go re-run: the daemons first, waited for while the panel still serves
    while read -r u; do [ -z "$u" ] || stop_unit "$u"; done < <(jobs_daemons)
    wait_daemons_stopped "$force"
    stop_unit whp-panel.service
    # ... and whp-privd: on a go host (go -> go re-run) it is still running, and
    # ensure-whp-go-luks-state.sh --migrate refuses while it is (PFIX-7: 410 needed
    # `whp-stack php` first). From php it is not running; the go target starts it in step 6.
    stop_unit whp-privd.service
    # (the whp-jobs daemons were stopped and waited for above)
    SWITCH_PHASE=converge
    # privd is stopped: merge the privd.json per-area defaults before it can be
    # (re)started (a failure fails the switch under the ERR trap: no privd.json
    # without areas, no PRIVD_E_POLICY). VM pass 4: a harness run left none.
    bash "$ENSURE_PRIVD_SCRIPT"
    # ensure-whp-privd.sh exits 0 with only a WARNING when privd.json is absent,
    # whp is not installed or no whp-privd is shipped: assert the result itself
    # (a non-empty `areas` object), whatever its exit code said.
    if ! tr -d '[:space:]' < "$PRIVD_JSON" 2>/dev/null | grep -q '"areas":{"'; then
      echo "whp-stack: ERROR: $PRIVD_JSON has no areas after $ENSURE_PRIVD_SCRIPT (it exited 0: see its WARNING above); privd would answer PRIVD_E_POLICY. Fix that (whp privd-config merge-areas --dir /root/whp/configs/privd.d --config $PRIVD_JSON) and re-run: whp-stack go" >&2
      false
    fi
    bash "$GO_LUKS_STATE_SCRIPT" --migrate
    bash "$CONVERGE_SCRIPT"
  fi

  SWITCH_PHASE=post
  # 3. record the choice
  aw_load
  mkdir -p "$(dirname "$STACK_FILE")"
  printf '%s\n' "$to" | atomic_write_stdin "$STACK_FILE" 0644 || die "could not durably write $STACK_FILE"

  # 3b. re-render the stack-dependent Coraza logrotate config (D6); never fails the switch
  bash "$CORAZA_LOGROTATE_SCRIPT" || echo "whp-stack: WARNING: $CORAZA_LOGROTATE_SCRIPT failed; /etc/whp/logrotate-coraza.conf may still hold the other stack's rollup command" >&2

  # 4. cron set, then close the switch window
  bash "$CRONS_SCRIPT"
  resume_jobs

  # A job may have started while we were stopping the old stack. Do not abort
  # mid-switch; make it impossible to miss.
  busy="$(in_flight)"
  if [ -n "$busy" ]; then
    echo "whp-stack: WARNING: jobs appeared/are running during the switch:" >&2
    printf '  %s\n' "$busy" >&2
    echo "whp-stack: they were started against the old stack; check them after the switch" >&2
  fi

  # 5. file-manager peer policy (socket_group + peer_groups), then restart
  # (inside set_fm_peers)
  set_fm_peers "$to"

  # 6. persistence across reboot, then start the new stack
  while read -r u; do
    case "$u" in whp-jobs@*) continue ;; esac   # enabled = linked into the go target; keep it
    systemctl disable "$u" >/dev/null 2>&1 || true
  done < <(units_of "$from")
  systemctl disable "whp-stack-$from.target" >/dev/null 2>&1 || true
  systemctl enable "whp-stack-$to.target" >/dev/null 2>&1 || true
  if [ "$to" = php ]; then
    while read -r u; do systemctl enable "$u" >/dev/null 2>&1 || true; done < <(units_of php)
  fi
  # going to go: from here on no host PHP may run (prep Task 14).
  if [ "$to" = go ]; then
    record_go_since "$was_go"
    # PHP-exec collector: idempotent; first pass now so coverage starts at the go-mode start
    install_collector || echo "whp-stack: WARNING: could not install $COLLECTOR_CROND: no_php will be unverified once audit.log rotates" >&2
    collect_php_exec >/dev/null 2>&1 || echo "whp-stack: WARNING: first PHP-exec collection failed; cron retries every ${COLLECTOR_INTERVAL_MIN} min" >&2
  fi
  systemctl start "whp-stack-$to.target"

  # 7. verdict
  trap - ERR
  if [ "$to" = go ] && [ "$CUTOVER_GATED" = 1 ]; then
    # 8. the cutover gate ran: the post-converge checks (exit 4 on a FAIL,
    # stack left running) after the status line, whatever status said
    local src=0
    cmd_status || src=$?
    post_switch_checks
    return "$src"
  fi
  cmd_status
}

# quiesce_jobs / resume_jobs: open and close the switch window. The marker
# first (a cron job already being started sees it), then crond -- only if it
# was running, and only it is restarted. resume_jobs is idempotent and also
# runs from the EXIT trap, so a failed or refused switch never leaves crond
# stopped or the marker behind.
# Stopping crond does NOT stop cron children that are already running: a job
# started just before the window keeps running through it. What covers those
# are the `pgrep -u whp-jobs` refusals in the whp-jobs ensure script (run by
# the go converger) and in whp-stack-converge-php.sh -- do not remove them on
# the strength of this. (Not named here: php_mode_post_update_unchanged
# asserts this file never references the jobs ensure script.)
# atd (prep security review S1) is handled exactly like crond.
QUIESCED=0; CROND_STOPPED=0; ATD_STOPPED=0
# start_crond_verified: rapid stop/start cycles (restore, repeated switches)
# trip systemd's start limit and leave crond `failed (start-limit-hit)`; clear
# that first, then require it to be ACTIVE (fail closed).
start_crond_verified() {
  systemctl reset-failed "$CROND_UNIT" >/dev/null 2>&1 || true
  systemctl start "$CROND_UNIT" && systemctl is-active --quiet "$CROND_UNIT" >/dev/null
}
quiesce_jobs() {
  [ "$QUIESCED" = 0 ] || return 0
  mkdir -p "$(dirname "$SWITCHING")"
  : > "$SWITCHING"
  QUIESCED=1
  if systemctl is-active --quiet "$CROND_UNIT"; then
    systemctl stop "$CROND_UNIT"
    CROND_STOPPED=1
  fi
  if systemctl is-active --quiet "$ATD_UNIT"; then
    systemctl stop "$ATD_UNIT"
    ATD_STOPPED=1
  fi
}
resume_jobs() {
  [ "$QUIESCED" = 1 ] || return 0
  local rc=0
  if [ "$CROND_STOPPED" = 1 ]; then
    start_crond_verified || { echo "whp-stack: ERROR: $CROND_UNIT is not active after the restart; start it by hand (systemctl reset-failed $CROND_UNIT; systemctl start $CROND_UNIT)" >&2; rc=1; }
    CROND_STOPPED=0
  fi
  if [ "$ATD_STOPPED" = 1 ]; then
    systemctl start "$ATD_UNIT" || echo "whp-stack: WARNING: could not restart $ATD_UNIT; start it by hand" >&2
    ATD_STOPPED=0
  fi
  rm -f -- "$SWITCHING"
  QUIESCED=0
  return $rc
}

on_switch_error() {
  local line="$1"
  trap - ERR
  echo "whp-stack: switch to $SWITCH_TO FAILED (line $line); host may be half-switched. Current state:" >&2
  cmd_status >&2 || true
  if [ "$SWITCH_TO" = go ] && [ "${SWITCH_WAS_GO:-0}" = 1 ]; then
    # go -> go: the host was already on go, so `whp-stack php` (a copy-back) is the
    # wrong advice. Restart what this run stopped so the panel is not left down.
    local u
    for u in whp-privd.service whp-panel.service $(jobs_daemons); do
      systemctl start "$u" || echo "whp-stack: WARNING: could not restart $u; start it by hand" >&2
    done
    echo "whp-stack: the go units this run stopped were restarted (best effort); fix the error above, then re-run: whp-stack go" >&2
  else
    echo "whp-stack: recover with: whp-stack $SWITCH_FROM" >&2
  fi
  if [ "$SWITCH_TO" = go ] && [ -e "$AUDIT_RULES_FILE" ]; then
    echo "whp-stack: note: the go-mode no-php watch rules stay loaded and in $AUDIT_RULES_FILE until 'whp-stack php' removes them (by hand: auditctl -D -k $AUDIT_KEY; rm -f $AUDIT_RULES_FILE)" >&2
  fi
  exit 1
}

usage() {
  echo "usage: whp-stack status | go [--force] | php [--force] | ack-php-exec <time> <why> | collect-php-exec" >&2
  exit 2
}

case "${1:-}" in
  status) [ $# -eq 1 ] || usage; cmd_status ;;
  go|php) cmd_switch "$@" ;;
  ack-php-exec) shift; cmd_ack_php_exec "$@" ;;
  collect-php-exec) [ $# -eq 1 ] || usage; cmd_collect_php_exec ;;
  *) usage ;;
esac
