#!/usr/bin/env bash
# watch_phe.sh — one-shot phe108 recovery watcher.
# Runs every 15 min from crontab. When phe answers both ping AND ssh, notifies
# the manager tmux pane, writes a marker, and removes itself from crontab.
#
# Silent on failure (server still down) — just logs. So the log will fill with
# "still down" lines every 15 min while we wait.

set -uo pipefail

PHE_IP="100.74.71.38"
LOG="/data/cameron/agents_stuff/logs/watch_phe.log"
MARKER="/data/cameron/agents_stuff/.phe_is_up"
CRON_MARKER="watch_phe.sh"
NOW=$(date -Iseconds)

mkdir -p "$(dirname "$LOG")"

if ! timeout 3 ping -c 1 -W 2 "$PHE_IP" >/dev/null 2>&1; then
  echo "$NOW  still down (ping)" >> "$LOG"
  exit 0
fi

echo "$NOW  ping OK — testing ssh" >> "$LOG"

# Ping alone isn't enough — server could be booting with sshd not ready yet.
# Wait for ssh to succeed before declaring readiness.
if ! timeout 8 ssh -o BatchMode=yes -o ConnectTimeout=6 cameronsmith@"$PHE_IP" 'echo ready' >/dev/null 2>&1; then
  echo "$NOW  ping OK but ssh failed — continuing to wait" >> "$LOG"
  exit 0
fi

# Confirmed up: ping + ssh both work.
UPTIME=$(timeout 6 ssh -o BatchMode=yes cameronsmith@"$PHE_IP" 'uptime | head -c 100' 2>/dev/null || echo 'uptime unknown')
echo "$NOW  PHE UP: $UPTIME" >> "$LOG"
echo "$NOW" > "$MARKER"

# Notify manager pane (guarded with timeouts so a dead pane doesn't hang cron)
{
  timeout 5 tmux send-keys -t agents:manager C-u
  sleep 1
  timeout 5 tmux send-keys -t agents:manager -l "[watch_phe] PHE108 IS BACK ONLINE ($NOW). ping OK, ssh OK. Uptime: $UPTIME. Removing myself from cron. Ready to walk through the post-phe-recovery migration checklist whenever you're ready."
  sleep 1
  timeout 5 tmux send-keys -t agents:manager Enter
  sleep 1
  timeout 5 tmux send-keys -t agents:manager Enter
} 2>/dev/null || echo "$NOW  (tmux notify failed silently)" >> "$LOG"

# Self-disable
(crontab -l 2>/dev/null | grep -v "$CRON_MARKER") | crontab -
echo "$NOW  cron entry removed — watcher done" >> "$LOG"