claude
0088fda095
Guard now polls every 30s but renews every 15 min. Coupling them meant a wave that finished just after a renewal sat undetected for a full interval with the sprite pinned hot. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
67 lines
2.8 KiB
Bash
Executable File
67 lines
2.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# wave-guard.sh <sentinel-name> <task> [<task>...]
|
|
#
|
|
# Solves the real cause of the wave-3 loss: the sprite AUTO-SUSPENDS when idle
|
|
# ("When idle, sprites pause automatically" — /.sprite/llm.txt). Detached workers
|
|
# do NOT keep it awake; only services, live sessions, or a task lease do.
|
|
#
|
|
# Holds a sprite task lease (POST /v1/tasks, max expire 3600s) and renews it while
|
|
# workers run, so the box stays hot with no console attached. When the workers stop
|
|
# it writes the durable completion record AND releases the lease, so the sprite is
|
|
# free to suspend instead of burning money idling.
|
|
set -u
|
|
SENT="$1"; shift
|
|
TASKS=("$@")
|
|
L="$HOME/bookshelf/logs"; OUT="$L/$SENT"; LEASE="bookshelf-wave"
|
|
RENEW="${RENEW:-900}" # renew every 15 min against a 60 min lease
|
|
POLL="${POLL:-30}" # but CHECK for completion every 30s (see loop below)
|
|
GUARD_LOG="$L/wave-guard.log"
|
|
|
|
lease_hold() {
|
|
sprite-env curl -X DELETE "/v1/tasks/$LEASE" >/dev/null 2>&1
|
|
sprite-env curl -X POST /v1/tasks -H 'Content-Type: application/json' \
|
|
-d "{\"name\":\"$LEASE\",\"expire\":\"3600s\"}" >/dev/null 2>&1
|
|
}
|
|
lease_release() { sprite-env curl -X DELETE "/v1/tasks/$LEASE" >/dev/null 2>&1; }
|
|
log() { echo "[$(date -Is)] $*" >> "$GUARD_LOG"; }
|
|
|
|
trap 'lease_release; log "guard exiting, lease released"; exit 0' TERM INT
|
|
|
|
log "guard start: sentinel=$SENT tasks=${TASKS[*]} renew=${RENEW}s"
|
|
lease_hold; log "lease '$LEASE' acquired (3600s)"
|
|
last_renew=$(date +%s)
|
|
|
|
# Poll FREQUENTLY (POLL) but renew SLOWLY (RENEW). Coupling the two, as the first
|
|
# version did, meant a wave that finished right after a renewal sat undetected for a
|
|
# full RENEW interval with the sprite pinned hot the whole time.
|
|
while pgrep -f 'run-task\.sh|run-resume\.sh' >/dev/null; do
|
|
sleep "$POLL"
|
|
now=$(date +%s)
|
|
if [ $(( now - last_renew )) -ge "$RENEW" ]; then
|
|
lease_hold; last_renew=$now
|
|
log "lease renewed; workers still running"
|
|
fi
|
|
done
|
|
|
|
log "workers stopped; writing $SENT"
|
|
{
|
|
echo "=== $SENT written $(date -Is) ==="
|
|
echo "Workers finished. The orchestrator was NOT necessarily alive for this."
|
|
echo
|
|
for t in "${TASKS[@]}"; do
|
|
echo "--- $t ---"
|
|
grep -hE 'SUCCESS|GIVING UP|WALL CLOCK|QUOTA' "$L/$t.state" 2>/dev/null | tail -3
|
|
if [ -s "$L/$t.json" ]; then
|
|
jq -r '"cost=$" + ((.total_cost_usd//0)|tostring) + " turns=" + ((.num_turns//0)|tostring)' "$L/$t.json" 2>/dev/null
|
|
else
|
|
echo "!! $t.json is 0 bytes -> worker was KILLED, not finished (hazard #3/#5)"
|
|
fi
|
|
echo
|
|
done
|
|
echo "NEXT: orchestrator must independently verify before accepting:"
|
|
echo " cd ~/bookshelf && ./tasks/gw assembleDebug && ./tasks/gw testDebugUnitTest"
|
|
echo " git status --porcelain # boundary check: who touched what"
|
|
} > "$OUT"
|
|
|
|
lease_release; log "lease released; sprite may suspend"
|