jfmonty2/bookshelf
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

orchestration: verify lease acquisition in wave-guard; F3 prompt for wave 4 finish

Browse Source 
wave-guard.sh post-mortem: lease_hold() did DELETE-then-POST with both results
discarded and logged "lease renewed" unconditionally, so a failed re-POST left
the box with no lease while the log claimed it was protected. That matches the
wave-4 loss exactly (last "renewal" 11:39, workers dead 11:46, reboot 12:55).

Now: every acquire is verified against GET /v1/tasks before it is believed, a
failed acquire retries and is logged as a failure, and the lease is re-checked
every POLL rather than only every RENEW so a lease lost between renewals is
caught in seconds.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_016mTs3kQXQsQwonXpEq7aEw
This commit is contained in:
claude
2026-09-06 19:00:11 +00:00
parent cf36b27457
commit 36b46b4644
2 changed files with 142 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tasks/wave-guard.sh
+51 -15
View File
@@ -9,37 +9,73 @@
# workers run, so the box stays hot with no console attached. When the workers stop
# it writes the durable completion record AND releases the lease, so the sprite is
# free to suspend instead of burning money idling.
#
# WAVE 4 POST-MORTEM — why this file was rewritten:
# The previous lease_hold() did DELETE-then-POST with both results discarded, and
# logged "lease renewed" unconditionally. If that POST ever failed, the box was left
# with NO lease while the log claimed otherwise. That is exactly what wave 4 looks
# like: last "renewal" 11:39, workers dead by 11:46, box rebooted 12:55. So now:
# - every acquire is VERIFIED against GET /v1/tasks before it is believed
# - a failed acquire retries, and is logged as a failure, never as a success
# - the lease is re-checked every POLL (cheap), not only every RENEW, so a lease
# lost between renewals is noticed in seconds instead of never
set -u
SENT="$1"; shift
TASKS=("$@")
L="$HOME/bookshelf/logs"; OUT="$L/$SENT"; LEASE="bookshelf-wave"
RENEW="${RENEW:-900}" # renew every 15 min against a 60 min lease
POLL="${POLL:-30}" # but CHECK for completion every 30s (see loop below)
RENEW="${RENEW:-900}" # proactively renew every 15 min against a 60 min lease
POLL="${POLL:-30}" # but CHECK completion + lease health every 30s
GUARD_LOG="$L/wave-guard.log"
lease_hold() {
sprite-env curl -X DELETE "/v1/tasks/$LEASE" >/dev/null 2>&1
sprite-env curl -X POST /v1/tasks -H 'Content-Type: application/json' \
-d "{\"name\":\"$LEASE\",\"expire\":\"3600s\"}" >/dev/null 2>&1
}
lease_release() { sprite-env curl -X DELETE "/v1/tasks/$LEASE" >/dev/null 2>&1; }
log() { echo "[$(date -Is)] $*" >> "$GUARD_LOG"; }
lease_present() { sprite-env curl /v1/tasks 2>/dev/null | grep -q "\"$LEASE\""; }
# Acquire and VERIFY. Returns 0 only if the lease is actually live afterwards.
lease_acquire() {
local attempt
for attempt in 1 2 3; do
sprite-env curl -X DELETE "/v1/tasks/$LEASE" >/dev/null 2>&1
sprite-env curl -X POST /v1/tasks -H 'Content-Type: application/json' \
-d "{\"name\":\"$LEASE\",\"expire\":\"3600s\"}" >/dev/null 2>&1
if lease_present; then return 0; fi
log "WARNING: lease acquire attempt $attempt did not take; retrying"
sleep 5
done
return 1
}
lease_release() { sprite-env curl -X DELETE "/v1/tasks/$LEASE" >/dev/null 2>&1; }
trap 'lease_release; log "guard exiting, lease released"; exit 0' TERM INT
log "guard start: sentinel=$SENT tasks=${TASKS[*]} renew=${RENEW}s"
lease_hold; log "lease '$LEASE' acquired (3600s)"
log "guard start: sentinel=$SENT tasks=${TASKS[*]} renew=${RENEW}s poll=${POLL}s"
if lease_acquire; then
log "lease '$LEASE' acquired and VERIFIED (3600s)"
else
log "FATAL: could not acquire lease '$LEASE' -- the sprite may suspend mid-wave"
fi
last_renew=$(date +%s)
# Poll FREQUENTLY (POLL) but renew SLOWLY (RENEW). Coupling the two, as the first
# version did, meant a wave that finished right after a renewal sat undetected for a
# full RENEW interval with the sprite pinned hot the whole time.
while pgrep -f 'run-task\.sh|run-resume\.sh' >/dev/null; do
sleep "$POLL"
now=$(date +%s)
if [ $(( now - last_renew )) -ge "$RENEW" ]; then
lease_hold; last_renew=$now
log "lease renewed; workers still running"
if lease_acquire; then
log "lease renewed and VERIFIED; workers still running"
else
log "ERROR: scheduled renewal FAILED; lease is not held"
fi
last_renew=$now
elif ! lease_present; then
# Lost between renewals (expiry race, API blip, someone else's DELETE).
log "ERROR: lease disappeared between renewals -- re-acquiring now"
if lease_acquire; then
log "lease re-acquired and VERIFIED"
last_renew=$now
else
log "ERROR: emergency re-acquire FAILED; sprite is unprotected"
fi
fi
done