orchestration: wave 4 prompts; decouple guard poll from lease renewal

Guard now polls every 30s but renews every 15 min. Coupling them meant a wave that
finished just after a renewal sat undetected for a full interval with the sprite
pinned hot.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
2026-09-06 10:54:03 +00:00
parent 9bf5ff4d6c
commit 0088fda095
3 changed files with 165 additions and 3 deletions
+11 -3
View File
@@ -14,6 +14,7 @@ SENT="$1"; shift
TASKS=("$@")
L="$HOME/bookshelf/logs"; OUT="$L/$SENT"; LEASE="bookshelf-wave"
RENEW="${RENEW:-900}" # renew every 15 min against a 60 min lease
POLL="${POLL:-30}" # but CHECK for completion every 30s (see loop below)
GUARD_LOG="$L/wave-guard.log"
lease_hold() {
@@ -28,11 +29,18 @@ trap 'lease_release; log "guard exiting, lease released"; exit 0' TERM INT
log "guard start: sentinel=$SENT tasks=${TASKS[*]} renew=${RENEW}s"
lease_hold; log "lease '$LEASE' acquired (3600s)"
last_renew=$(date +%s)
# Poll FREQUENTLY (POLL) but renew SLOWLY (RENEW). Coupling the two, as the first
# version did, meant a wave that finished right after a renewal sat undetected for a
# full RENEW interval with the sprite pinned hot the whole time.
while pgrep -f 'run-task\.sh|run-resume\.sh' >/dev/null; do
sleep "$RENEW"
lease_hold
log "lease renewed; workers still running"
sleep "$POLL"
now=$(date +%s)
if [ $(( now - last_renew )) -ge "$RENEW" ]; then
lease_hold; last_renew=$now
log "lease renewed; workers still running"
fi
done
log "workers stopped; writing $SENT"