#!/usr/bin/env bash # service-worker.sh # # Runs ONE worker under the sprite service supervisor instead of `setsid nohup`. # # WHY THIS EXISTS (wave-4 post-mortem, second failure): # `setsid nohup ./tasks/run-task.sh ...` does NOT reliably outlive the orchestrator. # On 09-06 at 19:04 the orchestrator's session was torn down and the detached worker # AND its wave-guard died with it -- no reboot (uptime was continuous), so this was a # teardown kill, not the hazard-#5 suspend. `setsid` did not save it. # # /.sprite/llm.txt: services are "long-running processes that persist across reboots" # and "services and sessions keep sprites alive". A service therefore fixes BOTH # failure modes at once -- it survives orchestrator teardown AND holds the box awake -- # which makes the task-lease guard redundant. # # IDEMPOTENCE MATTERS HERE: the supervisor restarts services. Without the sentinel # guard below, a finished wave would be re-run on every restart and burn quota # redoing completed work. set -u NAME="$1"; PROMPT="$2"; SENT="$3" L="$HOME/bookshelf/logs"; DONE="$L/$SENT" SVC="${SVC_NAME:-bookshelf-worker}" log() { echo "[$(date -Is)] service-worker: $*" >> "$L/service-worker.log"; } # Already finished -> do not re-run. Idle briefly so a restart loop stays slow. if [ -f "$DONE" ]; then log "$SENT already exists; nothing to do (service can be deleted)" sleep 300 exit 0 fi log "starting worker $NAME (prompt=$PROMPT, sentinel=$SENT)" "$HOME/bookshelf/tasks/run-task.sh" "$NAME" "$PROMPT" rc=$? log "worker $NAME exited rc=$rc; writing $SENT" { echo "=== $SENT written $(date -Is) ===" echo "Worker finished under the service supervisor. The orchestrator was NOT" echo "necessarily alive for this. run-task.sh exit code: $rc" echo echo "--- $NAME ---" grep -hE 'SUCCESS|GIVING UP|WALL CLOCK|QUOTA' "$L/$NAME.state" 2>/dev/null | tail -5 if [ -s "$L/$NAME.json" ]; then jq -r '"cost=$" + ((.total_cost_usd//0)|tostring) + " turns=" + ((.num_turns//0)|tostring)' "$L/$NAME.json" 2>/dev/null jq -r '"is_error=" + ((.is_error//false)|tostring)' "$L/$NAME.json" 2>/dev/null else echo "!! $NAME.json is 0 bytes -> worker was KILLED, not finished (hazard #3)" fi echo echo "NEXT: orchestrator must independently verify before accepting:" echo " cd ~/bookshelf && ./tasks/gw assembleDebug && ./tasks/gw testDebugUnitTest" echo " git status --porcelain # what did the worker actually touch?" } > "$DONE" sync # Sentinel is durable now, so it is safe to stop ourselves. This lets the sprite # suspend instead of idling hot on the owner's dime. log "stopping service $SVC so the sprite can suspend" sprite-env services stop "$SVC" >/dev/null 2>&1 sleep 60