Heal zombie running projects stuck on Review forever.

If the event log already ends with pipeline_complete, flip meta to complete on project/status fetch so Progress no longer spins on a dead worker.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/routers/admin.py

@@ -497,13 +497,23 @@ async def list_running_pipelines(request: Request):
             if meta.status not in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING):
                 continue
 
-            # Sweeper: if the execution is in a terminal Cloud Run state,
-            # the worker is already gone. Flip status → error so the UI
-            # stops lying. Skip the sweep when execution_name is missing
-            # (worker may still be enqueueing).
+            # Sweeper: if the execution is in a terminal Cloud Run / local
+            # state, the worker is already gone. Flip status → error so the
+            # UI stops lying. Also heal projects whose event log already
+            # ends with pipeline_complete (finished, meta never flipped).
+            healed = proj_svc.heal_if_pipeline_finished(storage, uid, meta.id)
+            if healed is not None:
+                continue
+
             exec_state = "unknown"
             if meta.execution_name:
                 exec_state = job_runner.get_execution_state(meta.execution_name)
+            elif not job_runner.use_cloud_run_jobs():
+                # Local zombie: no execution_name but a dead pid file, or
+                # no live proc — treat as failed after the stale window.
+                exec_state = job_runner.get_execution_state(
+                    f"local/projects/{meta.id}"
+                )
             if exec_state in ("succeeded", "failed", "cancelled"):
                 # Allow a short grace period so we don't race the worker
                 # writing its own terminal status. updated may be stale

backend/routers/pipeline.py

@@ -509,8 +509,16 @@ async def events(project_id: str, request: Request):
 
 @router.get("/pipeline/{project_id}/status")
 async def status(project_id: str, request: Request):
-    """Polling fallback — returns current project state."""
-    _, meta = await resolve_or_404(request, project_id)
+    """Polling fallback — returns current project state.
+
+    Also heals zombie ``running``/``queued`` projects whose event log
+    already ends with ``pipeline_complete`` (worker died after finishing).
+    """
+    storage = get_storage(request)
+    owner_user_id, meta = await resolve_or_404(request, project_id)
+    healed = proj_svc.heal_if_pipeline_finished(storage, owner_user_id, project_id)
+    if healed is not None:
+        meta = healed
     return {
         "status": meta.status,
         "summary": meta.summary,
@@ -519,6 +527,7 @@ async def status(project_id: str, request: Request):
         "placement_status": meta.placement_status,
         "placement_state": meta.placement_state,
         "placement_running": (meta.placement_status or "draft") in ("queued", "running"),
+        "healed": healed is not None,
     }
 
 

backend/routers/projects.py

@@ -140,8 +140,10 @@ async def list_projects(request: Request):
 
 @router.get("/projects/{project_id}")
 async def get_project(project_id: str, request: Request):
-    _, meta = await resolve_or_404(request, project_id)
-    return meta.model_dump()
+    storage = get_storage(request)
+    owner_user_id, meta = await resolve_or_404(request, project_id)
+    healed = proj_svc.heal_if_pipeline_finished(storage, owner_user_id, project_id)
+    return (healed or meta).model_dump()
 
 
 @router.delete("/projects/{project_id}")

backend/services/projects.py

@@ -281,6 +281,51 @@ def mark_stale_running(
         return None
 
 
+def heal_if_pipeline_finished(
+    storage: StorageBackend, user_id: str, project_id: str,
+) -> ProjectMeta | None:
+    """If meta says queued/running but events already ended with
+    ``pipeline_complete``, flip status to ``complete``.
+
+    Covers zombies where the worker wrote the terminal event (and often
+    the report) then died before the meta transition — e.g. container
+    rebuild mid-shutdown. Returns updated meta, or ``None`` if no heal.
+    """
+    meta = get_project(storage, user_id, project_id)
+    if meta is None or meta.status not in (STATUS_RUNNING, STATUS_QUEUED):
+        return None
+
+    events_prefix = f"{_project_prefix(user_id, project_id)}/events/"
+    try:
+        keys = storage.list_prefix(events_prefix)
+    except Exception:
+        return None
+    event_keys = sorted(
+        k for k in keys if k.endswith(".json") and "/events/" in k
+    )
+    if not event_keys:
+        return None
+    try:
+        last = storage.read_json(event_keys[-1])
+    except Exception:
+        return None
+    if (last or {}).get("event") != "pipeline_complete":
+        return None
+
+    summary = (last.get("data") or {}).get("summary")
+    try:
+        return transition_status(
+            storage, user_id, project_id,
+            from_status={STATUS_RUNNING, STATUS_QUEUED},
+            to_status=STATUS_COMPLETE,
+            summary=summary if isinstance(summary, dict) else meta.summary,
+            cancel_requested=False,
+            pipeline_state=None,
+        )
+    except StatusConflict:
+        return None
+
+
 # --- CRUD ---
 
 

frontend/content/changelog.md

@@ -2,6 +2,14 @@
 
 What's new in Pinscope.
 
+## 2.28.4 — 2026-09-13 — Unstick zombie running pipelines
+
+A finished run whose worker died before flipping meta stayed `running`, so Progress showed Review spinning forever. Heal those projects when the event log already ends with `pipeline_complete`.
+
+- [Fixed] `heal_if_pipeline_finished` on project get / pipeline status.
+- [Fixed] Progress stepper marks all stages complete on `pipeline_complete`.
+- [Fixed] Admin sweeper also covers local zombies without `execution_name`.
+
 ## 2.28.3 — 2026-09-13 — Stop progress→report bounce on finished projects
 
 Opening a finished project no longer flashes Processing and dumps you on the report. Progress only auto-opens the report after a live run on that visit.

frontend/src/app/(app)/project/[id]/progress/page.tsx

@@ -20,7 +20,7 @@ import {
 import { PipelineStepper } from "@/components/progress/pipeline-stepper";
 import { PausedRunBanner } from "@/components/billing/paused-run-banner";
 import { usePipelineProgress } from "@/hooks/use-pipeline-progress";
-import { cancelPipeline, fetchProject, fetchReport, resumePipeline, reprocessPipeline } from "@/lib/api";
+import { cancelPipeline, fetchPipelineStatus, fetchProject, fetchReport, resumePipeline, reprocessPipeline } from "@/lib/api";
 import type { PauseCheckpoint } from "@/lib/types";
 import {
   AlertTriangle,
@@ -72,32 +72,41 @@ export default function ProgressPage({
   }
 
   // Decide whether this visit is a live run before opening SSE / auto-redirect.
+  // /status also heals zombies whose event log already ends with pipeline_complete.
   useEffect(() => {
     let cancelledFetch = false;
-    fetchProject(id)
-      .then((p) => {
+    (async () => {
+      try {
+        const st = await fetchPipelineStatus(id);
         if (cancelledFetch) return;
-        setProjectName(p.name);
+        const nameP = fetchProject(id).then((p) => {
+          if (!cancelledFetch) {
+            setProjectName(p.name);
+            if (p.pauseCheckpoint) setProjectCheckpoint(p.pauseCheckpoint);
+          }
+        }).catch(() => {});
+
         if (
-          p.status === "paused_insufficient_credits" ||
-          p.status === "paused_by_user"
+          st.status === "paused_insufficient_credits" ||
+          st.status === "paused_by_user"
         ) {
           setProjectPaused(true);
-          setProjectCheckpoint(p.pauseCheckpoint ?? null);
           setGate("paused");
+          await nameP;
           return;
         }
-        if (p.status === "running" || p.status === "queued") {
+        if (st.status === "running" || st.status === "queued") {
           setGate("live");
+          await nameP;
           return;
         }
-        // Finished / draft — progress is the wrong page; go to the project hub.
+        // Finished / draft / healed — progress is the wrong page.
         setGate("idle");
         router.replace(`/project/${id}`);
-      })
-      .catch(() => {
+      } catch {
         if (!cancelledFetch) setGate("live");
-      });
+      }
+    })();
     return () => {
       cancelledFetch = true;
     };

frontend/src/hooks/use-pipeline-progress.ts

@@ -99,6 +99,15 @@ export function usePipelineProgress(projectId: string | null) {
 
     if (eventType === "pipeline_complete" || event.lastEventId === "pipeline_complete") {
       setSummary(data.summary as Record<string, number>);
+      // Force every stage complete so a historical replay cannot leave
+      // "Review Design" spinning if the last substep was still running.
+      setSteps((prev) =>
+        prev.map((s) => ({
+          ...s,
+          status: "complete" as const,
+          substeps: s.substeps.map((ss) => ({ ...ss, status: "complete" as const })),
+        })),
+      );
       setDone(true);
       terminalRef.current = true;
       esRef.current?.close();