Heal zombie running projects stuck on Review forever.

If the event log already ends with pipeline_complete, flip meta to complete on project/status fetch so Progress no longer spins on a dead worker.

Co-authored-by: Cursor <cursoragent@cursor.com>

backend/routers/admin.py

@@ -497,13 +497,23 @@ async def list_running_pipelines(request: Request):
             if meta.status not in (proj_svc.STATUS_QUEUED, proj_svc.STATUS_RUNNING):
                 continue
 
-            # Sweeper: if the execution is in a terminal Cloud Run state,
-            # the worker is already gone. Flip status → error so the UI
-            # stops lying. Skip the sweep when execution_name is missing
-            # (worker may still be enqueueing).
+            # Sweeper: if the execution is in a terminal Cloud Run / local
+            # state, the worker is already gone. Flip status → error so the
+            # UI stops lying. Also heal projects whose event log already
+            # ends with pipeline_complete (finished, meta never flipped).
+            healed = proj_svc.heal_if_pipeline_finished(storage, uid, meta.id)
+            if healed is not None:
+                continue
+
             exec_state = "unknown"
             if meta.execution_name:
                 exec_state = job_runner.get_execution_state(meta.execution_name)
+            elif not job_runner.use_cloud_run_jobs():
+                # Local zombie: no execution_name but a dead pid file, or
+                # no live proc — treat as failed after the stale window.
+                exec_state = job_runner.get_execution_state(
+                    f"local/projects/{meta.id}"
+                )
             if exec_state in ("succeeded", "failed", "cancelled"):
                 # Allow a short grace period so we don't race the worker
                 # writing its own terminal status. updated may be stale

backend/routers/pipeline.py

@@ -509,8 +509,16 @@ async def events(project_id: str, request: Request):
 
 @router.get("/pipeline/{project_id}/status")
 async def status(project_id: str, request: Request):
-    """Polling fallback — returns current project state."""
-    _, meta = await resolve_or_404(request, project_id)
+    """Polling fallback — returns current project state.
+
+    Also heals zombie ``running``/``queued`` projects whose event log
+    already ends with ``pipeline_complete`` (worker died after finishing).
+    """
+    storage = get_storage(request)
+    owner_user_id, meta = await resolve_or_404(request, project_id)
+    healed = proj_svc.heal_if_pipeline_finished(storage, owner_user_id, project_id)
+    if healed is not None:
+        meta = healed
     return {
         "status": meta.status,
         "summary": meta.summary,
@@ -519,6 +527,7 @@ async def status(project_id: str, request: Request):
         "placement_status": meta.placement_status,
         "placement_state": meta.placement_state,
         "placement_running": (meta.placement_status or "draft") in ("queued", "running"),
+        "healed": healed is not None,
     }
 
 

backend/routers/projects.py

@@ -140,8 +140,10 @@ async def list_projects(request: Request):
 
 @router.get("/projects/{project_id}")
 async def get_project(project_id: str, request: Request):
-    _, meta = await resolve_or_404(request, project_id)
-    return meta.model_dump()
+    storage = get_storage(request)
+    owner_user_id, meta = await resolve_or_404(request, project_id)
+    healed = proj_svc.heal_if_pipeline_finished(storage, owner_user_id, project_id)
+    return (healed or meta).model_dump()
 
 
 @router.delete("/projects/{project_id}")