Unstick analysis running when the worker is dead.
A container rebuild left Emmaforo status=running with no process, so PCB start kept 409. Heal on GET now matches placement/PCB: complete if report.json exists, otherwise error.
This commit is contained in:
@@ -308,46 +308,68 @@ def mark_stale_running(
|
|||||||
def heal_if_pipeline_finished(
|
def heal_if_pipeline_finished(
|
||||||
storage: StorageBackend, user_id: str, project_id: str,
|
storage: StorageBackend, user_id: str, project_id: str,
|
||||||
) -> ProjectMeta | None:
|
) -> ProjectMeta | None:
|
||||||
"""If meta says queued/running but events already ended with
|
"""Unstick analysis ``queued``/``running`` when the worker is gone.
|
||||||
``pipeline_complete``, flip status to ``complete``.
|
|
||||||
|
|
||||||
Covers zombies where the worker wrote the terminal event (and often
|
- Last event ``pipeline_complete`` → ``complete``
|
||||||
the report) then died before the meta transition — e.g. container
|
- Dead worker + ``report.json`` → ``complete``
|
||||||
rebuild mid-shutdown. Returns updated meta, or ``None`` if no heal.
|
- Dead worker otherwise → ``error`` (so PCB/placement can start)
|
||||||
"""
|
"""
|
||||||
meta = get_project(storage, user_id, project_id)
|
meta = get_project(storage, user_id, project_id)
|
||||||
if meta is None or meta.status not in (STATUS_RUNNING, STATUS_QUEUED):
|
if meta is None or meta.status not in (STATUS_RUNNING, STATUS_QUEUED):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
events_prefix = f"{_project_prefix(user_id, project_id)}/events/"
|
prefix = _project_prefix(user_id, project_id)
|
||||||
|
events_prefix = f"{prefix}/events/"
|
||||||
|
last = None
|
||||||
try:
|
try:
|
||||||
keys = storage.list_prefix(events_prefix)
|
event_keys = sorted(
|
||||||
|
k for k in storage.list_prefix(events_prefix)
|
||||||
|
if k.endswith(".json") and "/events/" in k
|
||||||
|
)
|
||||||
|
if event_keys:
|
||||||
|
last = storage.read_json(event_keys[-1])
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
last = None
|
||||||
event_keys = sorted(
|
|
||||||
k for k in keys if k.endswith(".json") and "/events/" in k
|
if (last or {}).get("event") == "pipeline_complete":
|
||||||
)
|
summary = (last.get("data") or {}).get("summary")
|
||||||
if not event_keys:
|
try:
|
||||||
return None
|
return transition_status(
|
||||||
|
storage, user_id, project_id,
|
||||||
|
from_status={STATUS_RUNNING, STATUS_QUEUED},
|
||||||
|
to_status=STATUS_COMPLETE,
|
||||||
|
summary=summary if isinstance(summary, dict) else meta.summary,
|
||||||
|
cancel_requested=False,
|
||||||
|
pipeline_state=None,
|
||||||
|
)
|
||||||
|
except StatusConflict:
|
||||||
|
return None
|
||||||
|
|
||||||
|
from backend.services import job_runner
|
||||||
|
|
||||||
|
exec_name = meta.execution_name or f"local/projects/{project_id}"
|
||||||
try:
|
try:
|
||||||
last = storage.read_json(event_keys[-1])
|
state = job_runner.get_execution_state(exec_name)
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
state = "unknown"
|
||||||
if (last or {}).get("event") != "pipeline_complete":
|
if state in ("pending", "running"):
|
||||||
return None
|
return None
|
||||||
|
|
||||||
summary = (last.get("data") or {}).get("summary")
|
if storage.exists(f"{prefix}/report.json"):
|
||||||
try:
|
try:
|
||||||
return transition_status(
|
return transition_status(
|
||||||
storage, user_id, project_id,
|
storage, user_id, project_id,
|
||||||
from_status={STATUS_RUNNING, STATUS_QUEUED},
|
from_status={STATUS_RUNNING, STATUS_QUEUED},
|
||||||
to_status=STATUS_COMPLETE,
|
to_status=STATUS_COMPLETE,
|
||||||
summary=summary if isinstance(summary, dict) else meta.summary,
|
cancel_requested=False,
|
||||||
cancel_requested=False,
|
pipeline_state=None,
|
||||||
pipeline_state=None,
|
)
|
||||||
)
|
except StatusConflict:
|
||||||
except StatusConflict:
|
return None
|
||||||
return None
|
return mark_stale_running(
|
||||||
|
storage, user_id, project_id,
|
||||||
|
f"Analysis worker terminated ({state})",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def heal_if_placement_stuck(
|
def heal_if_placement_stuck(
|
||||||
|
|||||||
@@ -170,7 +170,27 @@ def test_pcb_busy_helpers():
|
|||||||
assert draft.status in analysis
|
assert draft.status in analysis
|
||||||
|
|
||||||
|
|
||||||
def test_pcb_start_requires_board(tmp_path: Path):
|
def test_heal_dead_analysis_worker_unblocks_pcb(tmp_path: Path, monkeypatch):
|
||||||
|
from backend.services import job_runner, projects as proj_svc
|
||||||
|
from backend.services.storage import LocalStorageBackend
|
||||||
|
|
||||||
|
storage = LocalStorageBackend(tmp_path)
|
||||||
|
meta = proj_svc.create_project(storage, "usr_jwt", "board")
|
||||||
|
proj_svc.update_project(
|
||||||
|
storage, "usr_jwt", meta.id,
|
||||||
|
status=STATUS_RUNNING,
|
||||||
|
has_pcb=True,
|
||||||
|
has_bom=True,
|
||||||
|
has_netlist=True,
|
||||||
|
pcb_status="draft",
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(job_runner, "get_execution_state", lambda _n: "failed")
|
||||||
|
healed = proj_svc.heal_if_pipeline_finished(storage, "usr_jwt", meta.id)
|
||||||
|
assert healed is not None
|
||||||
|
assert healed.status == "error"
|
||||||
|
from backend.services.pcb_pipeline import analysis_busy
|
||||||
|
|
||||||
|
assert not analysis_busy(healed)
|
||||||
from fastapi.testclient import TestClient
|
from fastapi.testclient import TestClient
|
||||||
from backend.main import app
|
from backend.main import app
|
||||||
from backend.services.storage import LocalStorageBackend
|
from backend.services.storage import LocalStorageBackend
|
||||||
|
|||||||
Reference in New Issue
Block a user