"""Recovery path: when a turn produces no tool calls (model wrote findings as a JSON code block in prose instead of calling submit_review), the loop must NOT drop the work — it should nudge and force submit_review next turn. This regression was introduced by a system-prompt change that made Gemini default to text output for findings. The fix is in the review loop itself so it survives future prompt regressions. """ from __future__ import annotations from tests.paths import SIMPLE_PROJECT, TAXONOMY import json from pathlib import Path import pytest from pypdf import PdfWriter from backend.periscopex.utils import safe_mpn from backend.services import review_session as review_session from backend.services import validation as val from backend.services.llm.types import Completion, ToolCall, Usage from backend.services.storage import LocalStorageBackend GRAPH = SIMPLE_PROJECT / "design_graph.json" IC_MPNS = { "U1": "SPX3819M5-L-3-3/TR", "U2": "CH340E", "U3": "MSPM0G3507SPTR", } PREFIX = "users/local/projects/test" def _blank_pdf(path: Path) -> None: w = PdfWriter() w.add_blank_page(width=200, height=200) with path.open("wb") as fh: w.write(fh) def _usage(): return Usage(input_tokens=10, output_tokens=5, cache_creation_tokens=0, cache_read_tokens=0) @pytest.fixture def workspace(tmp_path): data = tmp_path / "data" (data / PREFIX).mkdir(parents=True) graph_path = data / PREFIX / "design_graph.json" graph_path.write_text(GRAPH.read_text()) report_path = data / PREFIX / "report.json" extracted = data / PREFIX / "extracted" extracted.mkdir() ds_dir = data / PREFIX / "uploads" / "datasheets" ds_dir.mkdir(parents=True) for mpn in IC_MPNS.values(): _blank_pdf(ds_dir / f"{safe_mpn(mpn)}.pdf") storage = LocalStorageBackend(data) return dict(data=data, graph=graph_path, report=report_path, extracted=extracted, ds_dir=ds_dir, storage=storage) def _recovery_script(ic_ref, n): """Turn 0: text only, no tool calls (the failure mode). Turn 1: submit_review under forced tool_choice (the recovery).""" if n == 0: return Completion( text="I'll write up findings as JSON below: [...]", tool_calls=[], # the bug: no tool calls usage=_usage(), stop_reason="end_turn", raw_assistant_blocks=[], ) # Turn 1 — forced submit_review under the recovery return Completion( text="", tool_calls=[ToolCall(id="t2", name="submit_review", input={ "findings": [{ "finding": "Recovered finding.", "why": "Recovered from a no-tool-call turn.", "status": "INFO", "source_page": 1, }], "checked_areas": ["recovery"], })], usage=_usage(), stop_reason="tool_use", raw_assistant_blocks=[], ) @pytest.mark.asyncio async def test_no_tool_calls_triggers_forced_submit_next_turn(workspace, monkeypatch): """A turn with zero tool calls should not drop the review — the next turn must be forced to submit_review and the resulting findings must land in the report.""" seen_tool_choices: list = [] class _Session: def __init__(self, ic_ref): self._ic = ic_ref self._n = 0 async def complete(self, messages, tools, tool_choice): seen_tool_choices.append((self._ic, self._n, tool_choice)) c = _recovery_script(self._ic, self._n) self._n += 1 return c async def close(self): pass class _Provider: name = "fake" async def create_session(self, model, system, max_tokens, **_kwargs): return _Session(_Provider._current_ic) _current_ic = None async def fake_cwf(stage, body): return await body(_Provider(), "fake-model") monkeypatch.setattr(val, "call_with_fallback", fake_cwf) monkeypatch.setattr(review_session, "call_with_fallback", fake_cwf) orig = val.review_ic_async async def wrapped(graph, cmap, ic_ref, pdf_path, **kw): _Provider._current_ic = ic_ref return await orig(graph, cmap, ic_ref, pdf_path, **kw) monkeypatch.setattr(val, "review_ic_async", wrapped) async def before_ic(ref): return True await val.validate_design_async( str(workspace["graph"]), str(workspace["report"]), str(workspace["extracted"]), pdf_dir=str(workspace["ds_dir"]), storage=workspace["storage"], before_ic=before_ic, project_prefix=PREFIX, run_meta={"git_commit": "testsha"}, ) # All three ICs should have recovered: each had a no-tool-call turn 0, # then submit_review under forced tool_choice on turn 1. report = json.loads(workspace["report"].read_text()) review = [f for f in report["findings"] if not f.get("rule_id")] assert len(review) == 3 assert sum(1 for f in review if f.get("status") == "INFO") == 3 # Verify the recovery actually forced submit_review on turn 1 for each IC. by_ic_turn = {(ic, n): tc for ic, n, tc in seen_tool_choices} for ic in ("U1", "U2", "U3"): # Turn 0 should be auto (model free to use any tool) assert by_ic_turn[(ic, 0)] == "auto", \ f"turn 0 for {ic} should be auto, got {by_ic_turn[(ic, 0)]!r}" # Turn 1 should be forced submit_review (the recovery) assert by_ic_turn[(ic, 1)] == {"name": "submit_review"}, \ f"turn 1 for {ic} should force submit_review, got {by_ic_turn[(ic, 1)]!r}" # Each trace should show the recovery: turn 0 has no tool calls, turn 1 # has submit_review. traces_dir = workspace["data"] / PREFIX / "review_traces" for ref in ("U1", "U2", "U3"): t = json.loads((traces_dir / f"{safe_mpn(ref)}.json").read_text()) assert len(t["turns"]) == 2 assert t["turns"][0]["tool_calls"] == [] assert t["turns"][1]["tool_calls"][0]["name"] == "submit_review" assert t["stop_reason"] == "submit_review"