ai: structured-output + reviewer agent for indicator summaries

Replaces the regex-based clean_summary / looks_like_leakage pipeline
that produced the 2026-05-29 valuation-read leak. Two layers of defence
in depth:

1. JSON-mode generation. The per-group and aggregate summary system
   prompts now require the model to emit a single object
   {"read": "..."}; response_format={"type":"json_object"} is passed
   through to the provider so the API enforces well-formed JSON. Prose
   outside the field is physically impossible. The "read" field is the
   only schema slot, so the model has nowhere to spill scratchpad
   into the envelope.

2. Reviewer agent. services/output_review.review_read() makes a second
   small LLM call that judges whether the candidate "read" string is
   publishable. It catches the residual failure mode — scratchpad
   INSIDE the field ("Let's see…", multi-question parentheticals,
   meta-commentary) — and returns a JSON verdict {"clean": bool,
   "reason": str}. Any failure (provider error, parse error, missing
   field) returns clean=false (fail-safe). Cost ~$0.0001/check; latency
   ~1-2 s in the hourly job, no user-facing latency.

The old regex scaffolding (_LEAK_PATTERNS, clean_summary,
looks_like_leakage, _TRAILING_QUOTE) is deleted entirely. It produced
false positives (chopped legitimate "The indicators are…" leaders) and
false negatives (never matched the chain-of-thought patterns the model
actually emits). The reviewer agent is strictly better on both.

On reviewer/parse rejection: don't persist a new IndicatorSummary; the
API's existing fallback to the previous good row continues to serve
the panel. Failures are logged as ind_summary.json_invalid /
ind_summary.reviewer_rejected so we can measure the rejection rate.

Reviewer cost is added to the row's recorded cost_usd so the monthly
budget cap covers the full pipeline.

Adds tests/test_output_review.py: 11 cases covering _extract_read
(JSON envelope handling — invalid JSON, missing field, wrong types,
empty values) and review_read (clean / unclean verdicts plus three
fail-safe paths for malformed reviewer responses).

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Giorgio Gilestro 2026-05-29 13:10:52 +02:00
parent 19d4854f50
commit 45fa31bb2b
4 changed files with 396 additions and 141 deletions

View file

@ -4,7 +4,7 @@ hourly stays comfortably under the monthly cap."""
from __future__ import annotations
import asyncio
import re
import json
import httpx
from sqlalchemy import desc, func, select
@ -35,6 +35,7 @@ from app.services.openrouter import (
llm_configured,
month_start,
)
from app.services.output_review import review_read
from app.services.translation import translate
@ -106,109 +107,41 @@ async def translate_summary_for_active_languages(session, summary_id: int) -> No
summary_id=summary_id, succeeded=succeeded, failed=failed)
# Strip known meta-commentary openers the model sometimes leaks despite the
# prompt's hard constraints. Each pattern matches one leading sentence.
_LEAK_PATTERNS = [
re.compile(p, re.IGNORECASE | re.DOTALL)
for p in (
# First-person meta — "I need to / I'll / I have to / I'm going to ..."
r"^i\s+(?:need|have|must|should|am going|'ll|will|shall|can|am)[^.]*\.\s*",
# "We need / we're / we are asked / we will ..."
r"^we\s+(?:need|are|'re|will|shall|can|should|must|have)[^.]*\.\s*",
r"^let\s+(?:me|us|'?s)[^.]*\.\s*",
r"^here[']s[^.]*\.\s*",
r"^sure[,!]?\s[^.]*\.\s*",
r"^looking at[^.]*\.\s*",
r"^based on[^.]*\.\s*",
r"^to (?:address|answer|write|summarise|summarize)[^.]*\.\s*",
r"^first[,]?\s[^.]*\.\s*",
r"^the (?:user|data shows|reader|task|request|reader sees|instructions?)[^.]*\.\s*",
r"^summary[:.]\s*",
r"^key\s*[:\-—]\s*",
r"^must\s+(?:be|cite|explain|avoid|give|stay|provide)[^.]*\.\s*",
r"^should\s+(?:be|give|cite|explain|avoid|provide)[^.]*\.\s*",
r"^avoid[^.]*\.\s*",
r"^cite\s+at\s+most[^.]*\.\s*",
r"^be\s+(?:speculative|specific|concise|brief)[^.]*\.\s*",
r"^stay\s+on[^.]*\.\s*",
r"^okay[,]?\s+",
r"^alright[,]?\s+",
r"^thinking[^.]*\.\s*",
# Prompt-leak prefixes — the model echoes example framing or rule
# headers from the system prompt.
r"^(?:good|bad|positive|negative)\s+example\s*[:\-—]\s*",
r"^example\s+(?:good|bad)\s*[:\-—]\s*",
r"^example\s*[:\-—]\s*",
r"^reference\s+style\s*[:\-—]\s*",
# Prompt label echoes (markdown-style or plain-text)
r"^(?:hard\s+)?constraints?\s*[:\-—][^.\n]*[.\n]\s*",
r"^key\s+observations?\s*[:\-—]\s*",
r"^observations?\s*[:\-—]\s*",
r"^focus\s+on[^.]*\.\s*",
r"^output\s+the\s+read[^.]*\.\s*",
r"^plain\s+prose[^.]*\.\s*",
r"^the\s+indicators?[^.]*\.\s*", # "The indicators include..." / "The indicators are..."
r"^indicators?\s*[:\-—]\s*",
r"^data\s*[:\-—]\s*",
r"^analysis\s*[:\-—]\s*",
r"^interpretation\s*[:\-—]\s*",
r"^read\s*[:\-—]\s*",
r"^note\s*[:\-—]\s*",
# Sometimes the response gets wrapped in literal quotes
r"^[\"'`]+",
)
]
# Defence-in-depth: read generation goes through JSON mode + a reviewer.
#
# 1. The system prompt instructs the model to emit {"read": "..."} only;
# response_format={"type":"json_object"} forces well-formed JSON at
# the API layer, so prose outside the field is impossible.
# 2. We extract `read`, then ask a second LLM call (services/output_review)
# whether the candidate text is publishable. Scratchpad INSIDE the
# field — "Let's see…", "X? Actually Y?" — is caught here.
# 3. Any failure at either stage (parse, missing field, reviewer veto,
# reviewer error) drops the candidate. The previous good
# IndicatorSummary stays visible.
#
# The old _LEAK_PATTERNS / clean_summary / looks_like_leakage regex
# scaffolding lived here previously. It produced false positives (e.g.
# chopping off a legitimate leading sentence like "The indicators are
# pricing…") and false negatives (it never caught the chain-of-thought
# patterns the model actually emits). The reviewer agent replaces it.
_TRAILING_QUOTE = re.compile(r"[\"'`]+\s*$")
# Tell-tale phrases that mean the model regurgitated the prompt as its
# "answer" — we'd rather show nothing than show this.
_LEAKAGE_FLAGS = (
"≤60 words", "60 words", "must be under", "must cite", "must explain",
"no meta-commentary", "no buy/sell", "horizon. ", "1-day moves",
"the instructions are", "instructions:", "constraints:", "hard constraints",
"good example", "bad example", "reference style",
)
def looks_like_leakage(text: str) -> bool:
"""Heuristic: after cleaning, if these phrases still appear, the output
is contaminated prompt-regurgitation and shouldn't be shown."""
low = text.lower()
return any(flag in low for flag in _LEAKAGE_FLAGS)
def clean_summary(text: str) -> str:
"""Strip leading meta-commentary. If cleaning removes nearly everything
(suggesting the model emitted reasoning then ran out of tokens), fall
back to the last non-empty paragraph of the raw output that's usually
where the actual answer ended up."""
raw = text.strip()
out = raw
# Up to 6 passes: handles compound leakage like
# "Constraints: <...>. The indicators are: <...>. <actual answer>"
for _ in range(6):
before = out
for pat in _LEAK_PATTERNS:
out = pat.sub("", out, count=1).lstrip()
if out == before:
break
if len(out) < 60 and len(raw) > 120:
# Cleaning ate too much; take the last non-empty paragraph of raw.
paragraphs = [p.strip() for p in re.split(r"\n\s*\n", raw) if p.strip()]
if paragraphs:
out = paragraphs[-1]
# Re-strip leaders from the recovered paragraph too.
for _ in range(2):
before = out
for pat in _LEAK_PATTERNS:
out = pat.sub("", out, count=1).lstrip()
if out == before:
break
# Trim any orphan closing quote/backtick from the wrap-strip above.
out = _TRAILING_QUOTE.sub("", out).rstrip()
return out
def _extract_read(raw: str) -> str | None:
"""Parse the model's JSON envelope and return the "read" field, or
None if the body isn't valid JSON / the field is missing / the field
isn't a string. Conservative: on any deviation from the schema we
drop the candidate rather than try to salvage it."""
try:
parsed = json.loads(raw)
except json.JSONDecodeError:
return None
if not isinstance(parsed, dict):
return None
read = parsed.get("read")
if not isinstance(read, str):
return None
read = read.strip()
return read or None
@ -228,19 +161,20 @@ async def _generate_one(
[{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}],
max_tokens=800, # DeepSeek sometimes spends 300+ on internal reasoning
response_format={"type": "json_object"},
)
except Exception as e:
session.add(AICall(model=active_model(), status="error", error=str(e)[:500]))
log.warning("ind_summary.failed", group=group, error=str(e)[:120])
return None
cleaned = clean_summary(result.content)
if looks_like_leakage(cleaned) or len(cleaned) < 40:
# Model regurgitated the prompt or produced nothing usable.
# Don't persist — keep the last good summary visible. Log it so
# we can see the rate of failures over time.
log.warning("ind_summary.leakage_detected",
group=group, preview=cleaned[:120])
candidate = _extract_read(result.content)
if candidate is None or len(candidate) < 40:
# JSON envelope malformed, "read" field missing/wrong type, or
# the candidate is too short to be a real read. Don't persist;
# the last good summary stays visible.
log.warning("ind_summary.json_invalid",
group=group, preview=result.content[:160])
session.add(AICall(
model=result.model,
prompt_tokens=result.prompt_tokens,
@ -250,6 +184,23 @@ async def _generate_one(
))
return None
verdict = await review_read(client, candidate)
if not verdict.clean:
# Reviewer caught scratchpad / meta-commentary / partial text
# INSIDE the read field. Drop the candidate; the previous good
# summary continues to serve.
log.warning("ind_summary.reviewer_rejected",
group=group, reason=verdict.reason,
preview=candidate[:120])
session.add(AICall(
model=result.model,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=(result.cost_usd or 0.0) + (verdict.cost_usd or 0.0),
status="leaked",
))
return None
summary = IndicatorSummary(
group_name=group,
generated_at=utcnow(),
@ -257,17 +208,19 @@ async def _generate_one(
tone=tone,
analysis=analysis,
prompt_version=PROMPT_VERSION,
content=cleaned,
content=candidate,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=result.cost_usd,
# Include the reviewer's cost in the row's recorded spend so the
# monthly budget tracking covers the full pipeline cost.
cost_usd=(result.cost_usd or 0.0) + (verdict.cost_usd or 0.0),
)
session.add(summary)
session.add(AICall(
model=result.model,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=result.cost_usd,
cost_usd=(result.cost_usd or 0.0) + (verdict.cost_usd or 0.0),
status="ok",
))
return summary
@ -338,6 +291,7 @@ async def run() -> None:
await translate_summary_for_active_languages(session, summary.id)
# One aggregate read across all groups, stored under __all__.
# Same JSON-mode + reviewer-agent path as per-group reads.
agg_system = build_aggregate_summary_system_prompt(tone, analysis)
agg_user = build_aggregate_summary_user_prompt(groups)
agg_summary: IndicatorSummary | None = None
@ -346,28 +300,53 @@ async def run() -> None:
client,
[{"role": "system", "content": agg_system},
{"role": "user", "content": agg_user}],
max_tokens=1500, # room for reasoning + 80-word output
max_tokens=1500,
response_format={"type": "json_object"},
)
agg_summary = IndicatorSummary(
group_name=AGGREGATE_GROUP_NAME,
generated_at=utcnow(),
model=result.model,
tone=tone,
analysis=analysis,
prompt_version=PROMPT_VERSION,
content=clean_summary(result.content),
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=result.cost_usd,
)
session.add(agg_summary)
session.add(AICall(
model=result.model,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=result.cost_usd, status="ok",
))
written += 1
candidate = _extract_read(result.content)
if candidate is None or len(candidate) < 40:
log.warning("ind_summary.agg_json_invalid",
tone=tone, preview=result.content[:160])
session.add(AICall(
model=result.model,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=result.cost_usd, status="leaked",
))
else:
verdict = await review_read(client, candidate)
full_cost = (result.cost_usd or 0.0) + (verdict.cost_usd or 0.0)
if not verdict.clean:
log.warning("ind_summary.agg_reviewer_rejected",
tone=tone, reason=verdict.reason,
preview=candidate[:120])
session.add(AICall(
model=result.model,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=full_cost, status="leaked",
))
else:
agg_summary = IndicatorSummary(
group_name=AGGREGATE_GROUP_NAME,
generated_at=utcnow(),
model=result.model,
tone=tone,
analysis=analysis,
prompt_version=PROMPT_VERSION,
content=candidate,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=full_cost,
)
session.add(agg_summary)
session.add(AICall(
model=result.model,
prompt_tokens=result.prompt_tokens,
completion_tokens=result.completion_tokens,
cost_usd=full_cost, status="ok",
))
written += 1
except Exception as e:
session.add(AICall(
model=active_model(), status="error",

View file

@ -296,12 +296,25 @@ question via the chat sidebar.
def build_summary_system_prompt(tone: str, analysis: str) -> str:
"""A lean, focused system prompt for the per-indicator-group hourly
summary. INTERPRETATION not description the reader has the table
next to this paragraph; they don't need numbers recited at them."""
next to this paragraph; they don't need numbers recited at them.
Output is JSON-mode: the model must emit a single object
{"read": "..."}. The wrapper makes scratchpad outside the field
physically impossible the API enforces well-formed JSON, and the
only schema slot is the publishable read. Scratchpad inside the
field is caught by the reviewer agent (services/output_review)."""
tone_block = _TONE[_resolve_tone(tone)]
analysis_block = _ANALYSIS.get(analysis.upper(), _ANALYSIS["SPECULATIVE"])
return f"""You write a TINY interpretation (≤60 words, 2-3 sentences) \
of ONE indicator group for a strategic markets dashboard.
# Output format (strict)
Return ONLY a single JSON object with exactly one field:
{{"read": "<your 2-3 sentence interpretation>"}}
Nothing outside that JSON object. No preamble. No markdown fences. \
No additional fields. The "read" string is what the user sees verbatim, \
so it must already be the finished, publishable text never your thinking.
# What this is for
The reader is looking at the table of numbers right next to your text. \
They can see the values. They CANNOT see the meaning. Your job is to \
@ -316,19 +329,20 @@ Even at 2-3 sentences, contrast what the underlying factors justify \
they don't diverge, say so in one clause. Never just describe the move \
without placing it on this axis.
# Hard constraints
# Hard constraints on the "read" string
- Plain prose, ONE paragraph. No markdown, no headers, no lists, no labels.
- Open IMMEDIATELY with substance. NEVER start with: "I need to", "I'll", \
"We need to", "We are asked", "Here's", "Let me", "Let's", "Sure", "Looking \
at", "Based on", "Summary:", "The data shows", "First", "To address". No \
meta-commentary at all.
- No rhetorical questions, no "X? Actually Y?" self-corrections, no \
parenthetical asides that question your own numbers. The text is the \
finished read, not the thinking.
- Cite at most 2-3 specific numbers and ONLY when they anchor an \
interpretation. Don't list moves; explain them.
- Multi-week / multi-month horizon. 1-day moves under 2% are noise skip.
- No buy/sell language. No predictions. No watch list. No TL;DR. No date \
header. No "system temperature" line that belongs to the full daily log.
- Output the read directly. Do NOT include phrases like "Example", "Good \
example", "Bad example", "Reference", or any meta-framing of your output.
{tone_block}
@ -350,13 +364,22 @@ def build_summary_user_prompt(group_name: str, quotes: list[dict]) -> str:
def build_aggregate_summary_system_prompt(tone: str, analysis: str) -> str:
"""System prompt for the cross-group aggregate read shown on the dashboard.
Wider lens than a per-group summary synthesise across all groups."""
Wider lens than a per-group summary synthesise across all groups.
Same JSON-mode contract as build_summary_system_prompt: output is
{"read": "..."} only; the field is the publishable text verbatim."""
tone_block = _TONE[_resolve_tone(tone)]
analysis_block = _ANALYSIS.get(analysis.upper(), _ANALYSIS["SPECULATIVE"])
return f"""You write a single SHORT cross-asset INTERPRETATION (≤80 \
words, 2-4 sentences) for the dashboard header. The reader is glancing \
give them the meaning of the whole tape, not a recap.
# Output format (strict)
Return ONLY a single JSON object with exactly one field:
{{"read": "<your 2-4 sentence cross-asset interpretation>"}}
Nothing outside that JSON object. No preamble. No markdown fences. \
No additional fields. The "read" string is what the user sees verbatim.
# What this is for
The reader can see every indicator on the dashboard below this paragraph. \
Your job is NOT to summarise the moves. It is to explain what the moves, \
@ -371,19 +394,19 @@ crowd is actually doing (irrational: positioning, narrative momentum, \
flows). At least one of the 2-4 sentences must name this gap or, if the \
two cohere, explicitly say so.
# Hard constraints
# Hard constraints on the "read" string
- Plain prose, ONE paragraph. No markdown, headers, lists, or labels.
- Open IMMEDIATELY with substance. NEVER start with: "I need to", "I'll", \
"We need to", "Here's", "Let me", "Looking at", "Based on", "Sure", "Summary:", \
"The data shows", "Across the board". No meta-commentary.
- No rhetorical questions, no "X? Actually Y?" self-corrections, no \
parenthetical asides that question your own numbers.
- Identify the single most important **cross-asset implication**: e.g. \
"rates and credit disagree", "equities outrun fundamentals", "geopolitical \
risk premium is in commodities but not vol". Cite no more than 3 specific \
numbers, and only as anchors for the interpretation.
- Multi-week / multi-month horizon. 1-day moves under 2% are noise.
- No buy/sell language. No predictions of specific levels.
- Output the read directly. Do NOT include phrases like "Example", "Good \
example", "Bad example", "Reference", or any meta-framing of your output.
{tone_block}

View file

@ -0,0 +1,107 @@
"""Second-pass reviewer agent for AI-generated reads.
The per-group and aggregate indicator summaries are generated in JSON
mode and the publishable text comes out of a single "read" field, but a
misbehaving model can still slip chain-of-thought INSIDE the field
("Let's see…", "X? Actually Y?", multi-question parentheticals). This
module makes a small second LLM call that judges the candidate read as
clean / unclean. Cost is ~$0.0001 per check; latency ~1-2 s in the
hourly job. No user-facing latency.
The reviewer is deliberately a tiny, JSON-shaped classifier same
JSON-mode mechanism as the generator, so the verdict can't be lost in
prose. If parsing fails or the call errors, the row is rejected
(fail-safe: the previously cached good summary stays visible).
"""
from __future__ import annotations
import json
from dataclasses import dataclass
import httpx
from app.logging import get_logger
from app.services.openrouter import call_llm
log = get_logger("output_review")
_SYSTEM_PROMPT = """\
You are a strict editor for a financial-markets dashboard. The author
was asked to produce a short interpretive read for human readers.
You receive their proposed read and decide if it is publishable as-is.
Mark CLEAN only if the text reads like a finished interpretation a
reader could see on a public dashboard without confusion.
Mark UNCLEAN if the text contains ANY of:
- Chain-of-thought / scratchpad markers used as thinking phrases like
"Let me", "Let's see", "we need to", "actually" (correcting itself),
"wait", "hmm", "or rather", "I should".
- Self-questioning parentheticals: "Q1 2026? Actually Q4 2025?",
"is it X or Y?", any place where the author appears to be working
out the answer in front of the reader.
- Multiple rhetorical questions or any question that interrupts the
declarative voice. A clean interpretive read is assertive.
- Meta-commentary about the task, output format, word limits, or
instructions e.g. "as required by the constraints", "the prompt
asks", "let me address each".
- Partial / truncated content. Starts mid-word, mid-number, mid-clause.
- Visible internal numbers without clear meaning ("change 1y +5.9%?"),
raw column names ("as_of 2026-01-01"), or any debug-like fragments.
- Anything other than the finished, publishable interpretation.
Return ONLY a JSON object with this exact shape:
{"clean": true | false, "reason": "<≤20 words, plain text>"}
No preamble, no markdown fences, no other fields.
"""
@dataclass(frozen=True)
class Verdict:
clean: bool
reason: str
cost_usd: float | None # cost of the review call itself, for the ledger
async def review_read(client: httpx.AsyncClient, candidate: str) -> Verdict:
"""Ask the LLM whether `candidate` is a publishable read.
Returns Verdict(clean, reason, cost). Any error provider failure,
JSON parse failure, missing field, wrong type yields a CONSERVATIVE
verdict (clean=False) so the caller drops the candidate. The
previously cached good summary stays visible on the dashboard."""
if not candidate or not candidate.strip():
return Verdict(clean=False, reason="empty candidate", cost_usd=0.0)
messages = [
{"role": "system", "content": _SYSTEM_PROMPT},
# Sent as a fenced user turn so the model can't confuse the
# candidate with instructions, even if the candidate happens to
# contain prompt-like prose.
{"role": "user", "content": f"Candidate read:\n```\n{candidate}\n```"},
]
try:
result = await call_llm(
client, messages,
max_tokens=120,
response_format={"type": "json_object"},
)
except Exception as e:
log.warning("review.call_failed", error=str(e)[:200])
return Verdict(clean=False, reason=f"reviewer error: {str(e)[:80]}",
cost_usd=None)
try:
parsed = json.loads(result.content)
except json.JSONDecodeError:
log.warning("review.parse_failed", preview=result.content[:200])
return Verdict(clean=False, reason="reviewer returned non-JSON",
cost_usd=result.cost_usd)
clean = parsed.get("clean")
reason = parsed.get("reason") or ""
if not isinstance(clean, bool):
return Verdict(clean=False, reason="reviewer omitted bool 'clean'",
cost_usd=result.cost_usd)
return Verdict(clean=clean, reason=str(reason)[:200], cost_usd=result.cost_usd)