feedback: thumb up/down on logs + reviewer self-score 0-10
Two unrelated features bundled because they ship together and share
migration 0028.
Strategic-log feedback (thumb up/down):
- New strategic_log_feedback table with UNIQUE(log_id, user_id) so each
user has one vote per log, flippable in place (up -> down -> clear).
UI shows aggregate counts only.
- app/services/log_feedback.py: set_vote, get_counts, sign/verify
feedback tokens (same itsdangerous pattern as auth.sign_pending,
30-day TTL for email links).
- POST /api/log/{id}/feedback: web vote, auth required, returns counts
+ the requesting user's own vote.
- GET /feedback?token=...&vote=...: email-link target, no auth, signed
token encodes (user, log, vote), renders feedback_thanks.html.
- partials/log.html: thumbs row below content, JS-driven swap via the
POST endpoint. Dashboard latest-log card and /log page both render
this partial via htmx, so the buttons appear in all three surfaces.
- digest emails: a "How was today's read?" row above the unsub footer,
with signed-token URLs against the latest StrategicLog at send time.
Plain-text fallback included.
Reviewer self-score (0-10):
- _SYSTEM_PROMPT asks for an integer score with anchors (10 exemplary,
5 borderline, 0 unfit). Verdict gains score: int | None.
- Deterministic-layer hits get score=0 (hard rule, no nuance);
error rows get None; LLM rows get the model's score clamped 0..10.
- ReviewerVerdict.score, StrategicLog.reviewer_score, and
IndicatorSummary.reviewer_score all new SMALLINT NULL columns.
- ai_log_job + indicator_summary_job persist verdict.score onto their
content rows when committing the row alongside content.
Tests:
- tests/test_strategic_log_feedback.py: vote, flip, clear, aggregate
across users, invalid vote, token round-trip + tamper + garbage +
'clear' not signable for email path.
- tests/test_output_review.py: score parsing, clamping (>10, <0),
missing/non-numeric -> None, deterministic-layer score=0.
Full suite: 427 passed (was 412), 5 skipped, no regressions.
Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
parent
f3ac65f8f7
commit
8946dee2e0
14 changed files with 962 additions and 14 deletions
|
|
@ -325,3 +325,109 @@ async def test_review_portfolio_rider_active_when_flag_enabled(monkeypatch):
|
|||
surface="portfolio",
|
||||
)
|
||||
assert "# Surface: portfolio commentary" in seen_systems[0]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Reviewer self-score (0-10)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_parses_score_from_llm_json(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
def handler(_req):
|
||||
return httpx.Response(200, json={
|
||||
"choices": [{"message": {"content":
|
||||
'{"clean": true, "reason": "exemplary", "score": 9}'},
|
||||
"finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 12, "cost": 0.00007},
|
||||
})
|
||||
async with httpx.AsyncClient(transport=_mock_post(handler)) as client:
|
||||
v = await review_read(client, "Markets pricing tighter policy.")
|
||||
assert v.clean is True
|
||||
assert v.score == 9
|
||||
assert v.layer == "llm"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_score_clamped_to_0_10(monkeypatch):
|
||||
"""A model returning 17 or -3 is buggy but must not blow up — clamp."""
|
||||
_configure(monkeypatch)
|
||||
def handler(_req):
|
||||
return httpx.Response(200, json={
|
||||
"choices": [{"message": {"content":
|
||||
'{"clean": false, "reason": "x", "score": 17}'},
|
||||
"finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 8, "cost": 0.00003},
|
||||
})
|
||||
async with httpx.AsyncClient(transport=_mock_post(handler)) as client:
|
||||
v = await review_read(client, "Some content.")
|
||||
assert v.score == 10
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_score_negative_clamped_to_0(monkeypatch):
|
||||
_configure(monkeypatch)
|
||||
def handler(_req):
|
||||
return httpx.Response(200, json={
|
||||
"choices": [{"message": {"content":
|
||||
'{"clean": false, "reason": "x", "score": -3}'},
|
||||
"finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 8, "cost": 0.00003},
|
||||
})
|
||||
async with httpx.AsyncClient(transport=_mock_post(handler)) as client:
|
||||
v = await review_read(client, "Some content.")
|
||||
assert v.score == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_missing_score_yields_none(monkeypatch):
|
||||
"""Older mocked responses don't carry score; verdict still valid,
|
||||
score is None."""
|
||||
_configure(monkeypatch)
|
||||
def handler(_req):
|
||||
return httpx.Response(200, json={
|
||||
"choices": [{"message": {"content":
|
||||
'{"clean": true, "reason": "ok"}'},
|
||||
"finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 6, "cost": 0.00002},
|
||||
})
|
||||
async with httpx.AsyncClient(transport=_mock_post(handler)) as client:
|
||||
v = await review_read(client, "Plain state-level prose.")
|
||||
assert v.clean is True
|
||||
assert v.score is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_score_non_numeric_yields_none(monkeypatch):
|
||||
"""Defensive: a string or null in the score field doesn't poison the
|
||||
verdict; score becomes None."""
|
||||
_configure(monkeypatch)
|
||||
def handler(_req):
|
||||
return httpx.Response(200, json={
|
||||
"choices": [{"message": {"content":
|
||||
'{"clean": true, "reason": "ok", "score": "high"}'},
|
||||
"finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 50, "completion_tokens": 6, "cost": 0.00002},
|
||||
})
|
||||
async with httpx.AsyncClient(transport=_mock_post(handler)) as client:
|
||||
v = await review_read(client, "Plain state-level prose.")
|
||||
assert v.clean is True
|
||||
assert v.score is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_review_deterministic_layer_score_is_zero(monkeypatch):
|
||||
"""A deterministic-layer hit is a hard reject by rule; the audit row
|
||||
carries score=0 (no nuance to score)."""
|
||||
_configure(monkeypatch)
|
||||
calls = []
|
||||
def handler(_req):
|
||||
calls.append(1)
|
||||
return httpx.Response(500, json={"error": "should not fire"})
|
||||
async with httpx.AsyncClient(transport=_mock_post(handler)) as client:
|
||||
v = await review_read(client, "You should buy the dip.")
|
||||
assert v.clean is False
|
||||
assert v.layer == "deterministic"
|
||||
assert v.score == 0
|
||||
assert calls == []
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue