giorgio/read.markets
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feedback: thumb up/down on logs + reviewer self-score 0-10

Browse source 
Two unrelated features bundled because they ship together and share
migration 0028.

Strategic-log feedback (thumb up/down):
- New strategic_log_feedback table with UNIQUE(log_id, user_id) so each
  user has one vote per log, flippable in place (up -> down -> clear).
  UI shows aggregate counts only.
- app/services/log_feedback.py: set_vote, get_counts, sign/verify
  feedback tokens (same itsdangerous pattern as auth.sign_pending,
  30-day TTL for email links).
- POST /api/log/{id}/feedback: web vote, auth required, returns counts
  + the requesting user's own vote.
- GET /feedback?token=...&vote=...: email-link target, no auth, signed
  token encodes (user, log, vote), renders feedback_thanks.html.
- partials/log.html: thumbs row below content, JS-driven swap via the
  POST endpoint. Dashboard latest-log card and /log page both render
  this partial via htmx, so the buttons appear in all three surfaces.
- digest emails: a "How was today's read?" row above the unsub footer,
  with signed-token URLs against the latest StrategicLog at send time.
  Plain-text fallback included.

Reviewer self-score (0-10):
- _SYSTEM_PROMPT asks for an integer score with anchors (10 exemplary,
  5 borderline, 0 unfit). Verdict gains score: int | None.
- Deterministic-layer hits get score=0 (hard rule, no nuance);
  error rows get None; LLM rows get the model's score clamped 0..10.
- ReviewerVerdict.score, StrategicLog.reviewer_score, and
  IndicatorSummary.reviewer_score all new SMALLINT NULL columns.
- ai_log_job + indicator_summary_job persist verdict.score onto their
  content rows when committing the row alongside content.

Tests:
- tests/test_strategic_log_feedback.py: vote, flip, clear, aggregate
  across users, invalid vote, token round-trip + tamper + garbage +
  'clear' not signable for email path.
- tests/test_output_review.py: score parsing, clamping (>10, <0),
  missing/non-numeric -> None, deterministic-layer score=0.

Full suite: 427 passed (was 412), 5 skipped, no regressions.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Giorgio Gilestro 2026-05-29 21:28:03 +02:00
parent f3ac65f8f7
commit 8946dee2e0
14 changed files with 962 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

53
app/services/digest_email.py
View file

@ -47,6 +47,7 @@ _DIGEST_HTML_TEMPLATE = """\
</div>
<div style="height:24px; line-height:24px; font-size:0;">&nbsp;</div>
<div style="border-top:1px solid {L_border};"></div>
{feedback_row}
<div style="height:14px; line-height:14px; font-size:0;">&nbsp;</div>
<div class="muted" style="font-size:11px; color:{L_muted};">
<a href="{unsubscribe_url}" style="color:{L_accent};">Unsubscribe in one click</a>
@ -70,6 +71,32 @@ def _strip_html_to_text(html_body: str) -> str:
return text.strip()
def _feedback_row_html(
feedback_up_url: str | None,
feedback_down_url: str | None,
light_accent: str,
light_muted: str,
) -> str:
"""Build the optional 'How was today's read?' row that sits between
the digest content and the unsubscribe footer. Empty string when no
feedback URLs were supplied (e.g. there's no latest log to vote on)."""
if not feedback_up_url or not feedback_down_url:
return ""
return (
'<div style="height:14px; line-height:14px; font-size:0;">&nbsp;</div>'
f'<div class="muted" style="font-size:12px; color:{light_muted};">'
"How was today&rsquo;s read? "
f'<a href="{feedback_up_url}" '
f'style="color:{light_accent}; text-decoration:none;">'
"&#x1F44D; Helpful</a>"
" &middot; "
f'<a href="{feedback_down_url}" '
f'style="color:{light_accent}; text-decoration:none;">'
"&#x1F44E; Not useful</a>"
"</div>"
)
def render_digest_email(
*,
kind: str,
@ -77,10 +104,17 @@ def render_digest_email(
content_html: str,
unsubscribe_url: str,
settings_url: str,
feedback_up_url: str | None = None,
feedback_down_url: str | None = None,
) -> tuple[str, str, str]:
"""Returns (subject, text_body, html_body) for a digest email.
`kind` is "daily" or "weekly". Anything else raises ValueError."""
`kind` is "daily" or "weekly". Anything else raises ValueError.
When ``feedback_up_url`` and ``feedback_down_url`` are both supplied,
a small thumb up/down row is rendered above the unsubscribe footer.
Both must be signed-token URLs pointing at /feedback (see
``app.services.log_feedback.sign_feedback_token``)."""
if kind == "daily":
label = "Daily"
subject = f"{branding.BRAND_NAME} · Daily — {date_str}"
@ -90,6 +124,12 @@ def render_digest_email(
else:
raise ValueError(f"unknown digest kind: {kind!r}")
feedback_row = _feedback_row_html(
feedback_up_url, feedback_down_url,
light_accent=branding.LIGHT["accent"],
light_muted=branding.LIGHT["muted"],
)
html_body = _DIGEST_HTML_TEMPLATE.format(
brand=branding.BRAND_NAME,
brand_upper=branding.BRAND_NAME.upper(),
@ -99,6 +139,7 @@ def render_digest_email(
content_html=content_html,
unsubscribe_url=unsubscribe_url,
settings_url=settings_url,
feedback_row=feedback_row,
**{f"L_{k.replace('-', '_')}": v for k, v in branding.LIGHT.items()},
**{f"D_{k.replace('-', '_')}": v for k, v in branding.DARK.items()},
)
@ -109,8 +150,16 @@ def render_digest_email(
"",
_strip_html_to_text(content_html),
"",
]
if feedback_up_url and feedback_down_url:
text_lines.extend([
f"Was this read useful? Helpful: {feedback_up_url}",
f" Not useful: {feedback_down_url}",
"",
])
text_lines.extend([
f"Unsubscribe: {unsubscribe_url}",
f"Manage preferences: {settings_url}",
]
])
text_body = "\n".join(text_lines)
return subject, text_body, html_body

177
app/services/log_feedback.py Normal file
View file

@ -0,0 +1,177 @@
"""Thumb up/down votes on strategic-log rows.
The model is one row per (log_id, user_id) — see
``app/models.py::StrategicLogFeedback``. The UI shows aggregate counts
only; user attribution is server-side state, not surface state. A user
can flip up → down (or vice versa) by re-submitting; "clear" deletes
the row.
For email-digest feedback links we sign a short payload with the same
itsdangerous serialiser pattern used elsewhere (``app/auth.py``). The
link is single-purpose: it identifies the (user, log, intended vote)
combination, lets the recipient click without being logged in, and
expires after EMAIL_FEEDBACK_TTL_SECONDS.
"""
from __future__ import annotations
from dataclasses import dataclass
from itsdangerous import BadSignature, SignatureExpired, URLSafeTimedSerializer
from sqlalchemy import delete, func, select
from sqlalchemy.ext.asyncio import AsyncSession
from app.config import get_settings
from app.db import utcnow
from app.models import StrategicLogFeedback
# Email-link tokens are valid for 30 days so a user can click the thumb
# in last week's Sunday digest. Long enough for slow readers; short
# enough that the signing-secret rotation can shake stale tokens off.
EMAIL_FEEDBACK_TTL_SECONDS = 30 * 24 * 60 * 60
# Discriminated enum of acceptable votes. "clear" is a sentinel that
# means "remove your existing vote, if any".
VALID_VOTES = ("up", "down", "clear")
class FeedbackError(ValueError):
"""Raised on bad input. Message is safe to surface to the user."""
@dataclass(frozen=True)
class FeedbackCounts:
up: int
down: int
user_vote: str | None # 'up' | 'down' | None
# ---------------------------------------------------------------------------
# Service
# ---------------------------------------------------------------------------
def _validate_vote(vote: str) -> str:
v = (vote or "").strip().lower()
if v not in VALID_VOTES:
raise FeedbackError(
f"vote must be one of {VALID_VOTES!r}; got {vote!r}"
)
return v
async def set_vote(
session: AsyncSession,
*,
log_id: int,
user_id: int,
vote: str,
) -> FeedbackCounts:
"""Insert / update / clear the user's vote on ``log_id``. Returns the
resulting aggregate counts plus the user's new vote state.
"clear" removes the row entirely; "up"/"down" upserts. The unique
index on (log_id, user_id) guarantees there's never more than one
row per pair."""
v = _validate_vote(vote)
if v == "clear":
await session.execute(
delete(StrategicLogFeedback)
.where(StrategicLogFeedback.log_id == log_id)
.where(StrategicLogFeedback.user_id == user_id)
)
await session.commit()
return await get_counts(session, log_id=log_id, user_id=user_id)
# Upsert by hand — SQLAlchemy's portable dialect doesn't expose
# ON DUPLICATE KEY UPDATE across MySQL+SQLite reliably for our test
# path, so SELECT then INSERT-or-UPDATE is the simplest correct shape.
existing = (await session.execute(
select(StrategicLogFeedback)
.where(StrategicLogFeedback.log_id == log_id)
.where(StrategicLogFeedback.user_id == user_id)
)).scalar_one_or_none()
if existing is None:
session.add(StrategicLogFeedback(
log_id=log_id, user_id=user_id, vote=v,
created_at=utcnow(), updated_at=utcnow(),
))
else:
existing.vote = v
existing.updated_at = utcnow()
await session.commit()
return await get_counts(session, log_id=log_id, user_id=user_id)
async def get_counts(
session: AsyncSession,
*,
log_id: int,
user_id: int | None = None,
) -> FeedbackCounts:
"""Aggregate up/down counts for one log, plus the requesting user's
own vote (if ``user_id`` is supplied). One indexed query for the
counts; a second indexed lookup for the personal vote."""
rows = (await session.execute(
select(
StrategicLogFeedback.vote,
func.count(StrategicLogFeedback.id),
)
.where(StrategicLogFeedback.log_id == log_id)
.group_by(StrategicLogFeedback.vote)
)).all()
counts = {vote: int(n) for vote, n in rows}
user_vote: str | None = None
if user_id is not None:
user_vote = (await session.execute(
select(StrategicLogFeedback.vote)
.where(StrategicLogFeedback.log_id == log_id)
.where(StrategicLogFeedback.user_id == user_id)
)).scalar_one_or_none()
return FeedbackCounts(
up=counts.get("up", 0),
down=counts.get("down", 0),
user_vote=user_vote,
)
# ---------------------------------------------------------------------------
# Email-link token helpers (parallel to app.auth.sign_pending)
# ---------------------------------------------------------------------------
def _feedback_serializer() -> URLSafeTimedSerializer:
s = get_settings()
secret = s.CASSANDRA_SESSION_SECRET or s.CASSANDRA_TOKEN or "dev-insecure-secret"
return URLSafeTimedSerializer(secret, salt="cassandra-log-feedback-v1")
def sign_feedback_token(user_id: int, log_id: int, vote: str) -> str:
"""Signed token for an email-digest thumb link. Encodes the intended
(user, log, vote) tuple. The recipient clicks the link without being
logged in; the receiving endpoint verifies the signature, applies the
vote, and shows a thank-you page."""
v = _validate_vote(vote)
if v == "clear":
raise FeedbackError("clear is not a valid email-link vote")
return _feedback_serializer().dumps({
"uid": int(user_id), "lid": int(log_id), "v": v,
})
def verify_feedback_token(token: str) -> dict | None:
"""Returns {"user_id": int, "log_id": int, "vote": "up"|"down"} on
valid + un-expired tokens, or None on bad signature / expired / bad
payload. The TTL is ``EMAIL_FEEDBACK_TTL_SECONDS``."""
try:
data = _feedback_serializer().loads(
token, max_age=EMAIL_FEEDBACK_TTL_SECONDS,
)
return {
"user_id": int(data["uid"]),
"log_id": int(data["lid"]),
"vote": str(data["v"]),
}
except (BadSignature, SignatureExpired, KeyError, TypeError, ValueError):
return None

48
app/services/output_review.py
View file

@ -123,8 +123,19 @@ Mark UNCLEAN if the text contains ANY of:
claim on a *named* instrument is not.
- Anything else other than the finished, publishable commentary.
Also assign a SCORE 0-10 to the candidate:
- 10 = exemplary editorial: sharp, well-grounded, no perimeter risk, clean prose.
- 7-9 = publishable as-is, varying degrees of polish.
- 4-6 = borderline: scratchpad leakage, mild perimeter drift, or weak structure,
but not yet outright unfit. (Anything ≤4 should usually be clean=false.)
- 1-3 = unfit: clear chain-of-thought, partial / truncated, or financial-advice drift.
- 0 = unfit by hard rule (deterministic catch territory).
Clean=true implies a score of ~7+; clean=false implies ~4 or lower. Use the
score to communicate confidence within the verdict.
Return ONLY a JSON object with this exact shape:
{"clean": true | false, "reason": "<≤20 words, plain text>"}
{"clean": true | false, "reason": "<≤20 words, plain text>", "score": 0-10}
No preamble, no markdown fences, no other fields.
"""
@ -171,6 +182,12 @@ class Verdict:
reason: str
cost_usd: float | None # cost of the review call itself, for the ledger
layer: str = "llm" # "deterministic" | "llm" | "error"
# Integer 0-10. None for error rows; 0 for deterministic-layer hits
# (rejected by hard rule, no nuance to score); 0-10 from the model
# on LLM-layer verdicts. Stored alongside the content row for
# future analysis — see StrategicLog.reviewer_score and
# IndicatorSummary.reviewer_score.
score: int | None = None
# Truncation cap for the audit log's candidate_text column. Generous enough
@ -199,6 +216,7 @@ async def _record_verdict(
reason=verdict.reason[:240] if verdict.reason else None,
layer=verdict.layer,
model=model,
score=verdict.score,
)
session.add(row)
await session.flush()
@ -237,7 +255,7 @@ async def review_read(
if not candidate or not candidate.strip():
verdict = Verdict(clean=False, reason="empty candidate", cost_usd=0.0,
layer="deterministic")
layer="deterministic", score=0)
await _record_verdict(session, surface=surface, candidate=candidate or "",
verdict=verdict, model=None)
return verdict
@ -250,6 +268,7 @@ async def review_read(
reason=f"lexicon:{hit.rule}: {hit.snippet}",
cost_usd=0.0,
layer="deterministic",
score=0,
)
log.info("review.deterministic_reject",
rule=hit.rule, snippet=hit.snippet, surface=surface)
@ -293,7 +312,7 @@ async def review_read(
except Exception as e:
log.warning("review.call_failed", error=str(e)[:200])
verdict = Verdict(clean=False, reason=f"reviewer error: {str(e)[:80]}",
cost_usd=None, layer="error")
cost_usd=None, layer="error", score=None)
await _record_verdict(session, surface=surface, candidate=candidate,
verdict=verdict, model=reviewer_model)
return verdict
@ -317,7 +336,7 @@ async def review_read(
except json.JSONDecodeError:
log.warning("review.parse_failed", preview=result.content[:200])
verdict = Verdict(clean=False, reason="reviewer returned non-JSON",
cost_usd=result.cost_usd, layer="error")
cost_usd=result.cost_usd, layer="error", score=None)
await _record_verdict(session, surface=surface, candidate=candidate,
verdict=verdict, model=reviewer_model)
return verdict
@ -326,13 +345,25 @@ async def review_read(
reason = parsed.get("reason") or ""
if not isinstance(clean, bool):
verdict = Verdict(clean=False, reason="reviewer omitted bool 'clean'",
cost_usd=result.cost_usd, layer="error")
cost_usd=result.cost_usd, layer="error", score=None)
await _record_verdict(session, surface=surface, candidate=candidate,
verdict=verdict, model=reviewer_model)
return verdict
# Score is optional and bounded; the verdict is still valid without it.
raw_score = parsed.get("score")
score: int | None
if isinstance(raw_score, bool):
# bool is a subclass of int — exclude it explicitly to avoid
# silently treating True/False as 1/0.
score = None
elif isinstance(raw_score, (int, float)):
score = max(0, min(10, int(raw_score)))
else:
score = None
verdict = Verdict(clean=clean, reason=str(reason)[:200],
cost_usd=result.cost_usd, layer="llm")
cost_usd=result.cost_usd, layer="llm", score=score)
await _record_verdict(session, surface=surface, candidate=candidate,
verdict=verdict, model=reviewer_model)
return verdict
@ -391,7 +422,7 @@ async def generate_with_review(
content=None,
verdict=Verdict(clean=False,
reason=f"generator error: {str(e)[:80]}",
cost_usd=None, layer="error"),
cost_usd=None, layer="error", score=None),
attempts=attempt,
)
@ -411,6 +442,7 @@ async def generate_with_review(
return ReviewedGeneration(
content=None,
verdict=last_verdict or Verdict(clean=False, reason="no attempts",
cost_usd=None, layer="error"),
cost_usd=None, layer="error",
score=None),
attempts=max_attempts,
)