news: auto-tag headlines + market-aware cadence + filter UI

- Move news_job from hourly to 3x/hour (cron 10,30,50), with a CadencePolicy
  gate that throttles to active hours (07-21 UTC weekdays at 20 min), off-hours
  (3 h), weekends (6 h). Keeps the daytime feed fresh without spamming RSS
  sources overnight.
- Tag each headline on ingestion via DeepSeek (BATCH_SIZE=25, max_tokens=4000,
  json.JSONDecoder().raw_decode + per-row regex recovery for resilient parsing).
  Vocabulary: 16 tags including new EU / USA / AI / Conflict. NULL tags are
  picked up automatically on the next news_job run, so back-tagging is implicit
  rather than a separate migration step.
- Tag UI: pill bar above the feed with off → include → exclude cycle on click;
  shift-click jumps straight to exclude. State persists in localStorage and is
  injected into /api/news requests via htmx:configRequest. Per-row chips sit to
  the right of the headline (new 5-column grid: age | source | title | tags |
  UTC) so vertical density stays high.
- Strategic log header bug: model was hallucinating "(Updated 21:30 UTC)" in
  future tense. Bumped PROMPT_VERSION 6→7, added explicit ban on time-of-day
  clauses, and supply the actual current UTC time in the user prompt so the
  model has no need to invent one.

Migration 0012 adds headlines.tags (JSON, nullable). Tests cover vocabulary
integrity, validation/normalisation, and the JSON-recovery parser (17 tests).
This commit is contained in:
Giorgio Gilestro 2026-05-21 23:25:03 +01:00
parent 6e7f57c6b2
commit 2013bfa8cc
15 changed files with 745 additions and 25 deletions

130
tests/test_news_tagging.py Normal file
View file

@ -0,0 +1,130 @@
"""Tests for the deterministic half of news_tagging: vocabulary filtering
and JSON-response parsing. The LLM call itself isn't exercised."""
from __future__ import annotations
import pytest
from app.services.news_tagging import (
MAX_TAGS_PER_HEADLINE,
TAG_LABELS,
TAG_VOCABULARY,
_parse_batch_response,
_validate_tags,
)
# ---------------------------------------------------------------------------
# Vocabulary integrity
# ---------------------------------------------------------------------------
def test_every_vocab_tag_has_a_label():
"""Display labels must cover every tag — missing keys would render
the raw machine-name in the UI."""
for t in TAG_VOCABULARY:
assert t in TAG_LABELS, f"missing label for {t}"
def test_other_is_the_fallback_tag():
assert "other" in TAG_VOCABULARY
# ---------------------------------------------------------------------------
# _validate_tags
# ---------------------------------------------------------------------------
def test_validate_drops_unknown_tags():
out = _validate_tags(["markets", "wibble", "tech"])
assert out == ["markets", "tech"]
def test_validate_normalises_spaces_to_hyphens():
"""Common drift: model returns 'monetary policy' instead of
'monetary-policy'. We normalise."""
out = _validate_tags(["monetary policy"])
assert out == ["monetary-policy"]
def test_validate_normalises_case():
out = _validate_tags(["MARKETS", "Geopolitics"])
assert out == ["markets", "geopolitics"]
def test_validate_caps_at_max_tags():
out = _validate_tags(["markets", "tech", "china", "economy", "energy"])
assert len(out) == MAX_TAGS_PER_HEADLINE
def test_validate_dedupes():
out = _validate_tags(["markets", "markets", "tech"])
assert out == ["markets", "tech"]
def test_validate_rejects_non_list():
assert _validate_tags(None) == []
assert _validate_tags("markets") == []
assert _validate_tags({"tag": "markets"}) == []
def test_validate_skips_non_string_entries():
out = _validate_tags(["markets", 42, None, "tech"])
assert out == ["markets", "tech"]
# ---------------------------------------------------------------------------
# _parse_batch_response
# ---------------------------------------------------------------------------
def test_parse_basic_json_array():
raw = '[{"id": 1, "tags": ["markets", "tech"]}, {"id": 2, "tags": ["china"]}]'
out = _parse_batch_response(raw, {1, 2})
assert out == {1: ["markets", "tech"], 2: ["china"]}
def test_parse_strips_leading_prose():
"""Models occasionally prepend 'Here is the output:' before the JSON."""
raw = 'Sure! Here are the tags:\n[{"id": 1, "tags": ["markets"]}]'
out = _parse_batch_response(raw, {1})
assert out == {1: ["markets"]}
def test_parse_strips_markdown_fences():
raw = "```json\n[{\"id\": 1, \"tags\": [\"tech\"]}]\n```"
out = _parse_batch_response(raw, {1})
assert out == {1: ["tech"]}
def test_parse_drops_unexpected_ids():
raw = '[{"id": 99, "tags": ["markets"]}, {"id": 1, "tags": ["tech"]}]'
out = _parse_batch_response(raw, {1, 2})
assert out == {1: ["tech"]}
def test_parse_empty_tags_falls_back_to_other():
"""An item whose tags list ends up empty after validation gets
tagged 'other' so the row is marked tagged, not left NULL."""
raw = '[{"id": 1, "tags": ["nonsense"]}]'
out = _parse_batch_response(raw, {1})
assert out == {1: ["other"]}
def test_parse_unparseable_returns_empty():
"""Garbage in → empty out. The caller leaves those rows untagged
so they get retried on the next run."""
assert _parse_batch_response("nope, no JSON here", {1}) == {}
assert _parse_batch_response("[invalid json", {1}) == {}
def test_parse_ignores_non_dict_items():
raw = '[{"id": 1, "tags": ["markets"]}, "lol", null, {"id": 2, "tags": ["tech"]}]'
out = _parse_batch_response(raw, {1, 2})
assert out == {1: ["markets"], 2: ["tech"]}
def test_parse_handles_string_id_coercion():
"""Some models render the id as a string. We coerce."""
raw = '[{"id": "1", "tags": ["markets"]}]'
out = _parse_batch_response(raw, {1})
assert out == {1: ["markets"]}