diff --git a/alembic/versions/0012_headlines_tags.py b/alembic/versions/0012_headlines_tags.py new file mode 100644 index 0000000..157b1f5 --- /dev/null +++ b/alembic/versions/0012_headlines_tags.py @@ -0,0 +1,29 @@ +"""headlines.tags — AI-assigned content tags per headline + +Adds a JSON column to `headlines` for semantic tags (markets, geopolitics, +tech, etc.) assigned at ingest time by `app/services/news_tagging.py`. +NULL means "not yet tagged" — picked up automatically by the next +news_job run. + +Revision ID: 0012 +Revises: 0011 +Create Date: 2026-05-18 +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + + +revision: str = "0012" +down_revision: Union[str, None] = "0011" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.add_column("headlines", sa.Column("tags", sa.JSON, nullable=True)) + + +def downgrade() -> None: + op.drop_column("headlines", "tags") diff --git a/app/jobs/news_job.py b/app/jobs/news_job.py index 0d8af20..e5e30e9 100644 --- a/app/jobs/news_job.py +++ b/app/jobs/news_job.py @@ -1,21 +1,36 @@ -"""Hourly news ingestion. Reads enabled feeds from the DB (not TOML — DB has -the authoritative enabled/failure state). Per-ticker Yahoo news pulled for -each symbol in the default portfolio group ('pie').""" +"""News ingestion + AI tagging. + +Cron fires every 20 minutes. NEWS_POLICY gates the actual work: +- Active window (07-21 UTC weekdays): always run (20-min gap) +- Off-hours weekday: skip until 3h since last success +- Weekend: skip until 6h since last success + +Each run does (a) fresh fetch of all enabled feeds + per-ticker Yahoo +news, (b) bulk INSERT IGNORE into headlines, (c) batch-tags any rows +still NULL via news_tagging. Untagged rows survive run failures and are +retried automatically next cycle. +""" from __future__ import annotations import asyncio import httpx -from sqlalchemy import desc, select +from sqlalchemy import desc, func, select, update from sqlalchemy.dialects.mysql import insert as mysql_insert from app.db import utcnow from app.jobs._helpers import job_lifecycle, log -from app.models import Feed, Headline, InstrumentMap, TickerUniverse +from app.models import Feed, Headline, InstrumentMap, JobRun, TickerUniverse +from app.services.cadence import NEWS_POLICY from app.services.news import dedupe, fetch_feed, fetch_yahoo_news +from app.services.news_tagging import ToTag, tag_titles AUTO_DISABLE_AT = 5 +# Cap on how many untagged headlines a single run will tag. Stops a +# backlog from blowing the cost ledger if the tagger has been failing +# for a while. +TAG_PER_RUN_LIMIT = 200 async def _process_feed(client: httpx.AsyncClient, feed: Feed) -> tuple[Feed, list]: @@ -38,6 +53,21 @@ async def run() -> None: if run.status == "skipped": return + # Market-aware cadence: skip this fire if too soon (off-hours / + # weekend). Active window still runs every 20 min. + last_success = (await session.execute( + select(func.max(JobRun.finished_at)).where( + JobRun.name == "news_job", + JobRun.status == "success", + ) + )).scalar() + should_run, reason = NEWS_POLICY.should_run(last_success) + if not should_run: + log.info("news_job.cadence_skip", reason=reason) + run.status = "skipped" + run.error = reason + return + feeds = ( await session.execute(select(Feed).where(Feed.enabled == True)) ).scalars().all() @@ -91,8 +121,35 @@ async def run() -> None: await session.execute(stmt) await session.commit() + + # Tag any headlines still NULL — fresh inserts from this run plus + # any that failed to tag on previous runs. Bounded by + # TAG_PER_RUN_LIMIT so a long outage doesn't blow the cost ledger. + untagged_rows = (await session.execute( + select(Headline.id, Headline.title) + .where(Headline.tags.is_(None)) + .order_by(desc(Headline.published_at)) + .limit(TAG_PER_RUN_LIMIT) + )).all() + tagged_count = 0 + if untagged_rows: + items = [ToTag(id=int(r.id), title=r.title) for r in untagged_rows] + tags_by_id = await tag_titles(items) + for hid, tags in tags_by_id.items(): + await session.execute( + update(Headline) + .where(Headline.id == hid) + .values(tags=tags) + ) + tagged_count = len(tags_by_id) + await session.commit() + run.items_written = len(headlines) - log.info("news_job.done", fetched=len(all_headlines), kept=len(headlines)) + log.info( + "news_job.done", + fetched=len(all_headlines), kept=len(headlines), + untagged_seen=len(untagged_rows), tagged=tagged_count, + ) if __name__ == "__main__": diff --git a/app/models.py b/app/models.py index f1591fb..8ee33d1 100644 --- a/app/models.py +++ b/app/models.py @@ -67,6 +67,10 @@ class Headline(Base): published_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False) fetched_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) fingerprint: Mapped[str] = mapped_column(String(40), nullable=False) # sha1 of normalised title + # Semantic content tags from app.services.news_tagging. NULL = not yet + # tagged; the next news_job run picks it up. Each entry is one of the + # values in news_tagging.TAG_VOCABULARY. + tags: Mapped[list[str] | None] = mapped_column(JSON, nullable=True) __table_args__ = ( UniqueConstraint("fingerprint", name="uq_headlines_fingerprint"), diff --git a/app/routers/api.py b/app/routers/api.py index e9300b0..84e8ad6 100644 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -212,6 +212,13 @@ async def indicators( # --- News -------------------------------------------------------------------- +def _split_tag_param(s: str | None) -> set[str]: + """Parse a comma-separated tags query param, lowercase + trim.""" + if not s: + return set() + return {t.strip().lower() for t in s.split(",") if t.strip()} + + @router.get("/news") async def news_list( request: Request, @@ -219,19 +226,39 @@ async def news_list( category: str | None = Query(None), since_hours: float = Query(24.0, ge=0.1, le=720.0), limit: int = Query(50, ge=1, le=500), + tags: str | None = Query(None, description="comma-separated include list"), + exclude_tags: str | None = Query(None, description="comma-separated exclude list"), as_: str | None = Query(default=None, alias="as"), ): + from app.services.news_tagging import TAG_LABELS, TAG_VOCABULARY + cutoff = utcnow() - timedelta(hours=since_hours) stmt = select(Headline).where(Headline.published_at >= cutoff) if category: stmt = stmt.where(Headline.category == category) - stmt = stmt.order_by(desc(Headline.published_at)).limit(limit) + # Fetch a wider window than `limit` because we tag-filter client-of-DB. + # JSON column filters in MariaDB are doable but messy; in-Python is + # simple at our scale. + stmt = stmt.order_by(desc(Headline.published_at)).limit(max(limit * 3, 200)) rows = (await session.execute(stmt)).scalars().all() + include = _split_tag_param(tags) + exclude = _split_tag_param(exclude_tags) + + def _keep(h: Headline) -> bool: + ts = set(h.tags or []) + if include and not (ts & include): + return False + if exclude and (ts & exclude): + return False + return True + + filtered = [h for h in rows if _keep(h)][:limit] + if as_ == "html": now = utcnow() items = [] - for h in rows: + for h in filtered: when = _as_utc(h.published_at) if h.published_at else None items.append({ "age": _fmt_age(now, h.published_at), @@ -240,11 +267,17 @@ async def news_list( "url": h.url, "iso": when.isoformat() if when else None, "utc_short": when.strftime("%d %b %H:%M") + "Z" if when else "", + "tags": h.tags or [], }) return templates.TemplateResponse( - request, "partials/news.html", {"headlines": items}, + request, "partials/news.html", + {"headlines": items, + "tag_vocabulary": TAG_VOCABULARY, + "tag_labels": TAG_LABELS, + "active_include": sorted(include), + "active_exclude": sorted(exclude)}, ) - return [HeadlineOut.model_validate(r, from_attributes=True) for r in rows] + return [HeadlineOut.model_validate(r, from_attributes=True) for r in filtered] # --- Strategic log ----------------------------------------------------------- diff --git a/app/scheduler_main.py b/app/scheduler_main.py index fcedc68..e20d15e 100644 --- a/app/scheduler_main.py +++ b/app/scheduler_main.py @@ -40,7 +40,11 @@ async def main() -> None: sched = AsyncIOScheduler(timezone="UTC") sched.add_job(market_job.run, CronTrigger(minute=5), name="market_job", id="market_job") - sched.add_job(news_job.run, CronTrigger(minute=10), name="news_job", id="news_job") + # 3x/hour: cron fires at xx:10, xx:30, xx:50. NEWS_POLICY inside the + # job throttles off-hours / weekends so most fires no-op when the + # markets are closed. + sched.add_job(news_job.run, CronTrigger(minute="10,30,50"), + name="news_job", id="news_job") # portfolio_job removed in Phase G — server no longer holds holdings. sched.add_job(indicator_summary_job.run, CronTrigger(minute=7), name="indicator_summary_job", id="indicator_summary_job") sched.add_job(ai_log_job.run, CronTrigger(minute=20), name="ai_log_job", id="ai_log_job") diff --git a/app/schemas.py b/app/schemas.py index b904dbe..f5c13e2 100644 --- a/app/schemas.py +++ b/app/schemas.py @@ -25,6 +25,7 @@ class HeadlineOut(BaseModel): title: str url: str published_at: datetime + tags: list[str] | None = None # populated by news_tagging; null = pending class JobStatus(BaseModel): diff --git a/app/services/cadence.py b/app/services/cadence.py index b3c6127..8db3900 100644 --- a/app/services/cadence.py +++ b/app/services/cadence.py @@ -29,6 +29,11 @@ class CadencePolicy: (7, 21), # EU/US (LSE open through NYSE close) # (0, 8), # Asia (Tokyo + HK/Shanghai) — uncomment to add ) + # Minimum gap between successful runs DURING the active window. The + # cron may fire more frequently than this — we just skip until enough + # time has passed since the last success. Default 0 means "run on + # every cron fire" (the original AI-job behaviour). + active_gap_h: float = 0.0 # Minimum gap between successful runs outside the active window. off_hours_gap_h: float = 4.0 weekend_gap_h: float = 12.0 @@ -44,7 +49,7 @@ class CadencePolicy: if now.weekday() >= 5: return self.weekend_gap_h if self.is_active_window(now): - return 0.0 # always run during the active window + return self.active_gap_h return self.off_hours_gap_h def should_run( @@ -55,8 +60,6 @@ class CadencePolicy: """Returns (should_run, reason). The reason is human-readable for logs and the job_runs.error column when a run is skipped.""" now = now or datetime.now(timezone.utc) - if self.is_active_window(now): - return True, "active window" min_gap = self.min_gap_hours(now) if last_success_at is None: return True, "no prior successful run" @@ -64,9 +67,27 @@ class CadencePolicy: if last_success_at.tzinfo is None: last_success_at = last_success_at.replace(tzinfo=timezone.utc) age_h = (now - last_success_at).total_seconds() / 3600.0 + if min_gap <= 0 and self.is_active_window(now): + return True, "active window" if age_h >= min_gap: - return True, f"off-hours but last run {age_h:.1f}h ago (≥ {min_gap}h)" - return False, f"off-hours throttled — last run {age_h:.1f}h ago (< {min_gap}h)" + band = "active" if self.is_active_window(now) else ( + "weekend" if now.weekday() >= 5 else "off-hours" + ) + return True, f"{band}: last run {age_h:.2f}h ago (≥ {min_gap:.2f}h)" + band = "active" if self.is_active_window(now) else ( + "weekend" if now.weekday() >= 5 else "off-hours" + ) + return False, f"{band} throttled — last run {age_h:.2f}h ago (< {min_gap:.2f}h)" +# AI jobs: run hot during the active window, throttle off-hours. DEFAULT_POLICY = CadencePolicy() + +# News + tagging: 3 runs/hour during the active window (20-min gap), +# every 3h off-hours, every 6h on weekends. Cron fires every 20 min; +# the policy gates whether each fire actually does work. +NEWS_POLICY = CadencePolicy( + active_gap_h=1.0 / 3.0, # 20 minutes + off_hours_gap_h=3.0, + weekend_gap_h=6.0, +) diff --git a/app/services/news_tagging.py b/app/services/news_tagging.py new file mode 100644 index 0000000..bd0fe1a --- /dev/null +++ b/app/services/news_tagging.py @@ -0,0 +1,290 @@ +"""AI-driven content tagging for headlines. + +Each headline gets 1-3 tags from a fixed vocabulary (markets, geopolitics, +tech, etc.). Tagging happens at ingest time inside `news_job` — only +rows whose `tags` column is still NULL are processed, so re-runs are +idempotent and recover from prior failures naturally. + +Implementation notes: + +- Titles only (not body) — they're informative enough and keep the + prompt + cost small. +- Batched: ~50 titles per LLM call. Returns JSON with one entry per + input id. Unknown / hallucinated tags are dropped against the + vocabulary; an empty tag list falls back to ["other"] so we can tell + "tagged but bland" from "not yet tagged" (NULL). +- Uses the existing call_llm dispatcher → DeepSeek-direct primary, + OpenRouter fallback, per Phase G provider config. +""" +from __future__ import annotations + +import json +import re +from dataclasses import dataclass + +import httpx + +from app.logging import get_logger +from app.services.openrouter import call_llm + + +log = get_logger("news_tagging") + + +# Frozen vocabulary. Keep ASCII-lowercase, hyphenated. If you add or +# remove a tag, also update the system prompt below and the test fixture. +TAG_VOCABULARY: tuple[str, ...] = ( + "markets", + "monetary-policy", + "economy", + "geopolitics", + "conflict", # wars, military actions, armed escalation + "energy", + "commodities", + "tech", + "ai", # AI-specific: model releases, capex, regulation + "crypto", + "corporate", + "regulation", + # Geographic emphasis tags — overlap freely with thematic ones. + "usa", + "eu", + "china", + "other", +) + +# Display labels for the toggle UI (Title Case + readable). Keys must +# match TAG_VOCABULARY exactly. +TAG_LABELS: dict[str, str] = { + "markets": "Markets", + "monetary-policy": "Monetary policy", + "economy": "Economy", + "geopolitics": "Geopolitics", + "conflict": "Conflict", + "energy": "Energy", + "commodities": "Commodities", + "tech": "Tech", + "ai": "AI", + "crypto": "Crypto", + "corporate": "Corporate", + "regulation": "Regulation", + "usa": "USA", + "eu": "EU", + "china": "China", + "other": "Other", +} + +_VOCAB_SET = frozenset(TAG_VOCABULARY) + +# Batch size for one LLM call. Small enough that one batch of output +# (50 items × ~30 tokens each = ~1500 tokens) fits well under any +# reasonable max_tokens, AND so a single batch failure only loses a +# small number of rows to next-cycle retry. +BATCH_SIZE = 25 + +# Max tags per headline. Stories often touch multiple themes; we cap at +# three so the UI chips don't blow up. +MAX_TAGS_PER_HEADLINE = 3 + + +_SYSTEM_PROMPT = """\ +You tag financial / business news headlines with ONE to THREE content tags \ +from a fixed vocabulary. You receive a JSON array of headlines, each with \ +an `id` and a `title`. Return a JSON array of objects: `{"id": ..., \ +"tags": ["...", "..."]}`. Output nothing else — no prose, no markdown, no \ +preamble. The first character of your response must be `[`. + +# Vocabulary (use ONLY these values, lowercase, hyphens not spaces) +## Thematic tags +- markets — direct market moves: stocks, bonds, FX, indices +- monetary-policy — central banks, rate decisions, QE/QT, Fed/ECB/BOJ +- economy — macro data: CPI, GDP, jobs, PMI, retail sales +- geopolitics — sanctions, diplomacy, chokepoints, elections, trade +- conflict — active wars, military strikes, armed escalation + (use ALONGSIDE geopolitics, not instead of) +- energy — oil, gas, OPEC, energy transition, utilities +- commodities — gold, copper, agri, industrial metals (non-energy) +- tech — Big Tech, chips, semiconductors, software, social media +- ai — AI-specific: model releases, AI capex, AI regulation + (overlap with tech freely) +- crypto — bitcoin, ethereum, stablecoins, crypto regulation +- corporate — earnings, M&A, layoffs, single-company news without + a clear sector fit above +- regulation — antitrust, securities regs, EU/SEC rulings, trade rules +## Geographic tags (overlap freely with thematic ones) +- usa — US-specific news, US policy, US-driven stories +- eu — EU / Eurozone / individual EU member states +- china — China-specific news +## Fallback +- other — last resort: entertainment, sport, weather, off-topic + +# Tagging discipline +- 1 to 3 tags per headline. Prefer 1-2; use 3 only when the story \ +genuinely spans multiple themes. +- Tags can OVERLAP. "China bans US chips" → ["china", "tech", "geopolitics"]. +- For armed conflict, combine: "Israel strikes Lebanon" → ["conflict", "geopolitics"]. +- For AI stories, prefer "ai" over generic "tech" if the headline is AI-centric. +- Geographic tags are additive: a US-focused tech story → ["tech", "usa"]. +- "other" is a last resort. If a headline is entertainment, sport, weather, \ +or otherwise off-topic for a macro dashboard, tag it "other". +- Order tags by relevance: most specific first. +""" + + +@dataclass(frozen=True) +class _ToTag: + id: int + title: str + + +def _validate_tags(raw: list) -> list[str]: + """Filter a model-returned tag list down to known vocabulary + cap.""" + if not isinstance(raw, list): + return [] + cleaned: list[str] = [] + seen: set[str] = set() + for t in raw: + if not isinstance(t, str): + continue + # Normalise: lowercase, replace spaces with hyphens (common drift). + norm = t.strip().lower().replace(" ", "-") + if norm in _VOCAB_SET and norm not in seen: + cleaned.append(norm) + seen.add(norm) + if len(cleaned) >= MAX_TAGS_PER_HEADLINE: + break + return cleaned + + +def _parse_batch_response(content: str, expected_ids: set[int]) -> dict[int, list[str]]: + """Parse the model's JSON output into {id: tags}. + + Robust to leading prose / code fences / trailing notes — uses + ``json.JSONDecoder.raw_decode`` to parse the first complete JSON + value starting at the first ``[``. Anything after that array is + ignored. If the first parse fails, we fall back to extracting + well-formed ``{"id": ..., "tags": [...]}`` objects via regex so a + single corrupt item doesn't lose the whole batch. + """ + out: dict[int, list[str]] = {} + if not content: + return out + + # Trim common preambles + code fences. + stripped = content.strip() + # First-`[` to last-position parse via raw_decode. + start = stripped.find("[") + if start == -1: + log.warning("news_tagging.unparseable", preview=content[:120]) + return out + try: + data, _end = json.JSONDecoder().raw_decode(stripped[start:]) + if isinstance(data, list): + for item in data: + _absorb(item, expected_ids, out) + return out + except json.JSONDecodeError: + pass # fall through to per-item recovery + + # Recovery path: scrape individual objects. Looks for shapes like + # `{"id": 123, "tags": ["a", "b"]}` and tolerates any garbage between. + matched = 0 + for m in re.finditer( + r'\{\s*"id"\s*:\s*"?(\d+)"?\s*,\s*"tags"\s*:\s*(\[[^\]]*\])\s*\}', + stripped, + ): + try: + item = {"id": int(m.group(1)), "tags": json.loads(m.group(2))} + except (ValueError, json.JSONDecodeError): + continue + if _absorb(item, expected_ids, out): + matched += 1 + if not out: + log.warning( + "news_tagging.json_error_unrecoverable", + preview=content[:200], + ) + elif matched < len(expected_ids): + log.info( + "news_tagging.json_partial_recovery", + recovered=matched, expected=len(expected_ids), + ) + return out + + +def _absorb(item, expected_ids: set[int], out: dict[int, list[str]]) -> bool: + """Place one well-formed item into the output dict if it matches an + expected id. Returns True if it landed.""" + if not isinstance(item, dict): + return False + try: + iid = int(item.get("id")) + except (TypeError, ValueError): + return False + if iid not in expected_ids or iid in out: + return False + tags = _validate_tags(item.get("tags")) + # Empty post-validation = model picked nothing in vocabulary. Fall + # back to "other" so the row is marked tagged (distinguishes + # "tagged poorly" from "not yet tagged"). + out[iid] = tags or ["other"] + return True + + +async def tag_batch( + client: httpx.AsyncClient, + items: list[_ToTag], +) -> dict[int, list[str]]: + """Tag one batch of (id, title) pairs. Returns {id: tags}. Items not + in the result remain untagged (NULL in the DB) and are retried on the + next news_job run.""" + if not items: + return {} + user_msg = ( + "# Headlines to tag\n```json\n" + + json.dumps( + [{"id": it.id, "title": it.title} for it in items], + ensure_ascii=False, + ) + + "\n```" + ) + try: + result = await call_llm( + client, + messages=[ + {"role": "system", "content": _SYSTEM_PROMPT}, + {"role": "user", "content": user_msg}, + ], + # Generous ceiling: ~30 tokens/item × 25 items + reasoning + # overhead for thinking models. Hitting the cap returns empty + # content (finish_reason=length) and triggers the fallback. + max_tokens=4000, + ) + except Exception as e: + log.warning("news_tagging.llm_failed", n=len(items), error=str(e)[:200]) + return {} + return _parse_batch_response(result.content, {it.id for it in items}) + + +async def tag_titles(items: list[_ToTag]) -> dict[int, list[str]]: + """Tag a list of titles, splitting into BATCH_SIZE chunks. Returns + {id: tags}. Failed batches contribute nothing — their items stay + untagged for next time.""" + if not items: + return {} + out: dict[int, list[str]] = {} + async with httpx.AsyncClient(follow_redirects=True, timeout=60) as client: + for i in range(0, len(items), BATCH_SIZE): + chunk = items[i:i + BATCH_SIZE] + batch_out = await tag_batch(client, chunk) + out.update(batch_out) + log.info( + "news_tagging.batch_complete", + requested=len(items), tagged=len(out), + ) + return out + + +# Public re-export for the news_job hook + callers that want to assemble +# their own (id, title) tuples without importing the private dataclass. +ToTag = _ToTag diff --git a/app/services/openrouter.py b/app/services/openrouter.py index b1cfbd2..4e25c31 100644 --- a/app/services/openrouter.py +++ b/app/services/openrouter.py @@ -26,7 +26,10 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" # framing aimed at young investors entering the trading world. NOVICE retuned # to be pedagogical (defining terms, anti-pattern teach-backs); INTERMEDIATE # kept terse but with light-touch educational nudges. See tasks/todo.md. -PROMPT_VERSION = 6 +# v7 (2026-05-18): Forbid "(Updated HH:MM UTC)" clauses in the date header — +# the model was hallucinating future times. The user prompt now carries the +# actual current UTC time so the model has accurate temporal context. +PROMPT_VERSION = 7 # --- Core: invariant across tone/analysis settings ---------------------------- @@ -49,7 +52,11 @@ cover the same event, read the gap in framing — that's the data. implications is filler. # Structure -- One-line date header + any anchor framing (e.g. "Week 11 since Hormuz"). +- One-line date header containing ONLY the date (e.g. `2026-05-18`) and \ +optional anchor framing on the same line (e.g. "Week 11 since Hormuz"). \ +**Never include a time-of-day clause like "(Updated 21:30 UTC)"** — \ +generation time is recorded as metadata elsewhere. Inventing a future or \ +arbitrary time in the header confuses readers. - Immediately after the date header — with **nothing** in between — write a \ TL;DR. Format it as: @@ -423,7 +430,12 @@ def build_user_prompt( """Assemble the user message from already-fetched-and-persisted data. If `previous_log` is a StrategicLog from earlier today, it's included as 'Update mode' context — the model will revise rather than restart.""" - parts = [f"# Strategic log request — {today.strftime('%Y-%m-%d')}"] + parts = [ + f"# Strategic log request — {today.strftime('%Y-%m-%d')}", + # Explicit current time so the model doesn't hallucinate one. The + # date header it writes MUST stay date-only (per system prompt). + f"Current time: {today.strftime('%Y-%m-%d %H:%M UTC')}", + ] if anchor: parts.append(f"Anchor reference date: {anchor}") if reference_line: diff --git a/app/static/css/cassandra.css b/app/static/css/cassandra.css index aa7845f..a029a0c 100644 --- a/app/static/css/cassandra.css +++ b/app/static/css/cassandra.css @@ -1143,15 +1143,17 @@ details[open] .pf-analysis__head-left::before { content: "▾ "; } .news-row { padding: 4px 12px; display: grid; - grid-template-columns: 50px 130px 1fr 110px; + /* age | source | title | tags-on-right | utc-time */ + grid-template-columns: 50px 130px minmax(0, 1fr) minmax(0, auto) 110px; gap: 12px; font-size: 12px; border-bottom: 1px solid var(--surface-2); - align-items: baseline; + align-items: center; } @media (max-width: 720px) { .news-row { grid-template-columns: 50px 100px 1fr; } - .news-row .local { display: none; } + .news-row .local, + .news-row__tags { display: none; } } .news-row:hover { background: color-mix(in srgb, var(--accent) 5%, transparent); } .news-row .age { color: var(--dim); text-align: right; } @@ -1166,6 +1168,61 @@ details[open] .pf-analysis__head-left::before { content: "▾ "; } white-space: nowrap; } +/* News tag chips on each row + the top-bar pill toggles */ +.news-row__tags { + display: inline-flex; + flex-wrap: nowrap; + gap: 3px; + justify-content: flex-end; + overflow: hidden; + max-width: 100%; +} +.tag-chip { + font-family: var(--font-mono); + font-size: 9px; + letter-spacing: 0.04em; + color: var(--muted); + background: var(--surface-2); + border: 1px solid var(--border); + padding: 0 4px; + white-space: nowrap; + text-transform: uppercase; + line-height: 1.5; +} + +.news-tags { + display: flex; + flex-wrap: wrap; + gap: 4px; + padding: 8px 12px; + border-bottom: 1px solid var(--border); + background: var(--surface-2); +} +.news-tag { + font-family: var(--font-mono); + font-size: 10.5px; + letter-spacing: 0.04em; + text-transform: uppercase; + color: var(--muted); + background: transparent; + border: 1px solid var(--border); + padding: 3px 8px; + cursor: pointer; +} +.news-tag:hover { color: var(--accent); border-color: var(--accent); } +.news-tag[data-state="include"] { + background: var(--accent); + color: var(--bg); + border-color: var(--accent); +} +.news-tag[data-state="exclude"] { + color: var(--negative); + border-color: var(--negative); + text-decoration: line-through; +} +.news-tag--clear { color: var(--dim); border-style: dashed; } +.news-tag--clear:hover { color: var(--negative); border-color: var(--negative); } + /* --- Empty / loading state ------------------------------------------- */ .empty { diff --git a/app/templates/base.html b/app/templates/base.html index 8dea40e..d6cd975 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -28,6 +28,69 @@ } document.body.addEventListener('htmx:configRequest', function (evt) { evt.detail.parameters.tone = currentTone(); + // News tag filters — only attach to /api/news requests. + if ((evt.detail.path || '').indexOf('/api/news') === 0) { + var inc = newsTags('include'); + var exc = newsTags('exclude'); + if (inc.length) evt.detail.parameters.tags = inc.join(','); + if (exc.length) evt.detail.parameters.exclude_tags = exc.join(','); + } + }); + + // News tag preference: include / exclude sets persisted in + // localStorage. Click cycles include → exclude → off; + // shift-click goes straight to exclude. + function newsTags(kind) { + try { + var raw = localStorage.getItem('cassandra.news.' + kind); + var arr = raw ? JSON.parse(raw) : []; + return Array.isArray(arr) ? arr : []; + } catch (e) { return []; } + } + function setNewsTags(kind, arr) { + try { localStorage.setItem('cassandra.news.' + kind, JSON.stringify(arr)); } + catch (e) {} + } + function refreshNewsPanels() { + document.querySelectorAll('[hx-get*="/api/news"]').forEach(function (el) { + if (window.htmx) window.htmx.trigger(el, 'tags-changed'); + }); + } + // Event delegation so HTMX-swapped pills work without rebinding. + document.addEventListener('click', function (e) { + var el = e.target.closest && e.target.closest('.news-tag'); + if (!el) return; + e.preventDefault(); + var tag = el.getAttribute('data-tag') || ''; + if (el.classList.contains('news-tag--clear')) { + setNewsTags('include', []); + setNewsTags('exclude', []); + refreshNewsPanels(); + return; + } + var inc = newsTags('include'); + var exc = newsTags('exclude'); + var inInc = inc.indexOf(tag); + var inExc = exc.indexOf(tag); + if (e.shiftKey) { + // Shift-click → toggle exclude membership; remove from include. + if (inInc >= 0) inc.splice(inInc, 1); + if (inExc >= 0) exc.splice(inExc, 1); + else exc.push(tag); + } else { + // Plain click → cycle: off → include → exclude → off. + if (inInc >= 0) { + inc.splice(inInc, 1); + exc.push(tag); + } else if (inExc >= 0) { + exc.splice(inExc, 1); + } else { + inc.push(tag); + } + } + setNewsTags('include', inc); + setNewsTags('exclude', exc); + refreshNewsPanels(); }); // Reflect the saved value in the toggle on load. var pill = document.getElementById('tone-toggle'); diff --git a/app/templates/dashboard.html b/app/templates/dashboard.html index 08fb039..d83aec6 100644 --- a/app/templates/dashboard.html +++ b/app/templates/dashboard.html @@ -77,7 +77,7 @@
loading…
diff --git a/app/templates/news.html b/app/templates/news.html index b7d435f..9d68491 100644 --- a/app/templates/news.html +++ b/app/templates/news.html @@ -9,7 +9,7 @@
loading…
diff --git a/app/templates/partials/news.html b/app/templates/partials/news.html index 2566c85..36b8a59 100644 --- a/app/templates/partials/news.html +++ b/app/templates/partials/news.html @@ -1,11 +1,30 @@ +{% if tag_vocabulary %} +
+ {% for tag in tag_vocabulary %} + + {% endfor %} + {% if active_include or active_exclude %} + + {% endif %} +
+{% endif %} + {% if not headlines %} -
no headlines in window
+
no headlines in window{% if active_include or active_exclude %} (after tag filter){% endif %}
{% else %} {% for h in headlines %}
{{ h.age }} {{ h.source }} {{ h.title }} + + {% for t in h.tags or [] %}{{ tag_labels.get(t, t) }}{% endfor %} + {% if h.iso %} {% else %} diff --git a/tests/test_news_tagging.py b/tests/test_news_tagging.py new file mode 100644 index 0000000..e4f9bff --- /dev/null +++ b/tests/test_news_tagging.py @@ -0,0 +1,130 @@ +"""Tests for the deterministic half of news_tagging: vocabulary filtering +and JSON-response parsing. The LLM call itself isn't exercised.""" +from __future__ import annotations + +import pytest + +from app.services.news_tagging import ( + MAX_TAGS_PER_HEADLINE, + TAG_LABELS, + TAG_VOCABULARY, + _parse_batch_response, + _validate_tags, +) + + +# --------------------------------------------------------------------------- +# Vocabulary integrity +# --------------------------------------------------------------------------- + + +def test_every_vocab_tag_has_a_label(): + """Display labels must cover every tag — missing keys would render + the raw machine-name in the UI.""" + for t in TAG_VOCABULARY: + assert t in TAG_LABELS, f"missing label for {t}" + + +def test_other_is_the_fallback_tag(): + assert "other" in TAG_VOCABULARY + + +# --------------------------------------------------------------------------- +# _validate_tags +# --------------------------------------------------------------------------- + + +def test_validate_drops_unknown_tags(): + out = _validate_tags(["markets", "wibble", "tech"]) + assert out == ["markets", "tech"] + + +def test_validate_normalises_spaces_to_hyphens(): + """Common drift: model returns 'monetary policy' instead of + 'monetary-policy'. We normalise.""" + out = _validate_tags(["monetary policy"]) + assert out == ["monetary-policy"] + + +def test_validate_normalises_case(): + out = _validate_tags(["MARKETS", "Geopolitics"]) + assert out == ["markets", "geopolitics"] + + +def test_validate_caps_at_max_tags(): + out = _validate_tags(["markets", "tech", "china", "economy", "energy"]) + assert len(out) == MAX_TAGS_PER_HEADLINE + + +def test_validate_dedupes(): + out = _validate_tags(["markets", "markets", "tech"]) + assert out == ["markets", "tech"] + + +def test_validate_rejects_non_list(): + assert _validate_tags(None) == [] + assert _validate_tags("markets") == [] + assert _validate_tags({"tag": "markets"}) == [] + + +def test_validate_skips_non_string_entries(): + out = _validate_tags(["markets", 42, None, "tech"]) + assert out == ["markets", "tech"] + + +# --------------------------------------------------------------------------- +# _parse_batch_response +# --------------------------------------------------------------------------- + + +def test_parse_basic_json_array(): + raw = '[{"id": 1, "tags": ["markets", "tech"]}, {"id": 2, "tags": ["china"]}]' + out = _parse_batch_response(raw, {1, 2}) + assert out == {1: ["markets", "tech"], 2: ["china"]} + + +def test_parse_strips_leading_prose(): + """Models occasionally prepend 'Here is the output:' before the JSON.""" + raw = 'Sure! Here are the tags:\n[{"id": 1, "tags": ["markets"]}]' + out = _parse_batch_response(raw, {1}) + assert out == {1: ["markets"]} + + +def test_parse_strips_markdown_fences(): + raw = "```json\n[{\"id\": 1, \"tags\": [\"tech\"]}]\n```" + out = _parse_batch_response(raw, {1}) + assert out == {1: ["tech"]} + + +def test_parse_drops_unexpected_ids(): + raw = '[{"id": 99, "tags": ["markets"]}, {"id": 1, "tags": ["tech"]}]' + out = _parse_batch_response(raw, {1, 2}) + assert out == {1: ["tech"]} + + +def test_parse_empty_tags_falls_back_to_other(): + """An item whose tags list ends up empty after validation gets + tagged 'other' so the row is marked tagged, not left NULL.""" + raw = '[{"id": 1, "tags": ["nonsense"]}]' + out = _parse_batch_response(raw, {1}) + assert out == {1: ["other"]} + + +def test_parse_unparseable_returns_empty(): + """Garbage in → empty out. The caller leaves those rows untagged + so they get retried on the next run.""" + assert _parse_batch_response("nope, no JSON here", {1}) == {} + assert _parse_batch_response("[invalid json", {1}) == {} + + +def test_parse_ignores_non_dict_items(): + raw = '[{"id": 1, "tags": ["markets"]}, "lol", null, {"id": 2, "tags": ["tech"]}]' + out = _parse_batch_response(raw, {1, 2}) + assert out == {1: ["markets"], 2: ["tech"]} + + +def test_parse_handles_string_id_coercion(): + """Some models render the id as a string. We coerce.""" + raw = '[{"id": "1", "tags": ["markets"]}]' + out = _parse_batch_response(raw, {1}) + assert out == {1: ["markets"]}