diff --git a/Dockerfile b/Dockerfile index 1123177..0fd7ec9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,31 +32,3 @@ COPY alembic.ini ./ # Default command is the web app; scheduler container overrides via `command:`. EXPOSE 8000 CMD ["uvicorn", "app.main:app", "--host", "0.0.0.0", "--port", "8000", "--workers", "1"] - - -# --------------------------------------------------------------------------- -# Test stage — same Python, same prod deps, plus dev extras (pytest + -# aiosqlite). Built and run only via docker-compose.test.yml; never shipped. -# --------------------------------------------------------------------------- -FROM python:3.13-slim AS test - -ENV PYTHONUNBUFFERED=1 \ - PYTHONDONTWRITEBYTECODE=1 \ - PATH="/opt/venv/bin:$PATH" \ - TZ=UTC \ - PIP_DISABLE_PIP_VERSION_CHECK=1 \ - PIP_NO_CACHE_DIR=1 - -COPY --from=builder /opt/venv /opt/venv -WORKDIR /app -COPY pyproject.toml ./ -COPY app ./app -COPY alembic ./alembic -COPY alembic.ini ./ -# tests/ is excluded by .dockerignore (prod-correct: never bake tests into -# a shipped image). docker-compose.test.yml bind-mounts ./tests:/app/tests -# at run time, so the suite is always available without baking it in. - -RUN /opt/venv/bin/pip install ".[dev]" - -CMD ["pytest", "tests/", "-v"] diff --git a/alembic/versions/0016_portfolio_sync_pepper_fp.py b/alembic/versions/0016_portfolio_sync_pepper_fp.py deleted file mode 100644 index 1a5f6c0..0000000 --- a/alembic/versions/0016_portfolio_sync_pepper_fp.py +++ /dev/null @@ -1,39 +0,0 @@ -"""portfolio_sync: add pepper_fp for orphan-blob detection. - -When PORTFOLIO_SYNC_PEPPER rotates (intentional or otherwise), any -existing wrapped blob becomes permanently unreadable. Today that -manifests as a GCM InvalidTag → 500 on the GET endpoint. We add a -short HKDF-derived fingerprint of the pepper so we can detect the -rotation case explicitly and surface it to the client as a clean -"stale" state (410), distinct from genuine corruption (500). - -Existing rows get pepper_fp=NULL on upgrade; the service treats NULL -as "orphaned" (always true: those rows were written before this -column existed, so we can't prove the pepper matches). The next -successful upsert refreshes the fingerprint. - -Revision ID: 0016 -Revises: 0015 -Create Date: 2026-05-25 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0016" -down_revision: Union[str, None] = "0015" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "portfolio_sync", - sa.Column("pepper_fp", sa.LargeBinary(length=8), nullable=True), - ) - - -def downgrade() -> None: - op.drop_column("portfolio_sync", "pepper_fp") diff --git a/alembic/versions/0017_email_digest.py b/alembic/versions/0017_email_digest.py deleted file mode 100644 index 38cfd4e..0000000 --- a/alembic/versions/0017_email_digest.py +++ /dev/null @@ -1,55 +0,0 @@ -"""email digests: User.email_digest_opt_in, User.digest_tone, email_sends table. - -Revision ID: 0017 -Revises: 0016 -Create Date: 2026-05-25 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0017" -down_revision: Union[str, None] = "0016" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "users", - sa.Column( - "email_digest_opt_in", sa.Boolean(), nullable=False, - server_default=sa.text("1"), - ), - ) - op.add_column( - "users", - sa.Column("digest_tone", sa.String(length=16), nullable=True), - ) - - op.create_table( - "email_sends", - sa.Column("id", sa.BigInteger(), autoincrement=True, primary_key=True), - sa.Column("user_id", sa.Integer(), nullable=False), - sa.Column("kind", sa.String(length=16), nullable=False), - sa.Column("sent_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("status", sa.String(length=16), nullable=False), - sa.Column("error", sa.String(length=255), nullable=True), - sa.ForeignKeyConstraint( - ["user_id"], ["users.id"], ondelete="CASCADE", - ), - ) - op.create_index( - "ix_email_sends_user_kind_sent", - "email_sends", - ["user_id", "kind", "sent_at"], - ) - - -def downgrade() -> None: - op.drop_index("ix_email_sends_user_kind_sent", table_name="email_sends") - op.drop_table("email_sends") - op.drop_column("users", "digest_tone") - op.drop_column("users", "email_digest_opt_in") diff --git a/alembic/versions/0018_polar_webhook.py b/alembic/versions/0018_polar_webhook.py deleted file mode 100644 index bc085a7..0000000 --- a/alembic/versions/0018_polar_webhook.py +++ /dev/null @@ -1,55 +0,0 @@ -"""polar webhook: User.polar_customer_id/subscription_id, polar_events table. - -Revision ID: 0018 -Revises: 0017 -Create Date: 2026-05-26 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0018" -down_revision: Union[str, None] = "0017" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "users", - sa.Column("polar_customer_id", sa.String(length=64), nullable=True), - ) - op.add_column( - "users", - sa.Column("polar_subscription_id", sa.String(length=64), nullable=True), - ) - op.create_unique_constraint( - "uq_users_polar_customer", "users", ["polar_customer_id"], - ) - - op.create_table( - "polar_events", - sa.Column("id", sa.BigInteger(), autoincrement=True, primary_key=True), - sa.Column("event_id", sa.String(length=128), nullable=False), - sa.Column("event_type", sa.String(length=64), nullable=False), - sa.Column("received_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("processed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("payload", sa.Text(), nullable=False), - sa.UniqueConstraint("event_id", name="uq_polar_events_event_id"), - ) - op.create_index( - "ix_polar_events_type_received", - "polar_events", - ["event_type", "received_at"], - ) - - -def downgrade() -> None: - op.drop_index("ix_polar_events_type_received", table_name="polar_events") - op.drop_table("polar_events") - op.drop_constraint("uq_users_polar_customer", "users", type_="unique") - op.drop_column("users", "polar_subscription_id") - op.drop_column("users", "polar_customer_id") diff --git a/alembic/versions/0019_stripe.py b/alembic/versions/0019_stripe.py deleted file mode 100644 index 3ea4018..0000000 --- a/alembic/versions/0019_stripe.py +++ /dev/null @@ -1,56 +0,0 @@ -"""stripe integration: users.stripe_customer_id / stripe_subscription_id, -stripe_events table. - -Revision ID: 0019 -Revises: 0018 -Create Date: 2026-05-26 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0019" -down_revision: Union[str, None] = "0018" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "users", - sa.Column("stripe_customer_id", sa.String(length=64), nullable=True), - ) - op.add_column( - "users", - sa.Column("stripe_subscription_id", sa.String(length=64), nullable=True), - ) - op.create_unique_constraint( - "uq_users_stripe_customer", "users", ["stripe_customer_id"], - ) - - op.create_table( - "stripe_events", - sa.Column("id", sa.BigInteger(), autoincrement=True, primary_key=True), - sa.Column("event_id", sa.String(length=128), nullable=False), - sa.Column("event_type", sa.String(length=64), nullable=False), - sa.Column("received_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("processed_at", sa.DateTime(timezone=True), nullable=True), - sa.Column("error", sa.Text(), nullable=True), - sa.Column("payload", sa.Text(), nullable=False), - sa.UniqueConstraint("event_id", name="uq_stripe_events_event_id"), - ) - op.create_index( - "ix_stripe_events_type_received", - "stripe_events", - ["event_type", "received_at"], - ) - - -def downgrade() -> None: - op.drop_index("ix_stripe_events_type_received", table_name="stripe_events") - op.drop_table("stripe_events") - op.drop_constraint("uq_users_stripe_customer", "users", type_="unique") - op.drop_column("users", "stripe_subscription_id") - op.drop_column("users", "stripe_customer_id") diff --git a/alembic/versions/0020_trial_end.py b/alembic/versions/0020_trial_end.py deleted file mode 100644 index c845673..0000000 --- a/alembic/versions/0020_trial_end.py +++ /dev/null @@ -1,31 +0,0 @@ -"""stripe trial: users.stripe_trial_end_at. - -Revision ID: 0020 -Revises: 0019 -Create Date: 2026-05-26 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0020" -down_revision: Union[str, None] = "0019" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "users", - sa.Column( - "stripe_trial_end_at", - sa.DateTime(timezone=True), - nullable=True, - ), - ) - - -def downgrade() -> None: - op.drop_column("users", "stripe_trial_end_at") diff --git a/alembic/versions/0021_csv_format_template.py b/alembic/versions/0021_csv_format_template.py deleted file mode 100644 index bc6ca0a..0000000 --- a/alembic/versions/0021_csv_format_template.py +++ /dev/null @@ -1,40 +0,0 @@ -"""csv format templates table — LLM-fallback parser cache. - -Revision ID: 0021 -Revises: 0020 -Create Date: 2026-05-27 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0021" -down_revision: Union[str, None] = "0020" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "csv_format_templates", - sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), - sa.Column("fingerprint", sa.String(length=64), nullable=False), - sa.Column("headers", sa.JSON(), nullable=False), - sa.Column("sample_row", sa.JSON(), nullable=False), - sa.Column("mapping", sa.JSON(), nullable=False), - sa.Column("preamble_rows", sa.Integer(), nullable=False, server_default=sa.text("0")), - sa.Column("delimiter", sa.String(length=1), nullable=False, server_default=","), - sa.Column("broker_label", sa.String(length=128), nullable=True), - sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("use_count", sa.Integer(), nullable=False, server_default=sa.text("1")), - sa.Column("last_used_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("llm_model", sa.String(length=64), nullable=True), - sa.Column("llm_cost_usd", sa.Float(), nullable=True), - sa.UniqueConstraint("fingerprint", name="uq_csv_format_templates_fingerprint"), - ) - - -def downgrade() -> None: - op.drop_table("csv_format_templates") diff --git a/app/cli.py b/app/cli.py index c780f0b..47152fb 100644 --- a/app/cli.py +++ b/app/cli.py @@ -11,9 +11,8 @@ Usage from the host:: two months, not one (avoids accidental erosion of an existing grant when re-running the command). -This is the manual lever for admin grants. The Stripe webhook applies -the same stacking rule via ``referral_service.convert_referral`` for -both sides of a referral conversion. +This is the manual lever for Phase D.2. In D.3 the Paddle webhook will +call the same helper for both sides of a referral conversion. """ from __future__ import annotations @@ -95,57 +94,6 @@ async def show_status(email: str) -> int: return 0 -async def send_test_digest(email: str, kind: str) -> int: - """Generate a digest and send it to the named user immediately, ignoring - opt-in state and idempotency. Useful for previewing copy in your own - inbox before a real run lands.""" - import httpx - - from app.jobs._market_context import ( - REFERENCE_LINE, - latest_quotes_by_group, - recent_headlines_by_bucket, - ) - from app.jobs.email_digest_job import _generate_variants, _send_one - from app.services.openrouter import llm_configured - - if kind not in ("daily", "weekly"): - print(f"error: kind must be 'daily' or 'weekly' (got {kind!r})", - file=sys.stderr) - return 2 - if not llm_configured(): - print("error: LLM provider not configured (set OPENROUTER_API_KEY)", - file=sys.stderr) - return 1 - - factory = get_session_factory() - async with factory() as session: - user = await _get_user_by_email(session, email) - if user is None: - print(f"error: no user with email {email!r}", file=sys.stderr) - return 1 - today = _utcnow() - quotes = await latest_quotes_by_group(session) - news = await recent_headlines_by_bucket( - session, hours=(168 if kind == "weekly" else 24), - ) - ctx = dict(today=today, quotes_by_group=quotes, - headlines_by_bucket=news, reference_line=REFERENCE_LINE) - async with httpx.AsyncClient(follow_redirects=True) as client: - variants = await _generate_variants(session, client, kind, ctx) - tone = (user.digest_tone or "INTERMEDIATE").upper() - content = (variants.get(tone) - or variants.get("INTERMEDIATE") - or next(iter(variants.values()), None)) - if content is None: - print("error: all LLM variants failed", file=sys.stderr) - return 1 - date_str = today.strftime("%Y-%m-%d") - await _send_one(user, kind, content, date_str, session) - print(f"sent {kind} digest to {email} (tone={tone})") - return 0 - - def build_parser() -> argparse.ArgumentParser: p = argparse.ArgumentParser(prog="app.cli", description="Cassandra admin CLI") sub = p.add_subparsers(dest="cmd", required=True) @@ -160,11 +108,6 @@ def build_parser() -> argparse.ArgumentParser: s = sub.add_parser("show-status", help="Print paid-tier status for a user") s.add_argument("email") - t = sub.add_parser("send-test-digest", - help="Send one digest immediately (bypasses opt-in/idempotency)") - t.add_argument("email") - t.add_argument("kind", choices=("daily", "weekly")) - return p @@ -179,8 +122,6 @@ async def _dispatch(args) -> int: return await revoke_credit(args.email) if args.cmd == "show-status": return await show_status(args.email) - if args.cmd == "send-test-digest": - return await send_test_digest(args.email, args.kind) return 2 finally: await get_engine().dispose() diff --git a/app/config.py b/app/config.py index 0aabb1d..70c0f1c 100644 --- a/app/config.py +++ b/app/config.py @@ -90,24 +90,6 @@ class Settings(BaseSettings): # by app.services.openrouter._resolve_tone. CASSANDRA_TONE: str = "INTERMEDIATE" # NOVICE | INTERMEDIATE CASSANDRA_ANALYSIS: str = "SPECULATIVE" # DRY | SPECULATIVE - BETA_MODE: bool = True # Shows a "BETA" pill in the app header. Flip to False at GA. - - # Polar (merchant-of-record). Webhook secret is base64-encoded with a - # `whsec_` prefix in the Polar dashboard; paste it verbatim into the - # env var. Empty = webhook endpoint refuses with 503 (so a misconfig - # is loud rather than silently accepting unsigned events). - POLAR_WEBHOOK_SECRET: str = "" - POLAR_API_KEY: str = "" - - # Stripe (merchant-on-record for read.markets after Polar/Paddle - # both declined the financial-media category). Test-mode keys are - # `sk_test_*` / `whsec_*`; live-mode keys are `sk_live_*` — swap at - # GA cutover. Empty values make the corresponding endpoints 503 so - # a misconfig is loud rather than silently accepting unsigned events. - STRIPE_API_KEY: str = "" - STRIPE_WEBHOOK_SECRET: str = "" - STRIPE_PRICE_MONTHLY: str = "" # price_xxx for £7/month subscription - STRIPE_PRICE_ANNUAL: str = "" # price_xxx for £70/year subscription # Config file locations (overridable for tests) BASELINE_TOML: Path = Field(default_factory=lambda: CONFIG_DIR / "default.toml") diff --git a/app/jobs/_helpers.py b/app/jobs/_helpers.py index fe4b6b1..211110f 100644 --- a/app/jobs/_helpers.py +++ b/app/jobs/_helpers.py @@ -23,21 +23,17 @@ async def job_lifecycle(name: str) -> AsyncIterator[tuple[AsyncSession, JobRun]] handles the bookkeeping. A MariaDB GET_LOCK(name, 0) is acquired to prevent concurrent runs of the - same job across processes. If the lock is busy, we skip the run. - The lock dance is MariaDB-specific; on SQLite (used in tests) it's a - no-op, since the single-process test runner can't race itself.""" + same job across processes. If the lock is busy, we skip the run.""" factory = get_session_factory() async with factory() as session: - bind = session.get_bind() - use_lock = bind is not None and bind.dialect.name == "mysql" - if use_lock: - got = (await session.execute( - text("SELECT GET_LOCK(:n, 0)"), {"n": f"cassandra_{name}"} - )).scalar() - if not got: - log.warning("job.skipped_locked", name=name) - yield session, JobRun(name=name, started_at=utcnow(), status="skipped") - return + # Try lock; skip if held. + got = (await session.execute( + text("SELECT GET_LOCK(:n, 0)"), {"n": f"cassandra_{name}"} + )).scalar() + if not got: + log.warning("job.skipped_locked", name=name) + yield session, JobRun(name=name, started_at=utcnow(), status="skipped") + return run = JobRun(name=name, started_at=utcnow(), status="running") session.add(run) await session.commit() @@ -57,7 +53,6 @@ async def job_lifecycle(name: str) -> AsyncIterator[tuple[AsyncSession, JobRun]] log.error("job.failed", name=name, error=str(e)) raise finally: - if use_lock: - await session.execute(text("SELECT RELEASE_LOCK(:n)"), - {"n": f"cassandra_{name}"}) - await session.commit() + await session.execute(text("SELECT RELEASE_LOCK(:n)"), + {"n": f"cassandra_{name}"}) + await session.commit() diff --git a/app/jobs/_market_context.py b/app/jobs/_market_context.py deleted file mode 100644 index 5dd591f..0000000 --- a/app/jobs/_market_context.py +++ /dev/null @@ -1,86 +0,0 @@ -"""Shared market-context helpers consumed by LLM-driven jobs. - -Both ai_log_job and email_digest_job pull "the latest tape" the same -way — most-recent quote per (group, symbol), last N hours of headlines -bucketed by category, and the running month's LLM spend. Moved here so -neither job depends on the other's internals. -""" -from __future__ import annotations - -from collections import defaultdict -from datetime import timedelta - -from sqlalchemy import desc, func, select - -from app.db import utcnow -from app.models import AICall, Headline, Quote -from app.services.openrouter import month_start - - -REFERENCE_LINE = ( - "S&P 7,501 (ATH) · VIX 18.0 · US 10y 4.45% · HY OAS 279bps · " - "Brent $109/bbl · Gold $4,651/oz · CPI 3.8% YoY" -) - - -async def latest_quotes_by_group(session) -> dict[str, list[dict]]: - """Latest quote per (group, symbol). Skips error rows where price is null.""" - sub = ( - select( - Quote.group_name, - Quote.symbol, - func.max(Quote.fetched_at).label("mx"), - ) - .group_by(Quote.group_name, Quote.symbol) - .subquery() - ) - stmt = ( - select(Quote) - .join( - sub, - (Quote.group_name == sub.c.group_name) - & (Quote.symbol == sub.c.symbol) - & (Quote.fetched_at == sub.c.mx), - ) - .order_by(Quote.group_name, Quote.symbol) - ) - rows = (await session.execute(stmt)).scalars().all() - by_group: dict[str, list[dict]] = defaultdict(list) - for q in rows: - by_group[q.group_name].append(dict( - symbol=q.symbol, source=q.source, label=q.label, - note="", price=q.price, currency=q.currency, - as_of=q.as_of, changes=q.changes, - )) - return by_group - - -async def recent_headlines_by_bucket(session, hours: float = 24) -> dict[str, list[dict]]: - """Last N hours of headlines, bucketed by category. Hard cap per - bucket to keep the prompt under ~40KB.""" - cutoff = utcnow() - timedelta(hours=hours) - stmt = ( - select(Headline) - .where(Headline.published_at >= cutoff) - .order_by(desc(Headline.published_at)) - .limit(400) - ) - rows = (await session.execute(stmt)).scalars().all() - by_bucket: dict[str, list[dict]] = defaultdict(list) - for h in rows: - if len(by_bucket[h.category]) >= 40: - continue - by_bucket[h.category].append(dict( - when=h.published_at.isoformat(), - source=h.source, title=h.title, - )) - return by_bucket - - -async def month_spend(session) -> float: - start = month_start() - total = (await session.execute( - select(func.coalesce(func.sum(AICall.cost_usd), 0.0)) - .where(AICall.called_at >= start) - )).scalar() - return float(total or 0.0) diff --git a/app/jobs/ai_log_job.py b/app/jobs/ai_log_job.py index bc8b488..7d63eb2 100644 --- a/app/jobs/ai_log_job.py +++ b/app/jobs/ai_log_job.py @@ -4,6 +4,8 @@ and a row in the cost ledger.""" from __future__ import annotations import asyncio +from collections import defaultdict +from datetime import timedelta import httpx from sqlalchemy import desc, func, select @@ -11,13 +13,7 @@ from sqlalchemy import desc, func, select from app.config import get_settings from app.db import utcnow from app.jobs._helpers import job_lifecycle, log -from app.jobs._market_context import ( - REFERENCE_LINE, - latest_quotes_by_group, - month_spend, - recent_headlines_by_bucket, -) -from app.models import AICall, JobRun, StrategicLog +from app.models import AICall, Headline, JobRun, Quote, StrategicLog from app.services.cadence import DEFAULT_POLICY from app.services.openrouter import ( PROMPT_VERSION, @@ -26,9 +22,79 @@ from app.services.openrouter import ( build_user_prompt, call_llm, llm_configured, + month_start, ) +REFERENCE_LINE = ( + "S&P 7,501 (ATH) · VIX 18.0 · US 10y 4.45% · HY OAS 279bps · " + "Brent $109/bbl · Gold $4,651/oz · CPI 3.8% YoY" +) + + +async def _latest_quotes_by_group(session) -> dict[str, list[dict]]: + """Latest quote per (group, symbol). Skips error rows where price is null.""" + sub = ( + select( + Quote.group_name, + Quote.symbol, + func.max(Quote.fetched_at).label("mx"), + ) + .group_by(Quote.group_name, Quote.symbol) + .subquery() + ) + stmt = ( + select(Quote) + .join( + sub, + (Quote.group_name == sub.c.group_name) + & (Quote.symbol == sub.c.symbol) + & (Quote.fetched_at == sub.c.mx), + ) + .order_by(Quote.group_name, Quote.symbol) + ) + rows = (await session.execute(stmt)).scalars().all() + by_group: dict[str, list[dict]] = defaultdict(list) + for q in rows: + by_group[q.group_name].append(dict( + symbol=q.symbol, source=q.source, label=q.label, + note="", price=q.price, currency=q.currency, + as_of=q.as_of, changes=q.changes, + )) + return by_group + + +async def _recent_headlines_by_bucket(session, hours: float = 24) -> dict[str, list[dict]]: + """Last N hours of headlines, bucketed by category. Hard cap per bucket + to keep the prompt under ~40KB.""" + cutoff = utcnow() - timedelta(hours=hours) + stmt = ( + select(Headline) + .where(Headline.published_at >= cutoff) + .order_by(desc(Headline.published_at)) + .limit(400) + ) + rows = (await session.execute(stmt)).scalars().all() + by_bucket: dict[str, list[dict]] = defaultdict(list) + for h in rows: + if len(by_bucket[h.category]) >= 40: + continue + by_bucket[h.category].append(dict( + when=h.published_at.isoformat(), + source=h.source, title=h.title, + )) + return by_bucket + + +async def _month_spend(session) -> float: + start = month_start() + total = (await session.execute( + select(func.coalesce(func.sum(AICall.cost_usd), 0.0)) + .where(AICall.called_at >= start) + )).scalar() + return float(total or 0.0) + + async def run() -> None: async with job_lifecycle("ai_log_job") as (session, jr): if jr.status == "skipped": @@ -53,7 +119,7 @@ async def run() -> None: jr.error = reason return - spent = await month_spend(session) + spent = await _month_spend(session) if spent >= s.OPENROUTER_MONTHLY_CAP_USD: log.warning("ai_log.cap_reached", spent=spent, cap=s.OPENROUTER_MONTHLY_CAP_USD) @@ -61,8 +127,8 @@ async def run() -> None: jr.error = f"monthly cost cap reached (${spent:.2f})" return - quotes = await latest_quotes_by_group(session) - news = await recent_headlines_by_bucket(session) + quotes = await _latest_quotes_by_group(session) + news = await _recent_headlines_by_bucket(session) if not quotes and not news: log.warning("ai_log.no_data_yet") jr.status = "skipped" @@ -103,7 +169,7 @@ async def run() -> None: for tone, analysis in variants: # Re-check cost cap between variants so a runaway run is # bounded. - spent = await month_spend(session) + spent = await _month_spend(session) if spent >= s.OPENROUTER_MONTHLY_CAP_USD: log.warning("ai_log.cap_reached_midrun", spent=spent, completed=written) diff --git a/app/jobs/email_digest_job.py b/app/jobs/email_digest_job.py deleted file mode 100644 index 1f38777..0000000 --- a/app/jobs/email_digest_job.py +++ /dev/null @@ -1,224 +0,0 @@ -"""Daily/weekly editorial email digest. - -Runs once a day at 06:30 UTC via the scheduler. On Sundays sends the -weekly recap to every opt-in user (free + paid). On other days sends -the daily digest to opt-in paid users only. - -Generates LLM content once per tone (NOVICE + INTERMEDIATE), then fans -out by SMTP. EmailSend audit rows guard against double-delivery if the -job is re-run within the same UTC day. -""" -from __future__ import annotations - -import asyncio -from datetime import datetime, timedelta - -import httpx -from sqlalchemy import select - -from app import branding -from app.config import get_settings -from app.db import utcnow -from app.jobs._helpers import job_lifecycle, log -from app.jobs._market_context import ( - REFERENCE_LINE, - latest_quotes_by_group, - month_spend, - recent_headlines_by_bucket, -) -from app.models import EmailSend, User -from app.routers.email import sign_unsubscribe_token -from app.services.access import paid_status -from app.services.email_service import render_digest_email, send_email -from app.services.openrouter import ( - PROMPT_VERSION, - build_daily_digest_prompt, - build_weekly_digest_prompt, - call_llm, - llm_configured, -) - - -def _now() -> datetime: - """Indirection so tests can monkeypatch the "current time" without - touching the system clock.""" - return utcnow() - - -async def _opt_in_recipients(session, *, paid_only: bool) -> list[User]: - stmt = select(User).where(User.email_digest_opt_in.is_(True)) - rows = (await session.execute(stmt)).scalars().all() - if paid_only: - rows = [u for u in rows if paid_status(u).active] - return rows - - -async def _already_sent_today(session, user_id: int, kind: str, today: datetime) -> bool: - """True if an EmailSend row exists for this user+kind on the same UTC - day, with status in ('sent','error'). 'error' counts because we don't - want to keep retrying a bad address inside the same daily slot.""" - day_start = today.replace(hour=0, minute=0, second=0, microsecond=0) - day_end = day_start + timedelta(days=1) - stmt = select(EmailSend.id).where( - EmailSend.user_id == user_id, - EmailSend.kind == kind, - EmailSend.sent_at >= day_start, - EmailSend.sent_at < day_end, - EmailSend.status.in_(("sent", "error")), - ) - return (await session.execute(stmt)).first() is not None - - -async def _generate_variants(session, client, kind: str, ctx: dict) -> dict[str, str]: - """Returns {tone: html_content}. Missing tone means generation failed - for that variant — skip recipients on that tone. - - Persists an AICall row per attempt so digest LLM spend counts toward - the monthly cost cap on subsequent runs.""" - from app.models import AICall - from app.services.openrouter import active_model - - builder = build_weekly_digest_prompt if kind == "weekly" else build_daily_digest_prompt - out: dict[str, str] = {} - for tone in ("NOVICE", "INTERMEDIATE"): - sys_, usr = builder(tone=tone, **ctx) - try: - result = await call_llm( - client, - [{"role": "system", "content": sys_}, - {"role": "user", "content": usr}], - ) - out[tone] = result.content - session.add(AICall( - model=result.model, - prompt_tokens=result.prompt_tokens, - completion_tokens=result.completion_tokens, - cost_usd=result.cost_usd, - status="ok", - )) - await session.commit() - log.info("digest.variant_ok", kind=kind, tone=tone, - prompt_tokens=result.prompt_tokens, - completion_tokens=result.completion_tokens) - except Exception as e: - session.add(AICall( - model=active_model(), status="error", - error=f"{kind}/{tone}: {str(e)[:480]}", - )) - await session.commit() - log.error("digest.variant_failed", kind=kind, tone=tone, - error=str(e)[:200]) - return out - - -def _kind_for_today(today: datetime) -> str: - """Sunday → weekly. Mon–Sat → daily.""" - return "weekly" if today.weekday() == 6 else "daily" - - -async def _send_one(user: User, kind: str, content_html: str, date_str: str, - session) -> None: - settings_url = f"{branding.SITE_URL}/settings" - unsubscribe_url = ( - f"{branding.SITE_URL}/email/unsubscribe" - f"?token={sign_unsubscribe_token(user.id)}" - ) - subject, text_body, html_body = render_digest_email( - kind=kind, date_str=date_str, - content_html=content_html, - unsubscribe_url=unsubscribe_url, - settings_url=settings_url, - ) - try: - await send_email(to=user.email, subject=subject, - text_body=text_body, html_body=html_body) - status_ = "sent" - err = None - except Exception as e: - status_ = "error" - err = str(e)[:255] - log.error("digest.send_failed", user_id=user.id, error=err) - session.add(EmailSend( - user_id=user.id, kind=kind, sent_at=_now(), - status=status_, error=err, - )) - await session.commit() - - -async def run() -> None: - async with job_lifecycle("email_digest_job") as (session, jr): - if jr.status == "skipped": - return - s = get_settings() - if not llm_configured(): - log.warning("digest.skipped_no_key", provider=s.LLM_PROVIDER) - jr.status = "skipped" - return - - today = _now() - kind = _kind_for_today(today) - date_str = today.strftime("%Y-%m-%d") - - recipients = await _opt_in_recipients( - session, paid_only=(kind == "daily"), - ) - fresh: list[User] = [] - for u in recipients: - if not await _already_sent_today(session, u.id, kind, today): - fresh.append(u) - if not fresh: - log.info("digest.no_fresh_recipients", kind=kind, - total=len(recipients)) - jr.status = "skipped" - return - - spent = await month_spend(session) - if spent >= s.OPENROUTER_MONTHLY_CAP_USD: - log.warning("digest.cap_reached", spent=spent, - cap=s.OPENROUTER_MONTHLY_CAP_USD) - jr.status = "skipped" - jr.error = f"monthly cost cap reached (${spent:.2f})" - return - - quotes = await latest_quotes_by_group(session) - news = await recent_headlines_by_bucket( - session, hours=(168 if kind == "weekly" else 24), - ) - ctx = dict( - today=today, - quotes_by_group=quotes, - headlines_by_bucket=news, - reference_line=REFERENCE_LINE, - ) - - async with httpx.AsyncClient(follow_redirects=True) as client: - variants = await _generate_variants(session, client, kind, ctx) - - if not variants: - log.warning("digest.all_variants_failed", kind=kind) - jr.status = "failed" - jr.error = "all variants failed" - return - - written = 0 - for u in fresh: - tone = (u.digest_tone or "INTERMEDIATE").upper() - # Fall back to INTERMEDIATE first (the more common tone) and then - # to whatever variant succeeded, so an asymmetric LLM failure - # doesn't silently skip the user. - content = (variants.get(tone) - or variants.get("INTERMEDIATE") - or next(iter(variants.values()), None)) - if content is None: - continue - await _send_one(u, kind, content, date_str, session) - await asyncio.sleep(0.1) - written += 1 - - jr.items_written = written - log.info("digest.done", kind=kind, written=written, - prompt_version=PROMPT_VERSION) - - -if __name__ == "__main__": - asyncio.run(run()) diff --git a/app/main.py b/app/main.py index fe987f5..a9dcb50 100644 --- a/app/main.py +++ b/app/main.py @@ -19,13 +19,9 @@ from app.db import get_session_factory from app.logging import configure_logging, get_logger from app.routers import api as api_router from app.routers import auth as auth_router -from app.routers import email as email_router from app.routers import pages as pages_router -from app.routers import polar_webhook as polar_webhook_router from app.routers import public as public_router -from app.routers import stripe_billing as stripe_router from app.routers import sync as sync_router -from app.routers import ticker_validate as ticker_validate_router from app.routers import universe as universe_router from app.services.feeds_bootstrap import bootstrap_feeds @@ -87,18 +83,9 @@ app.mount( ) app.include_router(auth_router.router, tags=["auth"]) -app.include_router(email_router.router, tags=["email"]) app.include_router(api_router.router, prefix="/api", tags=["api"]) app.include_router(universe_router.router, prefix="/api", tags=["universe"]) -app.include_router(ticker_validate_router.router, prefix="/api", tags=["ticker-validate"]) app.include_router(sync_router.router, tags=["portfolio-sync"]) -# Polar webhook (no bearer-token auth — authenticity via HMAC). Path -# `/api/polar/webhook` is set on the route itself so the URL Polar -# stores remains stable even if api_router's prefix ever moves. -app.include_router(polar_webhook_router.router, tags=["polar-webhook"]) -# Stripe billing (checkout, portal, webhook). Auth lives per-route: -# checkout + portal require_auth, webhook is signature-gated. -app.include_router(stripe_router.router, tags=["stripe-billing"]) # Public router (no auth dep) before pages_router so the marketing/legal # paths can never collide with future authenticated routes. app.include_router(public_router.router) diff --git a/app/models.py b/app/models.py index 665a8cd..bdc884a 100644 --- a/app/models.py +++ b/app/models.py @@ -22,23 +22,15 @@ from sqlalchemy import ( String, Text, UniqueConstraint, - text, ) from sqlalchemy.orm import Mapped, mapped_column, relationship from app.db import Base, utcnow -# Portable autoincrement primary-key type. SQLite only treats `INTEGER -# PRIMARY KEY` as a ROWID alias (the bit that auto-fills); plain BIGINT -# requires explicit values, which breaks our async tests. `with_variant` -# emits INTEGER on SQLite and keeps BIGINT everywhere else. -_PK = BigInteger().with_variant(Integer(), "sqlite") - - class Quote(Base): __tablename__ = "quotes" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) symbol: Mapped[str] = mapped_column(String(128), nullable=False) source: Mapped[str] = mapped_column(String(32), nullable=False) label: Mapped[str] = mapped_column(String(128), default="") @@ -69,7 +61,7 @@ class QuoteDaily(Base): class Headline(Base): __tablename__ = "headlines" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) source: Mapped[str] = mapped_column(String(64), nullable=False) category: Mapped[str] = mapped_column(String(32), nullable=False) title: Mapped[str] = mapped_column(String(512), nullable=False) @@ -107,7 +99,7 @@ class Feed(Base): class StrategicLog(Base): __tablename__ = "strategic_logs" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) generated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, index=True) model: Mapped[str] = mapped_column(String(64), nullable=False) anchor_date: Mapped[str | None] = mapped_column(String(16)) @@ -124,7 +116,7 @@ class IndicatorSummary(Base): """Short AI-generated read for one indicator group, regenerated hourly. The latest row per group_name is what the dashboard renders.""" __tablename__ = "indicator_summaries" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) group_name: Mapped[str] = mapped_column(String(64), nullable=False) generated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) model: Mapped[str] = mapped_column(String(64), nullable=False) @@ -142,7 +134,7 @@ class IndicatorSummary(Base): class AICall(Base): """Cost ledger for OpenRouter calls. Feeds the monthly cap check.""" __tablename__ = "ai_calls" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) called_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow, index=True) model: Mapped[str] = mapped_column(String(64), nullable=False) prompt_tokens: Mapped[int | None] = mapped_column(Integer) @@ -169,50 +161,23 @@ class User(Base): settings_json: Mapped[dict | None] = mapped_column(JSON) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) last_login_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - # Referral code is unique + URL-safe; generated on first need rather - # than at row creation so existing accounts get one the next time - # they hit /settings. + # Referrals (Phase D.1). The code is unique + URL-safe; generated on + # first need rather than at row creation so existing accounts get one + # the next time they hit /settings. referral_code: Mapped[str | None] = mapped_column(String(16), nullable=True) referred_by_user_id: Mapped[int | None] = mapped_column( ForeignKey("users.id", ondelete="SET NULL"), nullable=True, ) - # Paid-tier credit window. Null = no credit. When set and > now(), - # the user gets paid-tier features regardless of `tier`. Populated - # by admin CLI (manual grants) and by referral conversion (45 days - # per converted referral, both parties). + # Paid-tier credit window (Phase D.2). Null = no credit. When set and + # > now(), the user gets paid-tier features regardless of `tier`. + # Populated by admin CLI (manual grants) or Paddle webhook (D.3). credit_until: Mapped[datetime | None] = mapped_column( DateTime(timezone=True), nullable=True, ) - email_digest_opt_in: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True, server_default=text("1"), - ) - # NULL = use INTERMEDIATE at render time. Server-side mirror of the - # dashboard tone, decoupled because the dashboard pref is localStorage. - digest_tone: Mapped[str | None] = mapped_column(String(16)) - # Polar (MoR) linkage — populated by the polar_webhook handler the - # first time we see a subscription/order event for the user. The - # customer id is the stable join key; the subscription id is what - # we cancel against from /settings. - polar_customer_id: Mapped[str | None] = mapped_column(String(64), nullable=True) - polar_subscription_id: Mapped[str | None] = mapped_column(String(64), nullable=True) - # Stripe (merchant-on-record for read.markets). Populated on the - # first checkout.session.completed event via client_reference_id; - # used thereafter to match incoming subscription/invoice events - # back to this row. - stripe_customer_id: Mapped[str | None] = mapped_column(String(64), nullable=True) - stripe_subscription_id: Mapped[str | None] = mapped_column(String(64), nullable=True) - # Set when a subscription is in `trialing` state — drives the - # "Free trial — N days remaining" hint on /settings. Cleared on - # subscription.revoked or when status transitions out of trialing. - stripe_trial_end_at: Mapped[datetime | None] = mapped_column( - DateTime(timezone=True), nullable=True, - ) __table_args__ = ( UniqueConstraint("email", name="uq_users_email"), UniqueConstraint("referral_code", name="uq_users_referral_code"), - UniqueConstraint("polar_customer_id", name="uq_users_polar_customer"), - UniqueConstraint("stripe_customer_id", name="uq_users_stripe_customer"), ) @@ -239,21 +204,16 @@ class PortfolioSync(Base): updated_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) fetch_window_start: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) fetch_count: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - # 8-byte HKDF fingerprint of the pepper that wrapped this row. A - # mismatch against the current pepper means the row is orphaned - # (pepper was rotated) — distinct from genuine GCM corruption. - pepper_fp: Mapped[bytes | None] = mapped_column(LargeBinary(length=8)) class Referral(Base): """One row per captured (referrer, referred) pair. Created at signup when the new user supplied a valid `?ref=`. The conversion fields (`converted_at`, `credited_at`) stay null until the referred - user makes their first paid subscription — the Stripe webhook calls - ``referral_service.convert_referral`` to fill them in and extend - both parties' ``credit_until``.""" + user makes their first paid subscription — Phase D.3 fills them in + via the Paddle webhook.""" __tablename__ = "referrals" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) referrer_user_id: Mapped[int] = mapped_column( ForeignKey("users.id", ondelete="CASCADE"), nullable=False, ) @@ -275,7 +235,7 @@ class EmailOTP(Base): sent in the email; we store an argon2 hash, expiry, attempt count, and a used_at timestamp so a single code can't be reused or brute-forced.""" __tablename__ = "email_otps" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) email: Mapped[str] = mapped_column(String(255), nullable=False) code_hash: Mapped[str] = mapped_column(String(255), nullable=False) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) @@ -296,7 +256,7 @@ class InstrumentMap(Base): Multiple rows can share a shortName (e.g. SHEL on LSE in GBX vs SHEL on NYSE in USD); the resolver picks the right one per user.""" __tablename__ = "instrument_map" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) t212_ticker: Mapped[str] = mapped_column(String(64), nullable=False) t212_shortname: Mapped[str] = mapped_column(String(32), nullable=False) yahoo_ticker: Mapped[str | None] = mapped_column(String(32)) @@ -338,7 +298,7 @@ class TickerUniverse(Base): class JobRun(Base): """One row per scheduled-job invocation; powers /api/health + the ops footer.""" __tablename__ = "job_runs" - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) name: Mapped[str] = mapped_column(String(64), nullable=False) started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) @@ -347,121 +307,3 @@ class JobRun(Base): items_written: Mapped[int | None] = mapped_column(Integer) __table_args__ = (Index("ix_jobruns_name_started", "name", "started_at"),) - - -class EmailSend(Base): - """Audit row per digest email send. Used for idempotency (don't send - twice on the same UTC day) and for surfacing 'last delivery' on the - Settings page.""" - __tablename__ = "email_sends" - - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), nullable=False, - ) - kind: Mapped[str] = mapped_column(String(16), nullable=False) # "daily" | "weekly" - sent_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=utcnow, nullable=False, - ) - status: Mapped[str] = mapped_column(String(16), nullable=False) # "sent" | "skipped" | "error" - error: Mapped[str | None] = mapped_column(String(255)) - - __table_args__ = ( - Index("ix_email_sends_user_kind_sent", "user_id", "kind", "sent_at"), - ) - - -class PolarEvent(Base): - """Audit + idempotency table for inbound Polar (MoR) webhook deliveries. - - Polar uses the Standard Webhooks spec, which guarantees each delivery - carries a unique `webhook-id` header. We store that ID under a UNIQUE - constraint so a replay of the same event is a no-op (the INSERT fails - and the handler returns the prior result). - - `processed_at` distinguishes "delivered and handled" from "delivered - but the handler crashed mid-flight" — the latter rows are what an - operator looks at when investigating a stuck subscription.""" - __tablename__ = "polar_events" - - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) - event_id: Mapped[str] = mapped_column(String(128), nullable=False) - event_type: Mapped[str] = mapped_column(String(64), nullable=False) - received_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=utcnow, nullable=False, - ) - processed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - error: Mapped[str | None] = mapped_column(Text) - # Raw JSON body, kept for forensics. Truncated to 16 KiB to keep - # one bad request from blowing up the row. - payload: Mapped[str] = mapped_column(Text, nullable=False) - - __table_args__ = ( - UniqueConstraint("event_id", name="uq_polar_events_event_id"), - Index("ix_polar_events_type_received", "event_type", "received_at"), - ) - - -class StripeEvent(Base): - """Audit + idempotency table for inbound Stripe webhook deliveries. - - Same shape and purpose as PolarEvent — Stripe's `event.id` plays the - same role as Standard Webhooks' `webhook-id`. We keep the tables - distinct (rather than a single 'webhook_events' table) so an - operator can look at the audit trail per processor without filtering - on a `source` column.""" - __tablename__ = "stripe_events" - - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) - event_id: Mapped[str] = mapped_column(String(128), nullable=False) - event_type: Mapped[str] = mapped_column(String(64), nullable=False) - received_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=utcnow, nullable=False, - ) - processed_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) - error: Mapped[str | None] = mapped_column(Text) - payload: Mapped[str] = mapped_column(Text, nullable=False) - - __table_args__ = ( - UniqueConstraint("event_id", name="uq_stripe_events_event_id"), - Index("ix_stripe_events_type_received", "event_type", "received_at"), - ) - - -class CsvFormatTemplate(Base): - """Cached column-mapping for a single broker CSV format. - - Populated on the first upload of a previously-unseen format via the - LLM-fallback parser. Subsequent uploads of the same format - (identified by ``fingerprint``, a sha256 of the normalised header - row) replay ``mapping`` deterministically with no LLM call. - - The table holds the actual ``headers`` and one anonymous ``sample_row`` - from the originating upload — there is no ``user_id`` column, no link - back to the uploader. The sample exists so the operator has concrete - material to look at when hand-writing future native parsers; the - system never auto-generates or modifies parser code from this data. - """ - __tablename__ = "csv_format_templates" - - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) - fingerprint: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) - headers: Mapped[list[str]] = mapped_column(JSON, nullable=False) - sample_row: Mapped[list[str]] = mapped_column(JSON, nullable=False) - mapping: Mapped[dict[str, str | None]] = mapped_column(JSON, nullable=False) - preamble_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - delimiter: Mapped[str] = mapped_column(String(1), nullable=False, default=",") - broker_label: Mapped[str | None] = mapped_column(String(128)) - first_seen_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=utcnow, - ) - # use_count and last_used_at are application-managed: parse_with_llm - # increments use_count and sets last_used_at = utcnow() on every cache hit. - # No onupdate hook — we don't want unrelated writes (e.g. broker_label edits) - # to re-stamp last_used_at. - use_count: Mapped[int] = mapped_column(Integer, nullable=False, default=1) - last_used_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=utcnow, - ) - llm_model: Mapped[str | None] = mapped_column(String(64)) - llm_cost_usd: Mapped[float | None] = mapped_column(Float) diff --git a/app/routers/api.py b/app/routers/api.py index 5e06090..84e8ad6 100644 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -8,7 +8,6 @@ from __future__ import annotations import calendar as _cal import re from datetime import date, datetime, timedelta, timezone -from typing import Literal from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import HTMLResponse, JSONResponse @@ -20,7 +19,7 @@ from collections import defaultdict import httpx from pydantic import BaseModel, Field -from app.auth import require_token, maybe_current_user, CurrentUser +from app.auth import require_token from app.config import get_settings from app.db import get_session, utcnow from app.services.openrouter import ( @@ -37,7 +36,6 @@ from app.models import ( JobRun, Quote, StrategicLog, - User, ) from app.schemas import ( HealthOut, @@ -51,8 +49,7 @@ from app.schemas import ( router = APIRouter(dependencies=[Depends(require_token)]) JOB_NAMES = ("market_job", "news_job", "ai_log_job", "rollup_job", - "indicator_summary_job", "universe_flush_job", - "email_digest_job") + "indicator_summary_job", "universe_flush_job") JOB_STALE_HOURS = 2.0 # job is "warn" if its last success was >2h ago # Per-group expected freshness — bonds and intraday tape want daily data, @@ -231,18 +228,11 @@ async def news_list( limit: int = Query(50, ge=1, le=500), tags: str | None = Query(None, description="comma-separated include list"), exclude_tags: str | None = Query(None, description="comma-separated exclude list"), - principal: CurrentUser | None = Depends(maybe_current_user), as_: str | None = Query(default=None, alias="as"), ): from app.services.news_tagging import TAG_LABELS, TAG_VOCABULARY - from app.services.access import FREE_NEWS_WINDOW_HOURS, is_paid_active - effective_hours = since_hours - capped = not is_paid_active(principal) - if capped: - effective_hours = min(since_hours, FREE_NEWS_WINDOW_HOURS) - - cutoff = utcnow() - timedelta(hours=effective_hours) + cutoff = utcnow() - timedelta(hours=since_hours) stmt = select(Headline).where(Headline.published_at >= cutoff) if category: stmt = stmt.where(Headline.category == category) @@ -285,9 +275,7 @@ async def news_list( "tag_vocabulary": TAG_VOCABULARY, "tag_labels": TAG_LABELS, "active_include": sorted(include), - "active_exclude": sorted(exclude), - "capped": capped, - "window_hours": effective_hours}, + "active_exclude": sorted(exclude)}, ) return [HeadlineOut.model_validate(r, from_attributes=True) for r in filtered] @@ -322,54 +310,31 @@ def _resolve_tone_param(tone: str | None) -> str: return "INTERMEDIATE" -def _free_tier_hour_filter(): - """Free-tier cadence filter for the strategic log: restrict matches to - logs generated at one of the 6-hour boundary hours (00, 06, 12, 18 - UTC). The job itself runs at :20 every hour, so this effectively gives - free users a fresh log roughly every six hours.""" - from app.services.access import FREE_LOG_HOURS_UTC - # `func.extract` works on both MariaDB and SQLite. - return func.extract("hour", StrategicLog.generated_at).in_(FREE_LOG_HOURS_UTC) - - @router.get("/log/latest") async def log_latest( request: Request, session: AsyncSession = Depends(get_session), as_: str | None = Query(default=None, alias="as"), tone: str | None = Query(default=None), - principal: CurrentUser | None = Depends(maybe_current_user), ): - from app.services.access import is_paid_active - free_only = not is_paid_active(principal) wanted_tone = _resolve_tone_param(tone) - - stmt = ( + row = (await session.execute( select(StrategicLog) .where(StrategicLog.tone == wanted_tone) .order_by(desc(StrategicLog.generated_at)) .limit(1) - ) - if free_only: - stmt = stmt.where(_free_tier_hour_filter()) - row = (await session.execute(stmt)).scalar_one_or_none() + )).scalar_one_or_none() # Fallback during rollout: if the requested tone isn't produced yet, # serve whatever is latest rather than 404 the panel. if row is None: - fallback = ( - select(StrategicLog) - .order_by(desc(StrategicLog.generated_at)) - .limit(1) - ) - if free_only: - fallback = fallback.where(_free_tier_hour_filter()) - row = (await session.execute(fallback)).scalar_one_or_none() + row = (await session.execute( + select(StrategicLog).order_by(desc(StrategicLog.generated_at)).limit(1) + )).scalar_one_or_none() if as_ == "html": return templates.TemplateResponse( request, "partials/log.html", - {"log": _log_partial_payload(row), "tone": wanted_tone, - "paid": not free_only}, + {"log": _log_partial_payload(row), "tone": wanted_tone}, ) if row is None: @@ -384,46 +349,34 @@ async def log_by_date( session: AsyncSession = Depends(get_session), as_: str | None = Query(default=None, alias="as"), tone: str | None = Query(default=None), - principal: CurrentUser | None = Depends(maybe_current_user), ): """Canonical log for a given day = MAX(generated_at) within that day, - filtered by tone (NOVICE | INTERMEDIATE; default from settings). - Free-tier users only see logs generated at the 6-hour boundary slots.""" + filtered by tone (NOVICE | INTERMEDIATE; default from settings).""" try: target = datetime.strptime(day, "%Y-%m-%d").date() except ValueError: raise HTTPException(status_code=400, detail="day must be YYYY-MM-DD") - from app.services.access import is_paid_active - free_only = not is_paid_active(principal) wanted_tone = _resolve_tone_param(tone) - - stmt = ( + row = (await session.execute( select(StrategicLog) .where(func.date(StrategicLog.generated_at) == target) .where(StrategicLog.tone == wanted_tone) .order_by(desc(StrategicLog.generated_at)) .limit(1) - ) - if free_only: - stmt = stmt.where(_free_tier_hour_filter()) - row = (await session.execute(stmt)).scalar_one_or_none() + )).scalar_one_or_none() if row is None: - # Fallback: any tone for that day (still tier-filtered). - fallback = ( + # Fallback: any tone for that day. + row = (await session.execute( select(StrategicLog) .where(func.date(StrategicLog.generated_at) == target) .order_by(desc(StrategicLog.generated_at)) .limit(1) - ) - if free_only: - fallback = fallback.where(_free_tier_hour_filter()) - row = (await session.execute(fallback)).scalar_one_or_none() + )).scalar_one_or_none() if as_ == "html": return templates.TemplateResponse( request, "partials/log.html", - {"log": _log_partial_payload(row), "tone": wanted_tone, - "paid": not free_only}, + {"log": _log_partial_payload(row), "tone": wanted_tone}, ) if row is None: raise HTTPException(status_code=404, detail="No log on this date") @@ -779,22 +732,11 @@ async def _month_spend(session: AsyncSession) -> float: async def chat( body: ChatRequest, session: AsyncSession = Depends(get_session), - principal: CurrentUser | None = Depends(maybe_current_user), ): """Answer one user turn given the conversation so far. Grounded on the latest strategic log + market data + thesis-filtered headlines. Ephemeral — the conversation lives entirely in the client; the endpoint just records each call's cost in `ai_calls`.""" - # Paid-only feature. Free users get the static log but not the - # interactive chat (see /pricing). - from app.services.access import is_paid_active - if not is_paid_active(principal): - raise HTTPException( - status_code=402, - detail={"code": "paid_required", - "message": "Follow-up chat is a paid-tier feature."}, - ) - s = get_settings() if not s.OPENROUTER_API_KEY: raise HTTPException(status_code=503, detail="OPENROUTER_API_KEY not set") @@ -858,40 +800,3 @@ async def chat( "prompt_tokens": result.prompt_tokens, "completion_tokens": result.completion_tokens, } - - -# --------------------------------------------------------------------------- -# Settings — digest preferences -# --------------------------------------------------------------------------- - - -class DigestPrefsIn(BaseModel): - opt_in: bool - tone: Literal["NOVICE", "INTERMEDIATE"] - - -class DigestPrefsOut(BaseModel): - opt_in: bool - tone: str - - -@router.patch("/settings/digest", response_model=DigestPrefsOut) -async def patch_digest_prefs( - payload: DigestPrefsIn, - principal: CurrentUser = Depends(require_token), - session: AsyncSession = Depends(get_session), -) -> DigestPrefsOut: - if principal.user is None: - # Admin bearer-token path — no per-user row to persist to. - raise HTTPException(status_code=400, detail="no_user_context") - # require_token loads `principal.user` in its own short-lived session. - # By the time this handler runs, that session is closed; mutating the - # detached object and committing via `session` would persist nothing. - # Re-fetch in the active session before writing. - user = await session.get(User, principal.user.id) - if user is None: - raise HTTPException(status_code=404, detail="user_not_found") - user.email_digest_opt_in = payload.opt_in - user.digest_tone = payload.tone - await session.commit() - return DigestPrefsOut(opt_in=payload.opt_in, tone=payload.tone) diff --git a/app/routers/auth.py b/app/routers/auth.py index 28a7d4d..59733a9 100644 --- a/app/routers/auth.py +++ b/app/routers/auth.py @@ -37,7 +37,7 @@ from app.db import get_session, utcnow from app.logging import get_logger from app.services.auth_service import AuthError, get_or_create_user, get_user from app.services import otp_service, referral_service -from app.services.email_service import EmailSendError, send_otp, send_welcome_email +from app.services.email_service import EmailSendError, send_otp from app.templates_env import templates @@ -239,26 +239,10 @@ async def verify_submit( if user is None: # User row vanished between cookie issue and verify. Restart flow. return RedirectResponse(url="/login", status_code=303) - is_first_login = user.last_login_at is None user.last_login_at = utcnow() - # Default opt-in is set on User row creation; we don't touch it here. - # The one-time welcome email below explains the digest and the Settings - # opt-out path — re-applying a checkbox state on every login would - # silently re-subscribe users who explicitly opted out later. await session.commit() log.info("user.login", user_id=user.id, email=email) - # First-login welcome email — best effort. SMTP failure must not block - # the login itself; we log and continue. Idempotent because we commit - # last_login_at above before this point, so a retried verify won't - # re-trigger send. - if is_first_login: - try: - await send_welcome_email(email) - except Exception as e: # noqa: BLE001 - log.warning("welcome_email.send_failed", - user_id=user.id, error=str(e)[:200]) - resp = RedirectResponse(url="/", status_code=303) _set_session_cookie(resp, user.id) _clear_pending_cookie(resp) @@ -292,32 +276,9 @@ async def verify_resend( # --------------------------------------------------------------------------- -_LOGOUT_HTML = """ - -Signing out… - - -Signing out…""" - - @router.post("/logout") async def logout(request: Request): - resp = HTMLResponse(content=_LOGOUT_HTML) + resp = RedirectResponse(url="/login", status_code=303) resp.delete_cookie(SESSION_COOKIE_NAME, path="/") _clear_pending_cookie(resp) return resp diff --git a/app/routers/email.py b/app/routers/email.py deleted file mode 100644 index 429101b..0000000 --- a/app/routers/email.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Email-related public routes. - -Currently: -- GET /email/unsubscribe?token=... - -The token is `itsdangerous.URLSafeSerializer` over a small payload, -signed with CASSANDRA_SESSION_SECRET. No auth dependency: the whole -point of one-click unsubscribe is that the user does not have to -sign in. -""" -from __future__ import annotations - -from fastapi import APIRouter, Depends, Query, Request -from fastapi.responses import HTMLResponse -from itsdangerous import BadSignature, URLSafeSerializer -from sqlalchemy.ext.asyncio import AsyncSession - -from app import branding -from app.config import get_settings -from app.db import get_session -from app.logging import get_logger -from app.models import User - - -router = APIRouter() -log = get_logger("email_router") - -_SALT = "digest-unsubscribe-v1" - - -def _serializer() -> URLSafeSerializer: - s = get_settings() - if not s.CASSANDRA_SESSION_SECRET: - # In tests with no secret configured, fall back to a constant. - # An empty CASSANDRA_SESSION_SECRET in prod would also break login, - # so this branch is "best-effort dev fallback", not a real prod path. - return URLSafeSerializer("dev-only-empty-secret", salt=_SALT) - return URLSafeSerializer(s.CASSANDRA_SESSION_SECRET, salt=_SALT) - - -def sign_unsubscribe_token(user_id: int) -> str: - return _serializer().dumps({"uid": int(user_id), "purpose": "digest_optout"}) - - -def verify_unsubscribe_token(token: str) -> int | None: - try: - data = _serializer().loads(token) - except BadSignature: - return None - if not isinstance(data, dict): - return None - if data.get("purpose") != "digest_optout": - return None - try: - return int(data["uid"]) - except (KeyError, TypeError, ValueError): - return None - - -_CONFIRM_PAGE = """\ - - - - - Unsubscribed — {brand} - - - -
-
{brand}
-
email preferences
-

You're unsubscribed from email digests.

-

- You can re-enable digests any time from - Settings. -

-
- - -""" - - -@router.get("/email/unsubscribe", response_class=HTMLResponse) -async def unsubscribe( - request: Request, - token: str = Query(...), - session: AsyncSession = Depends(get_session), -): - uid = verify_unsubscribe_token(token) - if uid is not None: - user = await session.get(User, uid) - if user is not None and user.email_digest_opt_in: - user.email_digest_opt_in = False - await session.commit() - log.info("email.unsubscribe.ok", user_id=uid) - else: - log.info("email.unsubscribe.noop_or_unknown", user_id=uid) - else: - log.info("email.unsubscribe.bad_token") - - # Same confirmation page regardless — don't leak token validity. - return HTMLResponse(_CONFIRM_PAGE.format(brand=branding.BRAND_NAME)) diff --git a/app/routers/pages.py b/app/routers/pages.py index f7ef42b..a00bf56 100644 --- a/app/routers/pages.py +++ b/app/routers/pages.py @@ -11,8 +11,8 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.auth import CurrentUser, maybe_current_user, require_auth, require_token from app.config import get_settings, load_groups from app.db import get_session -from app.models import EmailSend, Referral, StrategicLog, User -from app.services.access import is_paid_active, paid_status +from app.models import Referral, StrategicLog, User +from app.services.access import paid_status from app.services.referral_service import assign_code_if_missing from app.templates_env import templates @@ -37,8 +37,7 @@ async def root_page( return templates.TemplateResponse( request, "dashboard.html", - {"groups": list(groups.keys()), "anchor": s.CASSANDRA_ANCHOR_DATE, - "cu": cu, "paid": is_paid_active(cu)}, + {"groups": list(groups.keys()), "anchor": s.CASSANDRA_ANCHOR_DATE, "cu": cu}, ) @@ -75,40 +74,41 @@ async def _resolve_log_date(session: AsyncSession, day: str | None) -> date: return datetime.now(timezone.utc).date() -def _log_page_context(target: date, paid: bool) -> dict: +def _log_page_context(target: date) -> dict: s = get_settings() return { "selected_iso": target.isoformat(), "selected_month": target.strftime("%Y-%m"), "current_tone": s.CASSANDRA_TONE.upper(), "current_analysis": s.CASSANDRA_ANALYSIS.upper(), - "paid": paid, } -@router.get("/log", response_class=HTMLResponse) +@router.get( + "/log", + response_class=HTMLResponse, + dependencies=[Depends(require_token)], +) async def log_page( request: Request, session: AsyncSession = Depends(get_session), - cu: CurrentUser = Depends(require_auth), ): target = await _resolve_log_date(session, None) - return templates.TemplateResponse( - request, "log.html", _log_page_context(target, is_paid_active(cu)), - ) + return templates.TemplateResponse(request, "log.html", _log_page_context(target)) -@router.get("/log/{day}", response_class=HTMLResponse) +@router.get( + "/log/{day}", + response_class=HTMLResponse, + dependencies=[Depends(require_token)], +) async def log_page_day( request: Request, day: str, session: AsyncSession = Depends(get_session), - cu: CurrentUser = Depends(require_auth), ): target = await _resolve_log_date(session, day) - return templates.TemplateResponse( - request, "log.html", _log_page_context(target, is_paid_active(cu)), - ) + return templates.TemplateResponse(request, "log.html", _log_page_context(target)) @router.get("/settings", response_class=HTMLResponse) @@ -117,10 +117,9 @@ async def settings_page( session: AsyncSession = Depends(get_session), principal: CurrentUser = Depends(require_auth), ): - """Per-user settings. Shows email, tier, Stripe subscription - management, email-digest preferences, cloud-sync status, portfolio - import, and the referral block (own code + invite link + counts of - pending / converted / actively-credited referrals).""" + """Per-user settings. Currently shows email, tier, and the referral + block (own code + invite link + counts of pending/converted + referrals). The Credit / Paddle pieces land in D.3.""" user = principal.user if user is None: # Bearer-token admin path — no per-user settings to show. @@ -133,9 +132,8 @@ async def settings_page( # Lazily assign a referral code on first visit. user = await assign_code_if_missing(session, user) - # Stats: how many people have signed up with their code so far, how - # many converted (paid), and how many of those credit grants are - # still live (referrer-side bonus runway not yet expired). + # Stats: how many people have signed up with their code so far, and + # how many of those converted (paid). D.3 will fill `converted_at`. pending_count = (await session.execute( select(func.count(Referral.id)) .where(Referral.referrer_user_id == user.id) @@ -146,57 +144,9 @@ async def settings_page( .where(Referral.referrer_user_id == user.id) .where(Referral.converted_at.is_not(None)) )).scalar() or 0 - # An "active credit" is a conversion whose credit window hasn't yet - # expired for the REFERRED user. We approximate by counting - # conversions in the last REFERRAL_CREDIT_DAYS days — simpler than - # joining against the referred user's credit_until, and matches the - # marketing copy ("45 days of paid access each"). - from datetime import timedelta - from app.services.referral_service import REFERRAL_CREDIT_DAYS - credit_horizon = datetime.now(timezone.utc) - timedelta(days=REFERRAL_CREDIT_DAYS) - active_credit_count = (await session.execute( - select(func.count(Referral.id)) - .where(Referral.referrer_user_id == user.id) - .where(Referral.credited_at.is_not(None)) - .where(Referral.credited_at >= credit_horizon) - )).scalar() or 0 - - # Days of credit the user themselves has on their own account (from - # any source: referrer bonus, admin grant, refund-as-credit). None - # if no credit or it has already expired. - own_credit_days: int | None = None - if user.credit_until is not None: - cu = user.credit_until - if cu.tzinfo is None: - cu = cu.replace(tzinfo=timezone.utc) - delta = cu - datetime.now(timezone.utc) - if delta.total_seconds() > 0: - own_credit_days = max(1, -(-int(delta.total_seconds()) // 86400)) invite_url = str(request.url_for("login_page")) + f"?ref={user.referral_code}" - last_email_send = (await session.execute( - select(EmailSend) - .where(EmailSend.user_id == user.id) - .order_by(desc(EmailSend.sent_at)) - .limit(1) - )).scalar_one_or_none() - - # Trial countdown — when the Stripe subscription is in its 14-day - # trial, show "N days remaining" on the tier row. Computed here - # rather than in the template because Jinja's date arithmetic is - # painful, and we already have to handle MariaDB's tz-naive - # round-trip via _aware-style normalisation. - trial_days_remaining: int | None = None - if user.stripe_trial_end_at is not None: - end = user.stripe_trial_end_at - if end.tzinfo is None: - end = end.replace(tzinfo=timezone.utc) - delta = end - datetime.now(timezone.utc) - if delta.total_seconds() > 0: - # Round up so the last hours of the trial still read "1 day". - trial_days_remaining = max(1, -(-int(delta.total_seconds()) // 86400)) - return templates.TemplateResponse( request, "settings.html", { @@ -204,10 +154,6 @@ async def settings_page( "invite_url": invite_url, "pending_count": int(pending_count), "converted_count": int(converted_count), - "active_credit_count": int(active_credit_count), - "own_credit_days": own_credit_days, "paid": paid_status(user), - "last_email_send": last_email_send, - "trial_days_remaining": trial_days_remaining, }, ) diff --git a/app/routers/polar_webhook.py b/app/routers/polar_webhook.py deleted file mode 100644 index a60acd4..0000000 --- a/app/routers/polar_webhook.py +++ /dev/null @@ -1,304 +0,0 @@ -"""Polar (merchant-of-record) webhook endpoint. - -Polar uses the Standard Webhooks spec (https://www.standardwebhooks.com). -Every delivery carries three headers: - - webhook-id — unique ID for THIS delivery (use for idempotency). - webhook-timestamp — Unix seconds at send time (use for replay defence). - webhook-signature — space-separated list of `v1,` - tokens. Verifying any one of them means the - payload is authentic. - -The signed content is the literal string `{id}.{timestamp}.{body}`, signed -with the raw secret bytes (the secret is base64-encoded after the -`whsec_` prefix). We verify in constant time and reject anything that -doesn't match — including stale deliveries older than ±5 minutes — before -parsing JSON or touching the database. - -Idempotency is keyed on `webhook-id` via a unique constraint on -`polar_events.event_id`. A second delivery of the same id finds the row -already there and returns 200 without re-running the handler — Polar -will retry on non-2xx, so we must always 2xx after a successful first -processing. - -The router is mounted without the app's bearer-token dependency: webhook -authenticity is established via the HMAC, not the token.""" -from __future__ import annotations - -import base64 -import hashlib -import hmac -import json -import time -from datetime import datetime, timezone -from typing import Any - -from fastapi import APIRouter, Depends, HTTPException, Request -from sqlalchemy import select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app.config import get_settings -from app.db import get_session, utcnow -from app.logging import get_logger -from app.models import PolarEvent, User - - -log = get_logger("polar_webhook") -router = APIRouter() - - -# Max clock skew we'll tolerate on the `webhook-timestamp` header. Standard -# Webhooks recommends ±5 min; anything older is almost certainly replay. -_TIMESTAMP_TOLERANCE_S = 300 -# Cap stored payload at 16 KiB so a hostile (or buggy) sender can't blow -# up a single row. -_PAYLOAD_STORE_MAX = 16 * 1024 - - -def _decode_secret(secret: str) -> bytes: - """Polar/Standard-Webhooks secrets are base64 with a `whsec_` prefix. - Returns the raw HMAC key. Raises ValueError on malformed input.""" - if not secret: - raise ValueError("empty webhook secret") - s = secret - if s.startswith("whsec_"): - s = s[len("whsec_"):] - return base64.b64decode(s) - - -def _compute_signature(key: bytes, signed_payload: str) -> str: - """Return `v1,` — the format a single signature token uses.""" - mac = hmac.new(key, signed_payload.encode("utf-8"), hashlib.sha256).digest() - return "v1," + base64.b64encode(mac).decode("ascii") - - -def verify_standard_webhook( - *, - secret: str, - msg_id: str, - msg_timestamp: str, - msg_signature: str, - body: bytes, - now: float | None = None, -) -> None: - """Verify a Standard Webhooks delivery. Raises HTTPException(401) on - any failure. No return value — success is "did not raise".""" - try: - key = _decode_secret(secret) - except (ValueError, base64.binascii.Error) as e: - raise HTTPException(status_code=500, detail=f"bad webhook secret: {e}") - - # Timestamp / replay window. - try: - ts = int(msg_timestamp) - except ValueError: - raise HTTPException(status_code=401, detail="invalid timestamp") - drift = abs((now if now is not None else time.time()) - ts) - if drift > _TIMESTAMP_TOLERANCE_S: - raise HTTPException(status_code=401, detail="stale timestamp") - - signed_payload = f"{msg_id}.{msg_timestamp}.{body.decode('utf-8')}" - expected = _compute_signature(key, signed_payload) - - # The header can carry several space-separated tokens (key rotation). - # Any match — in constant time — is success. - candidates = msg_signature.split() - if not any(hmac.compare_digest(expected, c) for c in candidates): - raise HTTPException(status_code=401, detail="bad signature") - - -# --------------------------------------------------------------------------- -# Event handlers -# --------------------------------------------------------------------------- - - -def _customer_id_from_payload(payload_data: dict[str, Any]) -> str | None: - """Polar nests the customer object under `customer`. Some events also - surface `customer_id` at the top of `data` — accept either.""" - cust = payload_data.get("customer") or {} - return cust.get("id") or payload_data.get("customer_id") - - -def _customer_email_from_payload(payload_data: dict[str, Any]) -> str | None: - cust = payload_data.get("customer") or {} - return cust.get("email") - - -async def _find_user(session: AsyncSession, data: dict[str, Any]) -> User | None: - """Locate the User row that owns this event. - - Strategy: join by stored Polar customer id first (the only stable - link once we've seen a user). Fall back to email — the first time - Polar fires an event for a brand-new customer, we won't have the id - yet, but the customer record on Polar's side was created with the - user's email by our checkout call.""" - cid = _customer_id_from_payload(data) - if cid: - row = (await session.execute( - select(User).where(User.polar_customer_id == cid) - )).scalar_one_or_none() - if row is not None: - return row - email = _customer_email_from_payload(data) - if email: - row = (await session.execute( - select(User).where(User.email == email) - )).scalar_one_or_none() - return row - return None - - -async def _grant_paid( - session: AsyncSession, user: User, data: dict[str, Any], -) -> None: - """Flip the user to the paid tier and persist the Polar IDs we now - know. Safe to call repeatedly: tier is idempotent and the IDs only - change if Polar issued new ones.""" - user.tier = "paid" - cid = _customer_id_from_payload(data) - if cid and user.polar_customer_id != cid: - user.polar_customer_id = cid - sub_id = data.get("id") # subscription event payloads put sub id at top - if sub_id and user.polar_subscription_id != sub_id: - user.polar_subscription_id = sub_id - - -async def _revoke_paid(session: AsyncSession, user: User) -> None: - """Drop the user back to the free tier. We deliberately leave the - polar_customer_id in place so a re-subscription matches them back to - the same row.""" - user.tier = "free" - user.polar_subscription_id = None - - -async def _handle_subscription_active( - session: AsyncSession, data: dict[str, Any], event_type: str, -) -> None: - user = await _find_user(session, data) - if user is None: - log.warning("polar.user_not_found", event_type=event_type, - customer_id=_customer_id_from_payload(data)) - return - await _grant_paid(session, user, data) - - -async def _handle_subscription_revoked( - session: AsyncSession, data: dict[str, Any], event_type: str, -) -> None: - user = await _find_user(session, data) - if user is None: - log.warning("polar.user_not_found", event_type=event_type, - customer_id=_customer_id_from_payload(data)) - return - await _revoke_paid(session, user) - - -async def _handle_no_state_change( - session: AsyncSession, data: dict[str, Any], event_type: str, -) -> None: - """For events we want to record in the audit table but where the - tier doesn't move — canceled (still active until period end), - uncanceled, past_due, order events, refund created. The PolarEvent - row is the record.""" - return None - - -# Map event type → handler. Anything not in this map is acknowledged -# (200) but ignored, on the principle that Polar may add new event types -# over time and we don't want to start 4xx-ing on unknown ones. -_HANDLERS = { - "subscription.created": _handle_subscription_active, - "subscription.active": _handle_subscription_active, - "subscription.updated": _handle_subscription_active, - "subscription.uncanceled": _handle_subscription_active, - "subscription.canceled": _handle_no_state_change, - "subscription.revoked": _handle_subscription_revoked, - "subscription.past_due": _handle_no_state_change, - "order.paid": _handle_no_state_change, - "order.refunded": _handle_no_state_change, - "refund.created": _handle_no_state_change, -} - - -# --------------------------------------------------------------------------- -# Endpoint -# --------------------------------------------------------------------------- - - -@router.post("/api/polar/webhook") -async def polar_webhook( - request: Request, - session: AsyncSession = Depends(get_session), -) -> dict[str, str]: - s = get_settings() - if not s.POLAR_WEBHOOK_SECRET: - # Loud failure rather than accepting an unsigned event. - raise HTTPException(status_code=503, detail="webhook not configured") - - msg_id = request.headers.get("webhook-id", "") - msg_ts = request.headers.get("webhook-timestamp", "") - msg_sig = request.headers.get("webhook-signature", "") - if not (msg_id and msg_ts and msg_sig): - raise HTTPException(status_code=400, detail="missing standard-webhooks headers") - - body = await request.body() - verify_standard_webhook( - secret=s.POLAR_WEBHOOK_SECRET, - msg_id=msg_id, - msg_timestamp=msg_ts, - msg_signature=msg_sig, - body=body, - ) - - try: - envelope = json.loads(body) - except json.JSONDecodeError: - raise HTTPException(status_code=400, detail="invalid JSON") - - event_type = envelope.get("type") or "unknown" - data = envelope.get("data") or {} - - # Idempotency: insert the audit row first. If the webhook-id was - # already delivered, the UNIQUE constraint short-circuits with a - # 200 (Polar will stop retrying). - body_text = body.decode("utf-8", errors="replace")[:_PAYLOAD_STORE_MAX] - audit = PolarEvent( - event_id=msg_id, - event_type=event_type, - received_at=utcnow(), - payload=body_text, - ) - session.add(audit) - try: - await session.flush() - except IntegrityError: - # Already processed — return 200 so Polar doesn't keep retrying. - await session.rollback() - log.info("polar.duplicate_delivery", event_id=msg_id, type=event_type) - return {"status": "duplicate"} - - handler = _HANDLERS.get(event_type) - if handler is None: - # Unknown but well-signed event — record it, ack 200. - audit.processed_at = utcnow() - await session.commit() - log.info("polar.event_unhandled", type=event_type, id=msg_id) - return {"status": "ignored"} - - try: - await handler(session, data, event_type) - except Exception as e: - # Mark as errored so an operator can see what's stuck, then - # commit + ack 200. We do NOT want Polar to retry an event that - # broke handler logic — the same code will break the same way. - # Operator gets paged from the error column instead. - audit.error = str(e)[:1024] - await session.commit() - log.exception("polar.handler_error", type=event_type, id=msg_id) - return {"status": "handler_error"} - - audit.processed_at = utcnow() - await session.commit() - log.info("polar.processed", type=event_type, id=msg_id) - return {"status": "ok"} diff --git a/app/routers/public.py b/app/routers/public.py index 33bd245..a040ccd 100644 --- a/app/routers/public.py +++ b/app/routers/public.py @@ -15,7 +15,6 @@ from fastapi import APIRouter, Depends, Request from fastapi.responses import HTMLResponse from app.auth import CurrentUser, maybe_current_user -from app.services.access import is_paid_active from app.templates_env import templates @@ -34,9 +33,7 @@ async def pricing_page( request: Request, cu: CurrentUser | None = Depends(maybe_current_user), ): - ctx = _ctx(request, cu) - ctx["paid"] = is_paid_active(cu) - return templates.TemplateResponse(request, "pricing.html", ctx) + return templates.TemplateResponse(request, "pricing.html", _ctx(request, cu)) @router.get("/about", response_class=HTMLResponse) diff --git a/app/routers/stripe_billing.py b/app/routers/stripe_billing.py deleted file mode 100644 index 60bc7f7..0000000 --- a/app/routers/stripe_billing.py +++ /dev/null @@ -1,431 +0,0 @@ -"""Stripe billing endpoints — checkout, webhook, customer portal. - -Stripe is the merchant-on-record for read.markets (after Polar/Paddle -both declined the financial-media category). We delegate payment UI to -Stripe-hosted Checkout and Customer Portal; the only state we keep on -our side is `users.stripe_customer_id` / `users.stripe_subscription_id` -so we can match incoming webhooks back to the right user. - -The Stripe SDK is sync; we wrap calls in `asyncio.to_thread` so the -event loop doesn't block while Stripe answers. For our request volume -this is more reliable than the SDK's nascent async surface. - -Routes -- POST /api/stripe/checkout — logged-in user upgrades. Body: {cadence}. -- POST /api/stripe/webhook — Stripe → us, signature-verified. -- POST /api/stripe/portal — logged-in user opens the customer portal. -""" -from __future__ import annotations - -import asyncio -import json -from typing import Any, Literal - -import stripe -from fastapi import APIRouter, Body, Depends, HTTPException, Request -from pydantic import BaseModel -from sqlalchemy import select -from sqlalchemy.exc import IntegrityError -from sqlalchemy.ext.asyncio import AsyncSession - -from app import branding -from app.auth import CurrentUser, require_auth -from app.config import get_settings -from app.db import get_session, utcnow -from app.logging import get_logger -from app.models import StripeEvent, User - - -log = get_logger("stripe_billing") -router = APIRouter() - - -# Cap stored payload at 16 KiB so a hostile (or buggy) sender can't -# blow up a single row. Same pattern as polar_webhook. -_PAYLOAD_STORE_MAX = 16 * 1024 - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _require_configured() -> None: - s = get_settings() - if not s.STRIPE_API_KEY: - raise HTTPException(status_code=503, detail="stripe not configured") - - -def _price_for(cadence: str) -> str: - s = get_settings() - if cadence == "monthly": - if not s.STRIPE_PRICE_MONTHLY: - raise HTTPException(status_code=503, detail="STRIPE_PRICE_MONTHLY not set") - return s.STRIPE_PRICE_MONTHLY - if cadence == "annual": - if not s.STRIPE_PRICE_ANNUAL: - raise HTTPException(status_code=503, detail="STRIPE_PRICE_ANNUAL not set") - return s.STRIPE_PRICE_ANNUAL - raise HTTPException(status_code=400, detail="cadence must be 'monthly' or 'annual'") - - -def _stripe_client() -> stripe.StripeClient: - """Per-call client so we read the secret at request time (lets us - rotate the key by editing .env + reloading without rebuilding any - cached client).""" - return stripe.StripeClient(get_settings().STRIPE_API_KEY) - - -# --------------------------------------------------------------------------- -# POST /api/stripe/checkout -# --------------------------------------------------------------------------- - - -class CheckoutRequest(BaseModel): - cadence: Literal["monthly", "annual"] - - -class CheckoutResponse(BaseModel): - url: str - - -@router.post("/api/stripe/checkout", response_model=CheckoutResponse) -async def create_checkout( - body: CheckoutRequest, - session: AsyncSession = Depends(get_session), - cu: CurrentUser = Depends(require_auth), -) -> CheckoutResponse: - _require_configured() - if cu.user is None: - # Admin bearer token has no User row — they shouldn't be buying. - raise HTTPException(status_code=400, detail="admin token cannot purchase") - - user = await session.get(User, cu.user.id) - if user is None: - raise HTTPException(status_code=404, detail="user_not_found") - - price_id = _price_for(body.cadence) - client = _stripe_client() - - # Pass `customer` if we already minted one for this user (avoids - # creating duplicate Stripe customers on repeat checkouts); - # otherwise let Stripe create it via `customer_email`. - create_kwargs: dict[str, Any] = { - "mode": "subscription", - "line_items": [{"price": price_id, "quantity": 1}], - "client_reference_id": str(user.id), - "success_url": f"{branding.SITE_URL}/settings?upgraded=1", - "cancel_url": f"{branding.SITE_URL}/pricing", - # Lets us paste in a referral coupon at checkout once the - # referral redemption flow ships. - "allow_promotion_codes": True, - } - # Per-cadence cooling-off treatment: - # - # - Annual gets a 14-day free trial. No money moves during the - # trial, so the Consumer Contracts Regulations 14-day refund - # question is moot (nothing paid = nothing to refund). Card is - # still required at checkout so Stripe can charge on day 15. - # - # - Monthly bills immediately (a 14-day trial on a £7/month plan - # would give away ~50% of cycle one). The Reg-36 waiver lives - # on our own /pricing page as a required tick-box (see - # pricing.html); we deliberately do NOT use Stripe's - # consent_collection.terms_of_service here because that's an - # account-wide setting and we want per-product control (and - # per-product Terms URLs) as we grow. - if body.cadence == "annual": - create_kwargs["subscription_data"] = {"trial_period_days": 14} - if user.stripe_customer_id: - create_kwargs["customer"] = user.stripe_customer_id - else: - create_kwargs["customer_email"] = user.email - - try: - sess = await asyncio.to_thread( - client.checkout.sessions.create, params=create_kwargs, - ) - except stripe.StripeError as e: - log.error("stripe.checkout.create_failed", user_id=user.id, error=str(e)) - raise HTTPException(status_code=502, detail=f"stripe error: {e.user_message or str(e)}") - - if not sess.url: - raise HTTPException(status_code=502, detail="stripe returned no checkout URL") - log.info("stripe.checkout.created", user_id=user.id, session_id=sess.id, - cadence=body.cadence) - return CheckoutResponse(url=sess.url) - - -# --------------------------------------------------------------------------- -# POST /api/stripe/portal -# --------------------------------------------------------------------------- - - -class PortalResponse(BaseModel): - url: str - - -@router.post("/api/stripe/portal", response_model=PortalResponse) -async def create_portal_session( - session: AsyncSession = Depends(get_session), - cu: CurrentUser = Depends(require_auth), -) -> PortalResponse: - _require_configured() - if cu.user is None: - raise HTTPException(status_code=400, detail="admin token has no portal") - - user = await session.get(User, cu.user.id) - if user is None or not user.stripe_customer_id: - raise HTTPException( - status_code=404, - detail="no_stripe_customer — start a subscription first", - ) - - client = _stripe_client() - try: - portal = await asyncio.to_thread( - client.billing_portal.sessions.create, - params={ - "customer": user.stripe_customer_id, - "return_url": f"{branding.SITE_URL}/settings", - }, - ) - except stripe.StripeError as e: - log.error("stripe.portal.create_failed", user_id=user.id, error=str(e)) - raise HTTPException(status_code=502, detail=f"stripe error: {e.user_message or str(e)}") - - return PortalResponse(url=portal.url) - - -# --------------------------------------------------------------------------- -# POST /api/stripe/webhook -# --------------------------------------------------------------------------- - - -async def _find_user( - session: AsyncSession, - *, - client_ref: str | None = None, - customer_id: str | None = None, -) -> User | None: - """Find the User row this event belongs to. - - `client_reference_id` is the most reliable join key — we set it - to `str(user.id)` at checkout creation. After the first event we - also know `stripe_customer_id`, which subsequent subscription / - invoice events arrive carrying.""" - if client_ref: - try: - uid = int(client_ref) - except ValueError: - uid = None - if uid is not None: - u = await session.get(User, uid) - if u is not None: - return u - if customer_id: - row = (await session.execute( - select(User).where(User.stripe_customer_id == customer_id) - )).scalar_one_or_none() - return row - return None - - -async def _grant_paid( - session: AsyncSession, - user: User, - *, - customer_id: str | None, - subscription_id: str | None, - trial_end: int | None = None, - status: str | None = None, -) -> None: - # Capture "first paid transition" before mutating — drives the - # referral-conversion call below. Skipping the convert lookup on - # every renewal event saves a DB roundtrip per webhook. - first_paid_transition = user.tier != "paid" - - user.tier = "paid" - if customer_id and user.stripe_customer_id != customer_id: - user.stripe_customer_id = customer_id - if subscription_id and user.stripe_subscription_id != subscription_id: - user.stripe_subscription_id = subscription_id - # Track trial_end so the settings page can show "N days remaining". - # Only populated when Stripe reports the sub as trialing — once the - # status flips to active (paid for real), we clear the trial marker. - if status == "trialing" and trial_end: - from datetime import datetime, timezone - user.stripe_trial_end_at = datetime.fromtimestamp(trial_end, tz=timezone.utc) - elif status == "active": - user.stripe_trial_end_at = None - - # Apply referral credit on the FIRST paid transition only. - # convert_referral is itself idempotent (no-op on missing or - # already-converted rows), so this guard is purely a perf hint. - if first_paid_transition: - from app.services.referral_service import convert_referral - await convert_referral(session, user) - - -async def _revoke_paid(user: User) -> None: - user.tier = "free" - user.stripe_subscription_id = None - user.stripe_trial_end_at = None - # Keep stripe_customer_id so a re-subscription matches this row. - - -async def _handle_checkout_completed( - session: AsyncSession, event_type: str, obj: dict[str, Any], -) -> None: - user = await _find_user( - session, - client_ref=obj.get("client_reference_id"), - customer_id=obj.get("customer"), - ) - if user is None: - log.warning("stripe.user_not_found", event_type=event_type) - return - # checkout.session.completed doesn't carry trial_end on the session - # object itself — the subscription.created event that fires right - # after will carry it. We grant paid here without trial info and - # let the subscription event fill in trial_end_at moments later. - await _grant_paid( - session, - user, - customer_id=obj.get("customer"), - subscription_id=obj.get("subscription"), - ) - - -async def _handle_subscription_event( - session: AsyncSession, event_type: str, obj: dict[str, Any], -) -> None: - """customer.subscription.created / .updated — flip to paid if the - Stripe-side status says the subscription is active/trialing; drop - to free if it's an end-state.""" - user = await _find_user(session, customer_id=obj.get("customer")) - if user is None: - log.warning("stripe.user_not_found", event_type=event_type, - customer_id=obj.get("customer")) - return - status = obj.get("status") - # Stripe statuses: trialing, active, past_due, canceled, unpaid, - # incomplete, incomplete_expired, paused. Treat trialing/active as - # paid; everything else holds tier the same until we get an explicit - # subscription.deleted (which fires after the final state lands). - if status in ("trialing", "active"): - await _grant_paid( - session, - user, - customer_id=obj.get("customer"), - subscription_id=obj.get("id"), - trial_end=obj.get("trial_end"), - status=status, - ) - - -async def _handle_subscription_deleted( - session: AsyncSession, event_type: str, obj: dict[str, Any], -) -> None: - user = await _find_user(session, customer_id=obj.get("customer")) - if user is None: - log.warning("stripe.user_not_found", event_type=event_type, - customer_id=obj.get("customer")) - return - await _revoke_paid(user) - - -async def _handle_audit_only( - session: AsyncSession, event_type: str, obj: dict[str, Any], -) -> None: - """invoice.paid / invoice.payment_failed / charge.refunded — we - record these in stripe_events for the audit log but the tier doesn't - move until subscription.deleted fires.""" - return None - - -_HANDLERS = { - "checkout.session.completed": _handle_checkout_completed, - "customer.subscription.created": _handle_subscription_event, - "customer.subscription.updated": _handle_subscription_event, - "customer.subscription.deleted": _handle_subscription_deleted, - "invoice.paid": _handle_audit_only, - "invoice.payment_failed": _handle_audit_only, - "charge.refunded": _handle_audit_only, -} - - -@router.post("/api/stripe/webhook") -async def stripe_webhook( - request: Request, - session: AsyncSession = Depends(get_session), -) -> dict[str, str]: - s = get_settings() - if not s.STRIPE_WEBHOOK_SECRET: - raise HTTPException(status_code=503, detail="stripe webhook not configured") - - sig = request.headers.get("stripe-signature", "") - if not sig: - raise HTTPException(status_code=400, detail="missing stripe-signature header") - - body = await request.body() - # construct_event handles HMAC verification + timestamp tolerance. - # We then re-parse the body as plain JSON for handler dispatch — - # the Stripe SDK's StripeObject doesn't expose dict.get(), and - # round-tripping through json gives us simple, typed-dict access. - try: - stripe.Webhook.construct_event( - payload=body, sig_header=sig, secret=s.STRIPE_WEBHOOK_SECRET, - ) - except stripe.SignatureVerificationError: - raise HTTPException(status_code=401, detail="bad signature") - except ValueError: - raise HTTPException(status_code=400, detail="invalid payload") - - envelope = json.loads(body) - event_id = envelope.get("id") or "" - event_type = envelope.get("type") or "unknown" - obj = (envelope.get("data") or {}).get("object") or {} - - if not event_id: - raise HTTPException(status_code=400, detail="event missing id") - - # Idempotency: insert audit row first. UNIQUE on event_id makes a - # replay of the same Stripe event id a no-op (Stripe retries on - # non-2xx, so always 2xx after first successful processing). - audit = StripeEvent( - event_id=event_id, - event_type=event_type, - received_at=utcnow(), - payload=body.decode("utf-8", errors="replace")[:_PAYLOAD_STORE_MAX], - ) - session.add(audit) - try: - await session.flush() - except IntegrityError: - await session.rollback() - log.info("stripe.duplicate_delivery", event_id=event_id, type=event_type) - return {"status": "duplicate"} - - handler = _HANDLERS.get(event_type) - if handler is None: - audit.processed_at = utcnow() - await session.commit() - log.info("stripe.event_unhandled", type=event_type, id=event_id) - return {"status": "ignored"} - - try: - await handler(session, event_type, obj) - except Exception as e: - audit.error = str(e)[:1024] - await session.commit() - log.exception("stripe.handler_error", type=event_type, id=event_id) - # Ack 200 — we don't want Stripe retrying a handler that broke - # the same way on every delivery. An operator triages from the - # `error` column. - return {"status": "handler_error"} - - audit.processed_at = utcnow() - await session.commit() - log.info("stripe.processed", type=event_type, id=event_id) - return {"status": "ok"} diff --git a/app/routers/sync.py b/app/routers/sync.py index 0fa1174..e6496e0 100644 --- a/app/routers/sync.py +++ b/app/routers/sync.py @@ -43,7 +43,6 @@ class SyncBlobOut(BaseModel): class SyncStatusOut(BaseModel): exists: bool - orphaned: bool = False updated_at: datetime | None = None @@ -74,8 +73,8 @@ async def get_status( principal: CurrentUser = Depends(require_paid), session: AsyncSession = Depends(get_session), ) -> SyncStatusOut: - exists, orphaned, updated_at = await svc.fetch_status(session, principal.id) - return SyncStatusOut(exists=exists, orphaned=orphaned, updated_at=updated_at) + exists, updated_at = await svc.fetch_status(session, principal.id) + return SyncStatusOut(exists=exists, updated_at=updated_at) @router.post("", response_model=SyncWriteOut) @@ -109,15 +108,6 @@ async def download_blob( ) try: result = await svc.fetch(session, principal.id) - except svc.SyncOrphanedError: - # Known state: pepper rotated. The frontend uses 410 to swap the - # restore form for a "stale — re-upload" CTA. Logged at INFO, - # not ERROR, because this isn't a server fault. - log.info("portfolio_sync.orphaned", user_id=principal.id) - raise HTTPException( - status_code=status.HTTP_410_GONE, - detail="stale_blob", - ) except svc.SyncCryptoError: log.error("portfolio_sync.unwrap_failed", user_id=principal.id) raise HTTPException( diff --git a/app/routers/ticker_validate.py b/app/routers/ticker_validate.py deleted file mode 100644 index 53bc783..0000000 --- a/app/routers/ticker_validate.py +++ /dev/null @@ -1,156 +0,0 @@ -"""Per-ticker validation + historical-price endpoints. - -These power the dashboard's "Add a position" form. Neither endpoint -persists holdings — they wrap the existing Yahoo chart fetcher and -optionally seed anonymous ticker_universe (validate only). - -Both endpoints are gated behind ``require_paid`` so they match the rest -of the import surface. -""" -from __future__ import annotations - -from datetime import datetime, timezone - -import httpx -from fastapi import APIRouter, Depends, HTTPException -from sqlalchemy.ext.asyncio import AsyncSession - -from app.auth import require_auth -from app.db import get_session, utcnow -from app.logging import get_logger -from app.models import Quote as QuoteModel -from app.services.access import require_paid -from app.services.market import ( - UA, YAHOO_CHART, Quote, _yahoo_range_covering, fetch_yahoo, -) -from app.services.ticker_universe import upsert_tickers - - -log = get_logger("ticker_validate") - -router = APIRouter(dependencies=[Depends(require_auth)]) - - -@router.get( - "/ticker/validate", - dependencies=[Depends(require_paid)], -) -async def validate_ticker( - symbol: str, - session: AsyncSession = Depends(get_session), -) -> dict: - """Live quote for one ticker. - - Returns ``{ok: true, symbol, price, currency, as_of}`` on success - or ``{ok: false, error}`` when the symbol isn't recognised. Seeds - ticker_universe + writes a Quote row as a side-effect on success - so the dashboard's /api/universe call picks it up on the next - refresh.""" - symbol = symbol.strip().upper()[:32] - if not symbol: - return {"ok": False, "error": "symbol required"} - - async with httpx.AsyncClient(follow_redirects=True, timeout=15) as client: - quote = await fetch_yahoo(client, symbol, symbol, "") - - if quote.error or quote.price is None: - log.info("ticker.validate.miss", symbol=symbol, - error=(quote.error or "no price")[:120]) - return {"ok": False, "error": "Symbol not recognised"} - - # Side-effect: seed the universe + write the quote so /api/universe - # has data on the next minute-cycle refresh. - await upsert_tickers(session, [symbol]) - session.add(QuoteModel( - symbol=quote.symbol, source=quote.source, label=quote.label, - group_name="universe", price=quote.price, currency=quote.currency, - as_of=quote.as_of, changes=quote.changes or None, error=None, - fetched_at=utcnow(), - )) - await session.commit() - - return { - "ok": True, - "symbol": quote.symbol, - "price": quote.price, - "currency": quote.currency, - "as_of": quote.as_of, - } - - -async def fetch_yahoo_historical( - client: httpx.AsyncClient, - symbol: str, - target_iso: str, -) -> tuple[float | None, str | None, str | None]: - """Fetch the close on ``target_iso`` or the nearest preceding trading - day's close (within the available history window). - - Returns ``(close, currency, actual_iso)`` or ``(None, None, None)`` - when no usable data exists. Raises on provider-level HTTP errors - (the caller wraps these into a friendly ``ok:false`` response). - """ - range_param = _yahoo_range_covering(target_iso) - r = await client.get( - YAHOO_CHART.format(symbol=symbol), - params={"interval": "1d", "range": range_param, - "includePrePost": "false"}, - headers=UA, - timeout=15, - ) - r.raise_for_status() - result = r.json().get("chart", {}).get("result") - if not result: - return None, None, None - res = result[0] - currency = (res.get("meta") or {}).get("currency") - timestamps = res.get("timestamp") or [] - closes = (res.get("indicators", {}).get("quote") or [{}])[0].get("close") or [] - series = [(t, c) for t, c in zip(timestamps, closes) if c is not None] - if not series: - return None, None, None - target_dt = datetime.strptime(target_iso, "%Y-%m-%d").replace(tzinfo=timezone.utc) - # Add a 24h buffer so the target day itself is included (Yahoo - # timestamps are at market open, not midnight). - cutoff_ts = int(target_dt.timestamp()) + 86400 - selected: tuple[int, float] | None = None - for t, c in series: - if t <= cutoff_ts: - selected = (t, c) - else: - break - if selected is None: - return None, None, None - actual_iso = datetime.fromtimestamp(selected[0], timezone.utc).strftime("%Y-%m-%d") - return selected[1], currency, actual_iso - - -@router.get( - "/ticker/historical", - dependencies=[Depends(require_paid)], -) -async def get_historical(symbol: str, date: str) -> dict: - """Historical daily close. If ``date`` is a non-trading day we walk - back to the last preceding trading day and surface ``actual_date`` - so the UI can show the user which date we actually used.""" - symbol = symbol.strip().upper()[:32] - if not symbol: - return {"ok": False, "error": "symbol required"} - try: - target = datetime.strptime(date, "%Y-%m-%d").date() - except ValueError: - raise HTTPException(status_code=400, detail="invalid date format (YYYY-MM-DD)") - if target > datetime.now(timezone.utc).date(): - raise HTTPException(status_code=400, detail="date cannot be in the future") - - async with httpx.AsyncClient(follow_redirects=True, timeout=15) as client: - try: - close, currency, actual = await fetch_yahoo_historical(client, symbol, date) - except Exception as e: - log.warning("ticker.historical.failed", symbol=symbol, - date=date, error=str(e)[:200]) - return {"ok": False, "error": "Couldn't fetch historical price"} - - if close is None: - return {"ok": False, "error": "No data for that date"} - return {"ok": True, "close": close, "currency": currency, "actual_date": actual} diff --git a/app/routers/universe.py b/app/routers/universe.py index a77585f..163e99d 100644 --- a/app/routers/universe.py +++ b/app/routers/universe.py @@ -189,7 +189,7 @@ async def get_sparkline( # --------------------------------------------------------------------------- -@router.post("/portfolio/parse", dependencies=[Depends(require_paid)]) +@router.post("/portfolio/parse") async def parse_portfolio( file: UploadFile = File(...), session: AsyncSession = Depends(get_session), @@ -210,57 +210,27 @@ async def parse_portfolio( try: pie = parse_t212_csv(raw) - except CSVImportError: - # Unrecognised format — try the LLM-fallback parser. It hits a - # global format-fingerprint cache first; only the very first - # upload of each broker format pays an LLM call. - from app.services.llm_csv_parser import LLMParseError, parse_with_llm - try: - pie = await parse_with_llm(raw, session) - except LLMParseError as e: - raise HTTPException(status_code=400, detail=str(e)) + except CSVImportError as e: + raise HTTPException(status_code=400, detail=str(e)) positions_out: list[dict] = [] yahoo_tickers: list[str] = [] unmapped: list[str] = [] for p in pie.positions: - if pie.tickers_resolved: - # LLM path: ``p.slice`` is already a Yahoo-ready ticker. We - # still do a best-effort InstrumentMap lookup so we can use - # the canonical name + currency when we happen to have one; - # but unlike the T212 path we never *drop* a position just - # because resolve_slice missed. - yahoo_ticker = p.slice.strip().upper() - if not yahoo_ticker: - unmapped.append(p.name or "?") - continue - resolved = await resolve_slice(session, yahoo_ticker) - name = (resolved.name if resolved else None) or p.name - currency = ( - (resolved.currency if resolved else None) - or p.currency or "USD" - ) - else: - # T212 path: ``p.slice`` is a shortcode that MUST round-trip - # through the InstrumentMap. Drop unmapped positions — the - # warnings block surfaces them to the user. - resolved = await resolve_slice(session, p.slice) - if resolved is None or not resolved.yahoo_ticker: - unmapped.append(p.slice or p.name or "?") - continue - yahoo_ticker = resolved.yahoo_ticker - name = resolved.name or p.name - currency = resolved.currency + resolved = await resolve_slice(session, p.slice) + if resolved is None or not resolved.yahoo_ticker: + unmapped.append(p.slice or p.name or "?") + continue positions_out.append({ - "yahoo_ticker": yahoo_ticker, + "yahoo_ticker": resolved.yahoo_ticker, "t212_slice": p.slice, - "name": name, + "name": resolved.name or p.name, "qty": p.quantity, "avg_cost": p.average_price, # @property — no call parens - "currency": currency, + "currency": resolved.currency, }) - yahoo_tickers.append(yahoo_ticker) + yahoo_tickers.append(resolved.yahoo_ticker) # Synchronous upsert: bypass the Redis buffer so the dashboard has # live prices immediately. The buffer + flush machinery remains for @@ -351,7 +321,7 @@ async def analyze_portfolio( is persisted. The ai_calls ledger row records tokens + cost, never holdings. - Gated behind ``require_paid``: free-tier users get 402. + Gated behind ``require_paid`` (Phase D.2): free-tier users get 402. Admin bearer-token bypasses the gate for testing.""" # Read JSON body manually so we can enforce a hard size cap. FastAPI's # default body limit is generous; we want tighter control here. diff --git a/app/scheduler_main.py b/app/scheduler_main.py index 54dec6d..e20d15e 100644 --- a/app/scheduler_main.py +++ b/app/scheduler_main.py @@ -13,7 +13,7 @@ from app.db import get_engine from app.logging import configure_logging, get_logger from app.jobs import ( market_job, news_job, ai_log_job, rollup_job, - indicator_summary_job, universe_flush_job, email_digest_job, + indicator_summary_job, universe_flush_job, ) @@ -58,11 +58,6 @@ async def main() -> None: sched.add_job(universe_flush_job.evict_run, CronTrigger(hour=0, minute=15), name="universe_evict_job", id="universe_evict_job") - # Editorial email digests: daily Mon-Sat for paid opt-in, weekly Sunday - # recap for everyone opt-in. Job decides which kind based on weekday. - sched.add_job(email_digest_job.run, - CronTrigger(hour=6, minute=30), - name="email_digest_job", id="email_digest_job") sched.start() log.info("scheduler.started", jobs=[j.id for j in sched.get_jobs()]) diff --git a/app/services/access.py b/app/services/access.py index 2f91f7a..9066f1d 100644 --- a/app/services/access.py +++ b/app/services/access.py @@ -2,12 +2,11 @@ Two sources can grant paid access: -1. ``user.tier in {"paid", "enterprise"}`` — set by the Stripe webhook - once a subscription is active. -2. ``user.credit_until > now()`` — non-subscription credit. Populated - by the admin CLI (``python -m app.cli grant-credit``) and by the - referral-conversion path (45 days per converted referral, both - parties). +1. ``user.tier in {"paid", "enterprise"}`` — set by Paddle webhook in + Phase D.3 once a subscription is active. +2. ``user.credit_until > now()`` — non-subscription credit. Currently + populated by the admin CLI (`python -m app.cli grant-credit`) and, in + D.3, by the referral-conversion path (3 months at 50% off). Either is sufficient. We use a single ``paid_status`` function so the Settings page can show *why* a user has paid access ("paid subscription" @@ -23,17 +22,6 @@ from fastapi import Depends, HTTPException, status from app.auth import CurrentUser, require_auth from app.models import User -# How many hours of news the free tier sees. Paid sees whatever the -# endpoint's `since_hours` param requests (up to its own max). -FREE_NEWS_WINDOW_HOURS = 6.0 - -# The strategic-log job runs at :20 every hour (during trading windows). -# Free-tier users only see logs generated at these UTC hours — so the -# log refreshes for them roughly every 6 hours (00:20, 06:20, 12:20, -# 18:20). Paid users see the absolute latest log. Filtering happens -# read-side; we don't generate per-tier rows. -FREE_LOG_HOURS_UTC: tuple[int, ...] = (0, 6, 12, 18) - def _utcnow() -> datetime: return datetime.now(timezone.utc) diff --git a/app/services/csv_import.py b/app/services/csv_import.py index cacd84d..770ff1e 100644 --- a/app/services/csv_import.py +++ b/app/services/csv_import.py @@ -37,10 +37,7 @@ _REQUIRED_FIELDS = ("slice", "quantity") @dataclass(frozen=True) class ParsedPosition: - slice: str # T212 shortcode (e.g. "SGLN") or a - # Yahoo-ready ticker (e.g. "VOD.L") - # when produced by the LLM path — - # see ParsedPie.tickers_resolved. + slice: str # T212 shortcode, e.g. "SGLN" name: str invested_value: float | None current_value: float | None @@ -49,10 +46,6 @@ class ParsedPosition: dividends_gained: float | None = None dividends_cash: float | None = None dividends_reinvested: float | None = None - currency: str | None = None # Populated by the LLM path from the - # mapped currency_col; the T212 path - # leaves it None and gets currency - # from the InstrumentMap row. @property def average_price(self) -> float | None: @@ -74,11 +67,6 @@ class ParsedPie: invested: float | None # totals from the Total row value: float | None result: float | None - tickers_resolved: bool = False # True when ``slice`` on each position - # is already a Yahoo-ready ticker - # (LLM path). False (default) means - # tickers must still be resolved via - # the T212 InstrumentMap. def _normalise_header(h: str) -> str: diff --git a/app/services/email_service.py b/app/services/email_service.py index d3ed9f7..274e526 100644 --- a/app/services/email_service.py +++ b/app/services/email_service.py @@ -18,8 +18,6 @@ convenient for local dev that doesn't want a mail server configured. """ from __future__ import annotations -import html as _html_lib -import re as _re from email.message import EmailMessage import aiosmtplib @@ -198,231 +196,3 @@ def render_otp_email(code: str, ttl_minutes: int) -> tuple[str, str, str]: async def send_otp(to: str, code: str, ttl_minutes: int) -> None: subject, text, html = render_otp_email(code, ttl_minutes) await send_email(to, subject, text, html_body=html) - - -# --------------------------------------------------------------------------- -# Welcome email — sent once on first successful login. -# --------------------------------------------------------------------------- - - -_WELCOME_HTML_TEMPLATE = """\ - - - - - - - - Welcome to {brand} - - - - - -
-
- ▰ {brand_upper} -
-
 
-
- Welcome to {brand}. -
-
 
-
- You’re signed in. The dashboard is at - {app_url_short} — - a rolling news feed, cross-asset indicator panels, and a written - strategic read of the session, all updated through the day. -
-
 
-
-
 
-
- About the email digest -
-
 
-
- We send one Sunday digest to every account — the week behind + - the week ahead. Paid subscribers also get a short daily digest - (Mon–Sat), each a ~600-word read of the session. - You’re opted in by default; you can switch the digest off - at any time on the - Settings page, - or use the one-click unsubscribe link in every digest email. -
-
 
-
-
 
-
- Sent automatically by {brand} · do not reply -
-
- - -""" - - -_WELCOME_TEXT_TEMPLATE = """\ -{brand_upper} — welcome - -You're signed in. The dashboard is at {app_url}: a rolling news feed, -cross-asset indicator panels, and a written strategic read of the -session, all updated through the day. - -About the email digest ----------------------- -We send one Sunday digest to every account (the week behind + the -week ahead). Paid subscribers also get a short daily digest (Mon-Sat), -each a ~600-word read of the session. - -You're opted in by default; switch it off any time at {settings_url}, -or use the one-click unsubscribe link in every digest email. - -— -Sent automatically by {brand} · do not reply -""" - - -def render_welcome_email() -> tuple[str, str, str]: - """Returns (subject, text_body, html_body) for the post-signup welcome. - - Single-shot email, sent the first time a user successfully verifies - an OTP. Explains the digest (which is opt-in by default) and how to - turn it off — replaces the old verify-page checkbox which appeared - on every login and was misleading.""" - subject = f"Welcome to {branding.BRAND_NAME}" - fmt = dict( - brand=branding.BRAND_NAME, - brand_upper=branding.BRAND_NAME.upper(), - app_url=branding.APP_URL, - app_url_short=branding.APP_URL.replace("https://", "").replace("http://", ""), - settings_url=f"{branding.APP_URL}/settings", - FONT_MONO=branding.FONT_MONO, - **{f"L_{k.replace('-', '_')}": v for k, v in branding.LIGHT.items()}, - **{f"D_{k.replace('-', '_')}": v for k, v in branding.DARK.items()}, - ) - text = _WELCOME_TEXT_TEMPLATE.format(**fmt) - html = _WELCOME_HTML_TEMPLATE.format(**fmt) - return subject, text, html - - -async def send_welcome_email(to: str) -> None: - subject, text, html = render_welcome_email() - await send_email(to, subject, text, html_body=html) - - -# --------------------------------------------------------------------------- -# Digest email rendering -# --------------------------------------------------------------------------- - - -_DIGEST_HTML_TEMPLATE = """\ - - - - - - - {brand} — {label} - - - - - -
-
- ▰ {brand_upper} · {label_upper} -
-
 
-
- {content_html} -
-
 
-
-
 
-
- Unsubscribe in one click - · Manage preferences -
-
- - -""" - - -def _strip_html_to_text(html_body: str) -> str: - """Best-effort HTML → plain text for the multipart fallback. We don't - need perfection — just readable prose for clients that won't render - HTML.""" - text = _re.sub(r"(?i)<(/(p|h[1-6]|li|ul|ol)|br\s*/?)>", "\n", html_body) - text = _re.sub(r"<[^>]+>", "", text) - text = _html_lib.unescape(text) - text = _re.sub(r"\n{3,}", "\n\n", text) - return text.strip() - - -def render_digest_email( - *, - kind: str, - date_str: str, - content_html: str, - unsubscribe_url: str, - settings_url: str, -) -> tuple[str, str, str]: - """Returns (subject, text_body, html_body) for a digest email. - - `kind` is "daily" or "weekly". Anything else raises ValueError.""" - if kind == "daily": - label = "Daily" - subject = f"{branding.BRAND_NAME} · Daily — {date_str}" - elif kind == "weekly": - label = "Weekly recap" - subject = f"{branding.BRAND_NAME} · Weekly recap — {date_str}" - else: - raise ValueError(f"unknown digest kind: {kind!r}") - - html_body = _DIGEST_HTML_TEMPLATE.format( - brand=branding.BRAND_NAME, - brand_upper=branding.BRAND_NAME.upper(), - label=label, - label_upper=label.upper(), - FONT_MONO=branding.FONT_MONO, - content_html=content_html, - unsubscribe_url=unsubscribe_url, - settings_url=settings_url, - **{f"L_{k.replace('-', '_')}": v for k, v in branding.LIGHT.items()}, - **{f"D_{k.replace('-', '_')}": v for k, v in branding.DARK.items()}, - ) - - text_lines = [ - f"{branding.BRAND_NAME} — {label}", - date_str, - "", - _strip_html_to_text(content_html), - "", - f"Unsubscribe: {unsubscribe_url}", - f"Manage preferences: {settings_url}", - ] - text_body = "\n".join(text_lines) - return subject, text_body, html_body diff --git a/app/services/llm_csv_parser.py b/app/services/llm_csv_parser.py deleted file mode 100644 index 7bb84af..0000000 --- a/app/services/llm_csv_parser.py +++ /dev/null @@ -1,431 +0,0 @@ -"""LLM-fallback CSV parser. - -When the deterministic Trading 212 parser (``csv_import.parse_t212_csv``) -raises ``CSVImportError`` on an unrecognised format, this service kicks -in: - -1. Detect the CSV dialect (delimiter, preamble offset). -2. Compute a fingerprint of the normalised header row. -3. Look up ``CsvFormatTemplate`` by fingerprint. On hit, replay the - cached column-mapping deterministically. On miss, ask the LLM for a - mapping, validate it, persist a new template, and apply it. - -The LLM sees only headers + the first 3-5 sample rows. It returns a -column-mapping JSON, never transcribed numbers. The system never -auto-promotes a learned format to a hand-written parser — the operator -does that by inspecting collected ``sample_row`` values. -""" -from __future__ import annotations - -import csv -import hashlib -import io -import json - -import httpx -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import utcnow -from app.logging import get_logger -from app.models import CsvFormatTemplate -from app.services.csv_import import CSVImportError, ParsedPie, ParsedPosition -from app.services.openrouter import LogResult, call_llm - -# --------------------------------------------------------------------------- -# Module-level constants -# --------------------------------------------------------------------------- - -# Cap for how many leading lines we'll scan looking for the header row. -# Real broker preambles are typically 1-10 lines. -_MAX_PREAMBLE_SCAN = 30 - -# Number of sample rows to send to the LLM and max token budget for the reply. -_LLM_SAMPLES = 5 -_LLM_MAX_TOKENS = 400 - -# Required and optional keys in the LLM-returned column mapping. -_REQUIRED_MAPPING_KEYS = ("ticker_col", "qty_col") -_OPTIONAL_MAPPING_KEYS = ("name_col", "cost_col", "currency_col") - -# Maximum CSV payload size accepted by parse_with_llm. -_MAX_CSV_BYTES = 1_048_576 - -log = get_logger("llm_csv_parser") - - -_SYSTEM_PROMPT = """\ -You are an expert at recognising broker portfolio CSV formats. - -You will be given the header row and 3-5 sample data rows from a CSV. -Identify which column contains each field. Return ONLY a single JSON -object, no prose, no markdown fences. - -Schema (use the EXACT header string from the input; use null if no -column is a good match): - -{ - "ticker_col": "
", - "qty_col": "
", - "name_col": "
", - "cost_col": "
", // average price per share - "currency_col": "
", - "broker_label": "" -} - -Rules: -- ticker_col and qty_col are required. If either is missing, return all nulls. -- Use the EXACT header string as it appears in the input — do not paraphrase. -- Output JSON ONLY. No prose, no code fences. -""" - - -class LLMParseError(CSVImportError): - """Raised when the LLM call fails or returns an unusable mapping. - - Inherits from ``CSVImportError`` so route-level error handling can - treat both deterministic and LLM-path failures uniformly when - desired.""" - - -def _fingerprint(headers: list[str]) -> str: - """Stable hash of the header row. - - Lowercases each header, strips surrounding whitespace, joins with - ``|`` (a character extremely unlikely to appear inside a real - header), and returns the sha256 hex digest. Whitespace/case drift - in the same broker's export does not change the fingerprint; - adding or removing a column does.""" - normalised = "|".join(h.strip().lower() for h in headers) - return hashlib.sha256(normalised.encode("utf-8")).hexdigest() - - -def _decode_raw(raw: bytes) -> str: - """Best-effort UTF-8 decode with BOM strip and lossy fallback.""" - return raw.decode("utf-8-sig", errors="replace") - - -def _looks_numeric(value: str) -> bool: - """True if ``value`` parses as a number after stripping common - decoration (thousands separators, currency symbols, percent signs).""" - s = value.strip().replace(",", "").replace("$", "").replace("€", "") - s = s.replace("£", "").replace("%", "").lstrip("-+") - if not s: - return False - try: - float(s) - return True - except ValueError: - return False - - -def _detect_dialect(raw: bytes) -> tuple[str, int]: - """Detect (delimiter, preamble_rows). - - ``preamble_rows`` is the number of lines BEFORE the row we identify - as the actual table header. The header row is the first line whose - tokens are all non-numeric (so "Symbol,Quantity" is a header but - "AAPL,100" is data). Falls back to assuming the first line is the - header if no clear non-numeric line is found within the scan - window. - - Raises ``LLMParseError`` on empty input.""" - if not raw or not raw.strip(): - raise LLMParseError("empty CSV") - - text = _decode_raw(raw) - # csv.Sniffer is happy with ~4KB. Anything more and it gets slow. - sample = text[:4096] - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") - delimiter = dialect.delimiter - except csv.Error: - # Most broker exports are comma-delimited; default rather than - # error out — the caller will still validate column shapes. - delimiter = "," - - rows = list(csv.reader(io.StringIO(text), delimiter=delimiter)) - # Build a flat list of (index, non_empty_tokens) for rows within scan limit - parsed = [] - for i, row in enumerate(rows): - if i >= _MAX_PREAMBLE_SCAN: - break - non_empty = [c.strip() for c in row if c.strip()] - parsed.append((i, non_empty)) - - # Find the first all-alpha candidate row that is followed by a data - # row (one that contains at least one numeric token). This - # distinguishes real header rows from preamble metadata rows that - # also happen to be all-text. - for idx, (i, non_empty) in enumerate(parsed): - if len(non_empty) < 2: - continue - all_alpha = all(not _looks_numeric(c) for c in non_empty) - if not all_alpha: - continue - # Check whether the next non-empty row looks like data (has a numeric) - for _, next_non_empty in parsed[idx + 1:]: - if not next_non_empty: - continue - if any(_looks_numeric(c) for c in next_non_empty): - return delimiter, i - # Next row is also all-alpha → keep scanning - break - return delimiter, 0 - - -def _validate_mapping( - mapping: dict, headers: list[str], first_row: list[str], -) -> None: - """Verify the LLM-returned mapping is sane. - - - ``ticker_col`` and ``qty_col`` are required (non-null). - - Every named column must exist in ``headers``. - - The value at ``qty_col`` on ``first_row`` must parse as a number. - - The value at ``cost_col`` on ``first_row`` (if present) must parse - as a number. - - Raises ``LLMParseError`` on any failure, with a message that names - the specific problem (helpful for log forensics and for the - user-facing 400).""" - for key in _REQUIRED_MAPPING_KEYS: - if not mapping.get(key): - raise LLMParseError( - f"LLM mapping missing required column: {key.replace('_col', '')}" - ) - - headers_set = set(headers) - for key in _REQUIRED_MAPPING_KEYS + _OPTIONAL_MAPPING_KEYS: - col = mapping.get(key) - if col is not None and col not in headers_set: - raise LLMParseError( - f"LLM mapping references unknown column: {col!r}" - ) - - # Numeric sanity check: qty and (if present) cost must parse on row 1. - header_index = {h: i for i, h in enumerate(headers)} - qty_col = mapping["qty_col"] - qty_value = first_row[header_index[qty_col]] if header_index[qty_col] < len(first_row) else "" - if not _looks_numeric(qty_value): - raise LLMParseError( - f"LLM mapping qty_col={qty_col!r} maps to non-numeric value {qty_value!r}" - ) - - cost_col = mapping.get("cost_col") - if cost_col is not None: - cost_value = first_row[header_index[cost_col]] if header_index[cost_col] < len(first_row) else "" - if cost_value and not _looks_numeric(cost_value): - raise LLMParseError( - f"LLM mapping cost_col={cost_col!r} maps to non-numeric value {cost_value!r}" - ) - - -def _parse_number(value: str) -> float | None: - """Permissive float parse: strips thousands separators, currency - symbols, percent signs. Returns None on failure (so callers can - decide whether to skip or raise).""" - s = value.strip().replace(",", "").replace("$", "") - s = s.replace("€", "").replace("£", "").replace("%", "") - if not s: - return None - try: - return float(s) - except ValueError: - return None - - -def _apply_mapping( - headers: list[str], - data_rows: list[list[str]], - mapping: dict, -) -> ParsedPie: - """Iterate ``data_rows`` and produce a ``ParsedPie``. - - Rows that lack a parseable quantity (blank, non-numeric, zero) are - silently skipped — broker exports often include summary or - placeholder rows after the position list. ``name_col`` falls back - to the ticker symbol when null.""" - idx = {h: i for i, h in enumerate(headers)} - ticker_col = mapping["ticker_col"] - qty_col = mapping["qty_col"] - name_col = mapping.get("name_col") - cost_col = mapping.get("cost_col") - currency_col = mapping.get("currency_col") - - positions: list[ParsedPosition] = [] - invested_total = 0.0 - invested_seen = False - - for row in data_rows: - if not any(c.strip() for c in row): - continue - ticker_raw = row[idx[ticker_col]] if idx[ticker_col] < len(row) else "" - ticker = ticker_raw.strip().upper() - if not ticker: - continue - qty_raw = row[idx[qty_col]] if idx[qty_col] < len(row) else "" - qty = _parse_number(qty_raw) - if qty is None or qty <= 0: - continue - avg_cost: float | None = None - if cost_col is not None and idx[cost_col] < len(row): - avg_cost = _parse_number(row[idx[cost_col]]) - invested_value: float | None = None - if avg_cost is not None: - invested_value = qty * avg_cost - invested_total += invested_value - invested_seen = True - name = "" - if name_col is not None and idx[name_col] < len(row): - name = row[idx[name_col]].strip() - if not name: - name = ticker - currency: str | None = None - if currency_col is not None and idx[currency_col] < len(row): - currency = row[idx[currency_col]].strip() or None - positions.append(ParsedPosition( - slice=ticker, - name=name, - invested_value=invested_value, - current_value=None, - result=None, - quantity=qty, - currency=currency, - )) - - return ParsedPie( - name=None, - positions=tuple(positions), - invested=(invested_total if invested_seen else None), - value=None, - result=None, - tickers_resolved=True, - ) - - -def _build_user_prompt(headers: list[str], samples: list[list[str]]) -> str: - lines = ["headers: " + json.dumps(headers)] - lines.append("samples:") - for s in samples[:_LLM_SAMPLES]: - lines.append(" " + ",".join(s)) - return "\n".join(lines) - - -async def _extract_mapping_via_llm( - client: httpx.AsyncClient, - headers: list[str], - samples: list[list[str]], -) -> tuple[dict, LogResult]: - """Single LLM call returning ``(mapping_dict, LogResult)``. - - The LLM is asked for a strict JSON object (no markdown). We attempt - to parse the returned content; ``LLMParseError`` wraps any failure - in a way callers can surface to the user.""" - messages = [ - {"role": "system", "content": _SYSTEM_PROMPT}, - {"role": "user", "content": _build_user_prompt(headers, samples)}, - ] - try: - result = await call_llm(client, messages, max_tokens=_LLM_MAX_TOKENS) - except Exception as e: - raise LLMParseError(f"LLM provider failed: {e}") from e - - content = (result.content or "").strip() - # Strip code fences if the model added them despite instructions. - if content.startswith("```"): - content = content.strip("`") - # Drop optional 'json' language tag. - if content.lstrip().lower().startswith("json"): - content = content.lstrip()[4:] - content = content.strip() - try: - mapping = json.loads(content) - except json.JSONDecodeError as e: - raise LLMParseError(f"LLM did not return valid JSON: {e}") from e - if not isinstance(mapping, dict): - raise LLMParseError("LLM JSON was not an object") - return mapping, result - - -async def parse_with_llm(raw: bytes, session: AsyncSession) -> ParsedPie: - """Cache-first LLM-fallback CSV parse. - - On cache hit, applies the stored mapping deterministically and - increments ``use_count``. On cache miss, calls the LLM, validates - the returned mapping against the first data row, and persists a - new ``CsvFormatTemplate``. Raises ``LLMParseError`` on any - failure; the caller (route layer) maps that to a 400.""" - if len(raw) > _MAX_CSV_BYTES: - raise LLMParseError("CSV too large (1 MB max)") - if not raw or not raw.strip(): - raise LLMParseError("empty CSV") - - delimiter, preamble_rows = _detect_dialect(raw) - text = _decode_raw(raw) - - reader = csv.reader(io.StringIO(text), delimiter=delimiter) - rows = list(reader) - if preamble_rows >= len(rows): - raise LLMParseError("no header row found in CSV") - headers = [c.strip() for c in rows[preamble_rows]] - data_rows = rows[preamble_rows + 1:] - if not headers: - raise LLMParseError("empty header row") - - first_data_row = next( - (r for r in data_rows if any(c.strip() for c in r)), None, - ) - if first_data_row is None: - raise LLMParseError("CSV contains a header but no data rows") - - fp = _fingerprint(headers) - existing = (await session.execute( - select(CsvFormatTemplate).where(CsvFormatTemplate.fingerprint == fp) - )).scalar_one_or_none() - - if existing is not None: - log.info("csv.format.cache_hit", fingerprint=fp, - broker_label=existing.broker_label, use_count=existing.use_count) - pie = _apply_mapping(headers, data_rows, existing.mapping) - if not pie.positions: - raise LLMParseError( - "cached mapping produced no positions — the broker may have " - "changed their CSV shape; ask the operator to evict the " - "stale template" - ) - existing.use_count += 1 - existing.last_used_at = utcnow() - await session.commit() - return pie - - log.info("csv.format.cache_miss", fingerprint=fp, - header_count=len(headers)) - samples = [r for r in data_rows[:_LLM_SAMPLES] if any(c.strip() for c in r)] - async with httpx.AsyncClient(follow_redirects=True, timeout=30) as client: - mapping, llm_log = await _extract_mapping_via_llm(client, headers, samples) - _validate_mapping(mapping, headers, first_data_row) - - pie = _apply_mapping(headers, data_rows, mapping) - if not pie.positions: - raise LLMParseError( - "LLM mapping validated but produced no positions — the file " - "may not contain portfolio data" - ) - - now = utcnow() - session.add(CsvFormatTemplate( - fingerprint=fp, - headers=headers, - sample_row=first_data_row, - mapping=mapping, - preamble_rows=preamble_rows, - delimiter=delimiter, - broker_label=mapping.get("broker_label"), - first_seen_at=now, - last_used_at=now, - use_count=1, - llm_model=llm_log.model, - llm_cost_usd=llm_log.cost_usd, - )) - await session.commit() - return pie diff --git a/app/services/openrouter.py b/app/services/openrouter.py index a542b98..759e9f5 100644 --- a/app/services/openrouter.py +++ b/app/services/openrouter.py @@ -30,8 +30,7 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions" # v7 (2026-05-18): Forbid "(Updated HH:MM UTC)" clauses in the date header — # the model was hallucinating future times. The user prompt now carries the # actual current UTC time so the model has accurate temporal context. -# v9 (2026-05-25): Adds daily + weekly digest prompt builders for email. -PROMPT_VERSION = 9 +PROMPT_VERSION = 8 # --- Core: invariant across tone/analysis settings ---------------------------- @@ -508,107 +507,6 @@ def build_user_prompt( return "\n".join(parts) -def _digest_tone_clause(tone: str) -> str: - if tone.upper() == "NOVICE": - return "Use plain English. Define any jargon on first use." - return "Write for a reader who already speaks markets fluently." - - -def build_daily_digest_prompt( - *, - tone: str, - today, - quotes_by_group: dict, - headlines_by_bucket: dict, - reference_line: str, -) -> tuple[str, str]: - """System + user prompt for the once-a-day editorial digest. - - Different from the hourly log: the daily digest reflects on the past - 24h and looks forward to the upcoming session. Longer, less - 'live-blogging,' more contextual. Target ~600 words.""" - system = ( - "You write the daily editorial digest for Read the Markets. " - f"Audience tone: {tone.upper()}. {_digest_tone_clause(tone)} " - "Cover: (1) what mattered yesterday, (2) what to watch in today's " - "EU and US sessions, (3) one cross-asset thread connecting them. " - "No predictions of price level, no buy/sell language. Target ~600 " - "words. Output HTML using only

,

,
    ,
  • , , " - " — no , , or wrapper, no inline styles." - ) - user = _digest_user_prompt( - today=today, quotes_by_group=quotes_by_group, - headlines_by_bucket=headlines_by_bucket, reference_line=reference_line, - ) - return system, user - - -def build_weekly_digest_prompt( - *, - tone: str, - today, - quotes_by_group: dict, - headlines_by_bucket: dict, - reference_line: str, -) -> tuple[str, str]: - """System + user prompt for the Sunday weekly recap + look-ahead. - - Sent to ALL opt-in users (free and paid). Target ~900 words.""" - system = ( - "You write the Sunday weekly digest for Read the Markets. " - f"Audience tone: {tone.upper()}. {_digest_tone_clause(tone)} " - "Cover: (1) the week behind — what moved and why, " - "(2) the week ahead — releases, earnings, central-bank meetings, " - "(3) the cross-asset story to keep in mind. " - "No predictions of price level, no buy/sell language. Target ~900 " - "words. Output HTML using only

    ,

    ,
      ,
    • , , " - " — no , , or wrapper, no inline styles." - ) - user = _digest_user_prompt( - today=today, quotes_by_group=quotes_by_group, - headlines_by_bucket=headlines_by_bucket, reference_line=reference_line, - ) - return system, user - - -def _digest_user_prompt( - *, - today, - quotes_by_group: dict, - headlines_by_bucket: dict, - reference_line: str, -) -> str: - """Shared user-message body used by both digest prompts. Same data - shape as the hourly user prompt; reformatted for the digest context.""" - today_str = today.strftime("%A %d %B %Y") if hasattr(today, "strftime") else str(today) - lines = [f"TODAY (UTC): {today_str}", "", f"REFERENCE: {reference_line}", ""] - - if headlines_by_bucket: - lines.append("HEADLINES BY CATEGORY") - for cat, items in headlines_by_bucket.items(): - lines.append(f" [{cat}]") - for h in items[:30]: - when = h.get("when", "") - src = h.get("source", "") - title = h.get("title", "") - lines.append(f" {when} · {src} · {title}") - lines.append("") - - if quotes_by_group: - lines.append("LATEST QUOTES BY GROUP") - for grp, items in quotes_by_group.items(): - lines.append(f" [{grp}]") - for q in items[:30]: - sym = q.get("symbol", "") - price = q.get("price", "") - lbl = q.get("label", "") - ccy = q.get("currency", "") - lines.append(f" {sym} ({lbl}) — {price} {ccy}") - lines.append("") - - return "\n".join(lines) - - def _provider_chain() -> list[str]: """Ordered list of providers to try: primary, then fallback (unless the fallback is unset, the same as primary, or has no API key).""" diff --git a/app/services/portfolio_sync.py b/app/services/portfolio_sync.py index c0bbbe9..15c9c41 100644 --- a/app/services/portfolio_sync.py +++ b/app/services/portfolio_sync.py @@ -44,16 +44,8 @@ RATE_LIMIT_MAX = 6 class SyncCryptoError(Exception): - """Outer-wrap decryption failed even though the pepper fingerprint - matched — i.e. genuine corruption or tampering. The router maps this - to a 500.""" - - -class SyncOrphanedError(Exception): - """The row was wrapped with a different pepper than the one currently - configured (typically: dev-time pepper rotation). The data is - permanently unrecoverable, but this is a *known* state, not a server - fault — the router maps this to a 410 Gone.""" + """Outer-wrap decryption failed — usually a pepper change or + bit-rotted row. The router maps this to a 500.""" def _utcnow() -> datetime: @@ -80,22 +72,6 @@ def _server_key(user_id: int) -> bytes: ).derive(_pepper_bytes()) -_FP_LEN = 8 - - -def current_pepper_fp() -> bytes: - """8-byte HKDF-derived fingerprint of the current pepper. Doesn't - leak the pepper itself (HKDF is one-way) and is short enough to make - accidental collisions across rotations effectively zero (2^-32 birthday - floor — fine for a few-row dev install).""" - return HKDF( - algorithm=hashes.SHA256(), - length=_FP_LEN, - salt=b"portfolio-sync-pepper-fp", - info=b"v1", - ).derive(_pepper_bytes()) - - def wrap(user_id: int, inner_blob: bytes) -> tuple[bytes, bytes]: """Encrypt the client-side ciphertext (`inner_blob`) for storage. Returns (outer_ct, outer_nonce). The nonce is random per write.""" @@ -105,15 +81,9 @@ def wrap(user_id: int, inner_blob: bytes) -> tuple[bytes, bytes]: def unwrap(user_id: int, outer_ct: bytes, outer_nonce: bytes) -> bytes: - """Inverse of wrap(). Raises SyncCryptoError if the GCM tag fails. - - AESGCM.decrypt takes (nonce, data, associated_data) — not - (data, nonce). The original implementation had the arguments - swapped, which meant restore-from-cloud always failed even when - the pepper was correct. - """ + """Inverse of wrap(). Raises SyncCryptoError if the GCM tag fails.""" try: - return AESGCM(_server_key(user_id)).decrypt(outer_nonce, outer_ct, None) + return AESGCM(_server_key(user_id)).decrypt(outer_ct, outer_nonce, None) except Exception as exc: # InvalidTag, malformed ciphertext, etc. raise SyncCryptoError("outer wrap unwrap failed") from exc @@ -121,7 +91,6 @@ def unwrap(user_id: int, outer_ct: bytes, outer_nonce: bytes) -> bytes: async def upsert(session: AsyncSession, user_id: int, inner_blob: bytes) -> datetime: """Insert or replace this user's sync row. Returns the new updated_at.""" outer_ct, outer_nonce = wrap(user_id, inner_blob) - fp = current_pepper_fp() now = _utcnow() row = await session.get(PortfolioSync, user_id) if row is None: @@ -132,7 +101,6 @@ async def upsert(session: AsyncSession, user_id: int, inner_blob: bytes) -> date version=1, created_at=now, updated_at=now, - pepper_fp=fp, ) session.add(row) else: @@ -141,34 +109,19 @@ async def upsert(session: AsyncSession, user_id: int, inner_blob: bytes) -> date row.updated_at = now # Bump version field forward if we ever change the wrap scheme. row.version = 1 - row.pepper_fp = fp await session.commit() return now -def _is_orphaned(row: PortfolioSync) -> bool: - """A row is orphaned when its stored pepper fingerprint is present - and differs from the current pepper's fingerprint. NULL fingerprint - (rows from before the pepper_fp column existed) is treated - optimistically: we don't know whether the pepper rotated, so we let - the fetch path probe with a real unwrap and self-heal on success. - Status returns orphaned=False for NULL so the user is offered the - Restore form; if unwrap then fails, the GET path returns 410 and the - UI flips to the stale state.""" - return row.pepper_fp is not None and row.pepper_fp != current_pepper_fp() - - async def fetch_status( session: AsyncSession, user_id: int, -) -> tuple[bool, bool, datetime | None]: - """Cheap existence check — does NOT decrypt. Returns - (exists, orphaned, updated_at). Used by the dashboard to decide - whether to show the restore prompt vs the "stale, re-upload" prompt. - """ +) -> tuple[bool, datetime | None]: + """Cheap existence check — does NOT decrypt. Used by the dashboard to + decide whether to show the restore prompt.""" row = await session.get(PortfolioSync, user_id) if row is None: - return False, False, None - return True, _is_orphaned(row), row.updated_at + return False, None + return True, row.updated_at async def fetch( @@ -176,36 +129,13 @@ async def fetch( ) -> tuple[bytes, datetime] | None: """Returns (inner_blob, updated_at) or None if sync disabled. - Raises SyncOrphanedError if the row's pepper fingerprint mismatches - the current pepper, OR if a fingerprint-less legacy row fails to - unwrap (which can only mean a pepper rotation, since the arg-order - bug fix landed alongside the fingerprint column). - - Raises SyncCryptoError if the fingerprint matched but the outer wrap - still failed (genuine corruption or tampering). - - On a successful unwrap of a fingerprint-less legacy row, the current - pepper's fingerprint is backfilled so subsequent status checks - correctly report healthy (and future rotations are detectable). + Raises SyncCryptoError if the row exists but the outer wrap is + unreadable (typically: pepper was rotated without re-encrypting). """ row = await session.get(PortfolioSync, user_id) if row is None: return None - if _is_orphaned(row): - raise SyncOrphanedError("pepper fingerprint mismatch") - legacy = row.pepper_fp is None - try: - inner = unwrap(user_id, row.outer_ciphertext, row.outer_nonce) - except SyncCryptoError: - if legacy: - # Legacy row + decrypt fails = pepper rotated before the - # fingerprint column existed. Same observable state as a - # post-fingerprint orphan; report it that way. - raise SyncOrphanedError("legacy row, decrypt failed") - raise - if legacy: - row.pepper_fp = current_pepper_fp() - await session.commit() + inner = unwrap(user_id, row.outer_ciphertext, row.outer_nonce) return inner, row.updated_at diff --git a/app/services/referral_service.py b/app/services/referral_service.py index 05ee579..5f663e6 100644 --- a/app/services/referral_service.py +++ b/app/services/referral_service.py @@ -1,18 +1,15 @@ -"""Referral-code generation, lookup, signup-time linkage, and -conversion-time credit grants. +"""Referral-code generation, lookup, and signup-time linkage. -The flow: +D.1 lays down the bookkeeping only — actual credit application happens +in D.3 when the Paddle webhook fires. The flow: 1. /login renders an "invited" banner when the URL carries `?ref=`. 2. The code travels through the email-OTP flow inside the pending cookie so it survives the GET /login → POST /login → /verify hops. 3. When the new user's row is first created (POST /login on an unknown email), `referred_by_user_id` is set and a `Referral` row is written. -4. On the referred user's first paid subscription, `convert_referral` - is called from the Stripe webhook: both parties get a credit-window - extension worth the promised "50% off for 3 months" (= 45 days of - full paid access via `users.credit_until`), and the Referral row's - `converted_at` + `credited_at` are stamped. +4. On the new user's first paid subscription (D.3), we read the + `Referral` row to apply discounts to both parties. The code itself is 8 characters from an unambiguous alphabet so users can read it off a phone screen or dictate it over the phone. @@ -20,7 +17,6 @@ can read it off a phone screen or dictate it over the phone. from __future__ import annotations import secrets -from datetime import timedelta from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession @@ -28,7 +24,6 @@ from sqlalchemy.ext.asyncio import AsyncSession from app.db import utcnow from app.logging import get_logger from app.models import Referral, User -from app.services.access import _aware log = get_logger("referral") @@ -40,12 +35,6 @@ log = get_logger("referral") _ALPHABET = "ABCDEFGHJKMNPQRSTUVWXYZ23456789" _CODE_LEN = 8 -# Value-equivalent of the public-facing "50% off for 3 months" promise, -# delivered as a credit-window extension. 50% × 3 months ≈ 1.5 months -# of free service ≈ 45 days. Pure-credit delivery means the mechanism -# is processor-agnostic and stacks cleanly when both parties refer. -REFERRAL_CREDIT_DAYS = 45 - def generate_code() -> str: """Cryptographically random 8-char code from the unambiguous alphabet.""" @@ -139,65 +128,3 @@ async def link_new_user( referrer_id=referrer.id, referred_id=new_user.id, ) return ref - - -def _extend_credit(user: User, days: int) -> None: - """Stack `days` of paid-tier credit onto `user.credit_until`. Anchors - at max(now, current credit_until) so granting twice gives twice the - runway — never shortens the window. Mirrors the cli.grant_credit - anchoring rule so manual + automatic grants compose.""" - now = utcnow() - anchor = max(now, _aware(user.credit_until) or now) - user.credit_until = anchor + timedelta(days=days) - - -async def convert_referral( - session: AsyncSession, referred_user: User, -) -> Referral | None: - """Stamp the Referral row for `referred_user` as converted and grant - both parties their credit. Idempotent — safe to call from every - subscription event: - - - Returns None if no Referral row exists for this user (direct - signup, no inviter). - - Returns the existing Referral (unchanged) if `converted_at` is - already set — this is a renewal or duplicate webhook delivery. - - Otherwise: extends both users' `credit_until` by - REFERRAL_CREDIT_DAYS and sets `converted_at` + `credited_at`. - - The caller is responsible for committing the session — this lets - the Stripe webhook compose the conversion inside its outer - audit-row transaction, so a mid-flight failure rolls back the - tier flip AND the conversion together. - - Self-referral cannot happen here in practice (link_new_user blocks - it at signup) but we guard anyway: if the row somehow names the - same user on both sides, we stamp the timestamps but only credit - once.""" - row = (await session.execute( - select(Referral).where(Referral.referred_user_id == referred_user.id) - )).scalar_one_or_none() - if row is None: - return None - if row.converted_at is not None: - return row - - referrer = await session.get(User, row.referrer_user_id) - now = utcnow() - - # Always credit the buyer; credit the referrer too unless they're - # the same row (defence-in-depth) or have been deleted. - _extend_credit(referred_user, REFERRAL_CREDIT_DAYS) - if referrer is not None and referrer.id != referred_user.id: - _extend_credit(referrer, REFERRAL_CREDIT_DAYS) - - row.converted_at = now - row.credited_at = now - log.info( - "referral.converted", - referral_id=row.id, - referrer_id=row.referrer_user_id, - referred_id=row.referred_user_id, - credit_days=REFERRAL_CREDIT_DAYS, - ) - return row diff --git a/app/static/css/cassandra.css b/app/static/css/cassandra.css index b4b6f6b..e5f1d79 100644 --- a/app/static/css/cassandra.css +++ b/app/static/css/cassandra.css @@ -566,53 +566,6 @@ table.dense tr.row-stale td { color: var(--dim); } .pf-actions button:disabled { opacity: 0.5; cursor: not-allowed; } .pf-actions .pf-secondary { color: var(--muted); } .pf-actions .pf-secondary:hover { color: var(--negative); border-color: var(--negative); } - -/* Settings-page action button — same visual language as .pf-actions - button so buttons across /settings (Manage subscription, future - actions) read as one family. Standalone class (not nested under a - parent) so it can be dropped onto any button anywhere on the page. */ -.settings-btn { - font-family: var(--font-mono); - font-size: 11px; - letter-spacing: 0.06em; - text-transform: uppercase; - background: var(--surface-2); - color: var(--accent); - border: 1px solid var(--border); - padding: 7px 14px; - cursor: pointer; - border-radius: 2px; - text-decoration: none; - display: inline-block; -} -.settings-btn:hover { border-color: var(--accent); } -.settings-btn:disabled { opacity: 0.5; cursor: not-allowed; } - -/* Icon-button variant for inline row actions (e.g. Manage subscription - gear in the Tier row). Square hit area, accent on hover, tooltip via - title attribute. */ -.settings-icon-btn { - background: transparent; - border: 1px solid transparent; - color: var(--muted); - width: 32px; - height: 32px; - padding: 0; - display: inline-flex; - align-items: center; - justify-content: center; - cursor: pointer; - border-radius: 3px; - flex-shrink: 0; - transition: color 80ms linear, border-color 80ms linear, background 80ms linear; -} -.settings-icon-btn:hover { - color: var(--accent); - border-color: var(--border); - background: var(--surface-2); -} -.settings-icon-btn:disabled { opacity: 0.5; cursor: not-allowed; } -.settings-icon-btn svg { display: block; } .pf-analysis { margin-top: 14px; background: var(--surface-2); @@ -728,25 +681,6 @@ details[open] .pf-analysis__head-left::before { content: "▾ "; } .log-page__cal { padding: 10px; } .log-page__content { min-height: 60vh; } .log-page__chat { padding: 8px; min-height: 60vh; display: flex; flex-direction: column; } -.log-page__chat--locked { opacity: 0.92; } -.chat-locked { - flex: 1; - display: flex; - flex-direction: column; - justify-content: center; - align-items: center; - text-align: center; - gap: 16px; - padding: 24px 18px; - color: var(--muted); - font-size: 13px; - line-height: 1.55; - border: 1px dashed var(--border); - border-radius: 4px; - margin: 8px 4px; -} -.chat-locked p { margin: 0; max-width: 280px; } -.chat-locked strong { color: var(--text); display: block; margin-bottom: 6px; } /* --- Calendar widget --------------------------------------------------- */ @@ -906,46 +840,16 @@ details[open] .pf-analysis__head-left::before { content: "▾ "; } letter-spacing: 0.06em; gap: 4px; } -.auth-card input[type="email"], -.auth-card input[type="password"], -.auth-card input[type="text"] { +.auth-card input[type="email"], .auth-card input[type="password"] { background: var(--bg); border: 1px solid var(--border); color: var(--text); font-family: var(--font-mono); - font-size: 16px; - padding: 12px 14px; + font-size: 13px; + padding: 8px 10px; outline: none; - border-radius: 3px; -} -/* The 6-digit OTP input wants to be visually loud — it's the only - thing the user is doing on that page. Bigger, more spacing, taller. */ -.auth-card input[name="code"] { - font-size: 24px; - padding: 16px 14px; - letter-spacing: 0.5em; - text-align: center; } .auth-card input:focus { border-color: var(--accent); } - -/* --- Modal text inputs (cloud-sync PIN modal, etc.) ---------------- */ -/* Same visual treatment as auth-card so prompts read as a coherent - family. Replaces the inline `style="padding:8px"` that left these - inputs feeling cramped. */ -.modal-input { - width: 100%; - background: var(--bg); - border: 1px solid var(--border); - color: var(--text); - font-family: var(--font-mono); - font-size: 16px; - padding: 12px 14px; - margin-bottom: 12px; - outline: none; - border-radius: 3px; - box-sizing: border-box; -} -.modal-input:focus { border-color: var(--accent); } .auth-card button { margin-top: 8px; background: transparent; @@ -1020,13 +924,7 @@ details[open] .pf-analysis__head-left::before { content: "▾ "; } margin-left: 8px; } -/* Sections are
      elements — collapsed by default to keep the - settings page scannable. Click the summary to expand. */ -.settings-section { - margin-top: 14px; - border-top: 1px solid var(--surface-2); - padding-top: 14px; -} +.settings-section { margin-top: 22px; } .settings-section__head { font-family: var(--font-mono); font-size: 11px; @@ -1034,30 +932,8 @@ details[open] .pf-analysis__head-left::before { content: "▾ "; } text-transform: uppercase; color: var(--accent); margin-bottom: 6px; - cursor: pointer; - list-style: none; - user-select: none; - display: flex; - align-items: center; - gap: 8px; - padding: 4px 0; } -/* Suppress the native disclosure marker (Webkit + Firefox). */ -.settings-section__head::-webkit-details-marker { display: none; } -.settings-section__head::marker { content: ""; } -.settings-section__head::before { - content: "▸"; - color: var(--accent); - display: inline-block; - transition: transform 120ms ease-out; - font-size: 10px; -} -.settings-section[open] > .settings-section__head::before { - transform: rotate(90deg); -} -.settings-section[open] > .settings-section__head { margin-bottom: 10px; } -.settings-section__head:hover { color: var(--text); } -.settings-section__head:hover::before { color: var(--text); } +.settings-section__head::before { content: "▸ "; color: var(--accent); } .settings-section__lede { color: var(--muted); font-size: 12.5px; @@ -1651,24 +1527,15 @@ details[open] .pf-analysis__head-left::before { content: "▾ "; } margin: 0 0 24px; } .hero__ctas { display: flex; flex-wrap: wrap; gap: 12px; } -/* Shared button shape — was previously scoped to .hero__ctas, which made - the pricing-card CTAs render as bare anchors. */ -.btn-primary, -.btn-secondary { +.hero__ctas .btn-primary, +.hero__ctas .btn-secondary { display: inline-block; padding: 10px 22px; border-radius: 3px; font-size: 13.5px; font-weight: 500; - line-height: 1.4; text-decoration: none; - text-align: center; - cursor: pointer; } -/* Block variant: full-width within parent, slightly taller — used inside - tier cards so each CTA spans the card and reads as the obvious action. */ -.btn-block { display: block; width: 100%; padding: 12px 22px; font-size: 14px; } - /* Qualify with `a` so we beat `a { color: var(--accent) }` and any :link/:visited UA defaults. Without `a.btn-primary` the cascade can resolve in favour of the visited-link color on some browsers and the @@ -1703,11 +1570,6 @@ a.btn-secondary:hover { color: var(--accent); border-color: var(--accent); } border-radius: 4px; padding: 22px 22px 24px; background: var(--surface); - /* Flex column so the screenshot thumbnail can dock to the bottom via - margin-top:auto — that's what lines the three thumbnails up across - cards regardless of body-text length. */ - display: flex; - flex-direction: column; } .feature-card__tag { font-family: var(--font-mono); @@ -1728,10 +1590,6 @@ a.btn-secondary:hover { color: var(--accent); border-color: var(--accent); } line-height: 1.6; color: var(--muted); margin: 0; - /* Grow to fill the flex column so the thumbnail below docks to the - bottom of the card. With grid-stretched equal-height cards, this is - what aligns the thumbnails across the three cards. */ - flex-grow: 1; } /* --- Section primitives reused across pricing/about/legal ---------- */ @@ -1798,749 +1656,55 @@ a.btn-secondary:hover { color: var(--accent); border-color: var(--accent); } .tier-grid { display: grid; - grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); - gap: 20px; - margin: 8px 0 40px; + grid-template-columns: repeat(auto-fit, minmax(260px, 1fr)); + gap: 18px; + margin: 8px 0 24px; } .tier-card { - position: relative; border: 1px solid var(--border); - border-radius: 6px; - padding: 28px 26px 28px; + border-radius: 4px; + padding: 22px 22px 26px; background: var(--surface); display: flex; flex-direction: column; } .tier-card--featured { border-color: var(--accent); - box-shadow: 0 0 0 1px var(--accent) inset, - 0 12px 32px rgba(15, 23, 42, 0.10); + box-shadow: 0 0 0 1px var(--accent) inset; } -[data-theme="dark"] .tier-card--featured { - box-shadow: 0 0 0 1px var(--accent) inset, - 0 12px 32px rgba(0, 0, 0, 0.45); -} -.tier-card__badge { - position: absolute; - top: -11px; - left: 24px; - background: var(--accent); - color: var(--bg); - font-family: var(--font-mono); - font-size: 10px; - letter-spacing: 0.10em; - text-transform: uppercase; - font-weight: 600; - padding: 4px 10px; - border-radius: 3px; -} -/* Tier name — the actual heading, not the small uppercase chip it used - to be. Pairs with .tier-card__tagline for a one-line value framing. */ .tier-card__name { - font-size: 26px; - font-weight: 700; - letter-spacing: -0.01em; - color: var(--text); - margin: 0 0 4px; - line-height: 1.1; -} -.tier-card__tagline { - font-size: 13px; + font-family: var(--font-mono); + font-size: 11px; color: var(--muted); - line-height: 1.5; - margin-bottom: 22px; + letter-spacing: 0.08em; + text-transform: uppercase; + margin-bottom: 8px; } .tier-card__price { - font-size: 40px; + font-size: 22px; font-weight: 700; color: var(--text); - line-height: 1; - margin-bottom: 8px; - letter-spacing: -0.02em; -} -.tier-card__price-unit { - font-size: 15px; - color: var(--muted); - font-weight: 400; - letter-spacing: 0; + margin-bottom: 4px; } .tier-card__price-hint { font-size: 12px; color: var(--muted); - line-height: 1.55; - margin-bottom: 20px; -} -.tier-card__divider { - height: 1px; - background: var(--border); - margin: 0 0 18px; -} -.tier-card__list-head { - font-family: var(--font-mono); - font-size: 10.5px; - color: var(--muted); - letter-spacing: 0.10em; - text-transform: uppercase; - margin-bottom: 12px; + margin-bottom: 18px; } .tier-card ul { list-style: none; padding: 0; - margin: 0 0 24px; + margin: 0 0 22px; flex: 1; } .tier-card li { font-size: 13.5px; color: var(--text); - line-height: 1.55; - padding: 8px 0 8px 22px; - position: relative; + padding: 6px 0; border-bottom: 1px solid var(--border); } .tier-card li:last-child { border-bottom: 0; } -.tier-card li::before { - content: "✓"; - position: absolute; - left: 0; - top: 8px; - color: var(--positive); - font-weight: 700; -} -.tier-card__cta { margin-top: 18px; } -/* Consent block above the Subscribe buttons (paid card, logged-in - free user). The Subscribe buttons render disabled; ticking the box - is what enables them. Wording covers ToS agreement (both cadences) - + the Reg 36 CCR 2013 waiver (monthly only). */ -.tier-card__consent { - display: flex; - gap: 10px; - align-items: flex-start; - margin-bottom: 14px; - padding: 12px 14px; - background: var(--surface-2); - border: 1px solid var(--border); - border-radius: 4px; - font-size: 12px; - line-height: 1.55; - color: var(--muted); - cursor: pointer; -} -.tier-card__consent input[type="checkbox"] { - flex-shrink: 0; - margin-top: 2px; - cursor: pointer; -} -.tier-card__consent a { - color: var(--accent); - text-decoration: underline; -} -.tier-card__consent strong { color: var(--text); } - -.tier-card__more { - margin-top: 14px; - padding-top: 14px; - border-top: 1px dashed var(--border); - font-size: 12px; - color: var(--muted); - line-height: 1.55; -} - -/* Side-by-side feature comparison table. Lives below the cards and - makes the deltas readable at a glance — the cards sell, the table - confirms. */ -.compare-table { - width: 100%; - border-collapse: collapse; - margin: 0 0 16px; - font-size: 13.5px; -} -.compare-table th, -.compare-table td { - text-align: left; - padding: 12px 14px; - border-bottom: 1px solid var(--border); - vertical-align: top; - line-height: 1.5; -} -.compare-table thead th { - font-family: var(--font-mono); - font-size: 10.5px; - letter-spacing: 0.10em; - text-transform: uppercase; - color: var(--muted); - font-weight: 600; - border-bottom: 1px solid var(--border); -} -.compare-table th[scope="row"] { - font-weight: 500; - color: var(--text); - width: 38%; -} -.compare-table td.compare-table__free { color: var(--muted); } -.compare-table td.compare-table__paid { color: var(--text); font-weight: 500; } -.compare-table td.compare-table__paid strong { color: var(--accent); font-weight: 600; } -.compare-table td.compare-table__none { color: var(--dim); } -@media (max-width: 520px) { - .compare-table th[scope="row"] { width: 50%; } - .compare-table th, .compare-table td { padding: 10px 8px; font-size: 13px; } -} - -/* BETA indicator pill in the app header — see app/templates/base.html. */ -.beta-chip { - display: inline-block; - margin-left: 8px; - padding: 2px 7px; - font-size: 10px; - font-weight: 700; - letter-spacing: 0.14em; - font-family: var(--font-mono); - color: var(--bg); - background: var(--accent); - border-radius: 2px; - vertical-align: middle; - user-select: none; -} - - -/* ----------------------------------------------------------------------------- - Landing-page screenshots: hero shot, thumbnails inside feature cards, gallery - strip, and a -based lightbox. See app/templates/landing.html. */ - -/* All clickable screenshots are ' + + 'style="flex:0 0 140px;">' + + '' + '' + 'or import a new CSV →' + '' + @@ -255,12 +212,6 @@ savePie(pie); mountAndRender(); } catch (e2) { - if (e2 && e2.name === 'StaleBlobError') { - // Pepper rotated since the blob was written — silently clean - // up and fall through to the empty state with a soft notice. - autoCleanStaleBlob(mount); - return; - } err.textContent = (e2 && e2.name === 'BadPinError') ? 'Incorrect PIN.' : (e2.message || 'Could not restore.'); @@ -270,16 +221,6 @@ } function renderPanel(mount, pie, enriched, agg) { - var panel = document.getElementById('portfolio-panel'); - if (panel) panel.classList.remove('pf-empty'); - // The empty-state path forces the add form visible. When we move - // back to a populated view we re-hide it — unless edit mode is on, - // in which case the form stays visible for ongoing edits. - var form = document.getElementById('pf-add-form'); - if (form && panel && !panel.classList.contains('pf-editing')) { - form.hidden = true; - } - const ccyPills = Object.keys(agg.by_currency) .sort((a, b) => agg.by_currency[b] - agg.by_currency[a]) .map(c => { @@ -308,10 +249,6 @@ '' + lastDisplay + fxBadge + '' + '' + signed(p._ppl) + '' + '' + pct(p._ppl_pct) + '' + - '' + - '' + - '' + ''; }).join(''); @@ -368,7 +305,6 @@ 'QtyAvg' + 'LastP/L' + '%' + - '' + '' + '' + rows + '' + '' + @@ -481,14 +417,7 @@ catch (e) { console.warn('sync status check failed', e); } } if (status && status.paid && status.exists) { - if (status.orphaned) { - // Pepper rotated since the blob was written — clean up - // silently and show the standard empty state with a soft - // "please re-upload" notice. - autoCleanStaleBlob(mount); - } else { - renderRestoreFromCloud(mount, status); - } + renderRestoreFromCloud(mount, status); } else { renderEmpty(mount); } @@ -503,7 +432,7 @@ } const base = pie.base_currency || 'GBP'; const fx = (universeCache && universeCache.fx) || null; - const enriched = pie.positions.map((p, i) => Object.assign(enrichPosition(p, base, fx), { _orig_idx: i })) + const enriched = pie.positions.map(p => enrichPosition(p, base, fx)) .sort((a, b) => (b._value || 0) - (a._value || 0)); const agg = aggregate(enriched); renderPanel(mount, pie, enriched, agg); diff --git a/app/static/js/portfolio_edit.js b/app/static/js/portfolio_edit.js deleted file mode 100644 index 1354fe2..0000000 --- a/app/static/js/portfolio_edit.js +++ /dev/null @@ -1,255 +0,0 @@ -/* Dashboard-native portfolio editing. - * - * Owns: the EDIT button toggle, the add-position form behaviour - * (ticker validation on blur, qty/cost inputs, date-mode historical - * lookup, Add click), and per-row delete via event delegation. - * - * Reads/writes the portfolio via window.CassandraPortfolio.loadPie / - * savePie / mountAndRender — the same surface portfolio.js exposes - * for the CSV-import preview. - */ -(function () { - 'use strict'; - - const panel = document.getElementById('portfolio-panel'); - const editBtn = document.getElementById('pf-edit-btn'); - const doneBtn = document.getElementById('pf-done-btn'); - const form = document.getElementById('pf-add-form'); - if (!panel || !editBtn || !doneBtn || !form) return; - - function enterEditMode() { - panel.classList.add('pf-editing'); - form.hidden = false; - editBtn.hidden = true; - doneBtn.hidden = false; - editBtn.setAttribute('aria-pressed', 'true'); - document.getElementById('pf-add-ticker').focus(); - } - - function exitEditMode() { - panel.classList.remove('pf-editing'); - // The form is edit-mode-only — always hide it on exit, including - // when the portfolio is empty. The empty state shows guidance text - // that nudges the user back to the Edit button. - form.hidden = true; - editBtn.hidden = false; - doneBtn.hidden = true; - editBtn.setAttribute('aria-pressed', 'false'); - } - - editBtn.addEventListener('click', enterEditMode); - doneBtn.addEventListener('click', exitEditMode); - - // ---- Ticker validation on blur ------------------------------------- - - const tickerInput = document.getElementById('pf-add-ticker'); - const tickerStatus = document.getElementById('pf-add-ticker-status'); - const costCurrencyEl = document.getElementById('pf-add-cost-currency'); - const submitBtn = document.getElementById('pf-add-submit'); - const warningEl = document.getElementById('pf-add-warning'); - - let validated = null; // {symbol, price, currency, as_of} or null - - function setStatus(el, text, kind) { - el.textContent = text; - el.className = 'pf-add-status' + (kind ? ' pf-add-status--' + kind : ''); - } - - function updateSubmitState() { - const qty = parseFloat(document.getElementById('pf-add-qty').value); - const cost = parseFloat(document.getElementById('pf-add-cost').value); - submitBtn.disabled = !( - validated && qty > 0 && cost > 0 && isFinite(qty) && isFinite(cost) - ); - } - - function clearDuplicateWarning() { - warningEl.hidden = true; - warningEl.textContent = ''; - } - - function showDuplicateWarning(existing) { - warningEl.hidden = false; - warningEl.textContent = - `Already in your portfolio (${existing.qty} shares @ ` + - `${existing.avg_cost.toFixed(2)}). Adding will create a duplicate row.`; - } - - async function validateTicker() { - const raw = tickerInput.value.trim().toUpperCase(); - if (!raw) { - validated = null; - setStatus(tickerStatus, '', ''); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - updateSubmitState(); - return; - } - setStatus(tickerStatus, 'checking…', 'pending'); - try { - const r = await fetch('/api/ticker/validate?symbol=' + encodeURIComponent(raw)); - if (!r.ok) throw new Error('HTTP ' + r.status); - const j = await r.json(); - if (j.ok) { - validated = j; - setStatus( - tickerStatus, - '✓ ' + j.price.toFixed(2) + ' ' + (j.currency || ''), - 'ok', - ); - costCurrencyEl.textContent = j.currency || ''; - // Duplicate detection. - const pie = window.CassandraPortfolio.loadPie(); - const existing = pie && (pie.positions || []).find( - p => (p.yahoo_ticker || '').toUpperCase() === j.symbol - ); - if (existing) showDuplicateWarning(existing); - else clearDuplicateWarning(); - } else { - validated = null; - setStatus(tickerStatus, '✗ ' + (j.error || 'not recognised'), 'err'); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - } - } catch (e) { - validated = null; - setStatus(tickerStatus, '✗ couldn\'t validate — try again', 'err'); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - } - updateSubmitState(); - } - - tickerInput.addEventListener('blur', validateTicker); - document.getElementById('pf-add-qty').addEventListener('input', updateSubmitState); - document.getElementById('pf-add-cost').addEventListener('input', updateSubmitState); - - // ---- Add button → localStorage merge ------------------------------- - - function resetForm() { - tickerInput.value = ''; - document.getElementById('pf-add-qty').value = ''; - document.getElementById('pf-add-cost').value = ''; - document.getElementById('pf-add-date').value = ''; - validated = null; - setStatus(tickerStatus, '', ''); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - updateSubmitState(); - tickerInput.focus(); - } - - function addPosition() { - if (submitBtn.disabled) return; - const qty = parseFloat(document.getElementById('pf-add-qty').value); - const cost = parseFloat(document.getElementById('pf-add-cost').value); - const sym = validated.symbol; - - const pie = window.CassandraPortfolio.loadPie() || { - pie_name: null, - base_currency: 'GBP', - positions: [], - totals: {invested: 0, value: 0, result: 0}, - warnings: [], - }; - pie.positions = pie.positions || []; - pie.positions.push({ - yahoo_ticker: sym, - t212_slice: sym, // shared shape with CSV path - name: validated.name || sym, - qty: qty, - avg_cost: cost, - currency: validated.currency || 'USD', - }); - window.CassandraPortfolio.savePie(pie); - window.CassandraPortfolio.mountAndRender(); - resetForm(); - } - - submitBtn.addEventListener('click', addPosition); - - // Submit on Enter from any input within the form. - form.addEventListener('keydown', function (e) { - if (e.key === 'Enter' && !submitBtn.disabled) { - e.preventDefault(); - addPosition(); - } - }); - - // ---- Calendar-icon → historical lookup ----------------------------- - - const dateBtn = document.getElementById('pf-add-date-btn'); - const dateInput = document.getElementById('pf-add-date'); - const dateStatus = document.getElementById('pf-add-date-status'); - const costInput = document.getElementById('pf-add-cost'); - - dateBtn.addEventListener('click', function () { - if (!validated) { - setStatus(dateStatus, 'enter a valid ticker first', 'err'); - return; - } - dateInput.hidden = !dateInput.hidden; - if (!dateInput.hidden) { - dateInput.focus(); - if (typeof dateInput.showPicker === 'function') dateInput.showPicker(); - } else { - setStatus(dateStatus, '', ''); - } - }); - - async function fetchHistorical() { - if (!validated) { - setStatus(dateStatus, 'enter a valid ticker first', 'err'); - return; - } - const d = dateInput.value; - if (!d) { - setStatus(dateStatus, '', ''); - return; - } - setStatus(dateStatus, 'looking up…', 'pending'); - try { - const url = '/api/ticker/historical?symbol=' + - encodeURIComponent(validated.symbol) + - '&date=' + encodeURIComponent(d); - const r = await fetch(url); - if (r.status === 400) { - const j = await r.json().catch(() => ({detail: 'invalid date'})); - setStatus(dateStatus, '✗ ' + (j.detail || 'invalid date'), 'err'); - updateSubmitState(); - return; - } - const j = await r.json(); - if (j.ok) { - costInput.value = j.close.toFixed(2); - const tag = (j.actual_date && j.actual_date !== d) - ? '✓ from ' + j.actual_date - : '✓'; - setStatus(dateStatus, tag, 'ok'); - // Hide the date picker after a successful fill — keeps the row clean. - dateInput.hidden = true; - } else { - setStatus(dateStatus, '✗ ' + (j.error || 'no data'), 'err'); - } - } catch (e) { - setStatus(dateStatus, '✗ couldn\'t fetch — try again', 'err'); - } - updateSubmitState(); - } - - dateInput.addEventListener('change', fetchHistorical); - - // ---- Per-row delete (event delegation) ----------------------------- - - panel.addEventListener('click', function (e) { - const btn = e.target.closest('.pf-row-del'); - if (!btn) return; - const idx = parseInt(btn.dataset.idx, 10); - if (!Number.isInteger(idx)) return; - const pie = window.CassandraPortfolio.loadPie(); - if (!pie || !pie.positions || idx < 0 || idx >= pie.positions.length) return; - pie.positions.splice(idx, 1); - window.CassandraPortfolio.savePie(pie); - window.CassandraPortfolio.mountAndRender(); - }); -})(); diff --git a/app/templates/base.html b/app/templates/base.html index 9fdb0d1..fa05eb4 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -4,29 +4,6 @@ {% block title %}{{ BRAND_NAME }}{% endblock %} - {# Cross-user contamination guard. - - localStorage is browser-wide; if User A uploads a portfolio and User B - logs in on the same browser, the stale `cassandra.pie` would otherwise - render as User B's holdings. We stamp the logged-in user's id in - localStorage on every authenticated page load and wipe per-user keys - if the id changed since last time. Theme stays — it's cosmetic. #} - {# Apply saved theme before stylesheet renders to avoid a flash. #} -
      Strategic Log - - {% if paid %}refreshed hourly @ :20 UTC{% else %}refreshed every 6 hours · hourly on Paid{% endif %} - + generated hourly @ :20 UTC
      Flash News - - {% if paid %}last 24h · ingest hourly @ :10 UTC{% else %}last 6h · full 24h on Paid{% endif %} - + last 24h · ingest hourly @ :10 UTC

      This page is part of, and qualifies, the - Terms and Conditions. + Terms of Service.

      @@ -95,7 +95,7 @@ The service is provided “as is” without warranties of any kind. To the maximum extent permitted by applicable law, the operator excludes liability for any loss arising from use of, or reliance on, - the content. See the Terms and Conditions for the + the content. See the Terms of Service for the full limitation of liability.

      diff --git a/app/templates/landing.html b/app/templates/landing.html index 2732a68..e275377 100644 --- a/app/templates/landing.html +++ b/app/templates/landing.html @@ -11,8 +11,8 @@ tune out the high-frequency noise that comes from treating markets like a casino. We aggregate cross-asset news and macro signals, then write a plain-English read of what the underlying fundamentals - justify versus what the crowd is doing. Refreshed through the - trading day. A media service, not a financial one. + justify versus what the crowd is doing. Refreshed hourly. A media + service, not a financial one.

      {% if cu and (cu.user or cu.is_admin) %} @@ -25,17 +25,6 @@
      -
      - -
      -
      News, aggregated
      @@ -47,13 +36,6 @@ stuff is easy to find. Ingestion follows the trading calendar — off-hours stay quiet.

      -
      @@ -65,17 +47,10 @@ explains what the move means, not what it was. Anchored in earnings, policy, valuation — not chart patterns.

      -
      -
      The strategic read
      +
      The hourly read

      Rational vs irrational, every paragraph

      We tie the day’s headlines and the cross-asset signals into @@ -86,40 +61,15 @@ intermediate. This is editorial commentary on public data — not a forecast and not advice on any investment decision.

      - -
      -
      - -
      -

      More views

      -
      -

      - Paid users can also drop a portfolio CSV from their broker - (Trading 212 today, more brokers planned) for an AI sense-check on - concentration, regime fit, and currency exposure. Holdings stay in - your browser by default; opt in to encrypted cloud sync to restore - on another device. + Paid users can also drop a Trading 212 pie CSV for an AI + sense-check on concentration, regime fit, and currency exposure. + Holdings stay in your browser by default; opt in to encrypted cloud + sync to restore on another device.

      @@ -150,34 +100,4 @@ - - - -

      -       - - - {% endblock %} diff --git a/app/templates/log.html b/app/templates/log.html index 8abee4c..3e56727 100644 --- a/app/templates/log.html +++ b/app/templates/log.html @@ -30,7 +30,6 @@
      loading log…
      - {% if paid %} - {% else %} - - {% endif %} -{% if paid %}{% endif %} + {% endblock %} diff --git a/app/templates/partials/news.html b/app/templates/partials/news.html index 5f19f4d..36b8a59 100644 --- a/app/templates/partials/news.html +++ b/app/templates/partials/news.html @@ -33,10 +33,3 @@ {% endfor %} {% endif %} -{% if capped %} -
      - Free tier — showing the last {{ window_hours|int }} hours of news. - Upgrade - for the full 24-hour feed plus daily and weekly email digests. -
      -{% endif %} diff --git a/app/templates/pricing.html b/app/templates/pricing.html index 93f1562..8d88352 100644 --- a/app/templates/pricing.html +++ b/app/templates/pricing.html @@ -6,262 +6,66 @@

      Pricing

      - Two tiers. The core editorial is free today — a rolling - 6-hour news feed, the cross-asset indicator panels, and a strategic - log refreshed every six hours. Paid stretches the news feed to a - full 24 hours, runs the strategic log hourly, unlocks the follow-up - chat against past logs, adds portfolio import with AI analysis, and - turns on the daily email digest on top of the Sunday recap everyone - gets. + Two tiers. The news aggregator and the hourly macro interpretation + are free for everyone — we want the read out where people can use + it. The paid tier extends the same editorial commentary to the + specific tickers in a portfolio you upload — an educational read + of public data, not advice on whether to hold them.

      -

      Free

      -
      The core editorial — news, indicators, and a strategic log every 6 hours.
      +
      Free
      £0
      -
      No card needed.
      -
      -
      What you get
      +
      Forever. No card needed.
        -
      • News feed — headlines from the last 6 hours, auto-tagged by theme, click-to-filter
        • -
      • Cross-asset indicator panels (equities, rates, FX, commodities, credit, …) with a one-paragraph AI read on each tab
        • -
      • Strategic log — a single editorial interpretation of the day, refreshed every 6 hours
        • -
      • Two reading levels: Novice (defines jargon) or Intermediate (terse, for fluent readers)
        • -
      • Sunday weekly digest by email — week behind + week ahead, one-click unsubscribe
        • +
      • News aggregator — auto-tagged by theme
        • +
      • Cross-asset macro signals across every asset class
        • +
      • Hourly AI interpretation of the news + the tape
        • +
      • Per-group cross-asset summaries
        • +
      • Novice / Intermediate reading levels
        • +
      • Portfolio import & analysis
        • +
      • Encrypted cloud sync
      -
      - Need the full-day news feed, hourly strategic log, follow-up chat, daily digests, or portfolio analysis? See Paid → -
      {% if cu and (cu.user or cu.is_admin) %} - Open dashboard + Open dashboard {% else %} - Sign up free + Sign up free {% endif %}
      -
      Best value
      -

      Paid

      -
      Full-day news feed, hourly strategic log, follow-up chat, and AI portfolio analysis.
      -
      £7 / month
      -
      - Or £70 / year — two months free, and - starts with a 14-day free trial (cancel during - the trial and you are not charged). Monthly plans start - immediately. Prices in GBP, VAT where applicable. -
      -
      -
      Everything in Free, plus
      +
      Paid
      +
      Coming soon
      +
      Checkout opens with our payments rollout.
        -
      • News feed: headlines from the last 24 hours instead of 6 — a whole session in view, nothing rolls off
        • -
      • Strategic log refreshed every hour instead of every six — track intraday moves as they unfold
        • -
      • Follow-up chat on any past log — ask the model a question against the day’s full context
        • -
      • Daily email digest (Mon–Sat) — ~600-word read of the session ahead, on top of the Sunday recap
        • -
      • Portfolio import from a broker CSV (Trading 212 supported today; more brokers planned)
        • -
      • AI portfolio read — diversification, sector and currency concentration, macro-regime fit on your holdings
        • -
      • Optional encrypted cloud sync — PIN-derived encryption in your browser, second-layer wrap on the server, no plaintext holdings server-side
        • +
      • Everything in Free
        • +
      • Portfolio import (Trading 212 CSV)
        • +
      • AI commentary on diversification, sector and currency concentration, and macro-regime context for the holdings you upload
        • +
      • Optional encrypted cloud sync across devices
        • +
      • Priority email when something material changes (later)
      -

      +

      + {% if cu and (cu.user or cu.is_admin) %} + Manage account + {% else %} + Sign up — paid unlocks soon + {% endif %} +
      +

      The portfolio feature does not produce buy, sell or hold - recommendations and does not consider your wider finances, debts, + recommendations. It does not consider your wider finances, debts, tax position or objectives. It is not regulated investment advice or a personal recommendation under FSMA / FCA COBS.

      -
      - {% if paid %} - Manage subscription - {% elif cu and cu.user %} - - - - {% else %} - Sign in to subscribe - {% endif %} -
      - - -
      -

      Free vs Paid at a glance

      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
      FeatureFreePaid
      News feed — headlines from the last…6 hours24 hours
      Strategic log refreshEvery 6 hoursEvery hour
      Cross-asset indicator panels✓✓
      Follow-up chat on past logs—Included
      Email digestSunday onlySunday + daily Mon–Sat
      Portfolio import (broker CSV)—Included
      AI portfolio read—Included
      Encrypted cloud sync—Included
      -
      - -
      - -
      -
      Invite a friend
      -
      Both of you get 45 days of paid access
      -
      - Share your personal invite link from Settings. The credit applies when they start a paid plan. -
      -
      - -
      - - - -

      Invite a friend

      -

      - Every account gets an 8-character referral code and matching invite - link, both shown on your Settings page. When - someone signs up through your link and starts a paid plan, - both of you get 45 days of paid access credited - to your account. -

      -

      How it works

      -
        -
      1. Sign up. Your code and link go live in Settings.
        2. -
      2. Share. Send the link, or read the code — the alphabet drops 0/O and 1/I/L so it dictates cleanly.
        3. -
      3. They sign up. The referral is recorded against your account when they verify their email.
        4. -
      4. They subscribe. 45 days of paid access lands on both accounts — usable any time over the next month and a half.
        5. -
      -

      The fine print

      -
        -
      • One referral per new account — whichever link they used first.
        • -
      • No self-referral.
        • -
      • Credits stack: if you already have a credit window running, the new 45 days extend from its end, not from today.
        • -
      • Credits aren’t refundable for cash — see Terms & Conditions § 6.
        • -
      • Pending signups, conversions, and active credits are visible on the Settings page.
        • -
      -       - - -

      How the data is handled

      diff --git a/app/templates/settings.html b/app/templates/settings.html index 20dfa57..ecdd25a 100644 --- a/app/templates/settings.html +++ b/app/templates/settings.html @@ -20,111 +20,53 @@

      Tier
      -
      -
      - - {% if paid and paid.active %} - {% if paid.source == "credit" %} - - Paid features active via credit · {{ paid.days_remaining }} day(s) remaining - (expires {{ paid.expires_at.strftime("%Y-%m-%d") }}). - - {% else %} - {% if trial_days_remaining %} - - Free trial — {{ trial_days_remaining }} - day{{ '' if trial_days_remaining == 1 else 's' }} remaining. - Cancel before the trial ends and you won’t be charged. - - {% else %} - Paid subscription active. - {% endif %} - {% endif %} - {% else %} +
      + + {% if paid and paid.active %} + {% if paid.source == "credit" %} - Free tier — upgrade for £7/month or £70/year. + Paid features active via credit · {{ paid.days_remaining }} day(s) remaining + (expires {{ paid.expires_at.strftime("%Y-%m-%d") }}). + {% else %} + Paid subscription active. {% endif %} -
      - {% if paid and paid.active and paid.source != "credit" and user.stripe_customer_id %} - + {% else %} + Paid features unlock with Paddle (D.3) or invite credits. {% endif %}
      - {% if paid and paid.active and paid.source != "credit" and user.stripe_customer_id %} - - {% endif %} - {# --- Import portfolio --------------------------------------------- #} - {# Open by default because /settings#import is the deep-link target - from the dashboard's "import a portfolio" CTA — if you arrive via - that link the section should already be expanded. #} -
      - Import portfolio (CSV) +
      +
      Import portfolio (Trading 212 CSV)

      - Drop a portfolio CSV from any broker — Trading 212 is recognised - natively and other formats (IBKR, Fidelity, Schwab…) are - auto-detected. We’ll parse it and show a preview before importing - anywhere. -
      T212 export path: - Investing → Your Pie → ··· → Export. + Export your pie from T212 + (Investing → Your Pie → ··· → Export) + and drop the CSV here. We’ll parse it and show a preview before + importing anywhere.

      ▱
      -
      Drop your broker's portfolio CSV here
      -
      or browse · max 1 MB · T212, IBKR and others auto-detected
      +
      Drop a T212 pie CSV here
      +
      or browse · max 1 MB
      -
      +
      {# --- Referral block ---------------------------------------------- #} -
      - Invite a friend +
      +
      Invite a friend

      Share your invite link. When your friend subscribes, you and - they each get 45 days of paid access credited - to your account. + they each get 50% off for 3 months.

      @@ -149,84 +91,14 @@
      Active credits
      -
      {{ active_credit_count }}
      - {% if own_credit_days %} -
      - +{{ own_credit_days }} day{{ '' if own_credit_days == 1 else 's' }} on your account -
      - {% endif %} +
      — (D.3)
      -
      - - {# --- Email digests block ------------------------------------------ #} -
      - Email digests -

      - Editorial commentary delivered to your inbox. Daily for paid (Mon–Sat) plus the Sunday recap; free tier gets the Sunday recap. -

      - -
      -
      Subscription
      -
      - -
      - One-click unsubscribe in every email. -
      -
      -
      - -
      -
      Reading level
      -
      -
      - - -
      -
      -
      - -
      -
      Last delivery
      -
      - {% if last_email_send %}{{ last_email_send.sent_at.strftime("%Y-%m-%d %H:%M") }} UTC — {{ last_email_send.status }}{% else %}—{% endif %} -
      -
      - -
      -
      - - + {# --- Cloud sync block --------------------------------------------- #} -
      - Cloud sync (encrypted) +
      +
      Cloud sync (encrypted)

      Manage the encrypted server-side copy of your portfolio. Sync is opted-in per import (see the Import section above). @@ -247,7 +119,7 @@ above to enable cloud sync.

      {% endif %} -
      + {# Future: Paddle subscription block, AI-spend ledger summary, etc. #} @@ -273,10 +145,10 @@
      + style="width:100%;padding:8px;margin-bottom:10px;" required> + style="width:100%;padding:8px;margin-bottom:10px;" required>
      -

      6. Refunds

      -

      - 14-day cooling-off period (UK / EU consumers). - The Consumer Contracts (Information, Cancellation and Additional - Charges) Regulations 2013 give consumers a 14-day right to cancel - digital service contracts. We honour this right in different ways - depending on your billing cadence. -

      -

      - 1a. Annual subscriptions. Every new annual - subscription begins with a 14-day free trial. No payment is taken - during the trial; cancel any time before the trial ends and you - are not charged. This is offered as a substitute for, and is at - least as generous as, the statutory 14-day refund right. After - the trial ends and the first payment is taken, the rules in - paragraph 2 below apply. -

      -

      - 1b. Monthly subscriptions. Monthly subscribers - receive immediate access to paid features at checkout and are - billed on day one. Before you can start a monthly subscription - you must tick a required consent box on the pricing page in - which you (i) agree to these Terms, (ii) give express consent to - begin performance immediately, and (iii) acknowledge that, under - Regulation 36 of the Consumer Contracts Regulations 2013, doing - so extinguishes your statutory 14-day right to cancel in respect - of digital content already delivered. The rules in paragraph 2 - below apply from the first day. -

      -

      - Cancellation after the cooling-off window. You can - cancel a paid subscription at any time. Cancellation takes effect at - the end of the current billing period; we do not pro-rate refunds - for the unused portion of a period you have already started, unless - a separate paragraph below applies. -

      -

      - Termination by us without fault on your part. If we - terminate or materially reduce a paid feature for reasons that are - not a breach by you (including a service shutdown), we will refund - the unused portion of any prepaid fees on a pro-rata basis. The same - applies under clause 8 if we terminate for a breach you did not - cause, and under clause 11 if you close your account because you do - not accept a material change to these Terms. -

      -

      - Service faults. Nothing in this clause limits your - statutory rights as a UK consumer under Part 1 of the Consumer - Rights Act 2015. If a paid feature is not supplied with reasonable - care and skill, you may be entitled to a repeat performance or a - price reduction (which can be a full refund) under that Act. -

      -

      - How to request a refund. Email - {{ OPERATOR_EMAIL }} from - the address tied to your account, with the order reference if you - have one. We aim to acknowledge within 5 working days. Refunds are - returned to the original payment method and typically arrive within - 14 days of approval, subject to your bank’s processing time. -

      -

      - Referral credits and any other non-cash credits applied to your - account are not refundable for cash. -

      -
      - -
      -

      7. Service availability

      +

      6. Service availability

      The Service is provided on a best-effort basis. There is no service level agreement: outages, data delays, and feature changes may occur @@ -172,7 +105,7 @@

      -

      8. Content & ownership

      +

      7. Content & ownership

      The Service’s code, design, indicator selection, and prompts are owned or licensed by {{ LEGAL_OPERATOR }}. To the extent any @@ -193,7 +126,7 @@

      -

      9. Suspension & termination

      +

      8. Suspension & termination

      We may suspend or terminate access without notice for violation of these Terms or for activity that risks the integrity, security, or @@ -205,13 +138,12 @@ respond, unless immediate suspension is necessary to protect the Service, its users, or any third party. If we terminate a paid subscription for a breach you did not cause, we will refund the - unused portion of any prepaid fees on a pro-rata basis (see - clause 6). + unused portion of any prepaid fees on a pro-rata basis.

      -

      10. No warranty

      +

      9. No warranty

      The Service is provided “as is” and “as available”, without warranties of any kind, express or implied, @@ -221,7 +153,7 @@

      -

      11. Limitation of liability

      +

      10. Limitation of liability

      To the maximum extent permitted by law, {{ LEGAL_OPERATOR }} is not liable for any indirect, incidental, special, consequential, or @@ -250,7 +182,7 @@

      -

      12. Changes

      +

      11. Changes

      These Terms may change. Material changes will be flagged in-app or by email. Continued use after a change means you accept the updated @@ -261,7 +193,7 @@

      -

      13. Governing law and jurisdiction

      +

      12. Governing law and jurisdiction

      These Terms are governed by the laws of England and Wales. Subject to any mandatory law of the consumer’s country of residence, @@ -273,7 +205,7 @@

      -

      14. Contact

      +

      13. Contact

      {{ OPERATOR_EMAIL }}

      diff --git a/app/templates/verify.html b/app/templates/verify.html index 1399fe5..ae62056 100644 --- a/app/templates/verify.html +++ b/app/templates/verify.html @@ -29,7 +29,8 @@ diff --git a/app/templates_env.py b/app/templates_env.py index 22cfdf6..fc92112 100644 --- a/app/templates_env.py +++ b/app/templates_env.py @@ -9,7 +9,6 @@ from fastapi.templating import Jinja2Templates from markupsafe import Markup, escape from app import branding -from app.config import get_settings from app.services.glossary import wrap_glossary @@ -76,4 +75,3 @@ templates.env.globals["TAGLINE"] = branding.TAGLINE templates.env.globals["LEGAL_OPERATOR"] = branding.LEGAL_OPERATOR templates.env.globals["OPERATOR_EMAIL"] = branding.OPERATOR_EMAIL templates.env.globals["OPERATOR_JURISDICTION"] = branding.OPERATOR_JURISDICTION -templates.env.globals["BETA_MODE"] = get_settings().BETA_MODE diff --git a/docker-compose.test.yml b/docker-compose.test.yml deleted file mode 100644 index 219930c..0000000 --- a/docker-compose.test.yml +++ /dev/null @@ -1,46 +0,0 @@ -# Ad-hoc test runner. -# -# STANDALONE — do not combine with docker-compose.yml. The `name:` field -# below puts the test container in its own Compose project (`cassandra-test`) -# so it CANNOT collide with the live prod stack on this host (containers, -# networks, volumes are all namespaced by project). -# -# Usage: -# # Run the full suite: -# docker compose -f docker-compose.test.yml run --rm test -# -# # Run a specific file or test: -# docker compose -f docker-compose.test.yml run --rm test pytest tests/test_email_digest_job.py -v -# docker compose -f docker-compose.test.yml run --rm test pytest -k unsubscribe -# -# # Open a shell in the test image (e.g. to poke around with pytest --pdb): -# docker compose -f docker-compose.test.yml run --rm test bash -# -# # Rebuild after a pyproject.toml change: -# docker compose -f docker-compose.test.yml build test -# -# Tests use an in-memory aiosqlite DB (see tests/conftest.py), so there is -# no MariaDB / Redis dependency and nothing touches the prod database. - -name: cassandra-test - -services: - test: - build: - context: . - target: test - # Same volume mounts as the dev override — edits on the host take effect - # on the next `run` without rebuilding the image. - volumes: - - ./app:/app/app - - ./tests:/app/tests - - ./alembic:/app/alembic - - ./alembic.ini:/app/alembic.ini:ro - - ./config:/app/config:ro - - ./pyproject.toml:/app/pyproject.toml:ro - environment: - # Sentinels so app.config can be imported without a real .env / DB. - # tests/conftest.py also sets these defensively. - DATABASE_URL: "sqlite+aiosqlite:///:memory:" - CASSANDRA_MOCK: "1" - PYTHONDONTWRITEBYTECODE: "1" diff --git a/docker-compose.yml b/docker-compose.yml index 8a7e03f..3038c98 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -43,12 +43,6 @@ services: REDIS_URL: redis://redis:6379/0 volumes: - ./config:/app/config:ro - # Mount app code + migrations from the host so edits take effect - # on a plain `docker compose restart app` — no image rebuild. - # Image still bakes a copy at build time as a fallback. - - ./app:/app/app - - ./alembic:/app/alembic - - ./alembic.ini:/app/alembic.ini:ro depends_on: db: condition: service_healthy @@ -68,9 +62,6 @@ services: REDIS_URL: redis://redis:6379/0 volumes: - ./config:/app/config:ro - - ./app:/app/app - - ./alembic:/app/alembic - - ./alembic.ini:/app/alembic.ini:ro depends_on: db: condition: service_healthy diff --git a/docs/paddle-merchant-summary.md b/docs/paddle-merchant-summary.md deleted file mode 100644 index e00be92..0000000 --- a/docs/paddle-merchant-summary.md +++ /dev/null @@ -1,146 +0,0 @@ -# Paddle merchant onboarding — Read the Markets - -Use this when filling Paddle's seller-application / business-description -fields. Framing is deliberately **media / publishing**, not financial -services — "trading", "signals", "advice" wording triggers rejection or -sends the application to extra compliance review. - ---- - -## Business description (one paragraph) - -**Read the Markets** is a UK-based subscription publishing service for -retail investors who want to *understand* what's moving in markets -without acting on tips or signals. The site aggregates public market -data (prices via Yahoo Finance) and public RSS news feeds, then -generates plain-English written commentary using a large language model. -Subscribers read; the service does not trade, hold client funds, or give -personal financial advice. Operated by Giorgio Gilestro, ICO-registered -as ZC098928. - -## What we sell - -A single B2C subscription that unlocks extended access to our written -market commentary and personal-portfolio analysis features. There is -one product, two billing cadences: - -- **Read the Markets — Paid plan, Monthly** — £7 GBP / month -- **Read the Markets — Paid plan, Annual** — £70 GBP / year - -A free tier exists indefinitely (no card required) and gives access to -the core editorial at reduced refresh cadence. Pricing in GBP; VAT -handled by Paddle as merchant of record. - -## What subscribers get on the Paid plan - -- 24-hour news headline window (free: 6 hours) -- Strategic interpretation log refreshed every hour during market hours - (free: every six hours) -- Daily written digest emailed Monday–Saturday -- The ability to ask follow-up questions to the AI about any past - published interpretation -- Optional upload of a personal portfolio CSV (currently Trading 212 - export) for an AI commentary on diversification and macro-regime fit - — purely descriptive, no buy/sell calls -- Optional end-to-end encrypted cloud sync of the portfolio file - -## What we explicitly do **not** do (regulatory framing) - -- **Not a financial-advice service.** We do not produce personalised - recommendations or consider a user's wider finances, debts, tax - position, or objectives. -- **No buy/sell/hold signals.** Output is editorial commentary on - public data. -- **No brokerage.** We never execute trades, hold client money, or - custody assets. -- **Not regulated under FSMA / FCA COBS.** This is explicitly stated on - the site disclaimer and in the portfolio-analysis feature - description. -- **No crypto trading, no margin/leverage products, no copy-trading, - no managed accounts.** -- **No tipster service.** All copy emphasises the difference between - "understanding markets" and "gambling on them." - -## Audience - -Retail readers in the UK and EU who want a daily macro briefing in -plain English. Comparable to a paid newsletter (e.g. Substack finance -writers) or a personal-finance magazine subscription, delivered as a -web app + email. - -## Refund & cancellation policy - -Published at §6: 14-day statutory -cooling-off (Consumer Contracts Regulations 2013), cancel-any-time -taking effect at end of billing period, pro-rata refund if we terminate -service through no fault of the user. Refund requests handled by email -at . - ---- - -## Comprehensive product overview (single-field answer) - -> Use this when Paddle asks **"Could you provide a comprehensive -> overview of your product?"** — one self-contained block, ~400 -> words, designed so the reviewer hits the "not a financial product" -> framing within the first two sentences. - -**Read the Markets** () is a UK-based -subscription publishing service that helps retail readers *understand* -what is moving in financial markets — through plain-English written -commentary, not through trading signals, advice, or recommendations. -It is best understood as a digital newsletter / news-and-media -subscription product, with an AI-content layer, comparable to a paid -financial newsletter on Substack or a digital news magazine. We do not -execute trades, hold client funds, custody any assets, or operate as a -broker or investment adviser. We are not authorised by the FCA and we -are not a regulated financial service. This positioning is explicit on -the live site disclaimer at and in -the body copy of every paid feature. - -The product works as follows. We aggregate public market data (stock, -FX, commodity, and rate quotes via Yahoo Finance) and public RSS news -feeds across the macro universe. A large language model (via -OpenRouter) then writes a short editorial interpretation of what the -underlying public data appears to be saying. The output is a written -article — a "strategic log" — refreshed through the trading day, plus -per-asset-class commentary panels and a daily / weekly written digest -delivered by email. Subscribers can also optionally upload a personal -portfolio holdings CSV (currently exported from Trading 212) to receive -a written sense-check of diversification, currency exposure, and -macro-regime fit on those holdings; this output is purely descriptive -and contains no buy, sell, or hold recommendations. - -A free tier exists indefinitely (no card required) and serves the core -editorial at a reduced refresh cadence (6-hour news window, strategic -log refreshed every six hours, weekly Sunday digest only). The Paid -plan extends those to a 24-hour news window, hourly strategic log -refresh, daily Mon–Sat email digest, the optional portfolio upload + -AI commentary, an interactive follow-up chat against any past -published article, and optional encrypted cloud sync of the portfolio -file. - -Pricing is in GBP, with Paddle as merchant of record handling VAT: -**£7 / month** or **£70 / year** (two months free). Subscribers can -cancel any time, taking effect at the end of the current billing -period. A 14-day statutory cooling-off period applies under the UK -Consumer Contracts Regulations 2013, plus pro-rata refunds where we -terminate service through no fault of the user — full refund policy at - §6. Operated by Giorgio Gilestro, -ICO-registered as ZC098928, contact . - ---- - -## Practical tips when completing the Paddle form - -- **Category dropdown:** pick **"Software / SaaS — Content & - publishing"** or **"Digital subscription — News & media"** if those - options exist. Avoid anything containing the words *financial - services*, *trading*, *investing tools*, or *fintech*. -- **Self-declaration:** describe the product as **media / publishing - with an AI-content angle** — not a financial service. -- **Linkable references for the reviewer:** - - Pricing & tier breakdown: - - Disclaimer (the legal "not advice" statement): - - Terms & Conditions (incl. §6 Refunds): - - Privacy notice (ICO ZC098928 surfaced here): diff --git a/docs/superpowers/plans/2026-05-25-beta-mode-and-paid-gap.md b/docs/superpowers/plans/2026-05-25-beta-mode-and-paid-gap.md deleted file mode 100644 index 328fe47..0000000 --- a/docs/superpowers/plans/2026-05-25-beta-mode-and-paid-gap.md +++ /dev/null @@ -1,2096 +0,0 @@ -# Beta Mode + Paid/Free Gap Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Ship the closed-beta launch package: a visible BETA chip in the app chrome, a 6-hour free-tier news cap, and email digests (daily for paid Mon–Sat, Sunday weekly for everyone), all gated by an opt-in flag. - -**Architecture:** Three coordinated changes layered onto existing FastAPI + APScheduler + MariaDB + OpenRouter + SMTP infrastructure. No new external services. New persisted state: two columns on `users` (`email_digest_opt_in`, `digest_tone`) and one new table (`email_sends`). One new job (`email_digest_job`), one new router (`email`), two new prompt builders. - -**Tech Stack:** FastAPI · SQLAlchemy 2.0 (async) · Alembic · APScheduler · aiosmtplib · itsdangerous · pytest + aiosqlite (tests) · Jinja2 templates · vanilla JS + HTMX. - -**Spec:** `docs/superpowers/specs/2026-05-25-beta-mode-and-paid-gap-design.md` - ---- - -## File Map - -**New files** -- `app/jobs/email_digest_job.py` — orchestrator (daily Mon–Sat for paid, weekly Sunday for all opt-in) -- `app/routers/email.py` — `GET /email/unsubscribe?token=…` -- `alembic/versions/0017_email_digest.py` — migration -- `tests/test_news_window.py` — news-cap regression -- `tests/test_email_digest_job.py` — job recipient selection, idempotency -- `tests/test_email_unsubscribe.py` — token roundtrip + endpoint -- `tests/test_email_render.py` — digest email renderer -- `tests/test_digest_prompts.py` — prompt builder unit tests -- `tests/test_settings_digest_api.py` — PATCH endpoint -- `tests/test_verify_subscribe.py` — sign-up checkbox - -**Modified files** -- `app/config.py` — add `BETA_MODE` flag -- `app/templates_env.py` — expose `BETA_MODE` to templates -- `app/templates/base.html` — render the chip -- `app/static/css/cassandra.css` — `.beta-chip` styling -- `app/services/access.py` — `FREE_NEWS_WINDOW_HOURS` constant -- `app/routers/api.py` — `news_list` clamps `since_hours` for non-paid; PATCH endpoint for digest prefs -- `app/templates/partials/news.html` — capped-footer marker -- `app/models.py` — `User.email_digest_opt_in`, `User.digest_tone`, `EmailSend` model -- `app/services/openrouter.py` — `build_daily_digest_prompt`, `build_weekly_digest_prompt`, bump `PROMPT_VERSION` -- `app/services/email_service.py` — `render_digest_email`, `send_digest` -- `app/scheduler_main.py` — register the digest job at 06:30 UTC -- `app/templates/settings.html` — Email digests section -- `app/templates/verify.html` — subscribe checkbox -- `app/routers/auth.py` — read `subscribe_to_digests` on verify POST -- `app/templates/pricing.html` — updated tier copy -- `app/cli.py` — `send-test-digest` command -- `app/main.py` — include new email router - ---- - -## Task 1: BETA chip - -**Files:** -- Modify: `app/config.py` -- Modify: `app/templates_env.py` -- Modify: `app/templates/base.html` (line ~140) -- Modify: `app/static/css/cassandra.css` - -- [ ] **Step 1: Add `BETA_MODE` to config** - -Open `app/config.py`. Find the `Settings` class. Add: - -```python - BETA_MODE: bool = True # Shows a "BETA" pill in the app header. Flip to False at GA. -``` - -Place it adjacent to other display/flag-style settings (alongside `CASSANDRA_TONE` or the LLM caps — whichever cluster fits). - -- [ ] **Step 2: Expose the flag to templates** - -Open `app/templates_env.py`. After the existing `templates.env.globals[...]` block (around line 75), add: - -```python -from app.config import get_settings as _get_settings # if not already imported -templates.env.globals["BETA_MODE"] = _get_settings().BETA_MODE -``` - -If `get_settings` is already imported in the file under another alias, reuse it. - -- [ ] **Step 3: Render the chip in the app header** - -Open `app/templates/base.html`. Find the brand link (line ~140): - -```html - {{ BRAND_NAME }} -``` - -Insert immediately after it: - -```html - {% if BETA_MODE %}BETA{% endif %} -``` - -- [ ] **Step 4: Style the chip** - -Open `app/static/css/cassandra.css`. Append at the bottom of the file: - -```css -/* BETA indicator pill in the app header — see app/templates/base.html. */ -.beta-chip { - display: inline-block; - margin-left: 8px; - padding: 2px 7px; - font-size: 10px; - font-weight: 700; - letter-spacing: 0.14em; - font-family: var(--font-mono); - color: var(--bg); - background: var(--accent); - border-radius: 2px; - vertical-align: middle; - user-select: none; -} -``` - -- [ ] **Step 5: Manual visual check** - -Run the app locally (`docker compose restart app` — the new volume mount picks up changes). Load any logged-in page and confirm the `BETA` pill renders next to the brand. Confirm the chip does NOT appear on `/pricing` (which uses `public_base.html`, not `base.html`). - -- [ ] **Step 6: Commit** - -```bash -git add app/config.py app/templates_env.py app/templates/base.html app/static/css/cassandra.css -git commit -m "beta: header chip flagged by BETA_MODE config (default on)" -``` - ---- - -## Task 2: Free-tier news window cap - -**Files:** -- Modify: `app/services/access.py` -- Modify: `app/routers/api.py` (lines 222-280, `news_list`) -- Modify: `app/templates/partials/news.html` -- Create: `tests/test_news_window.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_news_window.py`: - -```python -"""Free vs paid window clamp on /api/news.""" -from __future__ import annotations - -import asyncio -from datetime import datetime, timedelta, timezone - -import pytest - - -def _build_app(tmp_path): - from fastapi import FastAPI - from fastapi.testclient import TestClient - from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - - from app import db as db_mod - from app.auth import sign_session - from app.db import Base - from app.models import Headline, User - from app.routers import api as api_router - - engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/news.db") - factory = async_sessionmaker(engine, expire_on_commit=False) - db_mod._engine = engine - db_mod._session_factory = factory - - now = datetime.now(timezone.utc) - - async def _seed(): - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - async with factory() as s: - s.add(User(id=1, email="free@x", tier="free")) - s.add(User(id=2, email="paid@x", tier="paid")) - # Headlines: one 1h old, one 12h old, one 20h old. - for hours_old, title in ((1, "fresh"), (12, "mid"), (20, "old")): - s.add(Headline( - source="test", title=title, url=f"https://e/{title}", - category="general", - published_at=now - timedelta(hours=hours_old), - fetched_at=now, - tags=[], - )) - await s.commit() - - asyncio.run(_seed()) - - app = FastAPI() - app.include_router(api_router.router, prefix="/api") - client = TestClient(app) - return client, sign_session(1), sign_session(2) - - -@pytest.mark.skipif(False, reason="requires aiosqlite + httpx") -def test_free_user_clamped_to_6h(tmp_path): - client, free_sess, _ = _build_app(tmp_path) - r = client.get("/api/news?since_hours=24", - cookies={"cassandra_session": free_sess}) - assert r.status_code == 200 - titles = [h["title"] for h in r.json()] - assert "fresh" in titles - assert "mid" not in titles # 12h ago, beyond 6h - assert "old" not in titles - - -def test_paid_user_full_24h(tmp_path): - client, _, paid_sess = _build_app(tmp_path) - r = client.get("/api/news?since_hours=24", - cookies={"cassandra_session": paid_sess}) - assert r.status_code == 200 - titles = [h["title"] for h in r.json()] - assert {"fresh", "mid", "old"} <= set(titles) - - -def test_anonymous_clamped_to_6h(tmp_path): - client, _, _ = _build_app(tmp_path) - r = client.get("/api/news?since_hours=24") - assert r.status_code == 200 - titles = [h["title"] for h in r.json()] - assert "fresh" in titles - assert "mid" not in titles -``` - -- [ ] **Step 2: Run the failing test** - -```bash -pytest tests/test_news_window.py -v -``` - -Expected: FAIL — the clamp doesn't exist yet, so the free/anonymous calls return 12h and 20h headlines too. - -- [ ] **Step 3: Add the constant** - -Open `app/services/access.py`. Below the imports / above `_utcnow`, add: - -```python -# How many hours of news the free tier sees. Paid sees whatever the -# endpoint's `since_hours` param requests (up to its own max). -FREE_NEWS_WINDOW_HOURS = 6.0 -``` - -- [ ] **Step 4: Clamp the endpoint** - -Open `app/routers/api.py`. Find `news_list` (line 222). Update: - -```python -@router.get("/news") -async def news_list( - request: Request, - session: AsyncSession = Depends(get_session), - principal: CurrentUser | None = Depends(maybe_current_user), - category: str | None = Query(None), - since_hours: float = Query(24.0, ge=0.1, le=720.0), - limit: int = Query(50, ge=1, le=500), - tags: str | None = Query(None, description="comma-separated include list"), - exclude_tags: str | None = Query(None, description="comma-separated exclude list"), - as_: str | None = Query(default=None, alias="as"), -): - from app.services.news_tagging import TAG_LABELS, TAG_VOCABULARY - from app.services.access import FREE_NEWS_WINDOW_HOURS, is_paid_active - - effective_hours = since_hours - capped = not is_paid_active(principal) - if capped: - effective_hours = min(since_hours, FREE_NEWS_WINDOW_HOURS) - - cutoff = utcnow() - timedelta(hours=effective_hours) - # ... rest of the function unchanged through `filtered = ...` ... -``` - -Also add `maybe_current_user` to the imports at the top of `app/routers/api.py`: - -```python -from app.auth import maybe_current_user -``` - -(The existing `from app.auth import …` line may already import other names — extend it.) - -In the `as_ == "html"` branch, extend the template context with `capped` and `window_hours`: - -```python - return templates.TemplateResponse( - request, "partials/news.html", - {"headlines": items, - "tag_vocabulary": TAG_VOCABULARY, - "tag_labels": TAG_LABELS, - "active_include": sorted(include), - "active_exclude": sorted(exclude), - "capped": capped, - "window_hours": effective_hours}, - ) -``` - -The JSON branch is unchanged — the test asserts via the JSON shape. - -- [ ] **Step 5: Run the test, expect PASS** - -```bash -pytest tests/test_news_window.py -v -``` - -Expected: all three tests pass. - -- [ ] **Step 6: Add the partial-template footer** - -Open `app/templates/partials/news.html`. At the bottom of the file (after the last existing item rendering and any close tags), add: - -```html -{% if capped %} -
      - Free tier — showing the last {{ window_hours|int }} hours of news. - Upgrade - for the full 24-hour feed plus daily and weekly email digests. -
      -{% endif %} -``` - -- [ ] **Step 7: Run the full suite to catch regressions** - -```bash -pytest tests/ -x -q -``` - -Expected: no regressions (the existing `test_news_*` tests don't depend on the clamp). - -- [ ] **Step 8: Commit** - -```bash -git add app/services/access.py app/routers/api.py app/templates/partials/news.html tests/test_news_window.py -git commit -m "news: clamp free + anonymous to last 6h; paid keeps 24h" -``` - ---- - -## Task 3: DB model + Alembic migration - -**Files:** -- Modify: `app/models.py` -- Create: `alembic/versions/0017_email_digest.py` - -- [ ] **Step 1: Add columns to `User`** - -Open `app/models.py`. Find the `User` class (search for `class User(Base)`). Add two columns alongside the existing tier/credit fields: - -```python - email_digest_opt_in: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=True, server_default=text("1"), - ) - # NULL = use INTERMEDIATE at render time. Server-side mirror of the - # dashboard tone, decoupled because the dashboard pref is localStorage. - digest_tone: Mapped[str | None] = mapped_column(String(16)) -``` - -If `Boolean` or `text` aren't already imported, extend the SQLAlchemy import line at the top of the file: - -```python -from sqlalchemy import (..., Boolean, text, ...) -``` - -- [ ] **Step 2: Add the `EmailSend` model** - -In the same file, after the existing model definitions, add: - -```python -class EmailSend(Base): - """Audit row per digest email send. Used for idempotency (don't send - twice on the same UTC day) and for surfacing 'last delivery' on the - Settings page.""" - __tablename__ = "email_sends" - - id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - user_id: Mapped[int] = mapped_column( - ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True, - ) - kind: Mapped[str] = mapped_column(String(16), nullable=False) # "daily" | "weekly" - sent_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), default=utcnow, nullable=False, - ) - status: Mapped[str] = mapped_column(String(16), nullable=False) # "sent" | "skipped" | "error" - error: Mapped[str | None] = mapped_column(String(255)) - - __table_args__ = ( - Index("ix_email_sends_user_kind_sent", "user_id", "kind", "sent_at"), - ) -``` - -Add any missing imports (`BigInteger`, `ForeignKey`, `Index`) at the top. - -- [ ] **Step 3: Create the Alembic migration** - -Create `alembic/versions/0017_email_digest.py`: - -```python -"""email digests: User.email_digest_opt_in, User.digest_tone, email_sends table. - -Revision ID: 0017 -Revises: 0016 -Create Date: 2026-05-25 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0017" -down_revision: Union[str, None] = "0016" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.add_column( - "users", - sa.Column( - "email_digest_opt_in", sa.Boolean(), nullable=False, - server_default=sa.text("1"), - ), - ) - op.add_column( - "users", - sa.Column("digest_tone", sa.String(length=16), nullable=True), - ) - - op.create_table( - "email_sends", - sa.Column("id", sa.BigInteger(), autoincrement=True, primary_key=True), - sa.Column( - "user_id", sa.Integer(), nullable=False, - ), - sa.Column("kind", sa.String(length=16), nullable=False), - sa.Column("sent_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("status", sa.String(length=16), nullable=False), - sa.Column("error", sa.String(length=255), nullable=True), - sa.ForeignKeyConstraint( - ["user_id"], ["users.id"], ondelete="CASCADE", - ), - ) - op.create_index( - "ix_email_sends_user_kind_sent", - "email_sends", - ["user_id", "kind", "sent_at"], - ) - op.create_index( - op.f("ix_email_sends_user_id"), "email_sends", ["user_id"], - ) - - -def downgrade() -> None: - op.drop_index(op.f("ix_email_sends_user_id"), table_name="email_sends") - op.drop_index("ix_email_sends_user_kind_sent", table_name="email_sends") - op.drop_table("email_sends") - op.drop_column("users", "digest_tone") - op.drop_column("users", "email_digest_opt_in") -``` - -- [ ] **Step 4: Apply the migration to the dev DB** - -```bash -docker compose exec app alembic upgrade head -``` - -Expected output ends with: `Running upgrade 0016 -> 0017, email digests...` - -- [ ] **Step 5: Verify the columns exist** - -```bash -docker compose exec db mysql -ucassandra -p${MARIADB_PASSWORD} -e "DESCRIBE cassandra.users;" | grep -E "email_digest_opt_in|digest_tone" -docker compose exec db mysql -ucassandra -p${MARIADB_PASSWORD} -e "DESCRIBE cassandra.email_sends;" -``` - -Expected: both new columns on `users` and the full `email_sends` table appear. - -- [ ] **Step 6: Run the test suite — confirm no regressions from the model change** - -```bash -pytest tests/ -x -q -``` - -Expected: all existing tests still pass (no test depends on the old User shape). - -- [ ] **Step 7: Commit** - -```bash -git add app/models.py alembic/versions/0017_email_digest.py -git commit -m "db: add digest opt-in/tone on users, email_sends audit table" -``` - ---- - -## Task 4: Daily / Weekly digest prompts - -**Files:** -- Modify: `app/services/openrouter.py` -- Create: `tests/test_digest_prompts.py` - -- [ ] **Step 1: Inspect the existing prompt scaffolding** - -Read `app/services/openrouter.py` to confirm the shape of `build_system_prompt(tone, analysis)` and `build_user_prompt(...)`. The new functions will mirror that contract: return `(system_prompt, user_prompt)` strings ready for `call_llm`. - -- [ ] **Step 2: Write the failing test** - -Create `tests/test_digest_prompts.py`: - -```python -"""Unit tests for the daily / weekly digest prompt builders.""" -from __future__ import annotations - -from datetime import datetime, timezone - -from app.services.openrouter import ( - build_daily_digest_prompt, - build_weekly_digest_prompt, -) - - -def _ctx(): - return dict( - today=datetime(2026, 5, 25, 6, 30, tzinfo=timezone.utc), - quotes_by_group={"equities": [{"symbol": "SPX", "price": 7500.0, - "label": "S&P 500", "currency": "USD", - "source": "test", "note": "", - "as_of": None, "changes": {}}]}, - headlines_by_bucket={"general": [{"when": "2026-05-25T05:00:00+00:00", - "source": "FT", "title": "Brent slides"}]}, - reference_line="S&P 7,501 (ATH) · VIX 18.0 · US 10y 4.45%", - ) - - -def test_daily_prompt_tone_intermediate(): - sys_, usr = build_daily_digest_prompt(tone="INTERMEDIATE", **_ctx()) - assert "INTERMEDIATE" in sys_.upper() or "intermediate" in sys_.lower() - assert "Brent slides" in usr - assert "daily" in sys_.lower() - - -def test_daily_prompt_tone_novice_differs(): - sys_int, _ = build_daily_digest_prompt(tone="INTERMEDIATE", **_ctx()) - sys_nov, _ = build_daily_digest_prompt(tone="NOVICE", **_ctx()) - assert sys_int != sys_nov - - -def test_weekly_prompt_mentions_week(): - sys_, usr = build_weekly_digest_prompt(tone="INTERMEDIATE", **_ctx()) - assert "week" in sys_.lower() or "weekly" in sys_.lower() - assert "Brent slides" in usr - - -def test_prompts_return_strings(): - for fn in (build_daily_digest_prompt, build_weekly_digest_prompt): - sys_, usr = fn(tone="INTERMEDIATE", **_ctx()) - assert isinstance(sys_, str) and isinstance(usr, str) - assert len(sys_) > 50 and len(usr) > 50 -``` - -- [ ] **Step 3: Run the failing test** - -```bash -pytest tests/test_digest_prompts.py -v -``` - -Expected: FAIL — `ImportError: cannot import name 'build_daily_digest_prompt'`. - -- [ ] **Step 4: Implement the prompt builders** - -Open `app/services/openrouter.py`. After `build_user_prompt`, add: - -```python -def build_daily_digest_prompt( - *, - tone: str, - today, - quotes_by_group: dict, - headlines_by_bucket: dict, - reference_line: str, -) -> tuple[str, str]: - """System + user prompt for the once-a-day editorial digest. - - Different from the hourly log: the daily digest reflects on the past - 24h and looks forward to the upcoming session. Longer, less - 'live-blogging,' more contextual. Target ~600 words.""" - tone_clause = ( - "Use plain English. Define any jargon on first use." - if tone.upper() == "NOVICE" - else "Write for a reader who already speaks markets fluently." - ) - system = ( - "You write the daily editorial digest for Read the Markets. " - f"Audience tone: {tone.upper()}. {tone_clause} " - "Cover: (1) what mattered yesterday, (2) what to watch in today's " - "EU and US sessions, (3) one cross-asset thread connecting them. " - "No predictions of price level, no buy/sell language. Target ~600 " - "words. Output HTML using only

      ,

      ,
        ,
      • , , " - " — no , , or wrapper, no inline styles." - ) - user = _digest_user_prompt(today, quotes_by_group, headlines_by_bucket, reference_line) - return system, user - - -def build_weekly_digest_prompt( - *, - tone: str, - today, - quotes_by_group: dict, - headlines_by_bucket: dict, - reference_line: str, -) -> tuple[str, str]: - """System + user prompt for the Sunday weekly recap + look-ahead. - - Sent to ALL opt-in users (free and paid). Target ~900 words.""" - tone_clause = ( - "Use plain English. Define any jargon on first use." - if tone.upper() == "NOVICE" - else "Write for a reader who already speaks markets fluently." - ) - system = ( - "You write the Sunday weekly digest for Read the Markets. " - f"Audience tone: {tone.upper()}. {tone_clause} " - "Cover: (1) the week behind — what moved and why, " - "(2) the week ahead — releases, earnings, central-bank meetings, " - "(3) the cross-asset story to keep in mind. " - "No predictions of price level, no buy/sell language. Target ~900 " - "words. Output HTML using only

        ,

        ,
          ,
        • , , " - " — no , , or wrapper, no inline styles." - ) - user = _digest_user_prompt(today, quotes_by_group, headlines_by_bucket, reference_line) - return system, user - - -def _digest_user_prompt(today, quotes_by_group, headlines_by_bucket, reference_line): - """Shared user-message body used by both digest prompts. Same data - shape as the hourly user prompt; reformatted for the digest context.""" - today_str = today.strftime("%A %d %B %Y") if hasattr(today, "strftime") else str(today) - lines = [f"TODAY (UTC): {today_str}", "", f"REFERENCE: {reference_line}", ""] - - if headlines_by_bucket: - lines.append("HEADLINES BY CATEGORY") - for cat, items in headlines_by_bucket.items(): - lines.append(f" [{cat}]") - for h in items[:30]: - when = h.get("when", "") - src = h.get("source", "") - title = h.get("title", "") - lines.append(f" {when} · {src} · {title}") - lines.append("") - - if quotes_by_group: - lines.append("LATEST QUOTES BY GROUP") - for grp, items in quotes_by_group.items(): - lines.append(f" [{grp}]") - for q in items[:30]: - sym = q.get("symbol", "") - price = q.get("price", "") - lbl = q.get("label", "") - ccy = q.get("currency", "") - lines.append(f" {sym} ({lbl}) — {price} {ccy}") - lines.append("") - - return "\n".join(lines) -``` - -Bump the PROMPT_VERSION constant at the top of the file. Locate it (search for `PROMPT_VERSION =`) and increment by one — e.g., `8` → `9`. Add a short comment line above noting the bump is for digest prompts. - -- [ ] **Step 5: Run the tests, expect PASS** - -```bash -pytest tests/test_digest_prompts.py -v -``` - -Expected: 4 passed. - -- [ ] **Step 6: Commit** - -```bash -git add app/services/openrouter.py tests/test_digest_prompts.py -git commit -m "digest: daily + weekly prompt builders (NOVICE/INTERMEDIATE)" -``` - ---- - -## Task 5: Digest email rendering - -**Files:** -- Modify: `app/services/email_service.py` -- Create: `tests/test_email_render.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_email_render.py`: - -```python -"""Unit tests for render_digest_email.""" -from __future__ import annotations - -from app.services.email_service import render_digest_email - - -def test_daily_subject_and_bodies(): - subj, text, html = render_digest_email( - kind="daily", - date_str="2026-05-25", - content_html="

          Markets did stuff today.

          ", - unsubscribe_url="https://read.markets/email/unsubscribe?token=abc", - settings_url="https://read.markets/settings", - ) - assert "Daily" in subj - assert "2026-05-25" in subj - assert "Markets did stuff today" in html - assert "abc" in html # unsubscribe link landed - assert "/settings" in html - # Plain-text fallback strips HTML. - assert "

          " not in text - assert "Markets did stuff today" in text - - -def test_weekly_subject_says_recap(): - subj, _, _ = render_digest_email( - kind="weekly", - date_str="2026-05-25", - content_html="

          x

          ", - unsubscribe_url="https://x/u", - settings_url="https://x/s", - ) - assert "Weekly" in subj - assert "recap" in subj.lower() - - -def test_invalid_kind_raises(): - import pytest - with pytest.raises(ValueError): - render_digest_email( - kind="bogus", date_str="2026-05-25", - content_html="

          x

          ", - unsubscribe_url="u", settings_url="s", - ) -``` - -- [ ] **Step 2: Run the failing test** - -```bash -pytest tests/test_email_render.py -v -``` - -Expected: FAIL — `render_digest_email` doesn't exist. - -- [ ] **Step 3: Implement `render_digest_email`** - -Open `app/services/email_service.py`. Add at the bottom of the file: - -```python -# --------------------------------------------------------------------------- -# Digest email rendering -# --------------------------------------------------------------------------- - -import html as _html_lib -import re as _re - - -_DIGEST_HTML_TEMPLATE = """\ - - - - - - - {brand} — {label} - - - - - -
          -
          - ▰ {brand_upper} · {label_upper} -
          -
           
          -
          - {content_html} -
          -
           
          -
          -
           
          -
          - Unsubscribe in one click - · Manage preferences -
          -
          - - -""" - - -def _strip_html_to_text(html_body: str) -> str: - """Best-effort HTML → plain text for the multipart fallback. We don't - need perfection — just readable prose for clients that won't render - HTML.""" - # Block-level tags become double newlines,
          becomes single newline. - text = _re.sub(r"(?i)<(/(p|h[1-6]|li|ul|ol)|br\s*/?)>", "\n", html_body) - text = _re.sub(r"<[^>]+>", "", text) - text = _html_lib.unescape(text) - text = _re.sub(r"\n{3,}", "\n\n", text) - return text.strip() - - -def render_digest_email( - *, - kind: str, - date_str: str, - content_html: str, - unsubscribe_url: str, - settings_url: str, -) -> tuple[str, str, str]: - """Returns (subject, text_body, html_body) for a digest email. - - `kind` is "daily" or "weekly". Anything else raises ValueError.""" - if kind == "daily": - label = "Daily" - subject = f"{branding.BRAND_NAME} · Daily — {date_str}" - elif kind == "weekly": - label = "Weekly recap" - subject = f"{branding.BRAND_NAME} · Weekly recap — {date_str}" - else: - raise ValueError(f"unknown digest kind: {kind!r}") - - html_body = _DIGEST_HTML_TEMPLATE.format( - brand=branding.BRAND_NAME, - brand_upper=branding.BRAND_NAME.upper(), - label=label, - label_upper=label.upper(), - FONT_MONO=branding.FONT_MONO, - content_html=content_html, - unsubscribe_url=unsubscribe_url, - settings_url=settings_url, - **{f"L_{k.replace('-', '_')}": v for k, v in branding.LIGHT.items()}, - **{f"D_{k.replace('-', '_')}": v for k, v in branding.DARK.items()}, - ) - - text_lines = [ - f"{branding.BRAND_NAME} — {label}", - date_str, - "", - _strip_html_to_text(content_html), - "", - f"Unsubscribe: {unsubscribe_url}", - f"Manage preferences: {settings_url}", - ] - text_body = "\n".join(text_lines) - return subject, text_body, html_body -``` - -- [ ] **Step 4: Run the tests, expect PASS** - -```bash -pytest tests/test_email_render.py -v -``` - -Expected: 3 passed. - -- [ ] **Step 5: Commit** - -```bash -git add app/services/email_service.py tests/test_email_render.py -git commit -m "email: render_digest_email — multipart digest template" -``` - ---- - -## Task 6: Unsubscribe token + route - -**Files:** -- Create: `app/routers/email.py` -- Modify: `app/main.py` (router include) -- Create: `tests/test_email_unsubscribe.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_email_unsubscribe.py`: - -```python -"""Unsubscribe token roundtrip + endpoint.""" -from __future__ import annotations - -import asyncio - -import pytest - - -def _build_app(tmp_path, secret="testsecret"): - import os - os.environ["CASSANDRA_SESSION_SECRET"] = secret - - from fastapi import FastAPI - from fastapi.testclient import TestClient - from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - - from app import db as db_mod - from app.db import Base - from app.models import User - from app.routers import email as email_router - from app.config import get_settings - get_settings.cache_clear() # pick up the new env var - - engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/u.db") - factory = async_sessionmaker(engine, expire_on_commit=False) - db_mod._engine = engine - db_mod._session_factory = factory - - async def _seed(): - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - async with factory() as s: - s.add(User(id=42, email="u@x", tier="paid", email_digest_opt_in=True)) - await s.commit() - - asyncio.run(_seed()) - - app = FastAPI() - app.include_router(email_router.router) - return TestClient(app) - - -def test_sign_and_verify_token_roundtrip(tmp_path, monkeypatch): - monkeypatch.setenv("CASSANDRA_SESSION_SECRET", "rt-secret-32-bytes-or-so-padding-here") - from app.config import get_settings - get_settings.cache_clear() - from app.routers.email import sign_unsubscribe_token, verify_unsubscribe_token - tok = sign_unsubscribe_token(42) - assert verify_unsubscribe_token(tok) == 42 - assert verify_unsubscribe_token("garbage") is None - - -def test_get_unsubscribe_flips_flag(tmp_path): - client = _build_app(tmp_path) - from app.routers.email import sign_unsubscribe_token - tok = sign_unsubscribe_token(42) - r = client.get(f"/email/unsubscribe?token={tok}") - assert r.status_code == 200 - assert "unsubscribed" in r.text.lower() - - # DB state changed. - async def _check(): - from app import db as db_mod - async with db_mod._session_factory() as s: - from app.models import User - u = await s.get(User, 42) - assert u.email_digest_opt_in is False - asyncio.run(_check()) - - -def test_get_unsubscribe_invalid_token_returns_generic_page(tmp_path): - client = _build_app(tmp_path) - r = client.get("/email/unsubscribe?token=garbage") - # We don't 4xx — that would leak token validity. Show the generic page. - assert r.status_code == 200 - assert "unsubscribed" in r.text.lower() or "preferences" in r.text.lower() - - -def test_replay_is_idempotent(tmp_path): - client = _build_app(tmp_path) - from app.routers.email import sign_unsubscribe_token - tok = sign_unsubscribe_token(42) - r1 = client.get(f"/email/unsubscribe?token={tok}") - r2 = client.get(f"/email/unsubscribe?token={tok}") - assert r1.status_code == 200 - assert r2.status_code == 200 -``` - -- [ ] **Step 2: Run the failing test** - -```bash -pytest tests/test_email_unsubscribe.py -v -``` - -Expected: FAIL — `app.routers.email` doesn't exist. - -- [ ] **Step 3: Create the router** - -Create `app/routers/email.py`: - -```python -"""Email-related public routes. - -Currently: -- GET /email/unsubscribe?token=... - -The token is `itsdangerous.URLSafeSerializer` over a small payload, -signed with CASSANDRA_SESSION_SECRET. No auth dependency: the whole -point of one-click unsubscribe is that the user does not have to -sign in. -""" -from __future__ import annotations - -from fastapi import APIRouter, Depends, Request, Query -from fastapi.responses import HTMLResponse -from itsdangerous import BadSignature, URLSafeSerializer -from sqlalchemy.ext.asyncio import AsyncSession - -from app.config import get_settings -from app.db import get_session -from app.logging import get_logger -from app.models import User -from app.templates_env import templates - - -router = APIRouter() -log = get_logger("email_router") - -_SALT = "digest-unsubscribe-v1" - - -def _serializer() -> URLSafeSerializer: - s = get_settings() - if not s.CASSANDRA_SESSION_SECRET: - # In tests with no secret configured, fall back to a constant — - # NEVER reach production; settings validation should catch this. - return URLSafeSerializer("dev-only-empty-secret", salt=_SALT) - return URLSafeSerializer(s.CASSANDRA_SESSION_SECRET, salt=_SALT) - - -def sign_unsubscribe_token(user_id: int) -> str: - return _serializer().dumps({"uid": int(user_id), "purpose": "digest_optout"}) - - -def verify_unsubscribe_token(token: str) -> int | None: - try: - data = _serializer().loads(token) - except BadSignature: - return None - if not isinstance(data, dict): - return None - if data.get("purpose") != "digest_optout": - return None - try: - return int(data["uid"]) - except (KeyError, TypeError, ValueError): - return None - - -_CONFIRM_PAGE = """\ - - - - - Unsubscribed — {brand} - - - -
          -
          {brand}
          -
          email preferences
          -

          You're unsubscribed from email digests.

          -

          - You can re-enable digests any time from - Settings. -

          -
          - - -""" - - -@router.get("/email/unsubscribe", response_class=HTMLResponse) -async def unsubscribe( - request: Request, - token: str = Query(...), - session: AsyncSession = Depends(get_session), -): - from app import branding - uid = verify_unsubscribe_token(token) - if uid is not None: - user = await session.get(User, uid) - if user is not None and user.email_digest_opt_in: - user.email_digest_opt_in = False - await session.commit() - log.info("email.unsubscribe.ok", user_id=uid) - else: - log.info("email.unsubscribe.noop_or_unknown", user_id=uid) - else: - log.info("email.unsubscribe.bad_token") - - # Same confirmation page regardless — don't leak token validity. - return HTMLResponse(_CONFIRM_PAGE.format(brand=branding.BRAND_NAME)) -``` - -- [ ] **Step 4: Wire the router into the FastAPI app** - -Open `app/main.py`. Find the existing router includes (line ~85). Add: - -```python -from app.routers import email as email_router -# ... -app.include_router(email_router.router, tags=["email"]) -``` - -- [ ] **Step 5: Run the tests, expect PASS** - -```bash -pytest tests/test_email_unsubscribe.py -v -``` - -Expected: 4 passed. - -- [ ] **Step 6: Commit** - -```bash -git add app/routers/email.py app/main.py tests/test_email_unsubscribe.py -git commit -m "email: one-click unsubscribe endpoint w/ signed token" -``` - ---- - -## Task 7: Email digest job + idempotency - -**Files:** -- Create: `app/jobs/email_digest_job.py` -- Create: `tests/test_email_digest_job.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_email_digest_job.py`: - -```python -"""Recipient selection + idempotency for the digest job.""" -from __future__ import annotations - -import asyncio -from datetime import datetime, timezone, timedelta -from unittest.mock import AsyncMock, patch - -import pytest - - -def _bootstrap(tmp_path, today_weekday: int): - """Spin up an in-memory DB with three users: a paid opt-in, a paid - opt-out, a free opt-in. Returns the session factory + the test users.""" - from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - - from app import db as db_mod - from app.db import Base - from app.models import User - - engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/dj.db") - factory = async_sessionmaker(engine, expire_on_commit=False) - db_mod._engine = engine - db_mod._session_factory = factory - - async def _seed(): - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - async with factory() as s: - s.add(User(id=1, email="paid_in@x", tier="paid", email_digest_opt_in=True)) - s.add(User(id=2, email="paid_out@x", tier="paid", email_digest_opt_in=False)) - s.add(User(id=3, email="free_in@x", tier="free", email_digest_opt_in=True)) - await s.commit() - - asyncio.run(_seed()) - return factory - - -def _patch_today(weekday: int): - """Return a datetime whose weekday() == `weekday` (0=Mon, 6=Sun).""" - base = datetime(2026, 5, 25, 6, 30, tzinfo=timezone.utc) # Monday - return base + timedelta(days=(weekday - base.weekday()) % 7) - - -def _stub_generate(content="

          x

          "): - """Stub out the LLM call so the test never hits the network. Use a - SimpleNamespace so we don't have to know the real result class name.""" - from types import SimpleNamespace - async def _fake(_client, messages, **kwargs): - return SimpleNamespace( - content=content, model="stub", - prompt_tokens=10, completion_tokens=10, cost_usd=0.0, - ) - return _fake - - -@pytest.mark.skipif(False, reason="requires aiosqlite") -def test_daily_run_only_paid_opt_in(tmp_path): - _bootstrap(tmp_path, today_weekday=0) - from app.jobs import email_digest_job - with patch("app.jobs.email_digest_job._now", - return_value=_patch_today(0)), \ - patch("app.jobs.email_digest_job.send_email", - new=AsyncMock()) as send_mock, \ - patch("app.jobs.email_digest_job.call_llm", - new=AsyncMock(side_effect=_stub_generate())): - asyncio.run(email_digest_job.run()) - # Only user_id=1 (paid + opt-in) received an email. Two calls - # (one per tone), but only one recipient. - sent_to = {kwargs["to"] for _, _, kwargs in send_mock.mock_calls - if "to" in kwargs} - addresses_sent = {call.kwargs.get("to") for call in send_mock.await_args_list} - assert addresses_sent == {"paid_in@x"} - - -def test_weekly_run_includes_free_and_paid_opt_in(tmp_path): - _bootstrap(tmp_path, today_weekday=6) - from app.jobs import email_digest_job - with patch("app.jobs.email_digest_job._now", - return_value=_patch_today(6)), \ - patch("app.jobs.email_digest_job.send_email", - new=AsyncMock()) as send_mock, \ - patch("app.jobs.email_digest_job.call_llm", - new=AsyncMock(side_effect=_stub_generate())): - asyncio.run(email_digest_job.run()) - addresses_sent = {call.kwargs.get("to") for call in send_mock.await_args_list} - assert addresses_sent == {"paid_in@x", "free_in@x"} - - -def test_second_run_same_day_is_idempotent(tmp_path): - _bootstrap(tmp_path, today_weekday=0) - from app.jobs import email_digest_job - with patch("app.jobs.email_digest_job._now", - return_value=_patch_today(0)), \ - patch("app.jobs.email_digest_job.send_email", - new=AsyncMock()) as send_mock, \ - patch("app.jobs.email_digest_job.call_llm", - new=AsyncMock(side_effect=_stub_generate())): - asyncio.run(email_digest_job.run()) - first_count = len(send_mock.await_args_list) - asyncio.run(email_digest_job.run()) - second_count = len(send_mock.await_args_list) - assert first_count > 0 - assert second_count == first_count, "second run should not re-send" -``` - -- [ ] **Step 2: Run the failing test** - -```bash -pytest tests/test_email_digest_job.py -v -``` - -Expected: FAIL — `app.jobs.email_digest_job` doesn't exist yet. - -- [ ] **Step 3: Implement the job** - -Create `app/jobs/email_digest_job.py`: - -```python -"""Daily/weekly editorial email digest. - -Runs once a day at 06:30 UTC via the scheduler. On Sundays sends the -weekly recap to every opt-in user (free + paid). On other days sends -the daily digest to opt-in paid users only. - -Generates LLM content once per tone (NOVICE + INTERMEDIATE), then fans -out by SMTP. EmailSend audit rows guard against double-delivery if the -job is re-run within the same UTC day. -""" -from __future__ import annotations - -import asyncio -from collections import defaultdict -from datetime import datetime, timedelta, timezone - -import httpx -from sqlalchemy import desc, func, select - -from app import branding -from app.config import get_settings -from app.db import utcnow -from app.jobs._helpers import job_lifecycle, log -from app.jobs.ai_log_job import ( - REFERENCE_LINE, - _latest_quotes_by_group, - _recent_headlines_by_bucket, - _month_spend, -) -from app.models import EmailSend, User -from app.routers.email import sign_unsubscribe_token -from app.services.email_service import render_digest_email, send_email -from app.services.openrouter import ( - PROMPT_VERSION, - active_model, - build_daily_digest_prompt, - build_weekly_digest_prompt, - call_llm, - llm_configured, -) -from app.services.access import paid_status - - -# Indirection so tests can monkeypatch the "current time" without -# touching the system clock. -def _now() -> datetime: - return utcnow() - - -async def _opt_in_recipients(session, *, paid_only: bool) -> list[User]: - stmt = select(User).where(User.email_digest_opt_in.is_(True)) - rows = (await session.execute(stmt)).scalars().all() - if paid_only: - rows = [u for u in rows if paid_status(u).active] - return rows - - -async def _already_sent_today(session, user_id: int, kind: str, today: datetime) -> bool: - """True if an EmailSend row exists for this user+kind on the same UTC - day, with status in ('sent','error'). 'error' counts because we don't - want to keep retrying a bad address inside the same daily slot.""" - day_start = today.replace(hour=0, minute=0, second=0, microsecond=0) - day_end = day_start + timedelta(days=1) - stmt = select(EmailSend.id).where( - EmailSend.user_id == user_id, - EmailSend.kind == kind, - EmailSend.sent_at >= day_start, - EmailSend.sent_at < day_end, - EmailSend.status.in_(("sent", "error")), - ) - return (await session.execute(stmt)).first() is not None - - -async def _generate_variants(client, kind: str, ctx: dict) -> dict[str, str]: - """Returns {tone: html_content}. Missing tone means generation failed - for that variant — skip recipients on that tone.""" - builder = build_weekly_digest_prompt if kind == "weekly" else build_daily_digest_prompt - out: dict[str, str] = {} - for tone in ("NOVICE", "INTERMEDIATE"): - sys_, usr = builder(tone=tone, **ctx) - try: - result = await call_llm( - client, - [{"role": "system", "content": sys_}, - {"role": "user", "content": usr}], - ) - out[tone] = result.content - log.info("digest.variant_ok", kind=kind, tone=tone, - prompt_tokens=result.prompt_tokens, - completion_tokens=result.completion_tokens) - except Exception as e: - log.error("digest.variant_failed", kind=kind, tone=tone, - error=str(e)[:200]) - return out - - -def _kind_for_today(today: datetime) -> str | None: - """Sunday → weekly. Mon–Sat → daily. None means 'no run today'.""" - return "weekly" if today.weekday() == 6 else "daily" - - -async def _send_one(user: User, kind: str, content_html: str, date_str: str, - session) -> None: - settings_url = f"{branding.SITE_URL}/settings" - unsubscribe_url = ( - f"{branding.SITE_URL}/email/unsubscribe" - f"?token={sign_unsubscribe_token(user.id)}" - ) - subject, text_body, html_body = render_digest_email( - kind=kind, date_str=date_str, - content_html=content_html, - unsubscribe_url=unsubscribe_url, - settings_url=settings_url, - ) - try: - await send_email(to=user.email, subject=subject, - text_body=text_body, html_body=html_body) - status_ = "sent" - err = None - except Exception as e: - status_ = "error" - err = str(e)[:255] - log.error("digest.send_failed", user_id=user.id, error=err) - session.add(EmailSend( - user_id=user.id, kind=kind, sent_at=_now(), - status=status_, error=err, - )) - await session.commit() - - -async def run() -> None: - async with job_lifecycle("email_digest_job") as (session, jr): - if jr.status == "skipped": - return - s = get_settings() - if not llm_configured(): - log.warning("digest.skipped_no_key", provider=s.LLM_PROVIDER) - jr.status = "skipped" - return - - today = _now() - kind = _kind_for_today(today) - date_str = today.strftime("%Y-%m-%d") - - # Build the recipient list before LLM work — if it's empty, - # skip the LLM spend. - recipients = await _opt_in_recipients( - session, paid_only=(kind == "daily"), - ) - # Filter out anyone already sent today (idempotency). - fresh: list[User] = [] - for u in recipients: - if not await _already_sent_today(session, u.id, kind, today): - fresh.append(u) - if not fresh: - log.info("digest.no_fresh_recipients", kind=kind, - total=len(recipients)) - jr.status = "skipped" - return - - spent = await _month_spend(session) - if spent >= s.OPENROUTER_MONTHLY_CAP_USD: - log.warning("digest.cap_reached", spent=spent, - cap=s.OPENROUTER_MONTHLY_CAP_USD) - jr.status = "skipped" - jr.error = f"monthly cost cap reached (${spent:.2f})" - return - - quotes = await _latest_quotes_by_group(session) - news = await _recent_headlines_by_bucket( - session, hours=(168 if kind == "weekly" else 24), - ) - ctx = dict( - today=today, - quotes_by_group=quotes, - headlines_by_bucket=news, - reference_line=REFERENCE_LINE, - ) - - async with httpx.AsyncClient(follow_redirects=True) as client: - variants = await _generate_variants(client, kind, ctx) - - if not variants: - log.warning("digest.all_variants_failed", kind=kind) - jr.status = "failed" - jr.error = "all variants failed" - return - - written = 0 - for u in fresh: - tone = (u.digest_tone or "INTERMEDIATE").upper() - content = variants.get(tone) or variants.get("INTERMEDIATE") - if content is None: - # Both variants failed for this user's tone — skip. - continue - await _send_one(u, kind, content, date_str, session) - await asyncio.sleep(0.1) # gentle SMTP pacing - written += 1 - - jr.items_written = written - log.info("digest.done", kind=kind, written=written, - prompt_version=PROMPT_VERSION) - - -if __name__ == "__main__": - asyncio.run(run()) -``` - -- [ ] **Step 4: Run the tests, expect PASS** - -```bash -pytest tests/test_email_digest_job.py -v -``` - -Expected: 3 passed. - -- [ ] **Step 5: Commit** - -```bash -git add app/jobs/email_digest_job.py tests/test_email_digest_job.py -git commit -m "digest: daily/weekly job w/ EmailSend idempotency" -``` - ---- - -## Task 8: Scheduler registration - -**Files:** -- Modify: `app/scheduler_main.py` - -- [ ] **Step 1: Add the cron entry** - -Open `app/scheduler_main.py`. Locate the block where the other jobs are registered (line ~42 onward — `sched.add_job(ai_log_job.run, ...)` etc.). Add an import at the top and a new `sched.add_job` line: - -```python -from app.jobs import email_digest_job -# ... -sched.add_job( - email_digest_job.run, - CronTrigger(hour=6, minute=30), - name="email_digest_job", - id="email_digest_job", -) -``` - -- [ ] **Step 2: Verify the scheduler boots clean** - -```bash -docker compose restart scheduler -docker compose logs --tail=40 scheduler -``` - -Expected: `scheduler.started jobs=[..., 'email_digest_job']` in the log. - -- [ ] **Step 3: Commit** - -```bash -git add app/scheduler_main.py -git commit -m "scheduler: register email_digest_job at 06:30 UTC" -``` - ---- - -## Task 9: Settings page section + PATCH endpoint - -**Files:** -- Modify: `app/routers/api.py` (add PATCH /settings/digest) -- Modify: `app/templates/settings.html` -- Modify: `app/routers/pages.py` (settings_page context: pass last EmailSend) -- Create: `tests/test_settings_digest_api.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_settings_digest_api.py`: - -```python -"""PATCH /api/settings/digest persists opt-in + tone.""" -from __future__ import annotations - -import asyncio - -import pytest - - -def _build(tmp_path): - from fastapi import FastAPI - from fastapi.testclient import TestClient - from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - - from app import db as db_mod - from app.auth import sign_session - from app.db import Base - from app.models import User - from app.routers import api as api_router - - engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/s.db") - factory = async_sessionmaker(engine, expire_on_commit=False) - db_mod._engine = engine - db_mod._session_factory = factory - - async def _seed(): - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - async with factory() as s: - s.add(User(id=1, email="u@x", tier="paid", - email_digest_opt_in=True)) - await s.commit() - - asyncio.run(_seed()) - app = FastAPI() - app.include_router(api_router.router, prefix="/api") - return TestClient(app), sign_session(1) - - -def test_patch_round_trip(tmp_path): - client, sess = _build(tmp_path) - r = client.patch( - "/api/settings/digest", - json={"opt_in": False, "tone": "NOVICE"}, - cookies={"cassandra_session": sess}, - ) - assert r.status_code == 200, r.text - assert r.json() == {"opt_in": False, "tone": "NOVICE"} - - # Round-trip GET to confirm persistence. - async def _check(): - from app import db as db_mod - from app.models import User - async with db_mod._session_factory() as s: - u = await s.get(User, 1) - assert u.email_digest_opt_in is False - assert u.digest_tone == "NOVICE" - asyncio.run(_check()) - - -def test_patch_rejects_invalid_tone(tmp_path): - client, sess = _build(tmp_path) - r = client.patch( - "/api/settings/digest", - json={"opt_in": True, "tone": "PRO"}, # not in NOVICE|INTERMEDIATE - cookies={"cassandra_session": sess}, - ) - assert r.status_code == 422 - - -def test_patch_requires_auth(tmp_path): - client, _ = _build(tmp_path) - r = client.patch("/api/settings/digest", - json={"opt_in": True, "tone": "NOVICE"}) - assert r.status_code in (401, 303) -``` - -- [ ] **Step 2: Run the failing test** - -```bash -pytest tests/test_settings_digest_api.py -v -``` - -Expected: FAIL — endpoint not defined. - -- [ ] **Step 3: Add the endpoint** - -Open `app/routers/api.py`. Near the other settings-related code (or at the bottom of the file), add: - -```python -from pydantic import BaseModel, Field -from typing import Literal - - -class DigestPrefsIn(BaseModel): - opt_in: bool - tone: Literal["NOVICE", "INTERMEDIATE"] - - -class DigestPrefsOut(BaseModel): - opt_in: bool - tone: str - - -@router.patch("/settings/digest", response_model=DigestPrefsOut) -async def patch_digest_prefs( - payload: DigestPrefsIn, - principal: CurrentUser = Depends(require_auth), - session: AsyncSession = Depends(get_session), -) -> DigestPrefsOut: - if principal.user is None: - # Admin bearer-token path: nothing to persist. - raise HTTPException(status_code=400, detail="no_user_context") - principal.user.email_digest_opt_in = payload.opt_in - principal.user.digest_tone = payload.tone - await session.commit() - return DigestPrefsOut(opt_in=payload.opt_in, tone=payload.tone) -``` - -If `require_auth`, `CurrentUser`, or `HTTPException` aren't already imported, extend the import block at the top of the file. - -- [ ] **Step 4: Run the tests, expect PASS** - -```bash -pytest tests/test_settings_digest_api.py -v -``` - -Expected: 3 passed. - -- [ ] **Step 5: Add the Settings template section** - -Open `app/templates/settings.html`. Find a sensible insertion point (after the existing tier/credit section, before the cloud-sync section). Add: - -```html -
          -
          Email digests
          -
          - -
          - Free tier: Sunday weekly. Paid: daily Mon–Sat plus the Sunday recap. -
          -
          - - -
          -
          - Last delivery: - {% if last_email_send %}{{ last_email_send.sent_at.strftime("%Y-%m-%d %H:%M") }} UTC — {{ last_email_send.status }}{% else %}—{% endif %} -
          -
          -
          -
          - - -``` - -- [ ] **Step 6: Pass the last EmailSend into the template context** - -Open `app/routers/pages.py`. Find `settings_page` (line ~114). Inside the handler, before the existing `TemplateResponse(... "settings.html", ...)`, query the most recent EmailSend for the current user: - -```python -from sqlalchemy import desc, select -from app.models import EmailSend - -# ... -last_email_send = (await session.execute( - select(EmailSend) - .where(EmailSend.user_id == user.id) - .order_by(desc(EmailSend.sent_at)) - .limit(1) -)).scalar_one_or_none() -``` - -Add `"last_email_send": last_email_send` to the template context dict that's passed to `templates.TemplateResponse(..., "settings.html", {...})`. There may be two TemplateResponse calls in this handler (admin path + user path) — only the user path needs this. - -- [ ] **Step 7: Manual UI check** - -```bash -docker compose restart app -``` - -Open `/settings` in a browser, flip the checkbox, change the tone. Confirm "Saved." appears. Reload — selections persist. - -- [ ] **Step 8: Commit** - -```bash -git add app/routers/api.py app/routers/pages.py app/templates/settings.html tests/test_settings_digest_api.py -git commit -m "settings: digest opt-in + tone (PATCH /api/settings/digest + UI)" -``` - ---- - -## Task 10: Sign-up checkbox - -**Files:** -- Modify: `app/templates/verify.html` -- Modify: `app/routers/auth.py` -- Create: `tests/test_verify_subscribe.py` - -- [ ] **Step 1: Write the failing test** - -Create `tests/test_verify_subscribe.py`: - -```python -"""Verify-POST persists the subscribe_to_digests form field.""" -from __future__ import annotations - -import asyncio - -import pytest - - -def _build(tmp_path): - from fastapi import FastAPI - from fastapi.testclient import TestClient - from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - - from app import db as db_mod - from app.db import Base - from app.models import User - from app.routers import auth as auth_router - from app.auth import _pending_serializer - - engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/v.db") - factory = async_sessionmaker(engine, expire_on_commit=False) - db_mod._engine = engine - db_mod._session_factory = factory - - async def _seed(): - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - async with factory() as s: - s.add(User(id=10, email="newbie@x", tier="free", - email_digest_opt_in=True)) - await s.commit() - # Mark the OTP as "verified-able" — easiest is to monkeypatch - # otp_service.verify in the test, see below. - - asyncio.run(_seed()) - app = FastAPI() - app.include_router(auth_router.router) - pending = _pending_serializer().dumps({"email": "newbie@x", "uid": 10, "ref": None}) - return TestClient(app), pending - - -def test_verify_with_unchecked_subscribe_disables_opt_in(tmp_path, monkeypatch): - from app.services import otp_service - - async def _ok(*args, **kwargs): - return None - monkeypatch.setattr(otp_service, "verify", _ok) - - client, pending = _build(tmp_path) - r = client.post( - "/verify", - data={"code": "000000"}, # form field "subscribe_to_digests" omitted - cookies={"cassandra_pending": pending}, - follow_redirects=False, - ) - assert r.status_code == 303, r.text - - async def _check(): - from app import db as db_mod - from app.models import User - async with db_mod._session_factory() as s: - u = await s.get(User, 10) - assert u.email_digest_opt_in is False - asyncio.run(_check()) - - -def test_verify_with_checked_subscribe_keeps_opt_in(tmp_path, monkeypatch): - from app.services import otp_service - - async def _ok(*args, **kwargs): - return None - monkeypatch.setattr(otp_service, "verify", _ok) - - client, pending = _build(tmp_path) - r = client.post( - "/verify", - data={"code": "000000", "subscribe_to_digests": "on"}, - cookies={"cassandra_pending": pending}, - follow_redirects=False, - ) - assert r.status_code == 303 - - async def _check(): - from app import db as db_mod - from app.models import User - async with db_mod._session_factory() as s: - u = await s.get(User, 10) - assert u.email_digest_opt_in is True - asyncio.run(_check()) -``` - -- [ ] **Step 2: Run the failing test** - -```bash -pytest tests/test_verify_subscribe.py -v -``` - -Expected: FAIL — the verify handler doesn't read `subscribe_to_digests` yet, so the unchecked case still leaves opt-in `True` from the seed. - -- [ ] **Step 3: Update the verify template** - -Open `app/templates/verify.html`. Find the `
          ` block. Inside the form, after the code input and before the submit button, add: - -```html - -``` - -- [ ] **Step 4: Update the verify POST handler** - -Open `app/routers/auth.py`. Find `verify_submit` (line ~215). Add `subscribe_to_digests` to the form fields and persist it after a successful OTP verify: - -```python -@router.post("/verify") -async def verify_submit( - request: Request, - code: str = Form(...), - subscribe_to_digests: str | None = Form(default=None), - session: AsyncSession = Depends(get_session), -): - # ... existing OTP verification + pending lookup unchanged ... - - user = await get_user(session, pending["uid"]) - if user is None: - return RedirectResponse(url="/login", status_code=303) - user.last_login_at = utcnow() - # An unchecked HTML checkbox sends NO field; that means "opt out". - user.email_digest_opt_in = subscribe_to_digests is not None - await session.commit() - log.info("user.login", user_id=user.id, email=email) - # ... existing redirect + cookie setup unchanged ... -``` - -- [ ] **Step 5: Run the tests, expect PASS** - -```bash -pytest tests/test_verify_subscribe.py -v -``` - -Expected: 2 passed. - -- [ ] **Step 6: Commit** - -```bash -git add app/templates/verify.html app/routers/auth.py tests/test_verify_subscribe.py -git commit -m "auth: subscribe-to-digests checkbox on verify (default on)" -``` - ---- - -## Task 11: Pricing copy updates - -**Files:** -- Modify: `app/templates/pricing.html` - -- [ ] **Step 1: Rewrite the intro paragraph** - -Open `app/templates/pricing.html`. Replace lines 8-13 (the intro paragraph under the `Pricing` heading) with: - -```html -

          - Two tiers. The news aggregator and hourly AI interpretation are - available to everyone — paid extends the news window from - 6 hours to 24 hours and adds daily editorial by email, plus the - portfolio-import features. -

          -``` - -- [ ] **Step 2: Update the Free tier bullets** - -Find the Free `
            ` block. Replace it with: - -```html -
              -
            • News aggregator — last 6 hours, auto-tagged by theme
              • -
            • Cross-asset macro signals across every asset class
              • -
            • Hourly AI interpretation of the news + the tape
              • -
            • Per-group cross-asset summaries
              • -
            • Novice / Intermediate reading levels
              • -
            • Sunday weekly digest by email
              • -
            • Portfolio import & analysis
              • -
            • Encrypted cloud sync
              • -
            -``` - -- [ ] **Step 3: Update the Paid tier bullets** - -Find the Paid `
              ` block. Replace it with: - -```html -
                -
              • Everything in Free
                • -
              • News aggregator — full 24 hours
                • -
              • Portfolio import (Trading 212 CSV)
                • -
              • AI commentary on diversification, sector and currency concentration, and macro-regime context for the holdings you upload
                • -
              • Optional encrypted cloud sync across devices
                • -
              • Daily email digest (Mon–Sat) plus the Sunday recap
                • -
              -``` - -- [ ] **Step 4: Manual visual check** - -```bash -docker compose restart app -``` - -Load `/pricing` in a browser. Confirm both tier cards render the new bullets and the intro paragraph reads cleanly. - -- [ ] **Step 5: Commit** - -```bash -git add app/templates/pricing.html -git commit -m "pricing: free=6h news + Sunday digest; paid=24h + daily digest" -``` - ---- - -## Task 12: `send-test-digest` admin CLI - -**Files:** -- Modify: `app/cli.py` - -- [ ] **Step 1: Add the subcommand handler** - -Open `app/cli.py`. After `show_status`, add: - -```python -async def send_test_digest(email: str, kind: str) -> int: - """Generate a digest and send it to the named user immediately, ignoring - opt-in state and idempotency. Useful for previewing copy in your own - inbox before a real run lands.""" - import httpx - from app import branding - from app.db import get_session_factory - from app.jobs.ai_log_job import ( - REFERENCE_LINE, _latest_quotes_by_group, _recent_headlines_by_bucket, - ) - from app.jobs.email_digest_job import _send_one, _generate_variants - from app.services.openrouter import llm_configured - - if kind not in ("daily", "weekly"): - print(f"error: kind must be 'daily' or 'weekly' (got {kind!r})", - file=sys.stderr) - return 2 - if not llm_configured(): - print("error: LLM provider not configured (set OPENROUTER_API_KEY)", - file=sys.stderr) - return 1 - - factory = get_session_factory() - async with factory() as session: - user = await _get_user_by_email(session, email) - if user is None: - print(f"error: no user with email {email!r}", file=sys.stderr) - return 1 - today = _utcnow() - quotes = await _latest_quotes_by_group(session) - news = await _recent_headlines_by_bucket( - session, hours=(168 if kind == "weekly" else 24), - ) - ctx = dict(today=today, quotes_by_group=quotes, - headlines_by_bucket=news, reference_line=REFERENCE_LINE) - async with httpx.AsyncClient(follow_redirects=True) as client: - variants = await _generate_variants(client, kind, ctx) - tone = (user.digest_tone or "INTERMEDIATE").upper() - content = variants.get(tone) or variants.get("INTERMEDIATE") - if content is None: - print("error: all LLM variants failed", file=sys.stderr) - return 1 - date_str = today.strftime("%Y-%m-%d") - await _send_one(user, kind, content, date_str, session) - print(f"sent {kind} digest to {email} (tone={tone})") - return 0 -``` - -- [ ] **Step 2: Wire it into the argparse dispatcher** - -Find `build_parser()` (line ~97). Add a new subparser: - -```python - t = sub.add_parser("send-test-digest", - help="Send one digest immediately (bypasses opt-in/idempotency)") - t.add_argument("email") - t.add_argument("kind", choices=("daily", "weekly")) -``` - -Find `_dispatch(args)` (line ~114). Add a branch: - -```python - if args.cmd == "send-test-digest": - return await send_test_digest(args.email, args.kind) -``` - -(The exact form depends on the existing dispatcher's pattern — match it.) - -- [ ] **Step 3: Smoke-test the CLI** - -```bash -docker compose exec app python -m app.cli send-test-digest you@example.com daily -``` - -Expected output: `sent daily digest to you@example.com (tone=INTERMEDIATE)`. Check your inbox — the email should render with the new template + working unsubscribe link. - -- [ ] **Step 4: Commit** - -```bash -git add app/cli.py -git commit -m "cli: send-test-digest for previewing digest emails" -``` - ---- - -## Final Verification - -- [ ] **Step 1: Run the full suite** - -```bash -pytest tests/ -q -``` - -Expected: all tests pass. - -- [ ] **Step 2: Manual smoke** - -1. Visit `/news` while signed out — only ~6h of headlines, footer says "Free tier, upgrade for 24h". -2. Sign in as a free user — same 6h cap, footer present. -3. Sign in as a paid user (or admin) — 24h, no footer. -4. Visit `/settings` — Digest section present; toggle persists across reloads. -5. Visit `/pricing` — Free has Sunday-weekly bullet; Paid has 24h + daily bullets. -6. Sign up a new account — verify the "Email me the digest" checkbox is visible and on by default. -7. Send a test digest with the CLI — check inbox, click unsubscribe, confirm DB flag flipped, confirm Settings now shows opt-in=off. - -- [ ] **Step 3: Push** - -After the operator reviews and tags the work, push: - -```bash -git push -``` - -(Up to the operator — the spec says nothing about auto-push.) diff --git a/docs/superpowers/plans/2026-05-27-llm-csv-fallback-parser.md b/docs/superpowers/plans/2026-05-27-llm-csv-fallback-parser.md deleted file mode 100644 index 9b8ac1e..0000000 --- a/docs/superpowers/plans/2026-05-27-llm-csv-fallback-parser.md +++ /dev/null @@ -1,1732 +0,0 @@ -# LLM-fallback CSV Parser Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add an LLM-fallback CSV parser so non-T212 portfolio uploads succeed by extracting a column mapping (not data) via the LLM, caching that mapping globally by header fingerprint, and replaying it deterministically on every subsequent upload of the same broker format. - -**Architecture:** New service `app/services/llm_csv_parser.py` wraps `openrouter.call_llm` for one-time format discovery, persists results to a new `csv_format_templates` table, and produces the same `ParsedPie` shape as `parse_t212_csv`. The route `/api/portfolio/parse` in `app/routers/universe.py` gains a try/except fall-through: T212 first, LLM-cache lookup second, LLM call only on first encounter of a new format. The cache table stores headers + one anonymous data row + the JSON mapping; no `user_id` is ever recorded against the row. - -**Tech Stack:** FastAPI · SQLAlchemy 2.0 async · Alembic · MariaDB (prod) / aiosqlite (tests) · existing `openrouter.call_llm` (provider fallback + AICall ledger) - -**Spec:** `docs/superpowers/specs/2026-05-27-llm-csv-fallback-parser-design.md` - ---- - -## File Structure - -**Create:** -- `app/services/llm_csv_parser.py` — the new service; `parse_with_llm`, `LLMParseError`, helpers -- `alembic/versions/0021_csv_format_template.py` — hand-rolled migration (matches the style of `0020_trial_end.py`) -- `tests/test_llm_csv_parser.py` — unit + integration tests for the service -- `tests/fixtures/ibkr_sample.csv` — fabricated IBKR-shaped CSV (no real holdings) - -**Modify:** -- `app/models.py` — add `CsvFormatTemplate` class -- `app/routers/universe.py` — add `Depends(require_paid)` to `/portfolio/parse`; wrap `parse_t212_csv` in a try/except that falls through to `parse_with_llm` -- `app/templates/settings.html` — soften "Trading 212 CSV" copy to broker-agnostic - -**Reuse without modification:** -- `app/services/openrouter.py::call_llm, llm_configured, LogResult` -- `app/services/csv_import.py::ParsedPie, ParsedPosition, CSVImportError` -- `app/services/access.py::require_paid` -- `app/db::Base, utcnow` -- `tests/conftest.py` env setup (in-memory aiosqlite, `CASSANDRA_MOCK=1`) -- Session/engine bootstrap pattern from `tests/test_referral_conversion.py::_build_session_factory` - ---- - -## Test Conventions - -All tests must be runnable inside the test container: - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -v -``` - -DB-touching tests use the same `_build_session_factory(tmp_path)` pattern as `tests/test_referral_conversion.py` — a fresh per-test sqlite file, schema created via `Base.metadata.create_all`. Do NOT introduce a shared fixture across tests; per-test isolation matches the existing pattern. - -Network-touching tests (the LLM) MUST mock `app.services.openrouter.call_llm` — no real HTTP. Use `unittest.mock.AsyncMock`. - ---- - -### Task 1: Add `CsvFormatTemplate` model - -**Files:** -- Modify: `app/models.py` (append after the `Referral` class around line 270+) -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write the failing test** - -Create the test file with the import + schema test: - -```python -"""Unit + integration tests for the LLM-fallback CSV parser.""" -from __future__ import annotations - -import pytest - - -def test_csv_format_template_model_columns(): - """Model exposes every column the spec requires, with correct types.""" - from sqlalchemy import inspect - - from app.models import CsvFormatTemplate - - cols = {c.name: c for c in inspect(CsvFormatTemplate).columns} - assert "fingerprint" in cols - assert "headers" in cols - assert "sample_row" in cols - assert "mapping" in cols - assert "preamble_rows" in cols - assert "delimiter" in cols - assert "broker_label" in cols - assert "first_seen_at" in cols - assert "use_count" in cols - assert "last_used_at" in cols - assert "llm_model" in cols - assert "llm_cost_usd" in cols - # Crucially, no user attribution. - assert "user_id" not in cols - assert "first_seen_user_id" not in cols - # Fingerprint is the cache key. - assert cols["fingerprint"].unique is True - assert cols["fingerprint"].nullable is False -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py::test_csv_format_template_model_columns -v -``` - -Expected: FAIL with `ImportError: cannot import name 'CsvFormatTemplate'`. - -- [ ] **Step 3: Add the model in `app/models.py`** - -Append after the existing `Referral` class (around line 270+, before any trailing module helpers): - -```python -class CsvFormatTemplate(Base): - """Cached column-mapping for a single broker CSV format. - - Populated on the first upload of a previously-unseen format via the - LLM-fallback parser. Subsequent uploads of the same format - (identified by ``fingerprint``, a sha256 of the normalised header - row) replay ``mapping`` deterministically with no LLM call. - - The table holds the actual ``headers`` and one anonymous ``sample_row`` - from the originating upload — there is no ``user_id`` column, no link - back to the uploader. The sample exists so the operator has concrete - material to look at when hand-writing future native parsers; the - system never auto-generates or modifies parser code from this data. - """ - __tablename__ = "csv_format_templates" - - id: Mapped[int] = mapped_column(_PK, primary_key=True, autoincrement=True) - fingerprint: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) - headers: Mapped[list] = mapped_column(JSON, nullable=False) - sample_row: Mapped[list] = mapped_column(JSON, nullable=False) - mapping: Mapped[dict] = mapped_column(JSON, nullable=False) - preamble_rows: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - delimiter: Mapped[str] = mapped_column(String(1), nullable=False, default=",") - broker_label: Mapped[str | None] = mapped_column(String(128)) - first_seen_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=utcnow, - ) - use_count: Mapped[int] = mapped_column(Integer, nullable=False, default=1) - last_used_at: Mapped[datetime] = mapped_column( - DateTime(timezone=True), nullable=False, default=utcnow, - ) - llm_model: Mapped[str | None] = mapped_column(String(64)) - llm_cost_usd: Mapped[float | None] = mapped_column(Float) -``` - -- [ ] **Step 4: Run the test to verify it passes** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py::test_csv_format_template_model_columns -v -``` - -Expected: PASS. - -- [ ] **Step 5: Commit** - -```bash -git add app/models.py tests/test_llm_csv_parser.py -git commit -m "csv-parser: add CsvFormatTemplate model" -``` - ---- - -### Task 2: Add Alembic migration `0021` - -**Files:** -- Create: `alembic/versions/0021_csv_format_template.py` - -- [ ] **Step 1: Write the migration** - -Create `alembic/versions/0021_csv_format_template.py`: - -```python -"""csv format templates table — LLM-fallback parser cache. - -Revision ID: 0021 -Revises: 0020 -Create Date: 2026-05-27 -""" -from typing import Sequence, Union - -import sqlalchemy as sa -from alembic import op - - -revision: str = "0021" -down_revision: Union[str, None] = "0020" -branch_labels: Union[str, Sequence[str], None] = None -depends_on: Union[str, Sequence[str], None] = None - - -def upgrade() -> None: - op.create_table( - "csv_format_templates", - sa.Column("id", sa.BigInteger(), primary_key=True, autoincrement=True), - sa.Column("fingerprint", sa.String(length=64), nullable=False), - sa.Column("headers", sa.JSON(), nullable=False), - sa.Column("sample_row", sa.JSON(), nullable=False), - sa.Column("mapping", sa.JSON(), nullable=False), - sa.Column("preamble_rows", sa.Integer(), nullable=False, server_default="0"), - sa.Column("delimiter", sa.String(length=1), nullable=False, server_default=","), - sa.Column("broker_label", sa.String(length=128), nullable=True), - sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("use_count", sa.Integer(), nullable=False, server_default="1"), - sa.Column("last_used_at", sa.DateTime(timezone=True), nullable=False), - sa.Column("llm_model", sa.String(length=64), nullable=True), - sa.Column("llm_cost_usd", sa.Float(), nullable=True), - sa.UniqueConstraint("fingerprint", name="uq_csv_format_templates_fingerprint"), - ) - - -def downgrade() -> None: - op.drop_table("csv_format_templates") -``` - -- [ ] **Step 2: Verify the migration applies cleanly in the test container** - -```bash -docker compose -f docker-compose.test.yml run --rm test alembic upgrade head -docker compose -f docker-compose.test.yml run --rm test alembic downgrade -1 -docker compose -f docker-compose.test.yml run --rm test alembic upgrade head -``` - -Expected: each command exits 0; the second leaves us at `0020`, the third returns us to `0021`. - -If the test container doesn't have an alembic entrypoint, run the migration check via Python instead: - -```bash -docker compose -f docker-compose.test.yml run --rm test python -c " -from alembic.config import Config -from alembic import command -cfg = Config('alembic.ini') -command.upgrade(cfg, 'head') -command.downgrade(cfg, '-1') -command.upgrade(cfg, 'head') -print('OK') -" -``` - -Expected: prints `OK`. - -- [ ] **Step 3: Commit** - -```bash -git add alembic/versions/0021_csv_format_template.py -git commit -m "alembic: add 0021 csv_format_templates" -``` - ---- - -### Task 3: Create the fabricated IBKR test fixture - -**Files:** -- Create: `tests/fixtures/ibkr_sample.csv` - -- [ ] **Step 1: Write the fixture file** - -This is a fabricated IBKR-style activity statement — column names and shape are realistic, but the values are made up (no real holdings, no real account): - -```csv -Statement,Header,Field Name,Field Value -Statement,Data,BrokerName,Interactive Brokers LLC -Statement,Data,Title,Activity Statement -Statement,Data,Period,"January 1, 2026 - January 31, 2026" -Symbol,Quantity,Avg Price,Currency,Description -AAPL,100,150.25,USD,Apple Inc -MSFT,50,310.00,USD,Microsoft Corp -NVDA,40,425.50,USD,NVIDIA Corp -VOD.L,2000,0.74,GBP,Vodafone Group Plc -ASML.AS,10,650.00,EUR,ASML Holding NV -``` - -Note: lines 1-4 are a preamble (IBKR's exports often have multi-line headers). The actual data table starts at line 5 (`Symbol,Quantity,Avg Price,Currency,Description`). - -- [ ] **Step 2: Commit** - -```bash -git add tests/fixtures/ibkr_sample.csv -git commit -m "tests: add fabricated IBKR fixture for LLM parser" -``` - ---- - -### Task 4: `_fingerprint` helper - -**Files:** -- Create: `app/services/llm_csv_parser.py` (initial scaffold) -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write failing test** - -Append to `tests/test_llm_csv_parser.py`: - -```python -def test_fingerprint_stable_across_case_and_whitespace(): - from app.services.llm_csv_parser import _fingerprint - - a = _fingerprint(["Symbol", "Quantity", "Avg Price"]) - b = _fingerprint(["symbol", "quantity", "avg price"]) - c = _fingerprint([" SYMBOL ", "Quantity", " AVG PRICE"]) - assert a == b == c - - -def test_fingerprint_differs_for_different_columns(): - from app.services.llm_csv_parser import _fingerprint - - a = _fingerprint(["Symbol", "Quantity"]) - b = _fingerprint(["Symbol", "Quantity", "Avg Price"]) - assert a != b - - -def test_fingerprint_is_sha256_hex_64_chars(): - from app.services.llm_csv_parser import _fingerprint - - f = _fingerprint(["Symbol", "Quantity"]) - assert len(f) == 64 - assert all(c in "0123456789abcdef" for c in f) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k fingerprint -v -``` - -Expected: FAIL with `ImportError`. - -- [ ] **Step 3: Create the service scaffold + `_fingerprint`** - -Create `app/services/llm_csv_parser.py`: - -```python -"""LLM-fallback CSV parser. - -When the deterministic Trading 212 parser (``csv_import.parse_t212_csv``) -raises ``CSVImportError`` on an unrecognised format, this service kicks -in: - -1. Detect the CSV dialect (delimiter, preamble offset). -2. Compute a fingerprint of the normalised header row. -3. Look up ``CsvFormatTemplate`` by fingerprint. On hit, replay the - cached column-mapping deterministically. On miss, ask the LLM for a - mapping, validate it, persist a new template, and apply it. - -The LLM sees only headers + the first 3-5 sample rows. It returns a -column-mapping JSON, never transcribed numbers. The system never -auto-promotes a learned format to a hand-written parser — the operator -does that by inspecting collected ``sample_row`` values. -""" -from __future__ import annotations - -import hashlib - -from app.services.csv_import import CSVImportError - - -class LLMParseError(CSVImportError): - """Raised when the LLM call fails or returns an unusable mapping. - - Inherits from ``CSVImportError`` so route-level error handling can - treat both deterministic and LLM-path failures uniformly when - desired.""" - - -def _fingerprint(headers: list[str]) -> str: - """Stable hash of the header row. - - Lowercases each header, strips surrounding whitespace, joins with - ``|`` (a character extremely unlikely to appear inside a real - header), and returns the sha256 hex digest. Whitespace/case drift - in the same broker's export does not change the fingerprint; - adding or removing a column does.""" - normalised = "|".join(h.strip().lower() for h in headers) - return hashlib.sha256(normalised.encode("utf-8")).hexdigest() -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k fingerprint -v -``` - -Expected: 3 PASS. - -- [ ] **Step 5: Commit** - -```bash -git add app/services/llm_csv_parser.py tests/test_llm_csv_parser.py -git commit -m "csv-parser: add _fingerprint helper" -``` - ---- - -### Task 5: `_detect_dialect` helper - -**Files:** -- Modify: `app/services/llm_csv_parser.py` -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write failing tests** - -Append to `tests/test_llm_csv_parser.py`: - -```python -def test_detect_dialect_no_preamble_comma(): - from app.services.llm_csv_parser import _detect_dialect - - raw = b"Symbol,Quantity,Avg Price\nAAPL,100,150.25\nMSFT,50,310.00\n" - delimiter, preamble = _detect_dialect(raw) - assert delimiter == "," - assert preamble == 0 - - -def test_detect_dialect_with_preamble(): - from app.services.llm_csv_parser import _detect_dialect - - raw = ( - b"Statement,Header,Field Name,Field Value\n" - b"Statement,Data,BrokerName,Interactive Brokers LLC\n" - b"Statement,Data,Title,Activity Statement\n" - b"Statement,Data,Period,\"January 1, 2026 - January 31, 2026\"\n" - b"Symbol,Quantity,Avg Price,Currency,Description\n" - b"AAPL,100,150.25,USD,Apple Inc\n" - ) - delimiter, preamble = _detect_dialect(raw) - assert delimiter == "," - # The data-row header line is the FIFTH line (index 4); preamble = 4. - assert preamble == 4 - - -def test_detect_dialect_tab_delimited(): - from app.services.llm_csv_parser import _detect_dialect - - raw = b"Symbol\tQuantity\tAvg Price\nAAPL\t100\t150.25\n" - delimiter, preamble = _detect_dialect(raw) - assert delimiter == "\t" - assert preamble == 0 - - -def test_detect_dialect_empty_raises(): - from app.services.llm_csv_parser import LLMParseError, _detect_dialect - - with pytest.raises(LLMParseError): - _detect_dialect(b"") -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k detect_dialect -v -``` - -Expected: 4 FAIL with `ImportError` for `_detect_dialect`. - -- [ ] **Step 3: Implement `_detect_dialect`** - -Append to `app/services/llm_csv_parser.py`: - -```python -import csv -import io - - -# Cap for how many leading lines we'll scan looking for the header row. -# Real broker preambles are typically 1-10 lines. -_MAX_PREAMBLE_SCAN = 30 - - -def _decode_raw(raw: bytes) -> str: - """Best-effort UTF-8 decode with BOM strip and lossy fallback.""" - text = raw.decode("utf-8-sig", errors="replace") - return text - - -def _detect_dialect(raw: bytes) -> tuple[str, int]: - """Detect (delimiter, preamble_rows). - - ``preamble_rows`` is the number of lines BEFORE the row we identify - as the actual table header. The header row is the first line whose - tokens are all non-numeric (so "Symbol,Quantity" is a header but - "AAPL,100" is data). Falls back to assuming the first line is the - header if no clear non-numeric line is found within the scan - window. - - Raises ``LLMParseError`` on empty input.""" - if not raw or not raw.strip(): - raise LLMParseError("empty CSV") - - text = _decode_raw(raw) - # csv.Sniffer is happy with ~4KB. Anything more and it gets slow. - sample = text[:4096] - try: - dialect = csv.Sniffer().sniff(sample, delimiters=",;\t|") - delimiter = dialect.delimiter - except csv.Error: - # Most broker exports are comma-delimited; default rather than - # error out — the caller will still validate column shapes. - delimiter = "," - - reader = csv.reader(io.StringIO(text), delimiter=delimiter) - preamble = 0 - for i, row in enumerate(reader): - if i >= _MAX_PREAMBLE_SCAN: - break - if not row: - continue - # Skip rows that are obviously preamble: <2 tokens, or any token - # is purely numeric. The header row should have multiple - # alphabetical tokens. - non_empty = [c.strip() for c in row if c.strip()] - if len(non_empty) < 2: - continue - all_alpha = all(not _looks_numeric(c) for c in non_empty) - if all_alpha: - preamble = i - return delimiter, preamble - return delimiter, 0 - - -def _looks_numeric(value: str) -> bool: - """True if ``value`` parses as a number after stripping common - decoration (thousands separators, currency symbols, percent signs).""" - s = value.strip().replace(",", "").replace("$", "").replace("€", "") - s = s.replace("£", "").replace("%", "").lstrip("-+") - if not s: - return False - try: - float(s) - return True - except ValueError: - return False -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k detect_dialect -v -``` - -Expected: 4 PASS. - -- [ ] **Step 5: Commit** - -```bash -git add app/services/llm_csv_parser.py tests/test_llm_csv_parser.py -git commit -m "csv-parser: add _detect_dialect helper" -``` - ---- - -### Task 6: `_validate_mapping` helper - -**Files:** -- Modify: `app/services/llm_csv_parser.py` -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write failing tests** - -Append to `tests/test_llm_csv_parser.py`: - -```python -def test_validate_mapping_accepts_well_formed(): - from app.services.llm_csv_parser import _validate_mapping - - headers = ["Symbol", "Quantity", "Avg Price", "Currency"] - first_row = ["AAPL", "100", "150.25", "USD"] - mapping = { - "ticker_col": "Symbol", - "qty_col": "Quantity", - "cost_col": "Avg Price", - "currency_col": "Currency", - "name_col": None, - } - _validate_mapping(mapping, headers, first_row) # no raise - - -def test_validate_mapping_missing_ticker_raises(): - from app.services.llm_csv_parser import LLMParseError, _validate_mapping - - headers = ["Symbol", "Quantity"] - first_row = ["AAPL", "100"] - mapping = {"ticker_col": None, "qty_col": "Quantity"} - with pytest.raises(LLMParseError, match="ticker"): - _validate_mapping(mapping, headers, first_row) - - -def test_validate_mapping_missing_qty_raises(): - from app.services.llm_csv_parser import LLMParseError, _validate_mapping - - headers = ["Symbol", "Quantity"] - first_row = ["AAPL", "100"] - mapping = {"ticker_col": "Symbol", "qty_col": None} - with pytest.raises(LLMParseError, match="qty"): - _validate_mapping(mapping, headers, first_row) - - -def test_validate_mapping_unknown_column_raises(): - from app.services.llm_csv_parser import LLMParseError, _validate_mapping - - headers = ["Symbol", "Quantity"] - first_row = ["AAPL", "100"] - mapping = {"ticker_col": "Symbol", "qty_col": "NotARealColumn"} - with pytest.raises(LLMParseError, match="NotARealColumn"): - _validate_mapping(mapping, headers, first_row) - - -def test_validate_mapping_non_numeric_qty_raises(): - from app.services.llm_csv_parser import LLMParseError, _validate_mapping - - headers = ["Symbol", "Description"] - first_row = ["AAPL", "Apple Inc"] - # Mapping says qty is "Description", but "Apple Inc" can't parse as a number. - mapping = {"ticker_col": "Symbol", "qty_col": "Description"} - with pytest.raises(LLMParseError, match="numeric"): - _validate_mapping(mapping, headers, first_row) -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k validate_mapping -v -``` - -Expected: 5 FAIL with `ImportError`. - -- [ ] **Step 3: Implement `_validate_mapping`** - -Append to `app/services/llm_csv_parser.py`: - -```python -_REQUIRED_MAPPING_KEYS = ("ticker_col", "qty_col") -_OPTIONAL_MAPPING_KEYS = ("name_col", "cost_col", "currency_col") - - -def _validate_mapping( - mapping: dict, headers: list[str], first_row: list[str], -) -> None: - """Verify the LLM-returned mapping is sane. - - - ``ticker_col`` and ``qty_col`` are required (non-null). - - Every named column must exist in ``headers``. - - The value at ``qty_col`` on ``first_row`` must parse as a number. - - The value at ``cost_col`` on ``first_row`` (if present) must parse - as a number. - - Raises ``LLMParseError`` on any failure, with a message that names - the specific problem (helpful for log forensics and for the - user-facing 400).""" - for key in _REQUIRED_MAPPING_KEYS: - if not mapping.get(key): - raise LLMParseError( - f"LLM mapping missing required column: {key.replace('_col', '')}" - ) - - headers_set = set(headers) - for key in _REQUIRED_MAPPING_KEYS + _OPTIONAL_MAPPING_KEYS: - col = mapping.get(key) - if col is not None and col not in headers_set: - raise LLMParseError( - f"LLM mapping references unknown column: {col!r}" - ) - - # Numeric sanity check: qty and (if present) cost must parse on row 1. - header_index = {h: i for i, h in enumerate(headers)} - qty_col = mapping["qty_col"] - qty_value = first_row[header_index[qty_col]] if header_index[qty_col] < len(first_row) else "" - if not _looks_numeric(qty_value): - raise LLMParseError( - f"LLM mapping qty_col={qty_col!r} maps to non-numeric value {qty_value!r}" - ) - - cost_col = mapping.get("cost_col") - if cost_col is not None: - cost_value = first_row[header_index[cost_col]] if header_index[cost_col] < len(first_row) else "" - if cost_value and not _looks_numeric(cost_value): - raise LLMParseError( - f"LLM mapping cost_col={cost_col!r} maps to non-numeric value {cost_value!r}" - ) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k validate_mapping -v -``` - -Expected: 5 PASS. - -- [ ] **Step 5: Commit** - -```bash -git add app/services/llm_csv_parser.py tests/test_llm_csv_parser.py -git commit -m "csv-parser: add _validate_mapping helper" -``` - ---- - -### Task 7: `_apply_mapping` helper - -**Files:** -- Modify: `app/services/llm_csv_parser.py` -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write failing tests** - -Append to `tests/test_llm_csv_parser.py`: - -```python -def test_apply_mapping_builds_parsed_pie(): - from app.services.csv_import import ParsedPie, ParsedPosition - from app.services.llm_csv_parser import _apply_mapping - - headers = ["Symbol", "Quantity", "Avg Price", "Currency", "Description"] - data_rows = [ - ["AAPL", "100", "150.25", "USD", "Apple Inc"], - ["MSFT", "50", "310.00", "USD", "Microsoft Corp"], - ] - mapping = { - "ticker_col": "Symbol", - "qty_col": "Quantity", - "cost_col": "Avg Price", - "currency_col": "Currency", - "name_col": "Description", - } - - pie = _apply_mapping(headers, data_rows, mapping) - - assert isinstance(pie, ParsedPie) - assert len(pie.positions) == 2 - p0 = pie.positions[0] - assert isinstance(p0, ParsedPosition) - assert p0.slice == "AAPL" - assert p0.name == "Apple Inc" - assert p0.quantity == 100.0 - assert p0.invested_value == pytest.approx(15025.0) - # invested = qty * avg_cost = 100 * 150.25 = 15025 - assert pie.invested == pytest.approx(15025.0 + 50 * 310.00) - - -def test_apply_mapping_handles_missing_optional_columns(): - from app.services.llm_csv_parser import _apply_mapping - - headers = ["Symbol", "Quantity"] - data_rows = [["AAPL", "100"]] - mapping = { - "ticker_col": "Symbol", - "qty_col": "Quantity", - "cost_col": None, - "currency_col": None, - "name_col": None, - } - - pie = _apply_mapping(headers, data_rows, mapping) - p = pie.positions[0] - assert p.slice == "AAPL" - assert p.quantity == 100.0 - assert p.invested_value is None - assert p.name == "AAPL" # falls back to ticker when name_col absent - - -def test_apply_mapping_skips_blank_and_unparseable_rows(): - from app.services.llm_csv_parser import _apply_mapping - - headers = ["Symbol", "Quantity"] - data_rows = [ - ["AAPL", "100"], - ["", ""], # blank - ["MSFT", "not-a-number"], # bad qty - ["NVDA", "40"], - ] - mapping = {"ticker_col": "Symbol", "qty_col": "Quantity"} - - pie = _apply_mapping(headers, data_rows, mapping) - assert [p.slice for p in pie.positions] == ["AAPL", "NVDA"] -``` - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k apply_mapping -v -``` - -Expected: 3 FAIL with `ImportError`. - -- [ ] **Step 3: Implement `_apply_mapping`** - -Append to `app/services/llm_csv_parser.py`: - -```python -from app.services.csv_import import ParsedPie, ParsedPosition - - -def _parse_number(value: str) -> float | None: - """Permissive float parse: strips thousands separators, currency - symbols, percent signs. Returns None on failure (so callers can - decide whether to skip or raise).""" - s = value.strip().replace(",", "").replace("$", "") - s = s.replace("€", "").replace("£", "").replace("%", "") - if not s: - return None - try: - return float(s) - except ValueError: - return None - - -def _apply_mapping( - headers: list[str], - data_rows: list[list[str]], - mapping: dict, -) -> ParsedPie: - """Iterate ``data_rows`` and produce a ``ParsedPie``. - - Rows that lack a parseable quantity (blank, non-numeric, zero) are - silently skipped — broker exports often include summary or - placeholder rows after the position list. ``name_col`` falls back - to the ticker symbol when null.""" - idx = {h: i for i, h in enumerate(headers)} - ticker_col = mapping["ticker_col"] - qty_col = mapping["qty_col"] - name_col = mapping.get("name_col") - cost_col = mapping.get("cost_col") - - positions: list[ParsedPosition] = [] - invested_total = 0.0 - invested_seen = False - - for row in data_rows: - if not any(c.strip() for c in row): - continue - ticker_raw = row[idx[ticker_col]] if idx[ticker_col] < len(row) else "" - ticker = ticker_raw.strip().upper() - if not ticker: - continue - qty_raw = row[idx[qty_col]] if idx[qty_col] < len(row) else "" - qty = _parse_number(qty_raw) - if qty is None or qty <= 0: - continue - avg_cost: float | None = None - if cost_col is not None and idx[cost_col] < len(row): - avg_cost = _parse_number(row[idx[cost_col]]) - invested_value: float | None = None - if avg_cost is not None: - invested_value = qty * avg_cost - invested_total += invested_value - invested_seen = True - name = "" - if name_col is not None and idx[name_col] < len(row): - name = row[idx[name_col]].strip() - if not name: - name = ticker - positions.append(ParsedPosition( - slice=ticker, - name=name, - invested_value=invested_value, - current_value=None, - result=None, - quantity=qty, - )) - - return ParsedPie( - name=None, - positions=tuple(positions), - invested=(invested_total if invested_seen else None), - value=None, - result=None, - ) -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k apply_mapping -v -``` - -Expected: 3 PASS. - -- [ ] **Step 5: Commit** - -```bash -git add app/services/llm_csv_parser.py tests/test_llm_csv_parser.py -git commit -m "csv-parser: add _apply_mapping helper" -``` - ---- - -### Task 8: `_extract_mapping_via_llm` helper - -**Files:** -- Modify: `app/services/llm_csv_parser.py` -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write failing tests** - -Append to `tests/test_llm_csv_parser.py`: - -```python -@pytest.mark.asyncio -async def test_extract_mapping_via_llm_parses_valid_json(): - from unittest.mock import AsyncMock, MagicMock - from app.services.llm_csv_parser import _extract_mapping_via_llm - from app.services.openrouter import LogResult - - fake_result = LogResult( - content='{"ticker_col": "Symbol", "qty_col": "Quantity", ' - '"cost_col": "Avg Price", "currency_col": "Currency", ' - '"name_col": null, "broker_label": "IBKR Activity Statement"}', - model="deepseek/deepseek-v4-flash", - prompt_tokens=100, - completion_tokens=50, - cost_usd=0.0001, - ) - fake_client = MagicMock() - fake_call_llm = AsyncMock(return_value=fake_result) - - import app.services.llm_csv_parser as mod - mod.call_llm = fake_call_llm # monkeypatch - - headers = ["Symbol", "Quantity", "Avg Price", "Currency"] - samples = [["AAPL", "100", "150.25", "USD"]] - mapping, log = await _extract_mapping_via_llm(fake_client, headers, samples) - - assert mapping["ticker_col"] == "Symbol" - assert mapping["qty_col"] == "Quantity" - assert mapping["broker_label"] == "IBKR Activity Statement" - assert log.model == "deepseek/deepseek-v4-flash" - fake_call_llm.assert_awaited_once() - - -@pytest.mark.asyncio -async def test_extract_mapping_via_llm_malformed_json_raises(): - from unittest.mock import AsyncMock, MagicMock - from app.services.llm_csv_parser import LLMParseError, _extract_mapping_via_llm - from app.services.openrouter import LogResult - - fake_result = LogResult( - content="Sure thing — here is the mapping! ticker=Symbol", - model="deepseek/deepseek-v4-flash", - prompt_tokens=10, - completion_tokens=20, - cost_usd=0.00005, - ) - fake_client = MagicMock() - fake_call_llm = AsyncMock(return_value=fake_result) - - import app.services.llm_csv_parser as mod - mod.call_llm = fake_call_llm - - with pytest.raises(LLMParseError, match="JSON"): - await _extract_mapping_via_llm(fake_client, ["Symbol"], [["AAPL"]]) - - -@pytest.mark.asyncio -async def test_extract_mapping_via_llm_provider_failure_wraps(): - from unittest.mock import AsyncMock, MagicMock - from app.services.llm_csv_parser import LLMParseError, _extract_mapping_via_llm - - fake_client = MagicMock() - fake_call_llm = AsyncMock(side_effect=RuntimeError("provider down")) - - import app.services.llm_csv_parser as mod - mod.call_llm = fake_call_llm - - with pytest.raises(LLMParseError, match="provider"): - await _extract_mapping_via_llm(fake_client, ["Symbol"], [["AAPL"]]) -``` - -NOTE: If `pytest-asyncio` is not installed in the test container, the engineer must add `asyncio_mode = "auto"` to `pytest.ini` or use the existing decorator pattern from `tests/test_referral_conversion.py`. Check that file's top for `@pytest.mark.asyncio` usage and replicate it. - -- [ ] **Step 2: Run tests to verify they fail** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k extract_mapping_via_llm -v -``` - -Expected: 3 FAIL with `ImportError` for `_extract_mapping_via_llm`. - -- [ ] **Step 3: Implement `_extract_mapping_via_llm`** - -Append to `app/services/llm_csv_parser.py`: - -```python -import json - -import httpx - -from app.services.openrouter import LogResult, call_llm - - -# Hard caps on what we send to the LLM, so prompt cost stays bounded. -_LLM_SAMPLES = 5 -_LLM_MAX_TOKENS = 400 - - -_SYSTEM_PROMPT = """\ -You are an expert at recognising broker portfolio CSV formats. - -You will be given the header row and 3-5 sample data rows from a CSV. -Identify which column contains each field. Return ONLY a single JSON -object, no prose, no markdown fences. - -Schema (use the EXACT header string from the input; use null if no -column is a good match): - -{ - "ticker_col": "
              ", - "qty_col": "
              ", - "name_col": "
              ", - "cost_col": "
              ", // average price per share - "currency_col": "
              ", - "broker_label": "" -} - -Rules: -- ticker_col and qty_col are required. If either is missing, return all nulls. -- Use the EXACT header string as it appears in the input — do not paraphrase. -- Output JSON ONLY. No prose, no code fences. -""" - - -def _build_user_prompt(headers: list[str], samples: list[list[str]]) -> str: - lines = ["headers: " + json.dumps(headers)] - lines.append("samples:") - for s in samples[:_LLM_SAMPLES]: - lines.append(" " + ",".join(s)) - return "\n".join(lines) - - -async def _extract_mapping_via_llm( - client: httpx.AsyncClient, - headers: list[str], - samples: list[list[str]], -) -> tuple[dict, LogResult]: - """Single LLM call returning ``(mapping_dict, LogResult)``. - - The LLM is asked for a strict JSON object (no markdown). We attempt - to parse the returned content; ``LLMParseError`` wraps any failure - in a way callers can surface to the user.""" - messages = [ - {"role": "system", "content": _SYSTEM_PROMPT}, - {"role": "user", "content": _build_user_prompt(headers, samples)}, - ] - try: - result = await call_llm(client, messages, max_tokens=_LLM_MAX_TOKENS) - except Exception as e: - raise LLMParseError(f"LLM provider failed: {e}") from e - - content = (result.content or "").strip() - # Strip code fences if the model added them despite instructions. - if content.startswith("```"): - content = content.strip("`") - # Drop optional 'json' language tag. - if content.lstrip().lower().startswith("json"): - content = content.lstrip()[4:] - content = content.strip() - try: - mapping = json.loads(content) - except json.JSONDecodeError as e: - raise LLMParseError(f"LLM did not return valid JSON: {e}") from e - if not isinstance(mapping, dict): - raise LLMParseError("LLM JSON was not an object") - return mapping, result -``` - -- [ ] **Step 4: Run tests to verify they pass** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k extract_mapping_via_llm -v -``` - -Expected: 3 PASS. - -- [ ] **Step 5: Commit** - -```bash -git add app/services/llm_csv_parser.py tests/test_llm_csv_parser.py -git commit -m "csv-parser: add _extract_mapping_via_llm with provider-failure wrapping" -``` - ---- - -### Task 9: Public `parse_with_llm` orchestration - -**Files:** -- Modify: `app/services/llm_csv_parser.py` -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write the per-test session factory helper (copied from `test_referral_conversion.py`)** - -Append to the **top** of `tests/test_llm_csv_parser.py` (after the existing imports): - -```python -def _build_session_factory(tmp_path): - """Spin up a fresh in-memory schema and return (engine, factory). - Matches the pattern used in tests/test_referral_conversion.py.""" - from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine - - from app import db as db_mod - from app.db import Base - import app.models # noqa: F401 — registers models on Base.metadata - - engine = create_async_engine(f"sqlite+aiosqlite:///{tmp_path}/csv.db") - factory = async_sessionmaker(engine, expire_on_commit=False) - db_mod._engine = engine - db_mod._session_factory = factory - - async def _setup(): - async with engine.begin() as conn: - await conn.run_sync(Base.metadata.create_all) - - return engine, factory, _setup -``` - -Tests will `await setup()` themselves before using the factory. - -- [ ] **Step 2: Write failing tests for cache miss + cache hit** - -Append to `tests/test_llm_csv_parser.py`: - -```python -@pytest.mark.asyncio -async def test_parse_with_llm_cache_miss_inserts_template(tmp_path): - from unittest.mock import AsyncMock - from sqlalchemy import select - - from app.models import CsvFormatTemplate - from app.services.llm_csv_parser import parse_with_llm - from app.services.openrouter import LogResult - - _, factory, setup = _build_session_factory(tmp_path) - await setup() - - raw = ( - b"Symbol,Quantity,Avg Price,Currency\n" - b"AAPL,100,150.25,USD\n" - b"MSFT,50,310.00,USD\n" - ) - - import app.services.llm_csv_parser as mod - mod.call_llm = AsyncMock(return_value=LogResult( - content='{"ticker_col":"Symbol","qty_col":"Quantity",' - '"cost_col":"Avg Price","currency_col":"Currency",' - '"name_col":null,"broker_label":"Generic broker"}', - model="deepseek/deepseek-v4-flash", - prompt_tokens=120, completion_tokens=40, cost_usd=0.0002, - )) - - async with factory() as session: - pie = await parse_with_llm(raw, session) - - assert len(pie.positions) == 2 - assert pie.positions[0].slice == "AAPL" - - async with factory() as session: - rows = (await session.execute(select(CsvFormatTemplate))).scalars().all() - assert len(rows) == 1 - tmpl = rows[0] - assert tmpl.headers == ["Symbol", "Quantity", "Avg Price", "Currency"] - assert tmpl.sample_row == ["AAPL", "100", "150.25", "USD"] - assert tmpl.mapping["ticker_col"] == "Symbol" - assert tmpl.broker_label == "Generic broker" - assert tmpl.use_count == 1 - assert tmpl.llm_cost_usd == pytest.approx(0.0002) - # The crucial PII guarantee: - assert not hasattr(tmpl, "user_id"), "sample row must not be linked to a user" - - -@pytest.mark.asyncio -async def test_parse_with_llm_cache_hit_skips_llm(tmp_path): - from unittest.mock import AsyncMock - from sqlalchemy import select - - from app.db import utcnow - from app.models import CsvFormatTemplate - from app.services.llm_csv_parser import _fingerprint, parse_with_llm - - _, factory, setup = _build_session_factory(tmp_path) - await setup() - - headers = ["Symbol", "Quantity", "Avg Price", "Currency"] - fp = _fingerprint(headers) - - # Pre-populate a cache hit row. - async with factory() as session: - session.add(CsvFormatTemplate( - fingerprint=fp, - headers=headers, - sample_row=["AAPL", "100", "150.25", "USD"], - mapping={ - "ticker_col": "Symbol", "qty_col": "Quantity", - "cost_col": "Avg Price", "currency_col": "Currency", - "name_col": None, - }, - preamble_rows=0, - delimiter=",", - broker_label="Cached broker", - first_seen_at=utcnow(), - last_used_at=utcnow(), - use_count=1, - llm_model="seed", - llm_cost_usd=0.0, - )) - await session.commit() - - raw = ( - b"Symbol,Quantity,Avg Price,Currency\n" - b"NVDA,40,425.50,USD\n" - ) - - import app.services.llm_csv_parser as mod - mod.call_llm = AsyncMock(side_effect=AssertionError("call_llm must NOT be called on cache hit")) - - async with factory() as session: - pie = await parse_with_llm(raw, session) - - assert pie.positions[0].slice == "NVDA" - - async with factory() as session: - rows = (await session.execute(select(CsvFormatTemplate))).scalars().all() - assert len(rows) == 1 - assert rows[0].use_count == 2 - - -@pytest.mark.asyncio -async def test_parse_with_llm_stale_mapping_raises_but_does_not_evict(tmp_path): - from unittest.mock import AsyncMock - from sqlalchemy import select - - from app.db import utcnow - from app.models import CsvFormatTemplate - from app.services.llm_csv_parser import LLMParseError, _fingerprint, parse_with_llm - - _, factory, setup = _build_session_factory(tmp_path) - await setup() - - headers = ["Symbol", "Quantity"] - fp = _fingerprint(headers) - # Cached mapping says qty is in column "Symbol" — clearly wrong; will - # never produce a parseable row. - async with factory() as session: - session.add(CsvFormatTemplate( - fingerprint=fp, headers=headers, - sample_row=["AAPL", "100"], - mapping={"ticker_col": "Symbol", "qty_col": "Symbol"}, - preamble_rows=0, delimiter=",", broker_label=None, - first_seen_at=utcnow(), last_used_at=utcnow(), use_count=1, - llm_model="seed", llm_cost_usd=0.0, - )) - await session.commit() - - raw = b"Symbol,Quantity\nAAPL,100\nMSFT,50\n" - - import app.services.llm_csv_parser as mod - mod.call_llm = AsyncMock(side_effect=AssertionError("must not be called")) - - async with factory() as session: - with pytest.raises(LLMParseError): - await parse_with_llm(raw, session) - - # Stale template must NOT have been auto-deleted (operator owns eviction). - async with factory() as session: - rows = (await session.execute(select(CsvFormatTemplate))).scalars().all() - assert len(rows) == 1 -``` - -- [ ] **Step 3: Run tests to verify they fail** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -k parse_with_llm -v -``` - -Expected: 3 FAIL with `ImportError` for `parse_with_llm`. - -- [ ] **Step 4: Implement `parse_with_llm`** - -Append to `app/services/llm_csv_parser.py`: - -```python -from sqlalchemy import select -from sqlalchemy.ext.asyncio import AsyncSession - -from app.db import utcnow -from app.logging import get_logger -from app.models import CsvFormatTemplate - - -log = get_logger("llm_csv_parser") - - -# Hard cap shared with /api/portfolio/parse — bytes-level, mirrors T212 path. -_MAX_CSV_BYTES = 1_048_576 - - -async def parse_with_llm(raw: bytes, session: AsyncSession) -> ParsedPie: - """Cache-first LLM-fallback CSV parse. - - On cache hit, applies the stored mapping deterministically and - increments ``use_count``. On cache miss, calls the LLM, validates - the returned mapping against the first data row, and persists a - new ``CsvFormatTemplate``. Raises ``LLMParseError`` on any - failure; the caller (route layer) maps that to a 400.""" - if len(raw) > _MAX_CSV_BYTES: - raise LLMParseError("CSV too large (1 MB max)") - if not raw or not raw.strip(): - raise LLMParseError("empty CSV") - - delimiter, preamble_rows = _detect_dialect(raw) - text = _decode_raw(raw) - - reader = csv.reader(io.StringIO(text), delimiter=delimiter) - rows = list(reader) - if preamble_rows >= len(rows): - raise LLMParseError("no header row found in CSV") - headers = [c.strip() for c in rows[preamble_rows]] - data_rows = rows[preamble_rows + 1:] - if not headers: - raise LLMParseError("empty header row") - - first_data_row = next( - (r for r in data_rows if any(c.strip() for c in r)), None, - ) - if first_data_row is None: - raise LLMParseError("CSV contains a header but no data rows") - - fp = _fingerprint(headers) - existing = (await session.execute( - select(CsvFormatTemplate).where(CsvFormatTemplate.fingerprint == fp) - )).scalar_one_or_none() - - if existing is not None: - log.info("csv.format.cache_hit", fingerprint=fp, - broker_label=existing.broker_label, use_count=existing.use_count) - pie = _apply_mapping(headers, data_rows, existing.mapping) - if not pie.positions: - raise LLMParseError( - "cached mapping produced no positions — the broker may have " - "changed their CSV shape; ask the operator to evict the " - "stale template" - ) - existing.use_count += 1 - existing.last_used_at = utcnow() - await session.commit() - return pie - - log.info("csv.format.cache_miss", fingerprint=fp, - header_count=len(headers)) - samples = [r for r in data_rows[:_LLM_SAMPLES] if any(c.strip() for c in r)] - async with httpx.AsyncClient(follow_redirects=True, timeout=30) as client: - mapping, llm_log = await _extract_mapping_via_llm(client, headers, samples) - _validate_mapping(mapping, headers, first_data_row) - - pie = _apply_mapping(headers, data_rows, mapping) - if not pie.positions: - raise LLMParseError( - "LLM mapping validated but produced no positions — the file " - "may not contain portfolio data" - ) - - now = utcnow() - session.add(CsvFormatTemplate( - fingerprint=fp, - headers=headers, - sample_row=first_data_row, - mapping=mapping, - preamble_rows=preamble_rows, - delimiter=delimiter, - broker_label=mapping.get("broker_label"), - first_seen_at=now, - last_used_at=now, - use_count=1, - llm_model=llm_log.model, - llm_cost_usd=llm_log.cost_usd, - )) - await session.commit() - return pie -``` - -- [ ] **Step 5: Run tests to verify they pass** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py -v -``` - -Expected: every test passes (including everything from earlier tasks). - -- [ ] **Step 6: Commit** - -```bash -git add app/services/llm_csv_parser.py tests/test_llm_csv_parser.py -git commit -m "csv-parser: add public parse_with_llm with cache hit/miss orchestration" -``` - ---- - -### Task 10: Wire `parse_with_llm` into the route and add `require_paid` - -**Files:** -- Modify: `app/routers/universe.py:192-214` (the `parse_portfolio` route + decorator) -- Test: `tests/test_llm_csv_parser.py` - -- [ ] **Step 1: Write the route-level integration test (direct function call, no HTTP layer)** - -Calling `parse_portfolio` directly with a fake `UploadFile` and a real session sidesteps the pytest-asyncio + `TestClient` event-loop awkwardness. `Depends(require_paid)` is decorator-level and is not invoked when we call the function directly — which is what we want (paid gating is mechanical and trusted to FastAPI; we verify it by inspection in Step 2 of the next stage). - -Append to `tests/test_llm_csv_parser.py`: - -```python -@pytest.mark.asyncio -async def test_parse_portfolio_route_falls_through_to_llm(tmp_path, monkeypatch): - """End-to-end: T212 parser raises CSVImportError, LLM fallback runs, - response shape matches the existing JSON contract.""" - from io import BytesIO - from types import SimpleNamespace - from unittest.mock import AsyncMock - - from fastapi import UploadFile - - _, factory, setup = _build_session_factory(tmp_path) - await setup() - - import app.services.llm_csv_parser as mod - from app.services.openrouter import LogResult - mod.call_llm = AsyncMock(return_value=LogResult( - content='{"ticker_col":"Symbol","qty_col":"Quantity",' - '"cost_col":"Avg Price","currency_col":"Currency",' - '"name_col":"Description",' - '"broker_label":"IBKR Activity Statement"}', - model="deepseek/deepseek-v4-flash", - prompt_tokens=150, completion_tokens=60, cost_usd=0.0003, - )) - - # The route's inline Yahoo-fetch block would otherwise hit the network. - # Patch market.fetch to return a benign placeholder per ticker. - from app.services import market as market_mod - - async def _fake_fetch(client, symbol, label, group, anchor): - return SimpleNamespace( - symbol=symbol, source="test", label=label, - price=None, currency="USD", as_of="2026-05-27", - changes=None, error=None, - ) - monkeypatch.setattr(market_mod, "fetch", _fake_fetch) - - raw = open("tests/fixtures/ibkr_sample.csv", "rb").read() - upload = UploadFile(filename="ibkr.csv", file=BytesIO(raw)) - - from app.routers.universe import parse_portfolio - async with factory() as session: - result = await parse_portfolio(file=upload, session=session) - - assert result["base_currency"] == "GBP" - # At least the AAPL/MSFT/NVDA rows should be present; resolve_slice may - # filter some if there's no InstrumentMap row, which is fine for this - # test — we just want to confirm the LLM fallback ran end-to-end. - assert isinstance(result["positions"], list) - # LLM was called exactly once (cache miss). - assert mod.call_llm.await_count == 1 -``` - -- [ ] **Step 1b: Add a paid-gate inspection test (no HTTP needed)** - -```python -def test_parse_portfolio_route_requires_paid(): - """Static check that the /portfolio/parse route is gated by require_paid.""" - from app.routers.universe import router - from app.services.access import require_paid - - parse_route = next( - r for r in router.routes - if getattr(r, "path", "") == "/portfolio/parse" - ) - dep_callables = [d.dependency for d in parse_route.dependant.dependencies] - assert require_paid in dep_callables, ( - "The /portfolio/parse route must have Depends(require_paid)" - ) -``` - -- [ ] **Step 2: Run the test to verify it fails** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py::test_parse_portfolio_route_falls_through_to_llm -v -``` - -Expected: FAIL — probably with the T212 parser raising `CSVImportError` because the route does not yet have the LLM fallback wired up. The exact failure message confirms we need the wiring. - -- [ ] **Step 3: Wire `parse_with_llm` into the route** - -In `app/routers/universe.py`, find the `parse_portfolio` definition (search `async def parse_portfolio`). Make these two changes: - -**Change A: Add `require_paid` to the route decorator.** Find the existing line: - -```python -@router.post("/portfolio/parse") -async def parse_portfolio( - file: UploadFile = File(...), - session: AsyncSession = Depends(get_session), -) -> dict: -``` - -Replace with: - -```python -@router.post("/portfolio/parse", dependencies=[Depends(require_paid)]) -async def parse_portfolio( - file: UploadFile = File(...), - session: AsyncSession = Depends(get_session), -) -> dict: -``` - -**Change B: Add the LLM fallback.** Find the existing block: - -```python - try: - pie = parse_t212_csv(raw) - except CSVImportError as e: - raise HTTPException(status_code=400, detail=str(e)) -``` - -Replace with: - -```python - try: - pie = parse_t212_csv(raw) - except CSVImportError: - # Unrecognised format — try the LLM-fallback parser. It hits a - # global format-fingerprint cache first; only the very first - # upload of each broker format pays an LLM call. - from app.services.llm_csv_parser import LLMParseError, parse_with_llm - try: - pie = await parse_with_llm(raw, session) - except LLMParseError as e: - raise HTTPException(status_code=400, detail=str(e)) -``` - -- [ ] **Step 4: Run the integration test to verify it passes** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py::test_parse_portfolio_route_falls_through_to_llm -v -``` - -Expected: PASS. - -- [ ] **Step 5: Run the full test file + the existing T212 tests to confirm no regression** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/test_llm_csv_parser.py tests/test_csv_import.py -v -``` - -Expected: all PASS. Confirms the T212 happy path is untouched. - -- [ ] **Step 6: Commit** - -```bash -git add app/routers/universe.py tests/test_llm_csv_parser.py -git commit -m "universe: paid-gate + LLM fallback on /portfolio/parse" -``` - ---- - -### Task 11: UI copy tweaks - -**Files:** -- Modify: `app/templates/settings.html` (search the file for "Trading 212 CSV", "T212 pie CSV") - -- [ ] **Step 1: Update the section heading** - -Find: - -```html -Import portfolio (Trading 212 CSV) -``` - -Replace with: - -```html -Import portfolio (CSV) -``` - -- [ ] **Step 2: Update the drop-zone label** - -Find: - -```html -
              Drop a T212 pie CSV here
              -``` - -Replace with: - -```html -
              Drop your broker's portfolio CSV here
              -``` - -- [ ] **Step 3: Update the drop-zone hint** - -Find: - -```html -
              or browse · max 1 MB
              -``` - -Replace with: - -```html -
              or browse · max 1 MB · T212, IBKR and others auto-detected
              -``` - -- [ ] **Step 4: Soften the help paragraph** - -If there is a paragraph above or beside the drop-zone that begins with "Export your pie from T212", change its opening phrase from declarative to conditional. For example, if the current line is: - -```html -

              Export your pie from T212 as CSV ...

              -``` - -Replace with: - -```html -

              If you use Trading 212, export your pie as CSV ...

              -``` - -(Search the file for `Export your pie from T212` to locate the exact paragraph; preserve any surrounding markup.) - -- [ ] **Step 5: Commit** - -```bash -git add app/templates/settings.html -git commit -m "settings: soften import copy to be broker-agnostic" -``` - ---- - -### Task 12: Final regression run + manual smoke - -**Files:** -- (no code changes — verification only) - -- [ ] **Step 1: Full test suite** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/ -v -``` - -Expected: every existing test still passes; the new tests in `tests/test_llm_csv_parser.py` all pass. - -- [ ] **Step 2: Apply the migration against the dev DB and confirm the table exists** - -```bash -docker compose exec app alembic upgrade head -docker compose exec app python -c " -import asyncio -from sqlalchemy import inspect -from app.db import get_engine - -async def main(): - eng = get_engine() - async with eng.connect() as conn: - names = await conn.run_sync(lambda c: inspect(c).get_table_names()) - assert 'csv_format_templates' in names, names - print('csv_format_templates table present:', sorted(names)) -asyncio.run(main()) -" -``` - -Expected: prints the table name in the list. NOTE: this touches the prod DB on this host — only run when the user has explicitly approved this deploy. - -- [ ] **Step 3: Restart the app container** - -```bash -docker compose restart app -docker compose logs app --tail 30 | grep -E "(Uvicorn|startup complete|error)" || true -``` - -Expected: clean startup; no tracebacks. - -- [ ] **Step 4: Manual smoke — re-import a T212 CSV** - -Through the browser at `/settings`, drop a known T212 CSV. The dashboard should load as it always has. (Confirms zero regression on the happy path.) - -- [ ] **Step 5: Manual smoke — first IBKR-shaped upload** - -Through the browser at `/settings`, drop `tests/fixtures/ibkr_sample.csv` (or a real IBKR statement). The dashboard should load with the IBKR positions. Then query the DB to confirm the template was cached: - -```bash -docker compose exec app python -c " -import asyncio -from sqlalchemy import select -from app.db import get_session_factory -from app.models import CsvFormatTemplate - -async def main(): - factory = get_session_factory() - async with factory() as s: - rows = (await s.execute(select(CsvFormatTemplate))).scalars().all() - for r in rows: - print(r.fingerprint[:12], r.broker_label, 'use_count=', r.use_count, - 'cost=', r.llm_cost_usd) -asyncio.run(main()) -" -``` - -NOTE: this is a prod DB read; only run with explicit user approval. - -Expected: a single row, with `use_count=1` and a small positive `llm_cost_usd`. - -- [ ] **Step 6: Manual smoke — second IBKR-shaped upload (cache hit)** - -Drop the same fixture again. The dashboard should load identically, and the DB row should now show `use_count=2`. AICall ledger should NOT have a new row for this second upload (only the first paid the LLM cost). - -- [ ] **Step 7: Manual smoke — paid gating** - -In a free-tier browser session, attempt the upload. Expect a 402 response visible in network tools / surfaced as an "upgrade required" message in the UI. - ---- - -## Self-Review - -Spec coverage walkthrough: - -- **Trigger: transparent fallback** → Task 10 (route try/except) -- **Cache for reuse** → Task 9 (cache-hit branch in `parse_with_llm`) -- **Paid-only** → Task 10 Step 3 Change A (adds `Depends(require_paid)`) -- **LLM column-mapping only** → Tasks 6–8 (`_validate_mapping`, `_extract_mapping_via_llm`, no full-CSV extraction anywhere) -- **Global cache** → Tasks 1, 2 (no `user_id` column); Task 9 cache lookup is global -- **`sample_row` is a real first data row, anonymous** → Task 9 Step 4 (the `INSERT` uses `first_data_row`); Task 1 test asserts `user_id` absent -- **No self-heal / no auto-eviction** → Task 9 stale-mapping test asserts row is NOT deleted -- **No code authoring** → out of scope by construction (no code writes anywhere in the plan) -- **`fingerprint` = sha256(normalised headers)** → Task 4 -- **Preamble detection** → Task 5 -- **Drop-zone + heading copy softened** → Task 11 -- **Error handling** (LLM down → 502 in spirit; nonsense → 400; non-numeric qty → 400) → Tasks 6, 8, 10 (the route raises `HTTPException(400)` on `LLMParseError`; LLM provider failure is wrapped in `LLMParseError` by `_extract_mapping_via_llm`, which the route surfaces as 400 — note this is 400 in the implementation, not 502 as the spec implied; if you specifically want 502 for provider-down vs 400 for mapping-bad, split the exception types in Task 8 and branch in Task 10) -- **Migration 0021** → Task 2 -- **Fabricated IBKR fixture** → Task 3 -- **Test pattern matches `test_referral_conversion.py`** → Task 9 Step 1 (copies the factory pattern) - -One spec-vs-plan deviation worth flagging to the engineer: the spec error-handling table says "LLM provider down → 502". This plan returns 400 for all `LLMParseError` cases including provider-down, because the wrapping is uniform. If you want a 502/400 split, the simplest fix is to introduce a sibling `LLMProviderError(LLMParseError)` raised inside `_extract_mapping_via_llm`'s exception path, and branch on it in Task 10. Two-line change. Either behaviour is defensible — flagging so it's a conscious choice. diff --git a/docs/superpowers/plans/2026-05-27-manual-portfolio-composition.md b/docs/superpowers/plans/2026-05-27-manual-portfolio-composition.md deleted file mode 100644 index b909c16..0000000 --- a/docs/superpowers/plans/2026-05-27-manual-portfolio-composition.md +++ /dev/null @@ -1,1434 +0,0 @@ -# Manual Portfolio Composition Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Add dashboard-native portfolio editing: an EDIT button toggles inline edit mode with an add-position form (live ticker validation + avg-cost or bought-on-date toggle) and per-row delete; brand-new users see the same form in the empty state instead of being sent to /settings. - -**Architecture:** Two new paid-only endpoints (`/api/ticker/validate`, `/api/ticker/historical`) thinly wrap the existing `fetch_yahoo` to give the JS the data it needs. A new `portfolio_edit.js` owns edit-mode toggle + form behaviour and writes positions via the existing `window.CassandraPortfolio.savePie` API. `portfolio.js` is touched only to (a) emit × buttons hidden by CSS and (b) replace the empty-state CTA. No server-side portfolio persistence; localStorage stays authoritative. - -**Tech Stack:** FastAPI · `httpx.AsyncClient` · vanilla JS · Jinja2 templates · existing `app/services/market.fetch_yahoo` · existing `app/services/ticker_universe.upsert_tickers` - -**Spec:** `docs/superpowers/specs/2026-05-27-manual-portfolio-composition-design.md` - ---- - -## File Structure - -**Create:** -- `app/routers/ticker_validate.py` — both new endpoints (validate + historical) + the `fetch_yahoo_historical` helper. -- `tests/test_ticker_validate.py` — backend tests. -- `app/static/js/portfolio_edit.js` — edit-mode + add-position form behaviour. - -**Modify:** -- `app/main.py` — register the new router (one new line). -- `app/templates/dashboard.html` — EDIT button in panel header; hidden add-position form; new ` - -``` - -Replace with: - -```html -
              -
              - Portfolio - held locally · prices via /api/universe - - -
              -
              - -
              -
              loading…
              -
              -
              -
              - - - -``` - -- [ ] **Step 2: Commit** - -```bash -git add app/templates/dashboard.html -git commit -m "dashboard: scaffold portfolio edit-mode markup" -``` - -## Context for this task - -- No tests for this — pure markup. JS will hide/show via class toggle. -- The pencil SVG path is the standard "edit" icon (Feather-style). -- All form fields are present in the DOM at all times; `portfolio_edit.js` shows/hides via CSS class on `#portfolio-panel`. - ---- - -### Task 6: portfolio_edit.js — edit-mode toggle - -**Files:** -- Create: `app/static/js/portfolio_edit.js` - -- [ ] **Step 1: Create the scaffold with edit-mode toggle** - -Create `app/static/js/portfolio_edit.js`: - -```javascript -/* Dashboard-native portfolio editing. - * - * Owns: the EDIT button toggle, the add-position form behaviour - * (ticker validation on blur, qty/cost inputs, date-mode historical - * lookup, Add click), and per-row delete via event delegation. - * - * Reads/writes the portfolio via window.CassandraPortfolio.loadPie / - * savePie / mountAndRender — the same surface portfolio.js exposes - * for the CSV-import preview. - */ -(function () { - 'use strict'; - - const panel = document.getElementById('portfolio-panel'); - const editBtn = document.getElementById('pf-edit-btn'); - const doneBtn = document.getElementById('pf-done-btn'); - const form = document.getElementById('pf-add-form'); - if (!panel || !editBtn || !doneBtn || !form) return; - - function enterEditMode() { - panel.classList.add('pf-editing'); - form.hidden = false; - editBtn.hidden = true; - doneBtn.hidden = false; - editBtn.setAttribute('aria-pressed', 'true'); - document.getElementById('pf-add-ticker').focus(); - } - - function exitEditMode() { - panel.classList.remove('pf-editing'); - // Form stays visible when the pie is empty (empty-state UX handled - // by portfolio.js setting the pf-empty class on the panel). - if (!panel.classList.contains('pf-empty')) { - form.hidden = true; - } - editBtn.hidden = false; - doneBtn.hidden = true; - editBtn.setAttribute('aria-pressed', 'false'); - } - - editBtn.addEventListener('click', enterEditMode); - doneBtn.addEventListener('click', exitEditMode); -})(); -``` - -- [ ] **Step 2: Smoke-test in a browser** - -Restart the app (manual prod step — request user approval), navigate to the dashboard, click EDIT. Expected: form appears, focus lands in the ticker input; Done button appears; Edit button hides. Click Done: reverse. - -(No automated test; vanilla JS, no JS test framework in this repo.) - -- [ ] **Step 3: Commit** - -```bash -git add app/static/js/portfolio_edit.js -git commit -m "portfolio-edit: edit-mode toggle scaffold" -``` - ---- - -### Task 7: portfolio_edit.js — ticker validation on blur - -**Files:** -- Modify: `app/static/js/portfolio_edit.js` (append to the existing IIFE before the closing `})();`) - -- [ ] **Step 1: Add validation logic** - -Inside the IIFE in `app/static/js/portfolio_edit.js`, before the final `})();`, append: - -```javascript - // ---- Ticker validation on blur ------------------------------------- - - const tickerInput = document.getElementById('pf-add-ticker'); - const tickerStatus = document.getElementById('pf-add-ticker-status'); - const costCurrencyEl = document.getElementById('pf-add-cost-currency'); - const submitBtn = document.getElementById('pf-add-submit'); - const warningEl = document.getElementById('pf-add-warning'); - - let validated = null; // {symbol, price, currency, as_of} or null - - function setStatus(el, text, kind) { - el.textContent = text; - el.className = 'pf-add-status' + (kind ? ' pf-add-status--' + kind : ''); - } - - function updateSubmitState() { - const qty = parseFloat(document.getElementById('pf-add-qty').value); - const cost = parseFloat(document.getElementById('pf-add-cost').value); - submitBtn.disabled = !( - validated && qty > 0 && cost > 0 && isFinite(qty) && isFinite(cost) - ); - } - - function clearDuplicateWarning() { - warningEl.hidden = true; - warningEl.textContent = ''; - } - - function showDuplicateWarning(existing) { - warningEl.hidden = false; - warningEl.textContent = - `Already in your portfolio (${existing.qty} shares @ ` + - `${existing.avg_cost.toFixed(2)}). Adding will create a duplicate row.`; - } - - async function validateTicker() { - const raw = tickerInput.value.trim().toUpperCase(); - if (!raw) { - validated = null; - setStatus(tickerStatus, '', ''); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - updateSubmitState(); - return; - } - setStatus(tickerStatus, 'checking…', 'pending'); - try { - const r = await fetch('/api/ticker/validate?symbol=' + encodeURIComponent(raw)); - if (!r.ok) throw new Error('HTTP ' + r.status); - const j = await r.json(); - if (j.ok) { - validated = j; - setStatus( - tickerStatus, - '✓ ' + j.price.toFixed(2) + ' ' + (j.currency || ''), - 'ok', - ); - costCurrencyEl.textContent = j.currency || ''; - // Duplicate detection. - const pie = window.CassandraPortfolio.loadPie(); - const existing = pie && (pie.positions || []).find( - p => (p.yahoo_ticker || '').toUpperCase() === j.symbol - ); - if (existing) showDuplicateWarning(existing); - else clearDuplicateWarning(); - } else { - validated = null; - setStatus(tickerStatus, '✗ ' + (j.error || 'not recognised'), 'err'); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - } - } catch (e) { - validated = null; - setStatus(tickerStatus, '✗ couldn’t validate — try again', 'err'); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - } - updateSubmitState(); - } - - tickerInput.addEventListener('blur', validateTicker); - document.getElementById('pf-add-qty').addEventListener('input', updateSubmitState); - document.getElementById('pf-add-cost').addEventListener('input', updateSubmitState); -``` - -- [ ] **Step 2: Smoke-test in browser** - -Restart app. EDIT → type `AAPL`, tab out → expect green check with price + currency. Type bogus → red error. Add button stays disabled. - -- [ ] **Step 3: Commit** - -```bash -git add app/static/js/portfolio_edit.js -git commit -m "portfolio-edit: ticker validate on blur + duplicate warning" -``` - ---- - -### Task 8: portfolio_edit.js — Add button + localStorage merge - -**Files:** -- Modify: `app/static/js/portfolio_edit.js` (append inside the IIFE) - -- [ ] **Step 1: Add the submit handler** - -Append (inside the same IIFE): - -```javascript - // ---- Add button → localStorage merge ------------------------------- - - function resetForm() { - tickerInput.value = ''; - document.getElementById('pf-add-qty').value = ''; - document.getElementById('pf-add-cost').value = ''; - document.getElementById('pf-add-date').value = ''; - validated = null; - setStatus(tickerStatus, '', ''); - costCurrencyEl.textContent = ''; - clearDuplicateWarning(); - updateSubmitState(); - tickerInput.focus(); - } - - function addPosition() { - if (submitBtn.disabled) return; - const qty = parseFloat(document.getElementById('pf-add-qty').value); - const cost = parseFloat(document.getElementById('pf-add-cost').value); - const sym = validated.symbol; - - const pie = window.CassandraPortfolio.loadPie() || { - pie_name: null, - base_currency: 'GBP', - positions: [], - totals: {invested: 0, value: 0, result: 0}, - warnings: [], - }; - pie.positions = pie.positions || []; - pie.positions.push({ - yahoo_ticker: sym, - t212_slice: sym, // shared shape with CSV path - name: validated.name || sym, - qty: qty, - avg_cost: cost, - currency: validated.currency || 'USD', - }); - window.CassandraPortfolio.savePie(pie); - window.CassandraPortfolio.mountAndRender(); - resetForm(); - } - - submitBtn.addEventListener('click', addPosition); - - // Submit on Enter from any input within the form. - form.addEventListener('keydown', function (e) { - if (e.key === 'Enter' && !submitBtn.disabled) { - e.preventDefault(); - addPosition(); - } - }); -``` - -- [ ] **Step 2: Smoke-test in browser** - -EDIT → enter AAPL + qty 100 + cost 150.25 → Add. Row should appear in the table. Form clears, ticker input refocused. Refresh page; row persists (localStorage). - -- [ ] **Step 3: Commit** - -```bash -git add app/static/js/portfolio_edit.js -git commit -m "portfolio-edit: add button writes position to localStorage" -``` - ---- - -### Task 9: portfolio_edit.js — Bought-on-date mode + historical lookup - -**Files:** -- Modify: `app/static/js/portfolio_edit.js` (append inside the IIFE) - -- [ ] **Step 1: Add cost-mode toggle + historical lookup** - -Append: - -```javascript - // ---- Cost mode toggle + historical lookup -------------------------- - - const dateField = document.getElementById('pf-add-date-field'); - const dateInput = document.getElementById('pf-add-date'); - const dateStatus = document.getElementById('pf-add-date-status'); - const costInput = document.getElementById('pf-add-cost'); - - function onModeChange() { - const mode = document.querySelector( - 'input[name="pf-cost-mode"]:checked' - ).value; - if (mode === 'date') { - dateField.hidden = false; - costInput.readOnly = true; - costInput.placeholder = 'auto-filled from date'; - } else { - dateField.hidden = true; - costInput.readOnly = false; - costInput.placeholder = '150.25'; - setStatus(dateStatus, '', ''); - } - } - - document.querySelectorAll('input[name="pf-cost-mode"]').forEach(r => - r.addEventListener('change', onModeChange) - ); - - async function fetchHistorical() { - if (!validated) { - setStatus(dateStatus, 'enter a valid ticker first', 'err'); - return; - } - const d = dateInput.value; - if (!d) { - setStatus(dateStatus, '', ''); - costInput.value = ''; - updateSubmitState(); - return; - } - setStatus(dateStatus, 'looking up…', 'pending'); - try { - const url = '/api/ticker/historical?symbol=' + - encodeURIComponent(validated.symbol) + - '&date=' + encodeURIComponent(d); - const r = await fetch(url); - if (r.status === 400) { - const j = await r.json().catch(() => ({detail: 'invalid date'})); - setStatus(dateStatus, '✗ ' + (j.detail || 'invalid date'), 'err'); - costInput.value = ''; - updateSubmitState(); - return; - } - const j = await r.json(); - if (j.ok) { - costInput.value = j.close.toFixed(2); - const tag = (j.actual_date && j.actual_date !== d) - ? '✓ from ' + j.actual_date - : '✓'; - setStatus(dateStatus, tag, 'ok'); - } else { - setStatus(dateStatus, '✗ ' + (j.error || 'no data'), 'err'); - costInput.value = ''; - } - } catch (e) { - setStatus(dateStatus, '✗ couldn’t fetch — try again', 'err'); - costInput.value = ''; - } - updateSubmitState(); - } - - dateInput.addEventListener('blur', fetchHistorical); -``` - -- [ ] **Step 2: Smoke-test in browser** - -EDIT → AAPL validated → switch radio to "Bought on date" → cost field becomes read-only → pick a Friday → cost field auto-fills. Pick a Saturday → cost fills with previous Friday's close, "✓ from YYYY-MM-DD" tag visible. Pick today's date → fills with today's close. Pick a future date → red "date cannot be in the future". - -- [ ] **Step 3: Commit** - -```bash -git add app/static/js/portfolio_edit.js -git commit -m "portfolio-edit: bought-on-date mode + historical lookup" -``` - ---- - -### Task 10: portfolio.js — render × buttons + empty-state form - -**Files:** -- Modify: `app/static/js/portfolio.js` (two changes: `renderEmpty` and the row-render code) - -- [ ] **Step 1: Identify the row-render call site** - -```bash -grep -n "` rows for each position. (The skeleton is likely a `rows.map(...)` or string-concat. Locate it; this is where we add a per-row × cell.) - -- [ ] **Step 2: Modify each row to include an × delete cell** - -Inside the row-render loop, append a new `` to each row's HTML: - -```javascript -'' + - '' + -'' -``` - -(Use whichever loop variable holds the index. If positions are iterated with `.map((p, i) => ...)`, `i` is the index. The × cell goes at the END of each ``.) - -Also add a matching empty `` to the table header row so column counts line up. - -- [ ] **Step 3: Replace the empty-state render** - -Find `function renderEmpty(mount)` (~line 180). Replace its body: - -```javascript - function renderEmpty(mount) { - var notice = window._cassandraPortfolioBackupExpired - ? '
              Your encrypted cloud backup expired. ' + - 'Please re-upload your portfolio to refresh it.' + - '
              ' - : ''; - var panel = document.getElementById('portfolio-panel'); - if (panel) panel.classList.add('pf-empty'); - mount.innerHTML = - '
              ' + - notice + - 'Welcome — start by adding a position above, or ' + - 'import a CSV from your broker →' + - '
              '; - // When empty, the add form should be visible by default — the - // edit module toggles it via the pf-empty class. - var form = document.getElementById('pf-add-form'); - if (form) form.hidden = false; - } -``` - -Also, in the function that runs after a successful render (where the panel is NOT empty), remove the `pf-empty` class: - -```javascript -// After successful render of populated positions: -var panel = document.getElementById('portfolio-panel'); -if (panel) panel.classList.remove('pf-empty'); -``` - -- [ ] **Step 4: Add the × click handler in portfolio_edit.js** - -In `app/static/js/portfolio_edit.js`, append (inside the same IIFE): - -```javascript - // ---- Per-row delete (event delegation) ----------------------------- - - panel.addEventListener('click', function (e) { - const btn = e.target.closest('.pf-row-del'); - if (!btn) return; - const idx = parseInt(btn.dataset.idx, 10); - if (!Number.isInteger(idx)) return; - const pie = window.CassandraPortfolio.loadPie(); - if (!pie || !pie.positions || idx < 0 || idx >= pie.positions.length) return; - pie.positions.splice(idx, 1); - window.CassandraPortfolio.savePie(pie); - window.CassandraPortfolio.mountAndRender(); - }); -``` - -- [ ] **Step 5: Smoke-test** - -Restart app. Visit dashboard with a populated pie → no × visible. Click EDIT → × appears on each row. Click an × → that row vanishes. Refresh page → row stays gone. Visit dashboard with a CLEARED pie (open devtools → `localStorage.removeItem('cassandra.pie')`, refresh) → empty state shows the inline form + "Or import a CSV" link. - -- [ ] **Step 6: Commit** - -```bash -git add app/static/js/portfolio.js app/static/js/portfolio_edit.js -git commit -m "portfolio: render hidden × per row; empty state shows add form" -``` - ---- - -### Task 11: CSS for edit mode + form - -**Files:** -- Modify: `app/static/css/cassandra.css` (append a new block) - -- [ ] **Step 1: Append the edit-mode + form styles** - -Append to `app/static/css/cassandra.css`: - -```css -/* ---------- Dashboard portfolio edit mode ----------------------------- */ - -/* The EDIT button — same shape as .settings-icon-btn but smaller. */ -.pf-edit-btn, -.pf-done-btn { - display: inline-flex; - align-items: center; - gap: 4px; - background: transparent; - border: 1px solid var(--neu-dim, #444); - color: var(--text, #ccc); - padding: 2px 8px; - border-radius: 4px; - cursor: pointer; - font-size: 12px; - margin-left: auto; -} -.pf-edit-btn:hover, .pf-done-btn:hover { - background: var(--surface-2, #2a2a2a); - border-color: var(--accent, #5af); -} - -/* × button on each row — hidden by default, visible only in edit mode. */ -.pf-row-del-cell { width: 24px; text-align: center; } -.pf-row-del { - display: none; - background: transparent; - border: none; - color: var(--neu-dim, #888); - cursor: pointer; - font-size: 14px; - padding: 0 4px; -} -#portfolio-panel.pf-editing .pf-row-del { display: inline; } -#portfolio-panel.pf-editing .pf-row-del:hover { color: var(--err, #f55); } - -/* Add-position form. */ -.pf-add-form { - border: 1px solid var(--neu-dim, #333); - border-radius: 6px; - padding: 12px; - margin-bottom: 12px; - background: var(--surface-2, #1a1a1a); -} -.pf-add-row { - display: flex; - gap: 10px; - margin-bottom: 8px; - flex-wrap: wrap; - align-items: flex-end; -} -.pf-add-field { - display: flex; - flex-direction: column; - flex: 1 1 140px; - font-size: 12px; -} -.pf-add-label { - color: var(--neu-dim, #888); - margin-bottom: 2px; -} -.pf-add-field input[type="text"], -.pf-add-field input[type="number"], -.pf-add-field input[type="date"] { - background: var(--surface, #111); - border: 1px solid var(--neu-dim, #444); - color: var(--text, #ccc); - padding: 4px 6px; - border-radius: 3px; - font-family: inherit; - font-size: 13px; -} -.pf-add-cost-mode { gap: 16px; } -.pf-add-radio { - display: inline-flex; - gap: 4px; - align-items: center; - font-size: 12px; - cursor: pointer; -} -.pf-add-currency { - color: var(--neu-dim, #888); - font-size: 11px; - margin-top: 2px; -} -.pf-add-submit { - background: var(--accent, #5af); - color: var(--bg, #000); - border: none; - padding: 6px 14px; - border-radius: 3px; - cursor: pointer; - font-weight: 600; - align-self: flex-end; -} -.pf-add-submit:disabled { - background: var(--neu-dim, #444); - cursor: not-allowed; -} -.pf-add-status { - font-size: 11px; - margin-top: 2px; - min-height: 14px; -} -.pf-add-status--pending { color: var(--neu-dim, #888); } -.pf-add-status--ok { color: var(--ok, #6c6); } -.pf-add-status--err { color: var(--err, #f55); } -.pf-add-warning { - color: var(--warn, #fb3); - font-size: 11px; - margin-top: 8px; -} -``` - -- [ ] **Step 2: Smoke-test all visual states** - -Browser smoke: -- Light/dark theme rendering of the form. -- Disabled Add button looks visibly disabled. -- × buttons appear only in edit mode. -- Status text colours change correctly across pending/ok/err. - -- [ ] **Step 3: Commit** - -```bash -git add app/static/css/cassandra.css -git commit -m "css: portfolio edit-mode + add-position form styles" -``` - ---- - -### Task 12: Final regression + manual deploy verification - -**Files:** -- (no code changes — verification only) - -- [ ] **Step 1: Run the full test suite** - -```bash -docker compose -f docker-compose.test.yml run --rm test pytest tests/ 2>&1 | tail -5 -``` - -Expected: every test passes including the new `tests/test_ticker_validate.py`. Should land in the high-260s now. - -- [ ] **Step 2: Deploy (requires explicit user approval)** - -```bash -docker compose restart app -docker compose logs app --tail 30 | grep -E "(Uvicorn|startup complete|ERROR|Traceback)" -``` - -Expected: clean startup; no tracebacks. NOTE: this is a prod restart on this host — do not run without explicit user authorisation. - -- [ ] **Step 3: Manual smoke — populated portfolio** - -In a paid-tier browser session: - -1. Click EDIT → form appears, × buttons appear, focus lands in ticker input. -2. Type a known symbol → green check + price + currency. -3. Enter qty + cost → Add. New row appears. Form clears. -4. Repeat with date mode: pick a weekday → cost auto-fills. Pick a Saturday → cost fills with Friday's close, "from YYYY-MM-DD" tag shows. -5. Click × on any row → row vanishes; refresh confirms persistence. -6. Click Done → form hides, × hidden, dashboard returns to normal display. - -- [ ] **Step 4: Manual smoke — empty state** - -In devtools: `localStorage.removeItem('cassandra.pie')`. Refresh. - -Expected: the add-position form is visible at the top of the portfolio area; below it the message "Welcome — start by adding a position above, or import a CSV from your broker →". Add one position; the empty state disappears. - -- [ ] **Step 5: Manual smoke — paid gating** - -Open an incognito session as a free-tier user. Direct `GET /api/ticker/validate?symbol=AAPL` (devtools network tab or curl). Expect 402. - -- [ ] **Step 6: Manual smoke — error states** - -- Bad symbol `XYZNOTREAL` → red "Not recognised", Add stays disabled. -- Pick future date → red "date cannot be in the future" inline. -- Add a duplicate ticker → inline warning under the ticker field, but the Add button remains enabled (with valid qty+cost). - ---- - -## Self-Review - -**Spec coverage walkthrough:** - -- **Dashboard placement (not Settings)** → Task 5 puts everything inside `#portfolio-panel`. -- **EDIT button toggles edit mode** → Task 6 enterEditMode/exitEditMode. -- **Live prices keep updating during edit mode** → no change to portfolio.js's `setInterval(mountAndRender, ...)` cadence; edit-mode is purely visual layering. -- **On-blur, blocking ticker validation** → Task 7 (`blur` listener; Add button enabled only when `validated && qty > 0 && cost > 0`). -- **Avg-cost / bought-on-date toggle** → Task 9 with `onModeChange`. -- **Date is a UX helper only** → Task 8 `addPosition` stores only `qty + avg_cost + currency` (no date in localStorage). -- **Immediate per-row persist (no Save)** → Task 8 `addPosition` writes localStorage on each Add. -- **Two paid-only endpoints** → Tasks 1-4 with paid-gate inspection tests. -- **Side-effect: seed ticker_universe** → Task 1 endpoint code + Task 2 side-effect test. -- **Historical: walk back to preceding trading day** → Task 3 unit test (`test_fetch_yahoo_historical_walks_back_to_preceding_trading_day`). -- **Duplicate ticker → warning, allow proceed** → Task 7 `showDuplicateWarning`; Add button remains enabled. -- **× delete by index (handles duplicates correctly)** → Task 10's event delegation uses `data-idx` not by ticker. -- **Empty state shows the form** → Task 10 `renderEmpty` rewrite + `pf-empty` class. -- **No persistence of dates** → Task 8 (only `qty + avg_cost + currency` in the position object). -- **No multi-pie / no save-to-server** → out of scope; no tasks build these. - -**Type / signature consistency:** - -- `validate_ticker(symbol, session)` — used in Tasks 1, 2, 4. Consistent. -- `get_historical(symbol, date)` — Tasks 3, 4. No `session` parameter (no DB writes). Consistent. -- `fetch_yahoo_historical(client, symbol, target_iso) -> tuple[float|None, str|None, str|None]` — Tasks 3, all consistent. -- `window.CassandraPortfolio.{loadPie, savePie, mountAndRender}` — used in Tasks 7-10. Already exposed by existing portfolio.js. No new exports needed. - -**Spec deviation flagged:** - -- **Provider-down → 502 vs symbol-unknown → 200 ok:false distinction**: the spec preserves both. The plan collapses both into `{ok: false}` (200) because `fetch_yahoo` swallows exceptions and disentangling them post-hoc is fragile. UI difference: the user sees "Symbol not recognised" or "Couldn't validate, try again" depending on whether the JS got a network error vs a parsed JSON ok:false — close enough to the spec's intent. If the engineer prefers the spec's distinction, they can call an unsafe `httpx.AsyncClient` directly inside the endpoint and split exception types. Note in the commit message either way. - -No spec requirements have zero tasks. diff --git a/docs/superpowers/specs/2026-05-25-beta-mode-and-paid-gap-design.md b/docs/superpowers/specs/2026-05-25-beta-mode-and-paid-gap-design.md deleted file mode 100644 index fa628c9..0000000 --- a/docs/superpowers/specs/2026-05-25-beta-mode-and-paid-gap-design.md +++ /dev/null @@ -1,338 +0,0 @@ -# Beta Mode + Wider Paid/Free Gap — Design Spec - -**Date:** 2026-05-25 -**Status:** Draft — awaiting approval -**Scope:** Three coordinated changes that ship together because they all -target the closed-beta launch: (1) a visible BETA indicator in the app -chrome, (2) a free-tier cap on the news feed window, and (3) email -digests — daily for paid, Sunday-weekly for everyone. - -## 1. Goals - -- Set expectations for closed-beta testers that the product is still - evolving, without making the app look unfinished. -- Create a tangible reason to upgrade beyond portfolio import + sync. - Today the free tier carries nearly the entire editorial layer; after - this change, paid gets meaningfully more. -- Use existing infra (scheduler, SMTP, OpenRouter, settings page) - rather than introducing new dependencies. - -## 2. Non-goals - -- No payment / Paddle work. Pricing copy is updated, but checkout is - still gated behind the existing "Coming soon" CTA. -- No per-user personalised digests. Same content for all recipients. -- No timezone handling for users. Fixed 06:30 UTC daily send. -- No new analytics / metrics dashboards for opens or clicks beyond a - simple audit row per send. - -## 3. Component overview - -| Component | Change | -|-----------|--------| -| `app/templates/base.html` | BETA chip in header next to brand. | -| `app/static/css/cassandra.css` | `.beta-chip` styles. | -| `app/config.py` | `BETA_MODE: bool = True` env flag. | -| `app/services/access.py` | `FREE_NEWS_WINDOW_HOURS = 6` constant. | -| `app/routers/api.py:222-280` (`news_list`) | Soft-auth dep + free-tier window clamp. | -| `app/templates/partials/news.html` | "Showing last 6h — upgrade for 24h" footer when capped. | -| `app/models.py` | New columns on `User`: `email_digest_opt_in: bool`, `digest_tone: str | None`. New table `EmailSend(user_id, kind, sent_at, status, error)`. | -| `alembic/versions/0017_email_digest.py` | Migration. | -| `app/services/openrouter.py` | Add `build_daily_digest_prompt(tone)` and `build_weekly_digest_prompt(tone)`, bump `PROMPT_VERSION`. | -| `app/services/email_service.py` | Add `render_digest_email(kind, tone, content)` + `send_digest(user, kind, tone, html, text)`. | -| `app/jobs/email_digest_job.py` (new) | Daily-or-weekly orchestrator. Generates content once per tone, fans out to recipients. | -| `app/scheduler_main.py` | Register the new job at 06:30 UTC. | -| `app/templates/settings.html` | "Email digests" section: opt-in toggle, tone radio. | -| `app/templates/login.html` (post-OTP-verify flow) | Default-checked "send me the digest" checkbox. | -| `app/routers/auth.py` | OTP-verify handler reads the new `subscribe_to_digests` form field and sets `email_digest_opt_in` on the new `User`. | -| `app/routers/settings.py` (or wherever existing settings PATCH lives) | New `PATCH /api/settings/digest` endpoint: body `{opt_in: bool, tone: "NOVICE"|"INTERMEDIATE"}`. | -| `app/routers/email.py` (new) | `GET /email/unsubscribe?token=...` — HMAC-verified one-click off switch. | -| `app/templates/pricing.html` | Updated bullets — free gets weekly digest + last 6h news; paid gets daily digest + last 24h news. | - -## 4. Detailed design - -### 4.1 BETA chip - -`app/templates/base.html` only (not `public_base.html`). Insert -immediately after the brand link: - -```html -{{ BRAND_NAME }} -{% if BETA_MODE %}BETA{% endif %} -``` - -The `BETA_MODE` flag is injected into the template context globally — -add it to `app/templates_env.py` so every render gets it (analogous to -how `BRAND_NAME` is provided today). - -CSS: small uppercase pill with `var(--accent)` background and brand-bg -foreground; spaced ~8px from the brand link. Sizing matches the -existing top-right `.meta` chip so the visual weight is balanced. - -`BETA_MODE` defaults to `True` in `app/config.py`. Flip to `False` for -general launch — one-line change. - -### 4.2 Free-tier news cap (6h window) - -`app/routers/api.py`'s `news_list` (lines 222-280) currently has no -auth dep. Add `principal: CurrentUser | None = Depends(maybe_current_user)` -(soft-auth — keeps the endpoint reachable by anonymous visitors, -matching today's behaviour). - -Compute the effective window: - -```python -window = since_hours -if not is_paid_active(principal): - window = min(window, FREE_NEWS_WINDOW_HOURS) # 6.0 -cutoff = utcnow() - timedelta(hours=window) -``` - -`FREE_NEWS_WINDOW_HOURS = 6.0` lives in `app/services/access.py` as a -module-level constant. Tuning it later is one edit; promoting it to env -config is YAGNI for now. - -`is_paid_active` already accepts `CurrentUser | User | None` and admins -auto-pass — no special-casing needed here. - -When the cap is in effect, pass `capped: True` into the news partial so -it can render a soft footer: - -> Free tier — showing the last 6 hours of news. [Upgrade] for the full -> 24-hour feed plus daily and weekly email digests. - -When the user is paid (or admin), the footer is absent. When the -visitor is anonymous, the link goes to `/pricing`. - -### 4.3 Database changes - -```python -# app/models.py — User columns added: -email_digest_opt_in: Mapped[bool] = mapped_column(Boolean, nullable=False, - default=True, server_default=text("1")) -digest_tone: Mapped[str | None] = mapped_column(String(16)) # NOVICE | INTERMEDIATE -``` - -Defaults to opted-in (matches the chosen UX). `digest_tone` is nullable; -NULL is interpreted as INTERMEDIATE at render time. - -New table: - -```python -class EmailSend(Base): - __tablename__ = "email_sends" - id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True) - user_id: Mapped[int] = mapped_column(ForeignKey("users.id", ondelete="CASCADE"), - nullable=False, index=True) - kind: Mapped[str] = mapped_column(String(16), nullable=False) # "daily" | "weekly" - sent_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), - default=utcnow, nullable=False) - status: Mapped[str] = mapped_column(String(16), nullable=False) # "sent" | "skipped" | "error" - error: Mapped[str | None] = mapped_column(String(255)) - __table_args__ = ( - Index("ix_email_sends_user_kind_sent", "user_id", "kind", "sent_at"), - ) -``` - -Idempotency: the job queries `EmailSend` for the current day before -sending, so a job restart can't double-deliver. - -Migration: `alembic/versions/0017_email_digest.py`. - -### 4.4 Digest content generation - -Two new prompts in `app/services/openrouter.py`: - -- `build_daily_digest_prompt(tone)` — returns a `(system, user)` pair. - Pulls the same `quotes_by_group` + `headlines_by_bucket` data as - the hourly log but with a 24h headline window and a different - instruction set: less "current state" framing, more "what mattered in - the past day, what to watch today". Target length ~600 words. -- `build_weekly_digest_prompt(tone)` — 7-day headline window, weekly - recap + week-ahead anticipation. Target length ~900 words. - -Both reuse `call_llm()` and the existing cost-cap / ledger plumbing. -`PROMPT_VERSION` is bumped so the audit trail is unambiguous. - -For each digest run, the job generates **two** variants -(NOVICE + INTERMEDIATE) and stores them in memory for the fan-out -batch. There is no DB persistence of digest content — emails are the -artefact. If we later want to render them on a web archive, that's a -separate spec. - -Cost: at our current model pricing, two daily generations ≈ $0.04/day, -two weekly generations ≈ $0.06/week. Both well under the existing -`OPENROUTER_MONTHLY_CAP_USD` headroom. - -### 4.5 Email rendering and delivery - -`render_digest_email(kind, tone, html_body, text_body) -> (subject, -text, html)` in `email_service.py`. Wraps the LLM output in the same -multipart template family used for OTPs (light/dark palette, inline -styles, monospace stack, 520px max-width). Adds two footer rows: - -- "Don't want these? [Unsubscribe in one click](.../email/unsubscribe?token=...)" -- "Or change your preferences in [Settings](.../settings)" - -Subjects: - -- Daily: `"Read the Markets · Daily — {date}"` -- Weekly: `"Read the Markets · Weekly recap — {date}"` - -Fan-out: one SMTP send per recipient. Sequential with a small -`asyncio.sleep(0.1)` between sends to stay under common SMTP rate -limits. Failures are caught per-recipient, logged into `EmailSend`, -and don't block the rest of the batch. - -### 4.6 Sign-up opt-in checkbox - -The OTP-verify POST handler in `app/routers/auth.py` is where the user -is first established — that handler reads a `subscribe_to_digests` -form field and persists it into `User.email_digest_opt_in`. Default to -`True` if the field is absent (covers older clients or curl flows). -The verify template (`app/templates/verify.html`) gets a checkbox: - -```html - -``` - -Pre-existing users get `email_digest_opt_in=True` from the migration's -server-side default — but see §6 for the cutover plan. - -### 4.7 Settings page - -In `app/templates/settings.html`, add a section: - -``` -Email digests - [✓] Send me digests - Free tier: Sunday weekly. Paid: daily + Sunday. - Reading level: ( ) Novice (•) Intermediate - Last delivery: 2026-05-24 06:30 UTC — sent -``` - -The "Last delivery" row reads the most recent `EmailSend` row for this -user. If none, shows "—". - -Wire it via the existing settings JS pattern (look for the sync / -tone-toggle handlers in `static/js/`); the endpoints are -`PATCH /api/settings/digest` with body `{opt_in: bool, tone: str}`. - -### 4.8 One-click unsubscribe - -`GET /email/unsubscribe?token=`: - -- Token is `itsdangerous.URLSafeSerializer` over `{"uid": user_id, - "purpose": "digest_optout"}`, signed with `CASSANDRA_SECRET`. -- Handler verifies, flips `email_digest_opt_in=False`, renders a tiny - confirmation page ("You're unsubscribed. Re-enable any time in - [Settings](/settings)."). -- No auth required — that's the whole point of one-click unsubscribe. -- Replay-safe: re-running the same URL is idempotent (the column - is already false; the page renders the same confirmation). - -### 4.9 Scheduler integration - -`app/scheduler_main.py` already runs the hourly jobs. Add: - -```python -schedule_daily( # whatever helper exists, or apscheduler equivalent - "email_digest_job", - hour=6, minute=30, - target=email_digest_job.run, -) -``` - -The job itself decides what to do: - -```python -async def run(): - today = utcnow().date() - if today.weekday() == 6: # Sunday — weekly digest for everyone - await _run_weekly() - else: - await _run_daily() # paid only -``` - -`_run_weekly()` queries all users with `email_digest_opt_in=True`. -`_run_daily()` queries paid users with `email_digest_opt_in=True`. - -### 4.10 Pricing copy updates - -`app/templates/pricing.html` — modify the bullet lists. New copy: - -Free: -- "News aggregator — last 6 hours, auto-tagged by theme" (was: no time qualifier) -- "Cross-asset macro signals across every asset class" -- "Hourly AI interpretation of the news + the tape" -- "Per-group cross-asset summaries" -- "Novice / Intermediate reading levels" -- "**Sunday weekly digest by email**" *(new)* -- "❌ Portfolio import & analysis" -- "❌ Encrypted cloud sync" - -Paid (replaces the "Priority email when something material changes (later)" line): -- "Everything in Free" -- "**News aggregator — full 24 hours**" *(replaces the implicit 24h)* -- "Portfolio import (Trading 212 CSV)" -- "AI commentary on diversification, sector and currency concentration, …" -- "Optional encrypted cloud sync across devices" -- "**Daily email digest** (Mon–Sat) + Sunday weekly" *(replaces 'Priority email')* - -The intro paragraph at lines 8-13 needs to soften: - -> Two tiers. The news aggregator and hourly AI interpretation are -> available to everyone — paid extends the time window from 6h to 24h -> and adds daily editorial by email, plus the portfolio-import features. - -(Old copy said "free for everyone — we want the read out where people -can use it." That stance is moderated, not abandoned.) - -## 5. Error handling - -- **SMTP failure**: per-recipient try/except. Log to `EmailSend` with - `status="error"`, `error=str(exc)[:255]`. Job continues. Job-level - failure metrics surface via existing `JobRun` mechanism. -- **OpenRouter failure**: if the content generation fails for both - tones, the job records `JobRun.status="error"` and sends nothing. - Half-success (one tone) → send the variant that worked, skip the - other; users on the failed tone get nothing today (rather than wrong - content). -- **Cost-cap hit**: same pattern as the hourly job — skip the run with - a logged reason. -- **Unsubscribe token invalid / tampered**: render the same - confirmation page generically; do not leak whether the token was - valid (avoid enumeration). - -## 6. Cutover plan - -- Migration sets `email_digest_opt_in=True` for all existing users via - `server_default`. This is the user-requested default — they want - every paid beta-tester actually receiving the email. -- BETA mode is on from the first deploy. -- News cap is on from the first deploy. -- The first daily run lands at 06:30 UTC the morning after deploy. -- The first weekly run lands at 06:30 UTC the next Sunday. -- A pre-deploy admin CLI command (`python -m app.cli send-test-digest - --email me@…`) is added for the operator to dry-run a digest into - their own inbox before flipping the scheduler. - -## 7. Testing - -- Unit: `is_paid_active` window-clamping, opt-in flag round-trip, - token signing/verification. -- Integration: `tests/test_news_api.py` — anonymous + free vs paid - windowing. `tests/test_email_digest.py` — job runs, EmailSend rows - written, idempotency on re-run within the same day. -- Manual: send-test-digest CLI, click unsubscribe link, verify Settings - toggle round-trips. - -## 8. Open questions - -None at design time — all earlier ambiguities (chip scope, window -shape, content shape, send time, tone variants, opt-in default, -unsubscribe model) were resolved during brainstorming. diff --git a/docs/superpowers/specs/2026-05-27-llm-csv-fallback-parser-design.md b/docs/superpowers/specs/2026-05-27-llm-csv-fallback-parser-design.md deleted file mode 100644 index c946c8f..0000000 --- a/docs/superpowers/specs/2026-05-27-llm-csv-fallback-parser-design.md +++ /dev/null @@ -1,279 +0,0 @@ -# LLM-fallback CSV parser — Design Spec - -**Date:** 2026-05-27 -**Status:** Draft — pending implementation plan - -## Context - -Today the only supported broker import is Trading 212. `parse_t212_csv` expects -T212's exact column set (`Slice`, `Owned quantity`, etc.) and raises -`CSVImportError` on anything else. Every non-T212 user hits a wall at -onboarding. - -Rather than write a hand-rolled parser per broker (IBKR, Vanguard, Fidelity, -Schwab, eToro, Degiro, …) — and chase format drift forever — we use an LLM as -a transparent fallback. The LLM never sees holdings as data; it only looks at -**headers plus a handful of sample rows** and returns a JSON column-mapping. -Our existing Python code does the row iteration. - -The first time a broker format appears, the LLM produces a mapping. We -fingerprint the format (sha256 of normalized headers) and cache the mapping -in a new `csv_format_templates` table. Every subsequent upload of the same -format — by any user — replays the cached mapping deterministically, with no -LLM call. - -The cache row stores the header row and a single anonymous sample data row -(the first row from the originating upload, verbatim). No user identifier is -recorded — the row is not linked back to whoever uploaded it. The purpose of -the sample is to give the operator material to look at when designing future -native parsers; this collection is **passive learning only**, the system -never attempts to author or modify parser code automatically. - -Portfolio import is already advertised as a paid-only feature; we make that -explicit at the route level as part of this work. - -## Goals - -- Accept CSV exports from any broker, not just T212. -- Pay the LLM cost only once per **format**, not once per user. -- Never persist user holdings on the server (already a system-wide invariant). -- Surface the same response shape to the browser regardless of which parser - branch ran — no client changes beyond a copy tweak. - -## Non-goals - -- Per-broker UI customisation. The drop-zone stays generic. -- A human admin queue for reviewing LLM-discovered formats. Operator can - inspect rows directly in the DB if curious. -- **Auto-promoting learned formats to native parsers.** The operator will - hand-write any native parser by looking at the collected sample rows. The - system never writes or modifies code. -- Self-healing or auto-evicting stale cache entries. If a broker silently - changes their export shape under us, the cached mapping will start - producing parse errors; the operator deletes the row manually. We do not - invalidate cache entries automatically. -- Multi-stage / verification LLM passes. One call per first-time format. - -## Architecture - -``` -POST /api/portfolio/parse (paid-only) -├─ parse_t212_csv(raw) ── happy path, unchanged -│ └─ CSVImportError ↴ -│ -├─ parse_with_llm(raw, session) -│ ├─ detect delimiter + preamble offset -│ ├─ fingerprint = sha256(normalised headers) -│ ├─ SELECT csv_format_templates WHERE fingerprint=? -│ │ ├─ HIT → apply mapping (bump use_count/last_used_at after successful parse) -│ │ └─ MISS → openrouter.call_llm(headers + 3-5 sample rows) -│ │ → validate mapping -│ │ → INSERT csv_format_templates -│ │ → apply mapping -│ └─ returns ParsedPie (same shape as T212 path) -│ -└─ resolve_slice → upsert_tickers → inline Yahoo fetch → JSON response - (existing pipeline, unchanged) -``` - -### Why column-mapping, not full extraction - -We pass the LLM only **headers plus 3–5 sample rows**, not the full CSV. The -LLM returns column names, not transcribed numbers. Three benefits: - -1. **Safety** — LLMs hallucinate digits; they don't hallucinate column names - that aren't there. Mapping validation can verify every named column exists - in the actual header row. -2. **Cost** — prompt is ~1 KB regardless of portfolio size. -3. **Cacheability** — the mapping IS the cache. Replay is deterministic Python, - no LLM in the loop on re-imports. - -### Why global cache, not per-user - -The column structure of an IBKR Activity Statement is a property of IBKR, not -of any individual user. The cache row contains no user identifier — the -sample data row is stored verbatim but anonymously, with nothing linking it -to the uploader. Global cache is strictly better: faster onboarding for the -second IBKR user, and the collected samples form a small, useful corpus for -hand-writing native parsers later. - -## Data model - -New table `csv_format_templates`: - -| Column | Type | Notes | -|---|---|---| -| `id` | int PK | | -| `fingerprint` | `VARCHAR(64) UNIQUE NOT NULL` | sha256 hex of normalised header tuple | -| `headers` | JSON | List of strings — actual header row from the upload | -| `sample_row` | JSON | First data row from the originating upload, verbatim. Not linked to any user. | -| `mapping` | JSON | `{ticker_col, qty_col, name_col, cost_col, currency_col}` | -| `preamble_rows` | INT NOT NULL DEFAULT 0 | Non-data lines before the header row | -| `delimiter` | CHAR(1) NOT NULL DEFAULT ',' | | -| `broker_label` | VARCHAR(128) | LLM-identified label, e.g. "Interactive Brokers Activity Statement" | -| `first_seen_at` | DATETIME(tz) NOT NULL | When the format was first cached | -| `use_count` | INT NOT NULL DEFAULT 1 | Bumped on each successful cache hit | -| `last_used_at` | DATETIME(tz) NOT NULL | | -| `llm_model` | VARCHAR(64) | Provenance of the initial extraction | -| `llm_cost_usd` | FLOAT | Same | - -Migration: `alembic/versions/0021_csv_format_template.py` (based on `0020`). - -The full uploaded CSV is **not** stored — only the header row plus a single -data row (`sample_row`). No `user_id` column exists on this table; the sample -is anonymous by construction. This is a deliberate, narrow exception to the -otherwise-strict "no holdings persisted" invariant: we keep one row per -format so the operator has concrete material to look at when hand-writing a -future native parser. One anonymous row carries no portfolio context (no -totals, no other positions) and cannot be linked back to an account. - -## Components - -### `app/services/llm_csv_parser.py` — new - -Public surface: - -```python -async def parse_with_llm( - raw: bytes, - session: AsyncSession, -) -> ParsedPie: - """LLM-fallback CSV parser. - - Decodes raw bytes, detects delimiter and preamble offset, fingerprints - the header row, hits the csv_format_templates cache. On miss, calls - openrouter.call_llm with headers + 3-5 sample rows to extract a - column-mapping, validates it, persists a new template, and applies the - mapping. Returns the same ParsedPie shape as parse_t212_csv. - """ - -class LLMParseError(ValueError): - """Raised when the LLM call fails or returns an unusable mapping.""" -``` - -Internal helpers (not exported): - -- `_detect_dialect(raw: bytes) -> tuple[str, int]` — returns `(delimiter, preamble_rows)`. Uses Python's `csv.Sniffer` for delimiter, then walks rows until the first row whose tokens look like column headers (heuristic: all-strings, none parse as numbers). -- `_fingerprint(headers: list[str]) -> str` — lowercases, strips whitespace, joins with `|`, returns sha256 hex. -- `_extract_mapping_via_llm(client, headers, samples) -> dict` — builds the system prompt, calls `openrouter.call_llm`, parses the JSON envelope, raises `LLMParseError` on malformed output. -- `_validate_mapping(mapping, headers, first_row) -> None` — every named column must exist in `headers`; `qty_col`'s value on `first_row` must parse as a positive number; `cost_col` (if present) must parse as a number. Raises `LLMParseError` on failure. -- `_apply_mapping(rows, mapping) -> ParsedPie` — iterates remaining rows, builds `ParsedPosition` instances, computes totals from `qty * avg_cost` when explicit totals aren't present. - -Reuses without modification: - -- `app/services/openrouter.py::call_llm` — provider fallback chain + AICall ledger logging -- `app/services/csv_import.py::ParsedPie, ParsedPosition, CSVImportError` — same return type, same error hierarchy. `LLMParseError` inherits from `CSVImportError` so the route can catch both as one. - -### `app/routers/universe.py::parse_portfolio` — modified - -Two small changes: - -1. Add `Depends(require_paid)` to the route decorator. (Portfolio import has always been advertised as paid; this aligns the implementation.) -2. Wrap the existing `parse_t212_csv` call in a try/except that falls through to `parse_with_llm` on `CSVImportError`: - -```python -try: - pie = parse_t212_csv(raw) -except CSVImportError: - from app.services.llm_csv_parser import parse_with_llm, LLMParseError - try: - pie = await parse_with_llm(raw, session) - except LLMParseError as e: - raise HTTPException(status_code=400, detail=str(e)) -``` - -Everything below this point in the function — resolve_slice loop, upsert_tickers, inline Yahoo fetch, response build — is unchanged. `pie` has the same shape regardless of branch. - -### `app/models.py` — new model - -`CsvFormatTemplate` declared alongside the other tables. Columns as in the data model table above. - -### `app/templates/settings.html` — copy tweak - -- Section heading: "Import portfolio (Trading 212 CSV)" → "Import portfolio (CSV)" -- Drop-zone label: "Drop a T212 pie CSV here" → "Drop your broker's portfolio CSV here" -- Drop-zone hint: append " · T212, IBKR, and others auto-detected" after the size limit -- The "Export your pie from T212" instructions paragraph stays as a help link — T212 is still the best-documented happy path — but its phrasing softens to "If you use Trading 212…" - -## LLM prompt shape - -System prompt fixes the schema. User message contains headers + samples. - -``` -SYSTEM: You are an expert at recognising broker portfolio CSV formats. -You will be given the header row and 3-5 sample data rows from a CSV. -Identify which column contains each field. Return ONLY JSON, no prose. - -Schema: -{ - "ticker_col": "
              ", - "qty_col": "
              ", - "name_col": "
              ", - "cost_col": "
              ", // average price per share or unit cost - "currency_col": "
              ", - "broker_label": "" -} - -Rules: -- Use null when no column is a good match. -- ticker_col and qty_col are required; if either is missing return all nulls. -- Use the EXACT header string as it appears in the input. - -USER: headers: ["Symbol","Position","Avg Price","Currency"] -samples: - AAPL,100,150.00,USD - MSFT,50,300.00,USD - ... -``` - -The LLM never sees the entire file; it sees only the first ~5 data rows. -Token cost is bounded and uniform regardless of portfolio size. - -## Error handling - -| Failure | Response | Ledger | -|---|---|---| -| LLM provider down | 502 "couldn't parse — try again later" | AICall status=failed | -| LLM returns non-JSON | 400 "couldn't recognise as portfolio CSV" | AICall status=ok, no template stored | -| Mapping missing required columns (ticker/qty) | 400 same | AICall status=ok, no template stored | -| Mapping references non-existent column | 400 same | AICall status=ok, no template stored | -| Mapping validates but row parse fails on numerics | 400 same | template NOT stored | -| Cache hit but row parse fails (format drifted under us) | 400 with parse error | — | - -If a broker quietly changes their CSV shape such that a previously-good -cached mapping starts producing parse failures, the user sees an error and -the operator deletes the offending `csv_format_templates` row by hand. No -automatic eviction, no automatic retry. The cache is a learning store, not -a self-managing system. - -## Testing - -`tests/test_llm_csv_parser.py`: - -- **Fingerprint stability** — case/whitespace/BOM variants of the same headers hash to the same fingerprint. -- **Cache hit path** — pre-populate a `CsvFormatTemplate` row, mock `call_llm` to fail loudly, assert it is NOT called, assert positions come out correct, assert `use_count` is incremented. -- **Cache miss path** — mock `call_llm` to return a valid mapping JSON, assert a row is inserted with the upload's actual first data row as `sample_row` and no user_id anywhere, assert positions come out correct. -- **LLM returns malformed JSON** — raises `LLMParseError`, no template stored. -- **LLM maps to non-existent column** — raises `LLMParseError`, no template stored. -- **LLM maps qty to a non-numeric column** — raises `LLMParseError` on validation. -- **Stale cached mapping on parse failure** — pre-populate a template whose mapping no longer matches the file content, assert a 400 is returned and the template is NOT deleted automatically (operator owns eviction). -- **Integration** — POST a fabricated IBKR-shaped fixture to `/api/portfolio/parse`, assert ParsedPie round-trips, assert no second LLM call on a repeat upload. - -Existing `tests/test_csv_import.py` must still pass — the T212 happy path is unchanged. - -## Verification - -End-to-end manual check after deploy: - -1. Upload a T212 fixture → exists path stays unchanged (same dashboard load behaviour). -2. Upload a fabricated IBKR CSV → first upload calls LLM, returns positions, template row created in DB. -3. Re-upload the same IBKR CSV → second call has zero LLM cost (verify by counting `ai_calls` rows before/after), `use_count` increments to 2. -4. Inspect `csv_format_templates` row: confirm `headers` matches the upload's headers, `sample_row` is the first real data row, no `user_id` column exists on the table. -5. Upload random garbage (e.g. a screenshot renamed `.csv`) → 400 with clean error, no template stored, AICall row logged. -6. Free-tier account attempts import → 402 (paid gating). - -## Open questions for the implementation plan - -- Whether to read sample rows with `csv.reader` and re-encode them as text for the LLM (safer for embedded commas/quotes), or pass the raw first-N-lines verbatim. Default: the safer reader path. -- Whether to cap LLM-parsed portfolios at the same 1 MB limit as T212 (yes) and whether to add a separate cap on number-of-rows fed to the LLM as samples (yes, 5). -- Whether to log the fingerprint to the request log on cache hit/miss for operability. Default: yes, at INFO level, with `event_type="csv.format.cache_hit"` / `"csv.format.cache_miss"`. diff --git a/docs/superpowers/specs/2026-05-27-manual-portfolio-composition-design.md b/docs/superpowers/specs/2026-05-27-manual-portfolio-composition-design.md deleted file mode 100644 index a1a020b..0000000 --- a/docs/superpowers/specs/2026-05-27-manual-portfolio-composition-design.md +++ /dev/null @@ -1,312 +0,0 @@ -# Manual portfolio composition — Design Spec - -**Date:** 2026-05-27 -**Status:** Draft — pending implementation plan - -## Context - -Today the only way to populate the dashboard portfolio is to upload a CSV -(Trading 212 natively, anything else via the LLM-fallback parser landed in -the spec at `2026-05-27-llm-csv-fallback-parser-design.md`). That covers -users who already keep their holdings in a broker that can export. It does -not cover: - -- New users who want to try the dashboard with a handful of holdings without - hunting for an export feature in their broker. -- Existing users who bought something after their last export and want to - add a single position without re-importing. -- Users whose broker provides no usable export (some legacy ISAs, employer - share schemes, niche EU brokers). - -This feature adds a **dashboard-native edit mode**. A pencil-icon EDIT -button next to the portfolio heading toggles inline editing: each row -reveals a × delete button, and a small "Add a position" form appears at -the top of the portfolio block. Live prices continue updating while -editing. Brand-new users (no portfolio in localStorage) see the add form -directly in place of the existing "No portfolio loaded — import a CSV" -message. - -The CSV import flow in `/settings` is unchanged. - -## Goals - -- Let the user add a position to their dashboard portfolio in under 15 - seconds without leaving the page. -- Sanity-check tickers before they hit localStorage, so the dashboard never - shows symbols that won't price. -- Stay consistent with the existing data-flow invariant: the browser - localStorage owns the portfolio; the server persists no holdings. -- Accept either an average-cost-per-share or a "I bought on date X" input; - both produce the same stored shape (`avg_cost + qty`). - -## Non-goals - -- Multi-pie / named portfolios. Single localStorage pie remains the model. -- Persisting acquisition dates. The date input is a UX helper that - populates the cost field; only `avg_cost + qty` are stored. -- In-place edit of an existing row's qty or cost. Delete + re-add. -- Saving the portfolio to the server. localStorage stays the source of - truth. -- A separate manual-entry section on `/settings`. The feature lives on the - dashboard only. - -## Architecture - -``` -Dashboard (/) - ├─ Portfolio div - │ ├─ [✎ Edit] button (toggles edit mode in place) - │ ├─ Add-position form (visible only in edit mode, or always when empty) - │ │ ├─ Ticker input → on blur → GET /api/ticker/validate - │ │ ├─ Qty input - │ │ ├─ Cost mode toggle: ● Avg cost / ○ Bought on date - │ │ ├─ Cost field (number, or date-picker that auto-fills cost - │ │ │ via GET /api/ticker/historical) - │ │ └─ [Add] button (enabled only when validated + qty > 0 + cost > 0) - │ └─ Positions table - │ └─ Per-row [×] (visible only in edit mode) - └─ (everything else unchanged) -``` - -Two new **paid-only** endpoints are added; both wrap the existing -`app/services/market.fetch` machinery and do not persist holdings. The -edit-mode JavaScript merges new positions directly into the same -localStorage pie that the CSV import path writes. - -### Why dashboard-native, not Settings - -Editing a portfolio is a portfolio-management action. Putting it on the -dashboard puts the affordance where users naturally look — alongside the -thing they're editing. It also makes the empty-state CTA actionable in -place rather than redirecting to another page first. - -### Why two endpoints instead of one - -`/api/ticker/validate` returns the current quote — that's the live signal -the user wants for sanity reassurance ("yes, this symbol exists, and it's -worth $X today"). `/api/ticker/historical` returns the close on a chosen -date — only needed when the user picks "Bought on date" mode. Splitting -them keeps each endpoint's contract tight; combining would force the -common-case validate call to ship unused historical machinery. - -## Data flow - -### Validation flow (every add) - -1. User types `AAPL` and tabs out of the ticker field. -2. JS calls `GET /api/ticker/validate?symbol=AAPL`. -3. Server calls `market.fetch` for a single ticker. If a quote comes back, - returns `{ok: true, name: "Apple Inc", currency: "USD", price: 172.40, - as_of: "2026-05-27"}`. Side-effect: seeds anonymous `ticker_universe` - (same as the CSV path does). -4. JS shows a green inline check + `Apple Inc · $172.40 USD`. -5. Subsequent fields enable; Add button enables when all required filled. - -### Historical-price flow (only in "Bought on date" mode) - -1. User selects a date in the picker (e.g. `2024-01-15`). -2. On blur, JS calls `GET /api/ticker/historical?symbol=AAPL&date=2024-01-15`. -3. Server fetches the daily close for that date from Yahoo. If the date is - a non-trading day (weekend / holiday), uses the **last preceding trading - day** and returns its actual date. -4. Returns `{ok: true, close: 185.92, currency: "USD", actual_date: "2024-01-12"}`. -5. JS auto-fills the cost field with the close, shows a subtle "from 2024-01-12" - tag the user can dismiss. User can edit the auto-filled number. - -### Add flow - -1. User clicks **+ Add**. -2. JS reads localStorage pie (or creates a fresh `{positions: []}` shape if - none), appends a new position object: - ```json - { - "yahoo_ticker": "AAPL", - "t212_slice": "AAPL", - "name": "Apple Inc", - "qty": 100, - "avg_cost": 150.25, - "currency": "USD" - } - ``` -3. Writes back to localStorage. -4. Re-renders the positions table; the new row appears with live price - columns populating on the next `/api/universe` refresh. -5. Clears the form, focus returns to the ticker input for rapid serial entry. - -### Delete flow - -1. User clicks `×` on a row. -2. JS removes that position by index (not by ticker — so duplicates can be - removed independently). -3. Writes localStorage, re-renders. - -No undo. A confirmation dialog would be friction; the row can be re-added -in 10 seconds. - -## Server endpoints - -Both **require `Depends(require_paid)`** — matches the existing import -path. Anonymous / free-tier users get 402. - -### `GET /api/ticker/validate?symbol={t}` - -| Field | Meaning | -|---|---| -| `symbol` (query) | The ticker the user typed. Server uppercases + strips. Length capped at 32 chars. | - -Returns JSON: - -```json -{ - "ok": true, - "name": "Apple Inc", - "currency": "USD", - "price": 172.40, - "as_of": "2026-05-27" -} -``` - -or on unrecognised symbol: - -```json -{ "ok": false, "error": "Symbol not recognised" } -``` - -The endpoint returns 200 with `ok:false` for unrecognised symbols (so the -JS can render an inline error without parsing HTTP status). It returns 502 -for upstream provider failures (so the JS shows "Try again" rather than -"Not recognised"). - -Side effect: on `ok:true`, the symbol is upserted into anonymous -`ticker_universe` so the next `/api/universe` request includes its price. - -### `GET /api/ticker/historical?symbol={t}&date={YYYY-MM-DD}` - -| Field | Meaning | -|---|---| -| `symbol` (query) | Same shape as validate. | -| `date` (query) | ISO date the user picked. Server validates format + rejects future dates with 400. | - -Returns JSON: - -```json -{ - "ok": true, - "close": 185.92, - "currency": "USD", - "actual_date": "2024-01-12" -} -``` - -If the requested date is a non-trading day, the server walks back to the -last preceding day with a quote (up to a 7-day window) and returns -`actual_date` for transparency. - -On a symbol that's never had price data on the platform or before the -ticker's earliest available date: - -```json -{ "ok": false, "error": "No data for that date" } -``` - -## Client-side modules - -### `app/static/js/portfolio_edit.js` (NEW) - -A small module loaded by the dashboard template. Owns: - -- The `enterEditMode()` / `exitEditMode()` toggle. -- The add-position form's event wiring (validate-on-blur, mode toggle, - historical lookup, Add click). -- The × button click handler. -- A `mergePosition(pos)` helper that takes a position object, reads - localStorage, appends, writes back, dispatches a custom `portfolio:changed` - event for the existing portfolio.js renderer to pick up. - -### `app/static/js/portfolio.js` (MODIFIED, lightly) - -Three small additions: - -1. Empty-state CTA replaced. Instead of `"No portfolio loaded in this - browser. Import a portfolio CSV →"` linking to settings, the empty - state shows the inline add-position form (the same markup that edit - mode reveals) plus a small secondary link: `"Or import a CSV from your - broker →"` to `/settings#import`. -2. Listen for `portfolio:changed` and re-render. -3. Expose `window.CassandraPortfolio.merge(pos)` so `portfolio_edit.js` - can call it without circular module dependencies. - -### `app/templates/dashboard.html` (MODIFIED) - -Add: -- The `[✎ Edit]` button next to the portfolio heading. -- The add-position form markup (hidden by default; visible in edit mode - via a CSS class on the parent container, or always visible when the - pie is empty). -- A `