From 8a155ef157777d72a93a1d7c7367772c55ad58d6 Mon Sep 17 00:00:00 2001 From: Giorgio Gilestro Date: Sat, 16 May 2026 11:00:42 +0100 Subject: [PATCH] phase B (2/2): CSV upload endpoint + drag-drop UI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes Phase B. The full alternative-onboarding flow is now end-to-end: drop a T212 pie CSV → parser → InstrumentMap resolver → PortfolioSnapshot + Position rows, all without ever asking the user for broker credentials. - persist_pie() in app/services/csv_import.py: takes a ParsedPie, resolves each Slice via InstrumentMap, writes Portfolio + Snapshot + Position rows. Unmapped slices are still persisted using their CSV values and surfaced in the response for the UI to warn about. - POST /api/portfolios/upload: multipart endpoint accepting CSV file + optional portfolio_name + currency. 2 MiB cap. Returns import summary. - /upload page with drag-drop dropzone, file input fallback, and inline result panel showing invested/value/result + unmapped-slice warnings. - New "Import" link in the header nav. Verified end-to-end against the real T212 export: all 13 positions land with correct T212 tickers (incl. FPp_EQ for the Paris TotalEnergies listing the heuristic resolver picks), zero unmapped slices, totals reconcile to the penny. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/routers/api.py | 62 +++++++++++++- app/routers/pages.py | 6 ++ app/services/csv_import.py | 112 +++++++++++++++++++++++-- app/static/css/cassandra.css | 107 ++++++++++++++++++++++++ app/templates/base.html | 7 +- app/templates/upload.html | 157 +++++++++++++++++++++++++++++++++++ 6 files changed, 439 insertions(+), 12 deletions(-) create mode 100644 app/templates/upload.html diff --git a/app/routers/api.py b/app/routers/api.py index f41ea56..609a649 100644 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -9,7 +9,7 @@ import calendar as _cal import re from datetime import date, datetime, timedelta, timezone -from fastapi import APIRouter, Depends, HTTPException, Query, Request +from fastapi import APIRouter, Depends, File, Form, HTTPException, Query, Request, UploadFile from fastapi.responses import HTMLResponse, JSONResponse from sqlalchemy import desc, func, select from sqlalchemy.ext.asyncio import AsyncSession @@ -383,6 +383,66 @@ async def log_days( # --- Portfolios -------------------------------------------------------------- +# 2 MiB max for CSV uploads — T212 pies don't exceed a few KB in practice. +# Keeps the abuse vector small without rejecting legitimate exports. +_MAX_CSV_BYTES = 2 * 1024 * 1024 + + +@router.post("/portfolios/upload") +async def upload_portfolio_csv( + file: UploadFile = File(...), + portfolio_name: str | None = Form(default=None), + currency: str = Form(default="GBP"), + session: AsyncSession = Depends(get_session), +): + """Import a Trading 212 pie-export CSV. Parses, resolves each Slice to a + T212 ticker + Yahoo symbol via InstrumentMap, and persists a new + PortfolioSnapshot + Position rows. + + No user-id scoping yet — that lands in phase C. Until then, all uploads + land in the single shared portfolio identified by name.""" + from app.services.csv_import import CSVImportError, parse_t212_csv, persist_pie + + if not file.filename: + raise HTTPException(status_code=400, detail="No file uploaded") + if not file.filename.lower().endswith(".csv"): + raise HTTPException(status_code=400, detail="File must have .csv extension") + + raw = await file.read(_MAX_CSV_BYTES + 1) + if len(raw) > _MAX_CSV_BYTES: + raise HTTPException(status_code=413, detail=f"File exceeds {_MAX_CSV_BYTES} bytes") + if not raw: + raise HTTPException(status_code=400, detail="File is empty") + + try: + pie = parse_t212_csv(raw) + except CSVImportError as e: + raise HTTPException(status_code=400, detail=str(e)) + + try: + result = await persist_pie( + session, pie, + portfolio_name=portfolio_name, + currency=currency, + ) + except Exception as e: + # Roll back; surface a clean error + await session.rollback() + raise HTTPException(status_code=500, detail=f"Persist failed: {e}") + + return { + "portfolio_id": result.portfolio_id, + "snapshot_id": result.snapshot_id, + "portfolio_name": result.portfolio_name, + "is_new_portfolio": result.is_new_portfolio, + "positions": result.positions_written, + "unmapped": result.unmapped_slices, + "invested": pie.invested, + "value": pie.value, + "result": pie.result, + } + + @router.get("/portfolios") async def portfolios( request: Request, diff --git a/app/routers/pages.py b/app/routers/pages.py index 2a7ec11..156ebb6 100644 --- a/app/routers/pages.py +++ b/app/routers/pages.py @@ -33,6 +33,12 @@ async def news_page(request: Request): return templates.TemplateResponse(request, "news.html", {}) +@router.get("/upload", response_class=HTMLResponse) +async def upload_page(request: Request): + """Drag-drop CSV import. Posts to /api/portfolios/upload.""" + return templates.TemplateResponse(request, "upload.html", {}) + + async def _resolve_log_date(session: AsyncSession, day: str | None) -> date: """If `day` is YYYY-MM-DD use it; else fall back to the date of the most recent generated log; else today.""" diff --git a/app/services/csv_import.py b/app/services/csv_import.py index 63b51be..c6dd098 100644 --- a/app/services/csv_import.py +++ b/app/services/csv_import.py @@ -1,18 +1,19 @@ -"""Defensive parser for Trading 212 pie-export CSVs. +"""Defensive parser for Trading 212 pie-export CSVs + writer that persists +the parsed pie into PortfolioSnapshot/Position rows. -T212 has changed column order between exports historically; matching on header -NAME rather than column index makes this robust. We also explicitly skip the -'Total' aggregate row (it has slice='Total' and quantity='-'). - -Pure function — no DB, no HTTP. Persisting into PortfolioSnapshot/Position is -done by the upload endpoint after mapping each Slice to a Yahoo ticker via the -InstrumentMap service. +The parser is pure: no DB, no HTTP, no I/O. The writer (`persist_pie`) +takes a ParsedPie and resolves each position's Slice via InstrumentMap +to find its Yahoo ticker + canonical name before persisting. """ from __future__ import annotations import csv import io from dataclasses import dataclass +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from sqlalchemy.ext.asyncio import AsyncSession class CSVImportError(ValueError): @@ -197,3 +198,98 @@ def parse_t212_csv(content: str | bytes) -> ParsedPie: value=total.current_value if total else None, result=total.result if total else None, ) + + +# --- Persist parsed pie into portfolio/snapshot/positions ------------------- + + +@dataclass +class PersistResult: + portfolio_id: int + snapshot_id: int + positions_written: int + unmapped_slices: list[str] # slices we couldn't resolve to a Yahoo ticker + portfolio_name: str + is_new_portfolio: bool + + +async def persist_pie( + session: "AsyncSession", + pie: ParsedPie, + *, + portfolio_name: str | None = None, + source: str = "t212-csv", + currency: str = "GBP", +) -> PersistResult: + """Write a ParsedPie into Portfolio/PortfolioSnapshot/Position. + + - Portfolio is created on first sight of a given name; subsequent uploads + stack as new snapshots under the same portfolio. + - Each position's Slice is resolved to a T212 ticker + name via the + InstrumentMap. Unmapped slices still get stored using their raw CSV + values; we collect them in `unmapped_slices` for the UI to surface. + """ + # Late imports keep this module dependency-light for unit tests. + from sqlalchemy import select + + from app.db import utcnow + from app.models import Portfolio, PortfolioSnapshot, Position + from app.services.instrument_map import resolve_slice + + name = portfolio_name or pie.name or "Imported pie" + name = name.strip()[:64] + + portfolio = (await session.execute( + select(Portfolio).where(Portfolio.name == name) + )).scalar_one_or_none() + is_new = portfolio is None + if portfolio is None: + portfolio = Portfolio(name=name, source=source, currency=currency) + session.add(portfolio) + await session.flush() + + snap = PortfolioSnapshot( + portfolio_id=portfolio.id, + snapshot_at=utcnow(), + total_value=pie.value, + cash=None, + invested=pie.invested, + raw_json={ + "source": source, + "pie_name": pie.name, + "result": pie.result, + }, + ) + session.add(snap) + await session.flush() + + unmapped: list[str] = [] + for p in pie.positions: + resolved = await resolve_slice(session, p.slice) + if resolved and resolved.t212_ticker: + ticker = resolved.t212_ticker + position_name = resolved.name or p.name + else: + ticker = p.slice + position_name = p.name + unmapped.append(p.slice) + + session.add(Position( + snapshot_id=snap.id, + ticker=ticker, + name=position_name[:128] if position_name else None, + quantity=p.quantity, + average_price=p.average_price, + current_price=p.current_price, + ppl=p.result, + )) + + await session.commit() + return PersistResult( + portfolio_id=portfolio.id, + snapshot_id=snap.id, + positions_written=len(pie.positions), + unmapped_slices=unmapped, + portfolio_name=name, + is_new_portfolio=is_new, + ) diff --git a/app/static/css/cassandra.css b/app/static/css/cassandra.css index 1fb85dc..4b89ad3 100644 --- a/app/static/css/cassandra.css +++ b/app/static/css/cassandra.css @@ -591,6 +591,113 @@ table.dense tr.row-stale td { color: var(--dim); } .log-meta__row { display: flex; flex-wrap: wrap; align-items: center; gap: 0; margin-top: 6px; } .log-meta__row--dim { color: var(--dim); font-size: 10.5px; } +/* --- Upload page (drag-drop CSV) ------------------------------------- */ + +.dz { + border: 2px dashed var(--border); + background: var(--surface-2); + padding: 36px 20px; + text-align: center; + cursor: pointer; + transition: border-color 0.15s, background 0.15s; +} +.dz:hover, .dz--over { + border-color: var(--accent); + background: color-mix(in srgb, var(--accent) 6%, var(--surface-2)); +} +.dz__icon { + font-family: var(--font-mono); + font-size: 28px; + color: var(--accent); + letter-spacing: -2px; + margin-bottom: 6px; +} +.dz__label { + font-family: var(--font-mono); + font-size: 13px; + color: var(--text); + text-transform: uppercase; + letter-spacing: 0.08em; +} +.dz__hint { color: var(--muted); font-size: 11.5px; margin-top: 4px; } +.dz__hint a { color: var(--accent); } +.dz__filename { margin-top: 10px; color: var(--accent); font-size: 12px; font-family: var(--font-mono); min-height: 1em; } + +.form-row { display: grid; grid-template-columns: 180px 1fr; align-items: center; gap: 12px; padding: 6px 0; } +.form-row label { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: 0.08em; } +.form-row input[type="text"], .form-row select { + background: var(--bg); + border: 1px solid var(--border); + color: var(--text); + font-family: var(--font-mono); + font-size: 12px; + padding: 6px 8px; + outline: none; +} +.form-row input[type="text"]:focus, .form-row select:focus { border-color: var(--accent); } + +#submit-btn { + margin-top: 14px; + background: transparent; + border: 1px solid var(--accent); + color: var(--accent); + font-family: var(--font-mono); + font-size: 11px; + padding: 8px 18px; + text-transform: uppercase; + letter-spacing: 0.1em; + cursor: pointer; +} +#submit-btn:hover:not(:disabled) { background: var(--accent); color: var(--bg); } +#submit-btn:disabled { opacity: 0.4; cursor: not-allowed; } + +.result { + margin-top: 20px; + padding: 14px; + border: 1px solid var(--border); + border-left: 3px solid var(--accent); + background: color-mix(in srgb, var(--accent) 4%, transparent); + font-family: var(--font-sans); + font-size: 13px; +} +.result--err { border-left-color: var(--negative); background: color-mix(in srgb, var(--negative) 5%, transparent); } +.result__head { + font-family: var(--font-mono); + font-size: 11px; + text-transform: uppercase; + letter-spacing: 0.08em; + color: var(--accent); + margin-bottom: 10px; +} +.result--err .result__head { color: var(--negative); } +.result__tag { + display: inline-block; + margin-left: 6px; + font-size: 9px; + padding: 1px 5px; + border: 1px solid var(--accent); + color: var(--accent); +} +.result__grid { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 10px 18px; + margin-bottom: 10px; +} +.result__grid .k { + font-family: var(--font-mono); + font-size: 9.5px; + color: var(--muted); + text-transform: uppercase; + letter-spacing: 0.08em; +} +.result__grid .v { font-size: 17px; color: var(--text); font-variant-numeric: tabular-nums; margin-top: 2px; } +.result__grid .v.pos { color: var(--positive); } +.result__grid .v.neg { color: var(--negative); } +.result__row { color: var(--muted); font-size: 12px; margin-top: 6px; } +.result__warn { color: var(--alert); font-size: 12px; margin-top: 4px; } +.result__warn code { background: rgba(0,0,0,0.15); padding: 1px 4px; font-family: var(--font-mono); } + /* --- Chat sidebar ----------------------------------------------------- */ .chat-header { diff --git a/app/templates/base.html b/app/templates/base.html index bef4f18..e39c1fb 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -42,9 +42,10 @@
Cassandra
+ + + +
+ + + +{% endblock %}