phase B (1/4): CSV parser + InstrumentMap (T212 shortcode → Yahoo ticker)
First two slices of the multi-user roadmap (Phase B). Validates the
core onboarding mechanic against the user's real T212 export before
paying any auth/tenancy tax.
CSV parser (app/services/csv_import.py):
- Header-name matched (survives T212 reordering columns between
exports), tolerant of UTF-8 BOM, dash/N/A/empty markers, thousand-
separator commas, blank rows, zero-quantity stubs, missing Total row.
- Returns ParsedPie(name, positions, invested, value, result) with
derived avg_price + current_price per share in account currency.
- 14 tests covering happy path on the real CSV + 13 edge cases.
InstrumentMap (migration 0006 + app/services/instrument_map.py):
- Catalogue table mapping T212 ticker → Yahoo ticker, populated by
sync_from_t212() against the dev's read-only API key. Manual rows
(manual=True) are protected from auto-overwrite.
- Pure t212_ticker_to_yahoo() handles both suffix forms: single
trailing exchange letter (l/a/p/d/m/s/...) and country code (US,
DE, FR, IT, CA, ...). All 13 of the user's holdings + 15 case-
coverage tests pass.
- Live sync against T212 ingests 17,050 instruments (~2.2% unmappable
on exotic exchanges; can extend the suffix map later).
- resolve_slice() picks the right listing per shortName using a
UK-friendly currency preference (GBX > GBP > EUR > USD). Resolved
correctly for all 13 of the user's positions, including TTE on
Paris vs the NYSE dual-listing.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
6dac8a2c7f
commit
16e9f5f0cc
7 changed files with 840 additions and 0 deletions
49
alembic/versions/0006_instrument_map.py
Normal file
49
alembic/versions/0006_instrument_map.py
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
"""instrument_map — T212 shortcode → Yahoo ticker mapping
|
||||
|
||||
Revision ID: 0006
|
||||
Revises: 0005
|
||||
Create Date: 2026-05-16
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision: str = "0006"
|
||||
down_revision: Union[str, None] = "0005"
|
||||
branch_labels: Union[str, Sequence[str], None] = None
|
||||
depends_on: Union[str, Sequence[str], None] = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
op.create_table(
|
||||
"instrument_map",
|
||||
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
|
||||
# T212's internal ticker (e.g. "SGLNl_EQ", "EQNR_US_EQ"). Unique
|
||||
# globally because each T212 ticker is one listing.
|
||||
sa.Column("t212_ticker", sa.String(64), nullable=False),
|
||||
# T212's short name (e.g. "SGLN") — the value in the CSV's "Slice"
|
||||
# column. Not unique: many tickers share a shortName across listings.
|
||||
sa.Column("t212_shortname", sa.String(32), nullable=False),
|
||||
# Computed Yahoo Finance symbol. Nullable so unmappable rows still
|
||||
# land in the table (we may add manual mappings later).
|
||||
sa.Column("yahoo_ticker", sa.String(32)),
|
||||
sa.Column("name", sa.String(128), nullable=False),
|
||||
# T212's currencyCode (GBP, GBX, USD, EUR, CHF, …).
|
||||
sa.Column("currency", sa.String(8)),
|
||||
sa.Column("isin", sa.String(16)),
|
||||
sa.Column("instrument_type", sa.String(16)), # STOCK, ETF, …
|
||||
# True when the row was hand-edited; auto-sync won't overwrite.
|
||||
sa.Column("manual", sa.Boolean, nullable=False, server_default=sa.text("0")),
|
||||
sa.Column("last_verified_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.UniqueConstraint("t212_ticker", name="uq_imap_t212_ticker"),
|
||||
)
|
||||
op.create_index("ix_imap_shortname", "instrument_map", ["t212_shortname"])
|
||||
op.create_index("ix_imap_isin", "instrument_map", ["isin"])
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_index("ix_imap_isin", table_name="instrument_map")
|
||||
op.drop_index("ix_imap_shortname", table_name="instrument_map")
|
||||
op.drop_table("instrument_map")
|
||||
|
|
@ -187,6 +187,33 @@ class Position(Base):
|
|||
snapshot: Mapped[PortfolioSnapshot] = relationship(back_populates="positions")
|
||||
|
||||
|
||||
class InstrumentMap(Base):
|
||||
"""Maps T212's tickers/shortnames to Yahoo Finance tickers so we can
|
||||
refresh prices via Yahoo after a user uploads a T212 pie CSV.
|
||||
|
||||
Synced periodically from T212's /equity/metadata/instruments endpoint
|
||||
via the admin's read-only API key. Each row is one T212 listing.
|
||||
Multiple rows can share a shortName (e.g. SHEL on LSE in GBX vs
|
||||
SHEL on NYSE in USD); the resolver picks the right one per user."""
|
||||
__tablename__ = "instrument_map"
|
||||
id: Mapped[int] = mapped_column(BigInteger, primary_key=True, autoincrement=True)
|
||||
t212_ticker: Mapped[str] = mapped_column(String(64), nullable=False)
|
||||
t212_shortname: Mapped[str] = mapped_column(String(32), nullable=False)
|
||||
yahoo_ticker: Mapped[str | None] = mapped_column(String(32))
|
||||
name: Mapped[str] = mapped_column(String(128), nullable=False)
|
||||
currency: Mapped[str | None] = mapped_column(String(8))
|
||||
isin: Mapped[str | None] = mapped_column(String(16))
|
||||
instrument_type: Mapped[str | None] = mapped_column(String(16))
|
||||
manual: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
last_verified_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utcnow)
|
||||
|
||||
__table_args__ = (
|
||||
UniqueConstraint("t212_ticker", name="uq_imap_t212_ticker"),
|
||||
Index("ix_imap_shortname", "t212_shortname"),
|
||||
Index("ix_imap_isin", "isin"),
|
||||
)
|
||||
|
||||
|
||||
class JobRun(Base):
|
||||
"""One row per scheduled-job invocation; powers /api/health + the ops footer."""
|
||||
__tablename__ = "job_runs"
|
||||
|
|
|
|||
199
app/services/csv_import.py
Normal file
199
app/services/csv_import.py
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
"""Defensive parser for Trading 212 pie-export CSVs.
|
||||
|
||||
T212 has changed column order between exports historically; matching on header
|
||||
NAME rather than column index makes this robust. We also explicitly skip the
|
||||
'Total' aggregate row (it has slice='Total' and quantity='-').
|
||||
|
||||
Pure function — no DB, no HTTP. Persisting into PortfolioSnapshot/Position is
|
||||
done by the upload endpoint after mapping each Slice to a Yahoo ticker via the
|
||||
InstrumentMap service.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
from dataclasses import dataclass
|
||||
|
||||
|
||||
class CSVImportError(ValueError):
|
||||
"""Raised when the CSV is unparseable or missing required columns."""
|
||||
|
||||
|
||||
# Header name -> normalised key used in the parsed dict. Lowercase, ignore
|
||||
# leading/trailing whitespace, treat case-insensitively. Extra columns are
|
||||
# silently ignored.
|
||||
_HEADER_MAP = {
|
||||
"slice": "slice",
|
||||
"name": "name",
|
||||
"invested value": "invested_value",
|
||||
"value": "current_value",
|
||||
"result": "result",
|
||||
"owned quantity": "quantity",
|
||||
"dividends gained": "dividends_gained",
|
||||
"dividends cash": "dividends_cash",
|
||||
"dividends reinvested": "dividends_reinvested",
|
||||
}
|
||||
|
||||
# These must be present for the import to be meaningful at all.
|
||||
_REQUIRED_FIELDS = ("slice", "quantity")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedPosition:
|
||||
slice: str # T212 shortcode, e.g. "SGLN"
|
||||
name: str
|
||||
invested_value: float | None
|
||||
current_value: float | None
|
||||
result: float | None # P/L in pie currency
|
||||
quantity: float
|
||||
dividends_gained: float | None = None
|
||||
dividends_cash: float | None = None
|
||||
dividends_reinvested: float | None = None
|
||||
|
||||
@property
|
||||
def average_price(self) -> float | None:
|
||||
if self.invested_value is None or not self.quantity:
|
||||
return None
|
||||
return self.invested_value / self.quantity
|
||||
|
||||
@property
|
||||
def current_price(self) -> float | None:
|
||||
if self.current_value is None or not self.quantity:
|
||||
return None
|
||||
return self.current_value / self.quantity
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ParsedPie:
|
||||
name: str | None # from the Total row's Name column
|
||||
positions: tuple[ParsedPosition, ...]
|
||||
invested: float | None # totals from the Total row
|
||||
value: float | None
|
||||
result: float | None
|
||||
|
||||
|
||||
def _normalise_header(h: str) -> str:
|
||||
return h.strip().lower()
|
||||
|
||||
|
||||
def _parse_num(raw: str | None) -> float | None:
|
||||
"""Empty / 'N/A' / '-' / '—' → None. Otherwise float."""
|
||||
if raw is None:
|
||||
return None
|
||||
s = raw.strip()
|
||||
if not s or s in {"-", "—", "N/A", "n/a", "NA"}:
|
||||
return None
|
||||
# T212 occasionally exports with thousand-comma. Strip safely.
|
||||
s = s.replace(",", "")
|
||||
try:
|
||||
return float(s)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def parse_t212_csv(content: str | bytes) -> ParsedPie:
|
||||
"""Parse a T212 pie-export CSV.
|
||||
|
||||
Args:
|
||||
content: bytes or str containing the CSV (raw export file contents).
|
||||
|
||||
Returns:
|
||||
ParsedPie with positions list and aggregate totals.
|
||||
|
||||
Raises:
|
||||
CSVImportError: if the file is empty, missing required headers,
|
||||
or contains no usable rows.
|
||||
"""
|
||||
if isinstance(content, bytes):
|
||||
try:
|
||||
content = content.decode("utf-8-sig") # handle Excel BOM
|
||||
except UnicodeDecodeError:
|
||||
content = content.decode("latin-1")
|
||||
|
||||
reader = csv.reader(io.StringIO(content))
|
||||
try:
|
||||
header_row = next(reader)
|
||||
except StopIteration:
|
||||
raise CSVImportError("Empty CSV file")
|
||||
|
||||
# Map column index -> normalised field name. Unknown headers are ignored.
|
||||
field_by_index: dict[int, str] = {}
|
||||
for i, h in enumerate(header_row):
|
||||
key = _HEADER_MAP.get(_normalise_header(h))
|
||||
if key:
|
||||
field_by_index[i] = key
|
||||
|
||||
missing = [f for f in _REQUIRED_FIELDS if f not in field_by_index.values()]
|
||||
if missing:
|
||||
raise CSVImportError(
|
||||
f"CSV missing required column(s): {', '.join(missing)}. "
|
||||
f"Found headers: {header_row}"
|
||||
)
|
||||
|
||||
positions: list[ParsedPosition] = []
|
||||
total: ParsedPosition | None = None
|
||||
pie_name: str | None = None
|
||||
|
||||
for row_num, row in enumerate(reader, start=2):
|
||||
if not row or not any(cell.strip() for cell in row):
|
||||
continue # skip blank lines
|
||||
|
||||
record: dict[str, object] = {}
|
||||
for idx, field in field_by_index.items():
|
||||
raw = row[idx] if idx < len(row) else ""
|
||||
if field in {"slice", "name"}:
|
||||
record[field] = raw.strip()
|
||||
else:
|
||||
record[field] = _parse_num(raw)
|
||||
|
||||
slice_code = record.get("slice") or ""
|
||||
if not slice_code:
|
||||
continue # malformed; skip silently rather than abort
|
||||
|
||||
# The 'Total' row uses slice='Total' and quantity='-' — capture it
|
||||
# for aggregate totals but don't list it as a position.
|
||||
if slice_code.lower() == "total":
|
||||
pie_name = (record.get("name") or "").strip() or None
|
||||
total = ParsedPosition(
|
||||
slice=slice_code,
|
||||
name=pie_name or "Total",
|
||||
invested_value=record.get("invested_value"),
|
||||
current_value=record.get("current_value"),
|
||||
result=record.get("result"),
|
||||
quantity=0.0,
|
||||
dividends_gained=record.get("dividends_gained"),
|
||||
dividends_cash=record.get("dividends_cash"),
|
||||
dividends_reinvested=record.get("dividends_reinvested"),
|
||||
)
|
||||
continue
|
||||
|
||||
qty = record.get("quantity")
|
||||
if qty is None or qty == 0:
|
||||
# Position row with no usable quantity — skip rather than fail.
|
||||
continue
|
||||
|
||||
positions.append(ParsedPosition(
|
||||
slice=slice_code,
|
||||
name=(record.get("name") or "").strip(),
|
||||
invested_value=record.get("invested_value"),
|
||||
current_value=record.get("current_value"),
|
||||
result=record.get("result"),
|
||||
quantity=qty,
|
||||
dividends_gained=record.get("dividends_gained"),
|
||||
dividends_cash=record.get("dividends_cash"),
|
||||
dividends_reinvested=record.get("dividends_reinvested"),
|
||||
))
|
||||
|
||||
if not positions:
|
||||
raise CSVImportError(
|
||||
"CSV contained no parseable position rows. "
|
||||
"Expected at least one row with a Slice code and quantity."
|
||||
)
|
||||
|
||||
return ParsedPie(
|
||||
name=pie_name,
|
||||
positions=tuple(positions),
|
||||
invested=total.invested_value if total else None,
|
||||
value=total.current_value if total else None,
|
||||
result=total.result if total else None,
|
||||
)
|
||||
251
app/services/instrument_map.py
Normal file
251
app/services/instrument_map.py
Normal file
|
|
@ -0,0 +1,251 @@
|
|||
"""T212 shortcode → Yahoo Finance ticker resolution.
|
||||
|
||||
The CSV path gives us only T212's `Slice` (short name like "SGLN", "TTE").
|
||||
To refresh prices via Yahoo we need the proper Yahoo symbol (`SGLN.L`,
|
||||
`TTE.PA`, …). The mapping comes from T212's own catalogue
|
||||
(/equity/metadata/instruments), synced into the `instrument_map` table
|
||||
via the admin's read-only API key. The resolver then picks the right
|
||||
listing per user using currency preference.
|
||||
|
||||
This module has three responsibilities:
|
||||
1. **Pure translation** — turn a T212 ticker like `SGLNl_EQ` into a Yahoo
|
||||
symbol like `SGLN.L` from suffix rules. No DB, no HTTP.
|
||||
2. **Catalogue sync** — pull every T212 instrument and upsert into
|
||||
`instrument_map`. Hand-edited rows (`manual=True`) are never overwritten.
|
||||
3. **Slice resolution** — given a CSV `Slice` like "SHEL", find the best
|
||||
matching `instrument_map` row using configurable currency preference.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from datetime import timedelta
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.mysql import insert as mysql_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from app.db import utcnow
|
||||
from app.models import InstrumentMap
|
||||
from app.services.trading212 import Trading212
|
||||
|
||||
|
||||
# --- Pure translation: T212 ticker → Yahoo symbol ---------------------------
|
||||
|
||||
|
||||
# Single trailing letter before "_EQ" → exchange suffix.
|
||||
# These conventions come from observing T212's instrument catalogue.
|
||||
_T212_LETTER_TO_YAHOO = {
|
||||
"l": ".L", # London (LSE)
|
||||
"a": ".AS", # Amsterdam (Euronext)
|
||||
"p": ".PA", # Paris (Euronext)
|
||||
"d": ".DE", # Frankfurt (Xetra)
|
||||
"m": ".MI", # Milan (Borsa Italiana)
|
||||
"s": ".SW", # Swiss (SIX)
|
||||
"b": ".BR", # Brussels (Euronext)
|
||||
"i": ".IR", # Ireland (Euronext Dublin)
|
||||
"h": ".HE", # Helsinki (Nasdaq Nordic)
|
||||
"c": ".CO", # Copenhagen
|
||||
"o": ".OL", # Oslo
|
||||
}
|
||||
|
||||
# Country-code suffix (e.g. _US_EQ, _CA_EQ) → Yahoo suffix.
|
||||
_T212_COUNTRY_TO_YAHOO = {
|
||||
"US": "", # NYSE/NASDAQ — Yahoo bare ticker
|
||||
"CA": ".TO", # Toronto
|
||||
"DE": ".DE",
|
||||
"FR": ".PA",
|
||||
"GB": ".L",
|
||||
"IT": ".MI",
|
||||
"ES": ".MC",
|
||||
"NL": ".AS",
|
||||
"BE": ".BR",
|
||||
"IE": ".IR",
|
||||
"FI": ".HE",
|
||||
"CH": ".SW",
|
||||
"NO": ".OL",
|
||||
"DK": ".CO",
|
||||
"SE": ".ST",
|
||||
}
|
||||
|
||||
|
||||
def t212_ticker_to_yahoo(t212_ticker: str, short_name: str) -> str | None:
|
||||
"""Translate a T212 ticker like 'SGLNl_EQ' or 'EQNR_US_EQ' to its
|
||||
Yahoo Finance symbol. Returns None when the pattern isn't recognised.
|
||||
|
||||
Rules, in order:
|
||||
'XXXX_<CC>_EQ' (country code) → short_name + country suffix
|
||||
e.g. EQNR_US_EQ → EQNR
|
||||
SHEL_US_EQ → SHEL
|
||||
'XXXX<x>_EQ' (single trailing lowercase letter) → short_name + suffix
|
||||
e.g. SGLNl_EQ → SGLN.L
|
||||
SHELLa_EQ → SHELL.AS
|
||||
FPp_EQ → FP.PA
|
||||
"""
|
||||
if not t212_ticker.endswith("_EQ"):
|
||||
return None
|
||||
body = t212_ticker[:-3] # strip "_EQ"
|
||||
|
||||
# Country-code form: '..._XX' where XX is a 2-letter country code
|
||||
if len(body) >= 3 and body[-3] == "_" and body[-2:].isupper():
|
||||
cc = body[-2:]
|
||||
suffix = _T212_COUNTRY_TO_YAHOO.get(cc)
|
||||
if suffix is None:
|
||||
return None
|
||||
return f"{short_name}{suffix}"
|
||||
|
||||
# Single-letter form: '...x' where x is a recognised exchange letter
|
||||
if body and body[-1] in _T212_LETTER_TO_YAHOO:
|
||||
return f"{short_name}{_T212_LETTER_TO_YAHOO[body[-1]]}"
|
||||
|
||||
return None
|
||||
|
||||
|
||||
# --- Catalogue sync ---------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
fetched: int
|
||||
upserted: int
|
||||
unmappable: int
|
||||
skipped_manual: int
|
||||
|
||||
|
||||
async def sync_from_t212(
|
||||
session: AsyncSession,
|
||||
client: httpx.AsyncClient,
|
||||
t212: Trading212 | None = None,
|
||||
) -> SyncResult:
|
||||
"""Pull every T212 instrument and upsert into instrument_map. Hand-
|
||||
edited rows (manual=True) are never overwritten. Runs idempotently."""
|
||||
t212 = t212 or Trading212()
|
||||
instruments = await t212.instruments(client) or []
|
||||
|
||||
# Pre-fetch existing manual mappings so we know which t212_tickers
|
||||
# to skip on upsert.
|
||||
manual_tickers = {
|
||||
r.t212_ticker for r in (await session.execute(
|
||||
select(InstrumentMap).where(InstrumentMap.manual == True)
|
||||
)).scalars().all()
|
||||
}
|
||||
|
||||
now = utcnow()
|
||||
rows = []
|
||||
unmappable = 0
|
||||
skipped_manual = 0
|
||||
for inst in instruments:
|
||||
tkr = inst.get("ticker")
|
||||
sn = inst.get("shortName") or ""
|
||||
name = inst.get("name") or sn or tkr
|
||||
if not tkr:
|
||||
continue
|
||||
if tkr in manual_tickers:
|
||||
skipped_manual += 1
|
||||
continue
|
||||
yahoo = t212_ticker_to_yahoo(tkr, sn)
|
||||
if yahoo is None:
|
||||
unmappable += 1
|
||||
rows.append({
|
||||
"t212_ticker": tkr,
|
||||
"t212_shortname": sn,
|
||||
"yahoo_ticker": yahoo,
|
||||
"name": (name or sn)[:128],
|
||||
"currency": inst.get("currencyCode"),
|
||||
"isin": inst.get("isin"),
|
||||
"instrument_type": inst.get("type"),
|
||||
"manual": False,
|
||||
"last_verified_at": now,
|
||||
})
|
||||
|
||||
if not rows:
|
||||
return SyncResult(fetched=len(instruments), upserted=0,
|
||||
unmappable=unmappable, skipped_manual=skipped_manual)
|
||||
|
||||
# Bulk upsert. MySQL: ON DUPLICATE KEY UPDATE on t212_ticker unique key.
|
||||
# Chunk to avoid hitting MySQL's max_allowed_packet on 17k+ rows.
|
||||
chunk = 500
|
||||
upserted = 0
|
||||
for i in range(0, len(rows), chunk):
|
||||
stmt = mysql_insert(InstrumentMap).values(rows[i:i + chunk])
|
||||
stmt = stmt.on_duplicate_key_update(
|
||||
yahoo_ticker=stmt.inserted.yahoo_ticker,
|
||||
t212_shortname=stmt.inserted.t212_shortname,
|
||||
name=stmt.inserted.name,
|
||||
currency=stmt.inserted.currency,
|
||||
isin=stmt.inserted.isin,
|
||||
instrument_type=stmt.inserted.instrument_type,
|
||||
last_verified_at=stmt.inserted.last_verified_at,
|
||||
)
|
||||
await session.execute(stmt)
|
||||
upserted += len(rows[i:i + chunk])
|
||||
|
||||
await session.commit()
|
||||
return SyncResult(
|
||||
fetched=len(instruments), upserted=upserted,
|
||||
unmappable=unmappable, skipped_manual=skipped_manual,
|
||||
)
|
||||
|
||||
|
||||
# --- Resolution: CSV Slice → preferred Yahoo ticker -------------------------
|
||||
|
||||
|
||||
# Currency preference for users in a UK account. Listings denominated in the
|
||||
# user's account currency or its smaller-unit (GBX = pence) come first. EUR
|
||||
# ranks ABOVE USD because UK retail brokers (incl. T212) typically default
|
||||
# users to the London + Euronext listings; the NYSE dual-listing only wins
|
||||
# when no European listing exists (e.g. EQNR isn't on T212's Oslo book).
|
||||
_DEFAULT_CCY_PREFERENCE = ("GBX", "GBP", "EUR", "USD", "CHF", "JPY")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ResolvedInstrument:
|
||||
t212_ticker: str
|
||||
yahoo_ticker: str | None
|
||||
name: str
|
||||
currency: str | None
|
||||
isin: str | None
|
||||
|
||||
|
||||
async def resolve_slice(
|
||||
session: AsyncSession,
|
||||
slice_code: str,
|
||||
currency_preference: tuple[str, ...] = _DEFAULT_CCY_PREFERENCE,
|
||||
) -> ResolvedInstrument | None:
|
||||
"""Find the best Yahoo ticker for a given CSV Slice.
|
||||
|
||||
Picks the listing whose currency comes first in `currency_preference`.
|
||||
Manual mappings always win over auto-resolved ones."""
|
||||
if not slice_code:
|
||||
return None
|
||||
|
||||
rows = (await session.execute(
|
||||
select(InstrumentMap)
|
||||
.where(InstrumentMap.t212_shortname == slice_code)
|
||||
)).scalars().all()
|
||||
if not rows:
|
||||
return None
|
||||
|
||||
def rank(row: InstrumentMap) -> tuple[int, int]:
|
||||
manual_rank = 0 if row.manual else 1
|
||||
try:
|
||||
ccy_rank = currency_preference.index(row.currency or "")
|
||||
except ValueError:
|
||||
ccy_rank = len(currency_preference)
|
||||
return (manual_rank, ccy_rank)
|
||||
|
||||
rows.sort(key=rank)
|
||||
chosen = rows[0]
|
||||
return ResolvedInstrument(
|
||||
t212_ticker=chosen.t212_ticker,
|
||||
yahoo_ticker=chosen.yahoo_ticker,
|
||||
name=chosen.name,
|
||||
currency=chosen.currency,
|
||||
isin=chosen.isin,
|
||||
)
|
||||
|
||||
|
||||
def is_stale(row: InstrumentMap, max_age: timedelta = timedelta(days=14)) -> bool:
|
||||
"""True if the row hasn't been refreshed from T212 recently."""
|
||||
age = utcnow() - row.last_verified_at
|
||||
return age > max_age
|
||||
15
tests/fixtures/t212_pie_export.csv
vendored
Normal file
15
tests/fixtures/t212_pie_export.csv
vendored
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
"Slice","Name","Invested value","Value","Result","Owned quantity","Dividends gained","Dividends cash","Dividends reinvested"
|
||||
"SGLN","iShares Physical Gold",2325,2324.3,-0.7,"35.12084592","N/A","N/A","N/A"
|
||||
"IGLS","iShares UK Gilts 0-5yr (Dist)",2092.5,2089.84,-2.66,"16.59397303","N/A","N/A","N/A"
|
||||
"VEUR","Vanguard FTSE Developed Europe (Dist)",1511.25,1507.96,-3.29,"36.59200968","N/A","N/A","N/A"
|
||||
"HMCH","HSBC MSCI China (Dist)",1162.5,1160.88,-1.62,"197.68022163","N/A","N/A","N/A"
|
||||
"INRG","iShares Global Clean Energy Transition (Dist)",1162.5,1170.95,8.45,"125.20193861","N/A","N/A","N/A"
|
||||
"VFEM","Vanguard FTSE Emerging Markets (Dist)",581.25,581.11,-0.14,"9.55610357","N/A","N/A","N/A"
|
||||
"VJPN","Vanguard FTSE Japan (Dist)",581.25,582.61,1.36,"15.48657829","N/A","N/A","N/A"
|
||||
"BA","BAE Systems",578.34,563.12,-15.22,"30.43093922","N/A","N/A","N/A"
|
||||
"TTE","TotalEnergies",346.83,349.06,2.23,"5.08932089","N/A","N/A","N/A"
|
||||
"NATP","Future of Defence UCITS ETF (Acc)",348.75,347.1,-1.65,"23.59926918","N/A","N/A","N/A"
|
||||
"EQNR","Equinor",348.23,358.52,10.29,"12.14586234","N/A","N/A","N/A"
|
||||
"SHEL","Shell",347.01,350.47,3.46,"10.97091368","N/A","N/A","N/A"
|
||||
"BP","BP",231.34,233.41,2.07,"42.26932212","N/A","N/A","N/A"
|
||||
"Total","Defensive Ex-US 2026",11616.75,11619.33,2.58,"-","0","0","0"
|
||||
|
182
tests/test_csv_import.py
Normal file
182
tests/test_csv_import.py
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
"""Tests for app.services.csv_import.
|
||||
|
||||
Uses the real T212 pie-export sample at tests/fixtures/t212_pie_export.csv
|
||||
for the happy path, then synthetic fixtures for edge cases.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.services.csv_import import (
|
||||
CSVImportError,
|
||||
ParsedPie,
|
||||
ParsedPosition,
|
||||
parse_t212_csv,
|
||||
)
|
||||
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "t212_pie_export.csv"
|
||||
|
||||
|
||||
# --- Happy path: real T212 export ------------------------------------------
|
||||
|
||||
|
||||
def test_parses_real_t212_export():
|
||||
pie = parse_t212_csv(FIXTURE.read_bytes())
|
||||
assert isinstance(pie, ParsedPie)
|
||||
assert pie.name == "Defensive Ex-US 2026"
|
||||
assert pie.invested == pytest.approx(11616.75)
|
||||
assert pie.value == pytest.approx(11619.33)
|
||||
assert pie.result == pytest.approx(2.58)
|
||||
assert len(pie.positions) == 13
|
||||
|
||||
|
||||
def test_first_position_fields_resolved():
|
||||
pie = parse_t212_csv(FIXTURE.read_bytes())
|
||||
sgln = next(p for p in pie.positions if p.slice == "SGLN")
|
||||
assert sgln.name == "iShares Physical Gold"
|
||||
assert sgln.invested_value == pytest.approx(2325)
|
||||
assert sgln.current_value == pytest.approx(2324.3)
|
||||
assert sgln.result == pytest.approx(-0.7)
|
||||
assert sgln.quantity == pytest.approx(35.12084592)
|
||||
assert sgln.dividends_gained is None # 'N/A' in source
|
||||
assert sgln.average_price == pytest.approx(2325 / 35.12084592)
|
||||
assert sgln.current_price == pytest.approx(2324.3 / 35.12084592)
|
||||
|
||||
|
||||
def test_total_row_excluded_from_positions():
|
||||
pie = parse_t212_csv(FIXTURE.read_bytes())
|
||||
assert not any(p.slice.lower() == "total" for p in pie.positions)
|
||||
|
||||
|
||||
# --- Edge cases: defensive parsing -----------------------------------------
|
||||
|
||||
|
||||
def test_empty_file_raises():
|
||||
with pytest.raises(CSVImportError, match="Empty"):
|
||||
parse_t212_csv("")
|
||||
|
||||
|
||||
def test_missing_required_column_raises():
|
||||
# Only Name, no Slice or quantity
|
||||
csv = '"Name","Value"\n"iShares","100"\n'
|
||||
with pytest.raises(CSVImportError, match="missing required column"):
|
||||
parse_t212_csv(csv)
|
||||
|
||||
|
||||
def test_no_position_rows_raises():
|
||||
# Headers present but only the Total aggregate row.
|
||||
csv = (
|
||||
'"Slice","Name","Invested value","Value","Result","Owned quantity"\n'
|
||||
'"Total","Empty Pie",0,0,0,"-"\n'
|
||||
)
|
||||
with pytest.raises(CSVImportError, match="no parseable position"):
|
||||
parse_t212_csv(csv)
|
||||
|
||||
|
||||
def test_reordered_columns():
|
||||
"""T212 sometimes re-orders columns between exports. Header-name matching
|
||||
has to make that a non-issue."""
|
||||
csv = (
|
||||
'"Name","Owned quantity","Slice","Value","Invested value","Result"\n'
|
||||
'"Shell","10.5","SHEL","350","340","10"\n'
|
||||
'"Total","-","Total","350","340","10"\n'
|
||||
)
|
||||
pie = parse_t212_csv(csv)
|
||||
assert len(pie.positions) == 1
|
||||
p = pie.positions[0]
|
||||
assert p.slice == "SHEL"
|
||||
assert p.name == "Shell"
|
||||
assert p.quantity == pytest.approx(10.5)
|
||||
assert p.invested_value == pytest.approx(340)
|
||||
|
||||
|
||||
def test_unknown_columns_silently_ignored():
|
||||
csv = (
|
||||
'"Slice","Name","Owned quantity","Invested value",'
|
||||
'"Something T212 added later","Value"\n'
|
||||
'"SHEL","Shell","10","100","ignore me","105"\n'
|
||||
'"Total","-","-","100","-","105"\n'
|
||||
)
|
||||
pie = parse_t212_csv(csv)
|
||||
assert pie.positions[0].slice == "SHEL"
|
||||
assert pie.positions[0].current_value == pytest.approx(105)
|
||||
|
||||
|
||||
def test_handles_utf8_bom():
|
||||
"""Excel-saved CSVs often have a UTF-8 BOM. Should not break header
|
||||
matching for the first column."""
|
||||
csv = (
|
||||
""
|
||||
'"Slice","Name","Invested value","Value","Result","Owned quantity"\n'
|
||||
'"SHEL","Shell",100,105,5,"10"\n'
|
||||
'"Total","Test",100,105,5,"-"\n'
|
||||
).encode("utf-8")
|
||||
pie = parse_t212_csv(csv)
|
||||
assert pie.positions[0].slice == "SHEL"
|
||||
assert pie.name == "Test"
|
||||
|
||||
|
||||
def test_dash_and_na_become_none():
|
||||
csv = (
|
||||
'"Slice","Name","Invested value","Value","Result","Owned quantity",'
|
||||
'"Dividends gained","Dividends cash","Dividends reinvested"\n'
|
||||
'"SHEL","Shell",100,"-",,"10","N/A","—","n/a"\n'
|
||||
'"Total","P",100,100,0,"-","0","0","0"\n'
|
||||
)
|
||||
pie = parse_t212_csv(csv)
|
||||
p = pie.positions[0]
|
||||
assert p.current_value is None
|
||||
assert p.result is None
|
||||
assert p.dividends_gained is None
|
||||
assert p.dividends_cash is None
|
||||
assert p.dividends_reinvested is None
|
||||
|
||||
|
||||
def test_thousand_separator_commas_tolerated():
|
||||
csv = (
|
||||
'"Slice","Name","Invested value","Value","Result","Owned quantity"\n'
|
||||
'"BIG","Mega Holding","1,234,567.89","1,250,000",15432.11,"100"\n'
|
||||
'"Total","Big Pie","1,234,567.89","1,250,000",15432.11,"-"\n'
|
||||
)
|
||||
pie = parse_t212_csv(csv)
|
||||
assert pie.positions[0].invested_value == pytest.approx(1234567.89)
|
||||
assert pie.invested == pytest.approx(1234567.89)
|
||||
|
||||
|
||||
def test_blank_rows_skipped():
|
||||
csv = (
|
||||
'"Slice","Name","Invested value","Value","Result","Owned quantity"\n'
|
||||
'\n'
|
||||
'"SHEL","Shell",100,105,5,"10"\n'
|
||||
' \n'
|
||||
'"Total","P",100,105,5,"-"\n'
|
||||
)
|
||||
pie = parse_t212_csv(csv)
|
||||
assert len(pie.positions) == 1
|
||||
|
||||
|
||||
def test_zero_quantity_skipped():
|
||||
"""A position row with quantity=0 isn't a real holding — likely a stub
|
||||
left over from a fully-sold position. Don't fail; skip."""
|
||||
csv = (
|
||||
'"Slice","Name","Invested value","Value","Result","Owned quantity"\n'
|
||||
'"GONE","Sold Out",0,0,0,"0"\n'
|
||||
'"SHEL","Shell",100,105,5,"10"\n'
|
||||
'"Total","P",100,105,5,"-"\n'
|
||||
)
|
||||
pie = parse_t212_csv(csv)
|
||||
assert [p.slice for p in pie.positions] == ["SHEL"]
|
||||
|
||||
|
||||
def test_position_without_total_row_still_parses():
|
||||
csv = (
|
||||
'"Slice","Name","Invested value","Value","Result","Owned quantity"\n'
|
||||
'"SHEL","Shell",100,105,5,"10"\n'
|
||||
)
|
||||
pie = parse_t212_csv(csv)
|
||||
assert len(pie.positions) == 1
|
||||
assert pie.name is None
|
||||
assert pie.invested is None
|
||||
117
tests/test_instrument_map.py
Normal file
117
tests/test_instrument_map.py
Normal file
|
|
@ -0,0 +1,117 @@
|
|||
"""Pure-function tests for app.services.instrument_map.t212_ticker_to_yahoo.
|
||||
|
||||
Catalogue sync and DB-backed resolution are covered by integration tests
|
||||
against the live container, not here.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
pytest.importorskip("httpx")
|
||||
pytest.importorskip("sqlalchemy")
|
||||
|
||||
from app.services.instrument_map import t212_ticker_to_yahoo
|
||||
|
||||
|
||||
# --- Single-letter exchange suffix (the common LSE case) -------------------
|
||||
|
||||
|
||||
def test_lse_pence_listing():
|
||||
# iShares Physical Gold on LSE, quoted in GBX.
|
||||
assert t212_ticker_to_yahoo("SGLNl_EQ", "SGLN") == "SGLN.L"
|
||||
assert t212_ticker_to_yahoo("BAl_EQ", "BA") == "BA.L"
|
||||
assert t212_ticker_to_yahoo("BPl_EQ", "BP") == "BP.L"
|
||||
assert t212_ticker_to_yahoo("VEURl_EQ", "VEUR") == "VEUR.L"
|
||||
|
||||
|
||||
def test_amsterdam():
|
||||
# Shell dual-listed on Amsterdam.
|
||||
assert t212_ticker_to_yahoo("SHELLa_EQ", "SHELL") == "SHELL.AS"
|
||||
|
||||
|
||||
def test_paris():
|
||||
# TotalEnergies on Paris — T212 still uses the legacy FP shortcode in
|
||||
# the ticker, but the resolver uses whatever shortName T212 passes.
|
||||
assert t212_ticker_to_yahoo("FPp_EQ", "TTE") == "TTE.PA"
|
||||
assert t212_ticker_to_yahoo("FPp_EQ", "FP") == "FP.PA"
|
||||
|
||||
|
||||
def test_frankfurt():
|
||||
assert t212_ticker_to_yahoo("SAPd_EQ", "SAP") == "SAP.DE"
|
||||
|
||||
|
||||
def test_swiss():
|
||||
assert t212_ticker_to_yahoo("VFEMs_EQ", "VFEM") == "VFEM.SW"
|
||||
|
||||
|
||||
def test_milan():
|
||||
assert t212_ticker_to_yahoo("ENIm_EQ", "ENI") == "ENI.MI"
|
||||
|
||||
|
||||
# --- Country-code suffix (US + Canada + EU countries) ----------------------
|
||||
|
||||
|
||||
def test_us_listing_bare():
|
||||
# NYSE/NASDAQ — Yahoo uses the bare ticker.
|
||||
assert t212_ticker_to_yahoo("EQNR_US_EQ", "EQNR") == "EQNR"
|
||||
assert t212_ticker_to_yahoo("AAPL_US_EQ", "AAPL") == "AAPL"
|
||||
assert t212_ticker_to_yahoo("SHEL_US_EQ", "SHEL") == "SHEL"
|
||||
|
||||
|
||||
def test_country_code_with_uk():
|
||||
assert t212_ticker_to_yahoo("ABC_GB_EQ", "ABC") == "ABC.L"
|
||||
|
||||
|
||||
def test_country_code_with_germany():
|
||||
assert t212_ticker_to_yahoo("ALV_DE_EQ", "ALV") == "ALV.DE"
|
||||
|
||||
|
||||
def test_country_code_with_canada():
|
||||
assert t212_ticker_to_yahoo("RY_CA_EQ", "RY") == "RY.TO"
|
||||
|
||||
|
||||
# --- Edge cases ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unknown_letter_returns_none():
|
||||
assert t212_ticker_to_yahoo("FOOz_EQ", "FOO") is None
|
||||
|
||||
|
||||
def test_unknown_country_returns_none():
|
||||
assert t212_ticker_to_yahoo("FOO_ZZ_EQ", "FOO") is None
|
||||
|
||||
|
||||
def test_no_eq_suffix_returns_none():
|
||||
assert t212_ticker_to_yahoo("SGLN", "SGLN") is None
|
||||
assert t212_ticker_to_yahoo("SGLNl", "SGLN") is None
|
||||
|
||||
|
||||
def test_empty_strings():
|
||||
assert t212_ticker_to_yahoo("", "") is None
|
||||
assert t212_ticker_to_yahoo("_EQ", "") is None
|
||||
|
||||
|
||||
# --- Full user-portfolio mapping check -------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("t212_ticker,short,expected", [
|
||||
# Mappings derived from probing T212's instruments endpoint earlier
|
||||
# for the user's actual 13 holdings.
|
||||
("SGLNl_EQ", "SGLN", "SGLN.L"),
|
||||
("IGLSl_EQ", "IGLS", "IGLS.L"),
|
||||
("VEURl_EQ", "VEUR", "VEUR.L"),
|
||||
("HMCHl_EQ", "HMCH", "HMCH.L"),
|
||||
("INRGl_EQ", "INRG", "INRG.L"),
|
||||
("VFEMl_EQ", "VFEM", "VFEM.L"),
|
||||
("VJPNl_EQ", "VJPN", "VJPN.L"),
|
||||
("BAl_EQ", "BA", "BA.L"),
|
||||
("NATPl_EQ", "NATP", "NATP.L"),
|
||||
("BPl_EQ", "BP", "BP.L"),
|
||||
# Paris-listed TotalEnergies
|
||||
("FPp_EQ", "TTE", "TTE.PA"),
|
||||
# US-listed Equinor + Shell
|
||||
("EQNR_US_EQ", "EQNR", "EQNR"),
|
||||
("SHEL_US_EQ", "SHEL", "SHEL"),
|
||||
])
|
||||
def test_user_portfolio_mappings(t212_ticker, short, expected):
|
||||
assert t212_ticker_to_yahoo(t212_ticker, short) == expected
|
||||
Loading…
Add table
Add a link
Reference in a new issue