read.markets/app/services/instrument_map.py

"""T212 shortcode → Yahoo Finance ticker resolution.

The CSV path gives us only T212's `Slice` (short name like "SGLN", "TTE").
To refresh prices via Yahoo we need the proper Yahoo symbol (`SGLN.L`,
`TTE.PA`, …). The mapping comes from T212's own catalogue
(/equity/metadata/instruments), synced into the `instrument_map` table
via the admin's read-only API key. The resolver then picks the right
listing per user using currency preference.

This module has three responsibilities:
1. **Pure translation** — turn a T212 ticker like `SGLNl_EQ` into a Yahoo
   symbol like `SGLN.L` from suffix rules. No DB, no HTTP.
2. **Catalogue sync** — pull every T212 instrument and upsert into
   `instrument_map`. Hand-edited rows (`manual=True`) are never overwritten.
3. **Slice resolution** — given a CSV `Slice` like "SHEL", find the best
   matching `instrument_map` row using configurable currency preference.
"""
from __future__ import annotations

from dataclasses import dataclass
from datetime import timedelta

import httpx
from sqlalchemy import select
from sqlalchemy.dialects.mysql import insert as mysql_insert
from sqlalchemy.ext.asyncio import AsyncSession

from app.db import utcnow
from app.models import InstrumentMap
from app.services.trading212 import Trading212


# --- Pure translation: T212 ticker → Yahoo symbol ---------------------------


# Single trailing letter before "_EQ" → exchange suffix.
# These conventions come from observing T212's instrument catalogue.
_T212_LETTER_TO_YAHOO = {
    "l": ".L",      # London (LSE)
    "a": ".AS",     # Amsterdam (Euronext)
    "p": ".PA",     # Paris (Euronext)
    "d": ".DE",     # Frankfurt (Xetra)
    "m": ".MI",     # Milan (Borsa Italiana)
    "s": ".SW",     # Swiss (SIX)
    "b": ".BR",     # Brussels (Euronext)
    "i": ".IR",     # Ireland (Euronext Dublin)
    "h": ".HE",     # Helsinki (Nasdaq Nordic)
    "c": ".CO",     # Copenhagen
    "o": ".OL",     # Oslo
}

# Country-code suffix (e.g. _US_EQ, _CA_EQ) → Yahoo suffix.
_T212_COUNTRY_TO_YAHOO = {
    "US": "",       # NYSE/NASDAQ — Yahoo bare ticker
    "CA": ".TO",    # Toronto
    "DE": ".DE",
    "FR": ".PA",
    "GB": ".L",
    "IT": ".MI",
    "ES": ".MC",
    "NL": ".AS",
    "BE": ".BR",
    "IE": ".IR",
    "FI": ".HE",
    "CH": ".SW",
    "NO": ".OL",
    "DK": ".CO",
    "SE": ".ST",
}


def t212_ticker_to_yahoo(t212_ticker: str, short_name: str) -> str | None:
    """Translate a T212 ticker like 'SGLNl_EQ' or 'EQNR_US_EQ' to its
    Yahoo Finance symbol. Returns None when the pattern isn't recognised.

    Rules, in order:
      'XXXX_<CC>_EQ' (country code) → short_name + country suffix
                                       e.g. EQNR_US_EQ → EQNR
                                            SHEL_US_EQ → SHEL
      'XXXX<x>_EQ' (single trailing lowercase letter) → short_name + suffix
                                       e.g. SGLNl_EQ → SGLN.L
                                            SHELLa_EQ → SHELL.AS
                                            FPp_EQ → FP.PA
    """
    if not t212_ticker.endswith("_EQ"):
        return None
    body = t212_ticker[:-3]  # strip "_EQ"

    # Country-code form: '..._XX' where XX is a 2-letter country code
    if len(body) >= 3 and body[-3] == "_" and body[-2:].isupper():
        cc = body[-2:]
        suffix = _T212_COUNTRY_TO_YAHOO.get(cc)
        if suffix is None:
            return None
        return f"{short_name}{suffix}"

    # Single-letter form: '...x' where x is a recognised exchange letter
    if body and body[-1] in _T212_LETTER_TO_YAHOO:
        return f"{short_name}{_T212_LETTER_TO_YAHOO[body[-1]]}"

    return None


# --- Catalogue sync ---------------------------------------------------------


@dataclass
class SyncResult:
    fetched: int
    upserted: int
    unmappable: int
    skipped_manual: int


async def sync_from_t212(
    session: AsyncSession,
    client: httpx.AsyncClient,
    t212: Trading212 | None = None,
) -> SyncResult:
    """Pull every T212 instrument and upsert into instrument_map. Hand-
    edited rows (manual=True) are never overwritten. Runs idempotently."""
    t212 = t212 or Trading212()
    instruments = await t212.instruments(client) or []

    # Pre-fetch existing manual mappings so we know which t212_tickers
    # to skip on upsert.
    manual_tickers = {
        r.t212_ticker for r in (await session.execute(
            select(InstrumentMap).where(InstrumentMap.manual == True)
        )).scalars().all()
    }

    now = utcnow()
    rows = []
    unmappable = 0
    skipped_manual = 0
    for inst in instruments:
        tkr = inst.get("ticker")
        sn = inst.get("shortName") or ""
        name = inst.get("name") or sn or tkr
        if not tkr:
            continue
        if tkr in manual_tickers:
            skipped_manual += 1
            continue
        yahoo = t212_ticker_to_yahoo(tkr, sn)
        if yahoo is None:
            unmappable += 1
        rows.append({
            "t212_ticker": tkr,
            "t212_shortname": sn,
            "yahoo_ticker": yahoo,
            "name": (name or sn)[:128],
            "currency": inst.get("currencyCode"),
            "isin": inst.get("isin"),
            "instrument_type": inst.get("type"),
            "manual": False,
            "last_verified_at": now,
        })

    if not rows:
        return SyncResult(fetched=len(instruments), upserted=0,
                          unmappable=unmappable, skipped_manual=skipped_manual)

    # Bulk upsert. MySQL: ON DUPLICATE KEY UPDATE on t212_ticker unique key.
    # Chunk to avoid hitting MySQL's max_allowed_packet on 17k+ rows.
    chunk = 500
    upserted = 0
    for i in range(0, len(rows), chunk):
        stmt = mysql_insert(InstrumentMap).values(rows[i:i + chunk])
        stmt = stmt.on_duplicate_key_update(
            yahoo_ticker=stmt.inserted.yahoo_ticker,
            t212_shortname=stmt.inserted.t212_shortname,
            name=stmt.inserted.name,
            currency=stmt.inserted.currency,
            isin=stmt.inserted.isin,
            instrument_type=stmt.inserted.instrument_type,
            last_verified_at=stmt.inserted.last_verified_at,
        )
        await session.execute(stmt)
        upserted += len(rows[i:i + chunk])

    await session.commit()
    return SyncResult(
        fetched=len(instruments), upserted=upserted,
        unmappable=unmappable, skipped_manual=skipped_manual,
    )


# --- Resolution: CSV Slice → preferred Yahoo ticker -------------------------


# Currency preference for users in a UK account. Listings denominated in the
# user's account currency or its smaller-unit (GBX = pence) come first. EUR
# ranks ABOVE USD because UK retail brokers (incl. T212) typically default
# users to the London + Euronext listings; the NYSE dual-listing only wins
# when no European listing exists (e.g. EQNR isn't on T212's Oslo book).
_DEFAULT_CCY_PREFERENCE = ("GBX", "GBP", "EUR", "USD", "CHF", "JPY")


@dataclass
class ResolvedInstrument:
    t212_ticker: str
    yahoo_ticker: str | None
    name: str
    currency: str | None
    isin: str | None


async def resolve_slice(
    session: AsyncSession,
    slice_code: str,
    currency_preference: tuple[str, ...] = _DEFAULT_CCY_PREFERENCE,
) -> ResolvedInstrument | None:
    """Find the best Yahoo ticker for a given CSV Slice.

    Picks the listing whose currency comes first in `currency_preference`.
    Manual mappings always win over auto-resolved ones."""
    if not slice_code:
        return None

    rows = (await session.execute(
        select(InstrumentMap)
        .where(InstrumentMap.t212_shortname == slice_code)
    )).scalars().all()
    if not rows:
        return None

    def rank(row: InstrumentMap) -> tuple[int, int]:
        manual_rank = 0 if row.manual else 1
        try:
            ccy_rank = currency_preference.index(row.currency or "")
        except ValueError:
            ccy_rank = len(currency_preference)
        return (manual_rank, ccy_rank)

    rows.sort(key=rank)
    chosen = rows[0]
    return ResolvedInstrument(
        t212_ticker=chosen.t212_ticker,
        yahoo_ticker=chosen.yahoo_ticker,
        name=chosen.name,
        currency=chosen.currency,
        isin=chosen.isin,
    )


def is_stale(row: InstrumentMap, max_age: timedelta = timedelta(days=14)) -> bool:
    """True if the row hasn't been refreshed from T212 recently."""
    age = utcnow() - row.last_verified_at
    return age > max_age