add Eurostat + UK ONS sources; valuation/bubble/economy/bonds groups; aggregate read; market-open header

Three new data sources hooked into the existing SOURCES registry. All
open APIs, no keys:

  - EUROSTAT: prefix EUROSTAT:dataset?dim=val&... — current EU bond
    yields (Bund/OAT/BTP/EZ) and Eurozone economic indicators that
    FRED's OECD-mirror series stopped updating in 2022-2023.
  - ONS: prefix ONS:topic/cdid/dataset — current UK CPI, unemployment,
    GDP, industrial production. Replaces the 5+ month-stale FRED
    LRHUTTTTGBM156S mirror.

New indicator groups in default.toml feed the strategic/fundamental
lens we converged on: valuation (CAPE/Buffett anchors), bubble_watch
(SKEW/VVIX/RSP vs SPY/HYG vs TLT/IPO/crypto), economy (multi-region,
ALL current-or-stale-flagged), bonds (UK/EU/US/JPN sovereign yields).

Indicator panel now opens with an AI "read" interpretation per group
(generated hourly at :07 UTC alongside an aggregate cross-group read
shown in the dashboard header). The aggregate is grounded by a markets
strip — NYSE/LSE/Frankfurt/Tokyo/HK/Shanghai with open/closed LEDs and
next-open countdown, computed locally from each exchange's tz.

Other UX bits: indicator-row tooltips populated from TOML notes;
rows whose last observation is >90 days old get a 'stale' chip;
ghost symbols (in DB but no longer in TOML) filtered out of the
panel; Eurostat/ONS symbols display as short codes rather than the
full API path.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Giorgio Gilestro 2026-05-15 23:07:42 +01:00
parent a10409c02b
commit 1edf9cad41
15 changed files with 1156 additions and 10 deletions

View file

@ -16,6 +16,8 @@ from app.config import get_settings
YAHOO_CHART = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
FRED_API = "https://api.stlouisfed.org/fred/series/observations"
EUROSTAT_API = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset}"
ONS_API = "https://www.ons.gov.uk/{topic}/timeseries/{cdid}/{dataset}/data"
UA = {"User-Agent": "Mozilla/5.0 (cassandra) Python/httpx"}
@ -212,10 +214,225 @@ async def fetch_fred(
return Quote(symbol, "fred", label, note, None, None, None, error=str(e))
# --- Eurostat (no API key needed) -------------------------------------------
def _eurostat_time_to_iso(t: str) -> str:
"""Convert Eurostat time codes into ISO-style dates so they sort and
compare correctly. Accepts YYYY-MM, YYYY-Qn, YYYY, and YYYY-MM-DD."""
t = t.strip()
if len(t) == 4 and t.isdigit(): # annual: "2026"
return f"{t}-01-01"
if len(t) == 6 and t[4] == "Q": # quarterly: "2026Q1"
q = int(t[5])
return f"{t[:4]}-{(q - 1) * 3 + 1:02d}-01"
if len(t) == 7 and t[4] == "-": # monthly: "2026-03"
return f"{t}-01"
if len(t) == 10: # daily: "2026-03-15"
return t
return t # fall through; caller may flag
async def fetch_eurostat(
client: httpx.AsyncClient,
symbol: str,
label: str,
note: str,
anchor: str | None = None,
) -> Quote:
"""Fetch a Eurostat time series. `symbol` format:
DATASET?dim1=val1&dim2=val2
e.g. 'irt_lt_mcby_m?geo=DE&int_rt=MCBY' for German 10y bond yield.
Eurostat's API is open (no key), uses JSON-stat 2.0."""
import urllib.parse
try:
if "?" in symbol:
dataset, query = symbol.split("?", 1)
params = dict(urllib.parse.parse_qsl(query))
else:
dataset, params = symbol, {}
params.setdefault("format", "JSON")
params.setdefault("lang", "EN")
r = await client.get(
EUROSTAT_API.format(dataset=dataset),
params=params, headers=UA, timeout=20,
)
r.raise_for_status()
data = r.json()
time_cat = data["dimension"]["time"]["category"]
# JSON-stat 2.0: {"index": {timecode: pos}, "label": {timecode: human}}
time_index = time_cat["index"]
values = data.get("value") or {}
# Build (iso_date, value) pairs, sorted ascending in time.
rows: list[tuple[str, float]] = []
for tcode, pos in sorted(time_index.items(), key=lambda kv: kv[1]):
raw = values.get(str(pos))
if raw is None:
continue
try:
rows.append((_eurostat_time_to_iso(tcode), float(raw)))
except (TypeError, ValueError):
continue
if not rows:
raise ValueError("no observations")
last_date, last_val = rows[-1]
def _find_back(min_days: int) -> float | None:
ref = datetime.strptime(last_date, "%Y-%m-%d").date()
for d, v in reversed(rows[:-1]):
if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days:
return v
return None
prev_val = rows[-2][1] if len(rows) >= 2 else None
changes = {
"1d": _pct(prev_val, last_val),
"1m": _pct(_find_back(28), last_val),
"1y": _pct(_find_back(360), last_val),
}
anchor_used: str | None = None
if anchor:
anchor_d = _parse_date(anchor).date()
for d, v in reversed(rows):
if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d:
changes["anchor"] = _pct(v, last_val)
anchor_used = d
break
return Quote(
symbol=symbol, source="eurostat", label=label, note=note,
price=last_val, currency=None, as_of=last_date,
changes=changes, anchor_date=anchor_used,
)
except Exception as e:
return Quote(symbol, "eurostat", label, note, None, None, None, error=str(e))
# --- UK ONS (Office for National Statistics, no API key needed) -------------
_ONS_MONTH = {
"JAN": 1, "FEB": 2, "MAR": 3, "APR": 4, "MAY": 5, "JUN": 6,
"JUL": 7, "AUG": 8, "SEP": 9, "OCT": 10, "NOV": 11, "DEC": 12,
}
def _ons_date_to_iso(s: str) -> str | None:
"""ONS date formats: monthly '2026 MAR', quarterly '2026 Q1', annual '2025'."""
s = s.strip().upper()
parts = s.split()
try:
if len(parts) == 1 and parts[0].isdigit():
return f"{parts[0]}-01-01"
if len(parts) == 2:
year = int(parts[0])
tag = parts[1]
if tag in _ONS_MONTH:
return f"{year:04d}-{_ONS_MONTH[tag]:02d}-01"
if tag.startswith("Q") and tag[1:].isdigit():
q = int(tag[1:])
return f"{year:04d}-{(q - 1) * 3 + 1:02d}-01"
except (ValueError, IndexError):
pass
return None
async def fetch_ons(
client: httpx.AsyncClient,
symbol: str,
label: str,
note: str,
anchor: str | None = None,
) -> Quote:
"""Fetch a UK ONS time series. `symbol` format:
<topic_path>/<cdid>/<dataset>
e.g. 'economy/inflationandpriceindices/d7g7/mm23' for UK CPI YoY.
ONS publishes via www.ons.gov.uk; no auth, JSON when Accept header set."""
try:
parts = symbol.split("/")
if len(parts) < 3:
raise ValueError("ONS symbol must be topic/cdid/dataset")
dataset = parts[-1]
cdid = parts[-2]
topic = "/".join(parts[:-2])
r = await client.get(
ONS_API.format(topic=topic, cdid=cdid, dataset=dataset),
headers={**UA, "Accept": "application/json"},
timeout=20,
)
r.raise_for_status()
data = r.json()
# Use the most granular series available: months > quarters > years.
for key in ("months", "quarters", "years"):
raw_seq = data.get(key) or []
if raw_seq:
break
if not raw_seq:
raise ValueError("no observations")
rows: list[tuple[str, float]] = []
for entry in raw_seq:
iso = _ons_date_to_iso(entry.get("date", ""))
v = entry.get("value")
if iso is None or v in (None, "", "."):
continue
try:
rows.append((iso, float(v)))
except (TypeError, ValueError):
continue
if not rows:
raise ValueError("no parseable observations")
last_date, last_val = rows[-1]
def _find_back(min_days: int) -> float | None:
ref = datetime.strptime(last_date, "%Y-%m-%d").date()
for d, v in reversed(rows[:-1]):
if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days:
return v
return None
prev_val = rows[-2][1] if len(rows) >= 2 else None
changes = {
"1d": _pct(prev_val, last_val),
"1m": _pct(_find_back(28), last_val),
"1y": _pct(_find_back(360), last_val),
}
anchor_used: str | None = None
if anchor:
anchor_d = _parse_date(anchor).date()
for d, v in reversed(rows):
if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d:
changes["anchor"] = _pct(v, last_val)
anchor_used = d
break
return Quote(
symbol=symbol, source="ons", label=label, note=note,
price=last_val, currency=None, as_of=last_date,
changes=changes, anchor_date=anchor_used,
)
except Exception as e:
return Quote(symbol, "ons", label, note, None, None, None, error=str(e))
# --- Source registry ----------------------------------------------------------
FetcherFn = Callable[..., "Quote"]
SOURCES: dict[str, FetcherFn] = {"yahoo": fetch_yahoo, "FRED": fetch_fred}
SOURCES: dict[str, FetcherFn] = {
"yahoo": fetch_yahoo,
"FRED": fetch_fred,
"EUROSTAT": fetch_eurostat,
"ONS": fetch_ons,
}
def parse_symbol(symbol: str) -> tuple[FetcherFn, str]: