add Eurostat + UK ONS sources; valuation/bubble/economy/bonds groups; aggregate read; market-open header
Three new data sources hooked into the existing SOURCES registry. All
open APIs, no keys:
- EUROSTAT: prefix EUROSTAT:dataset?dim=val&... — current EU bond
yields (Bund/OAT/BTP/EZ) and Eurozone economic indicators that
FRED's OECD-mirror series stopped updating in 2022-2023.
- ONS: prefix ONS:topic/cdid/dataset — current UK CPI, unemployment,
GDP, industrial production. Replaces the 5+ month-stale FRED
LRHUTTTTGBM156S mirror.
New indicator groups in default.toml feed the strategic/fundamental
lens we converged on: valuation (CAPE/Buffett anchors), bubble_watch
(SKEW/VVIX/RSP vs SPY/HYG vs TLT/IPO/crypto), economy (multi-region,
ALL current-or-stale-flagged), bonds (UK/EU/US/JPN sovereign yields).
Indicator panel now opens with an AI "read" interpretation per group
(generated hourly at :07 UTC alongside an aggregate cross-group read
shown in the dashboard header). The aggregate is grounded by a markets
strip — NYSE/LSE/Frankfurt/Tokyo/HK/Shanghai with open/closed LEDs and
next-open countdown, computed locally from each exchange's tz.
Other UX bits: indicator-row tooltips populated from TOML notes;
rows whose last observation is >90 days old get a 'stale' chip;
ghost symbols (in DB but no longer in TOML) filtered out of the
panel; Eurostat/ONS symbols display as short codes rather than the
full API path.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
a10409c02b
commit
1edf9cad41
15 changed files with 1156 additions and 10 deletions
|
|
@ -16,6 +16,8 @@ from app.config import get_settings
|
|||
|
||||
YAHOO_CHART = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
|
||||
FRED_API = "https://api.stlouisfed.org/fred/series/observations"
|
||||
EUROSTAT_API = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset}"
|
||||
ONS_API = "https://www.ons.gov.uk/{topic}/timeseries/{cdid}/{dataset}/data"
|
||||
UA = {"User-Agent": "Mozilla/5.0 (cassandra) Python/httpx"}
|
||||
|
||||
|
||||
|
|
@ -212,10 +214,225 @@ async def fetch_fred(
|
|||
return Quote(symbol, "fred", label, note, None, None, None, error=str(e))
|
||||
|
||||
|
||||
# --- Eurostat (no API key needed) -------------------------------------------
|
||||
|
||||
|
||||
def _eurostat_time_to_iso(t: str) -> str:
|
||||
"""Convert Eurostat time codes into ISO-style dates so they sort and
|
||||
compare correctly. Accepts YYYY-MM, YYYY-Qn, YYYY, and YYYY-MM-DD."""
|
||||
t = t.strip()
|
||||
if len(t) == 4 and t.isdigit(): # annual: "2026"
|
||||
return f"{t}-01-01"
|
||||
if len(t) == 6 and t[4] == "Q": # quarterly: "2026Q1"
|
||||
q = int(t[5])
|
||||
return f"{t[:4]}-{(q - 1) * 3 + 1:02d}-01"
|
||||
if len(t) == 7 and t[4] == "-": # monthly: "2026-03"
|
||||
return f"{t}-01"
|
||||
if len(t) == 10: # daily: "2026-03-15"
|
||||
return t
|
||||
return t # fall through; caller may flag
|
||||
|
||||
|
||||
async def fetch_eurostat(
|
||||
client: httpx.AsyncClient,
|
||||
symbol: str,
|
||||
label: str,
|
||||
note: str,
|
||||
anchor: str | None = None,
|
||||
) -> Quote:
|
||||
"""Fetch a Eurostat time series. `symbol` format:
|
||||
DATASET?dim1=val1&dim2=val2
|
||||
e.g. 'irt_lt_mcby_m?geo=DE&int_rt=MCBY' for German 10y bond yield.
|
||||
Eurostat's API is open (no key), uses JSON-stat 2.0."""
|
||||
import urllib.parse
|
||||
|
||||
try:
|
||||
if "?" in symbol:
|
||||
dataset, query = symbol.split("?", 1)
|
||||
params = dict(urllib.parse.parse_qsl(query))
|
||||
else:
|
||||
dataset, params = symbol, {}
|
||||
params.setdefault("format", "JSON")
|
||||
params.setdefault("lang", "EN")
|
||||
|
||||
r = await client.get(
|
||||
EUROSTAT_API.format(dataset=dataset),
|
||||
params=params, headers=UA, timeout=20,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
|
||||
time_cat = data["dimension"]["time"]["category"]
|
||||
# JSON-stat 2.0: {"index": {timecode: pos}, "label": {timecode: human}}
|
||||
time_index = time_cat["index"]
|
||||
values = data.get("value") or {}
|
||||
|
||||
# Build (iso_date, value) pairs, sorted ascending in time.
|
||||
rows: list[tuple[str, float]] = []
|
||||
for tcode, pos in sorted(time_index.items(), key=lambda kv: kv[1]):
|
||||
raw = values.get(str(pos))
|
||||
if raw is None:
|
||||
continue
|
||||
try:
|
||||
rows.append((_eurostat_time_to_iso(tcode), float(raw)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
if not rows:
|
||||
raise ValueError("no observations")
|
||||
|
||||
last_date, last_val = rows[-1]
|
||||
|
||||
def _find_back(min_days: int) -> float | None:
|
||||
ref = datetime.strptime(last_date, "%Y-%m-%d").date()
|
||||
for d, v in reversed(rows[:-1]):
|
||||
if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days:
|
||||
return v
|
||||
return None
|
||||
|
||||
prev_val = rows[-2][1] if len(rows) >= 2 else None
|
||||
changes = {
|
||||
"1d": _pct(prev_val, last_val),
|
||||
"1m": _pct(_find_back(28), last_val),
|
||||
"1y": _pct(_find_back(360), last_val),
|
||||
}
|
||||
anchor_used: str | None = None
|
||||
if anchor:
|
||||
anchor_d = _parse_date(anchor).date()
|
||||
for d, v in reversed(rows):
|
||||
if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d:
|
||||
changes["anchor"] = _pct(v, last_val)
|
||||
anchor_used = d
|
||||
break
|
||||
|
||||
return Quote(
|
||||
symbol=symbol, source="eurostat", label=label, note=note,
|
||||
price=last_val, currency=None, as_of=last_date,
|
||||
changes=changes, anchor_date=anchor_used,
|
||||
)
|
||||
except Exception as e:
|
||||
return Quote(symbol, "eurostat", label, note, None, None, None, error=str(e))
|
||||
|
||||
|
||||
# --- UK ONS (Office for National Statistics, no API key needed) -------------
|
||||
|
||||
|
||||
_ONS_MONTH = {
|
||||
"JAN": 1, "FEB": 2, "MAR": 3, "APR": 4, "MAY": 5, "JUN": 6,
|
||||
"JUL": 7, "AUG": 8, "SEP": 9, "OCT": 10, "NOV": 11, "DEC": 12,
|
||||
}
|
||||
|
||||
|
||||
def _ons_date_to_iso(s: str) -> str | None:
|
||||
"""ONS date formats: monthly '2026 MAR', quarterly '2026 Q1', annual '2025'."""
|
||||
s = s.strip().upper()
|
||||
parts = s.split()
|
||||
try:
|
||||
if len(parts) == 1 and parts[0].isdigit():
|
||||
return f"{parts[0]}-01-01"
|
||||
if len(parts) == 2:
|
||||
year = int(parts[0])
|
||||
tag = parts[1]
|
||||
if tag in _ONS_MONTH:
|
||||
return f"{year:04d}-{_ONS_MONTH[tag]:02d}-01"
|
||||
if tag.startswith("Q") and tag[1:].isdigit():
|
||||
q = int(tag[1:])
|
||||
return f"{year:04d}-{(q - 1) * 3 + 1:02d}-01"
|
||||
except (ValueError, IndexError):
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
async def fetch_ons(
|
||||
client: httpx.AsyncClient,
|
||||
symbol: str,
|
||||
label: str,
|
||||
note: str,
|
||||
anchor: str | None = None,
|
||||
) -> Quote:
|
||||
"""Fetch a UK ONS time series. `symbol` format:
|
||||
<topic_path>/<cdid>/<dataset>
|
||||
e.g. 'economy/inflationandpriceindices/d7g7/mm23' for UK CPI YoY.
|
||||
ONS publishes via www.ons.gov.uk; no auth, JSON when Accept header set."""
|
||||
try:
|
||||
parts = symbol.split("/")
|
||||
if len(parts) < 3:
|
||||
raise ValueError("ONS symbol must be topic/cdid/dataset")
|
||||
dataset = parts[-1]
|
||||
cdid = parts[-2]
|
||||
topic = "/".join(parts[:-2])
|
||||
|
||||
r = await client.get(
|
||||
ONS_API.format(topic=topic, cdid=cdid, dataset=dataset),
|
||||
headers={**UA, "Accept": "application/json"},
|
||||
timeout=20,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
|
||||
# Use the most granular series available: months > quarters > years.
|
||||
for key in ("months", "quarters", "years"):
|
||||
raw_seq = data.get(key) or []
|
||||
if raw_seq:
|
||||
break
|
||||
if not raw_seq:
|
||||
raise ValueError("no observations")
|
||||
|
||||
rows: list[tuple[str, float]] = []
|
||||
for entry in raw_seq:
|
||||
iso = _ons_date_to_iso(entry.get("date", ""))
|
||||
v = entry.get("value")
|
||||
if iso is None or v in (None, "", "."):
|
||||
continue
|
||||
try:
|
||||
rows.append((iso, float(v)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if not rows:
|
||||
raise ValueError("no parseable observations")
|
||||
|
||||
last_date, last_val = rows[-1]
|
||||
|
||||
def _find_back(min_days: int) -> float | None:
|
||||
ref = datetime.strptime(last_date, "%Y-%m-%d").date()
|
||||
for d, v in reversed(rows[:-1]):
|
||||
if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days:
|
||||
return v
|
||||
return None
|
||||
|
||||
prev_val = rows[-2][1] if len(rows) >= 2 else None
|
||||
changes = {
|
||||
"1d": _pct(prev_val, last_val),
|
||||
"1m": _pct(_find_back(28), last_val),
|
||||
"1y": _pct(_find_back(360), last_val),
|
||||
}
|
||||
anchor_used: str | None = None
|
||||
if anchor:
|
||||
anchor_d = _parse_date(anchor).date()
|
||||
for d, v in reversed(rows):
|
||||
if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d:
|
||||
changes["anchor"] = _pct(v, last_val)
|
||||
anchor_used = d
|
||||
break
|
||||
|
||||
return Quote(
|
||||
symbol=symbol, source="ons", label=label, note=note,
|
||||
price=last_val, currency=None, as_of=last_date,
|
||||
changes=changes, anchor_date=anchor_used,
|
||||
)
|
||||
except Exception as e:
|
||||
return Quote(symbol, "ons", label, note, None, None, None, error=str(e))
|
||||
|
||||
|
||||
# --- Source registry ----------------------------------------------------------
|
||||
|
||||
FetcherFn = Callable[..., "Quote"]
|
||||
SOURCES: dict[str, FetcherFn] = {"yahoo": fetch_yahoo, "FRED": fetch_fred}
|
||||
SOURCES: dict[str, FetcherFn] = {
|
||||
"yahoo": fetch_yahoo,
|
||||
"FRED": fetch_fred,
|
||||
"EUROSTAT": fetch_eurostat,
|
||||
"ONS": fetch_ons,
|
||||
}
|
||||
|
||||
|
||||
def parse_symbol(symbol: str) -> tuple[FetcherFn, str]:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue