From 4e7e4981e3788662adf0a212b2983b60cc7a447a Mon Sep 17 00:00:00 2001 From: Giorgio Gilestro Date: Fri, 15 May 2026 23:13:58 +0100 Subject: [PATCH] add ECB Data Portal source; group-aware stale thresholds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ECB Statistical Data Warehouse joins as a 5th data source — open API, no key, daily euro-area yield curve data. Symbol format 'ECB:dataset/series_key', e.g. 'ECB:YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_10Y' for daily 10y AAA spot rate. Bonds tab adds ECB EZ 10y AAA + 2y AAA so there's at least some currently-fresh European sovereign data alongside the US Treasuries. Country-specific yields (Bund/OAT/BTP/Gilt/JGB) remain on Eurostat/FRED monthly mirrors — no free daily source exists for those. Stale threshold is now per-group instead of a flat 90 days. Daily-tape groups (bonds, rates, equity, etc.) flag stale after a week or three; monthly groups (economy, macro, valuation) stay at 60-90 days. The bonds tab will now correctly show 30-60 day-old country yields as stale next to the daily US/ECB ones. Co-Authored-By: Claude Opus 4.7 (1M context) --- app/routers/api.py | 28 +++++++++++-- app/services/market.py | 89 ++++++++++++++++++++++++++++++++++++++++++ config/default.toml | 7 +++- 3 files changed, 118 insertions(+), 6 deletions(-) diff --git a/app/routers/api.py b/app/routers/api.py index 90b80f4..f41ea56 100644 --- a/app/routers/api.py +++ b/app/routers/api.py @@ -55,6 +55,25 @@ router = APIRouter(dependencies=[Depends(require_token)]) JOB_NAMES = ("market_job", "news_job", "portfolio_job", "ai_log_job", "rollup_job") JOB_STALE_HOURS = 2.0 # job is "warn" if its last success was >2h ago +# Per-group expected freshness — bonds and intraday tape want daily data, +# macro/economy/valuation are monthly/quarterly by nature. Older than this +# many days from today → row gets a "stale" badge. +_STALE_DAYS_BY_GROUP = { + "bonds": 21, + "rates": 7, + "equity": 7, + "mag7": 7, + "commodities": 7, + "fx": 7, + "tech_ai": 7, + "financials": 7, + "bubble_watch":21, + "valuation": 60, + "macro": 60, + "economy": 90, +} +_STALE_DAYS_DEFAULT = 90 + # --- Small helpers ----------------------------------------------------------- @@ -158,17 +177,18 @@ async def indicators( .limit(1) )).scalar_one_or_none() - # Mark rows whose `as_of` is older than 90 days as stale so the UI - # can dim them — some FRED international series are months/years - # behind their primary source. + # Mark rows whose `as_of` is older than the group-specific threshold. + # Daily-tape groups (bonds, rates, equity, ...) flag stale earlier + # than monthly groups (economy, macro, valuation). today = utcnow().date() + threshold = _STALE_DAYS_BY_GROUP.get(group, _STALE_DAYS_DEFAULT) stale_symbols: set[str] = set() for r in rows: try: as_of_d = datetime.strptime(r.as_of, "%Y-%m-%d").date() if r.as_of else None except ValueError: as_of_d = None - if as_of_d and (today - as_of_d).days > 90: + if as_of_d and (today - as_of_d).days > threshold: stale_symbols.add(r.symbol) return templates.TemplateResponse( diff --git a/app/services/market.py b/app/services/market.py index 5f3ad28..84a4e50 100644 --- a/app/services/market.py +++ b/app/services/market.py @@ -18,6 +18,7 @@ YAHOO_CHART = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}" FRED_API = "https://api.stlouisfed.org/fred/series/observations" EUROSTAT_API = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset}" ONS_API = "https://www.ons.gov.uk/{topic}/timeseries/{cdid}/{dataset}/data" +ECB_API = "https://data-api.ecb.europa.eu/service/data/{dataset}/{series_key}" UA = {"User-Agent": "Mozilla/5.0 (cassandra) Python/httpx"} @@ -424,6 +425,93 @@ async def fetch_ons( return Quote(symbol, "ons", label, note, None, None, None, error=str(e)) +# --- ECB Data Portal (no API key, daily euro-area data) --------------------- + + +async def fetch_ecb( + client: httpx.AsyncClient, + symbol: str, + label: str, + note: str, + anchor: str | None = None, +) -> Quote: + """Fetch an ECB Statistical Data Warehouse series. `symbol` format: + / + e.g. 'YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_10Y' for 10y EZ AAA yield (daily). + ECB returns SDMX-JSON; we extract the single observation series.""" + try: + if "/" not in symbol: + raise ValueError("ECB symbol must be dataset/series_key") + dataset, series_key = symbol.split("/", 1) + r = await client.get( + ECB_API.format(dataset=dataset, series_key=series_key), + params={"format": "jsondata"}, + headers={**UA, "Accept": "application/json"}, + timeout=20, + ) + r.raise_for_status() + data = r.json() + + # SDMX-JSON: dataSets[0].series[key].observations is { "": [value, ...] } + # where indexes into structure.dimensions.observation[0].values for time. + ds = (data.get("dataSets") or [{}])[0] + series_map = ds.get("series") or {} + if not series_map: + raise ValueError("no series in response") + ser = next(iter(series_map.values())) + obs = ser.get("observations") or {} + times = [ + v["id"] + for v in data["structure"]["dimensions"]["observation"][0]["values"] + ] + + rows: list[tuple[str, float]] = [] + for idx_str in sorted(obs, key=int): + v = obs[idx_str][0] + if v is None: + continue + t = times[int(idx_str)] + iso = _eurostat_time_to_iso(t) # reuse: ECB time codes match Eurostat + try: + rows.append((iso, float(v))) + except (TypeError, ValueError): + continue + + if not rows: + raise ValueError("no observations") + last_date, last_val = rows[-1] + + def _find_back(min_days: int) -> float | None: + ref = datetime.strptime(last_date, "%Y-%m-%d").date() + for d, v in reversed(rows[:-1]): + if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days: + return v + return None + + prev_val = rows[-2][1] if len(rows) >= 2 else None + changes = { + "1d": _pct(prev_val, last_val), + "1m": _pct(_find_back(28), last_val), + "1y": _pct(_find_back(360), last_val), + } + anchor_used: str | None = None + if anchor: + anchor_d = _parse_date(anchor).date() + for d, v in reversed(rows): + if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d: + changes["anchor"] = _pct(v, last_val) + anchor_used = d + break + + return Quote( + symbol=symbol, source="ecb", label=label, note=note, + price=last_val, currency=None, as_of=last_date, + changes=changes, anchor_date=anchor_used, + ) + except Exception as e: + return Quote(symbol, "ecb", label, note, None, None, None, error=str(e)) + + # --- Source registry ---------------------------------------------------------- FetcherFn = Callable[..., "Quote"] @@ -432,6 +520,7 @@ SOURCES: dict[str, FetcherFn] = { "FRED": fetch_fred, "EUROSTAT": fetch_eurostat, "ONS": fetch_ons, + "ECB": fetch_ecb, } diff --git a/config/default.toml b/config/default.toml index 4f1468c..563c678 100644 --- a/config/default.toml +++ b/config/default.toml @@ -171,8 +171,11 @@ bonds = [ {symbol="^FVX", label="US 5y Treasury yield", note="belly of the curve"}, {symbol="^TNX", label="US 10y Treasury yield", note="benchmark long rate"}, {symbol="^TYX", label="US 30y Treasury yield", note="long-end / term premium proxy"}, - # Eurozone — Maastricht convergence yields via Eurostat (monthly) - {symbol="EUROSTAT:irt_lt_mcby_m?geo=EA&int_rt=MCBY", label="Eurozone 10y aggregate", note="EZ avg 10y govt bond yield"}, + # Eurozone — daily AAA yield curve via ECB Data Portal (most current) + {symbol="ECB:YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_10Y", label="Eurozone 10y AAA (ECB, daily)", note="EZ AAA-rated 10y spot rate — ECB daily"}, + {symbol="ECB:YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_2Y", label="Eurozone 2y AAA (ECB, daily)", note="EZ AAA short-end, daily"}, + # Eurozone country yields — Maastricht convergence series via Eurostat (monthly) + {symbol="EUROSTAT:irt_lt_mcby_m?geo=EA&int_rt=MCBY", label="Eurozone 10y aggregate", note="EZ avg 10y govt bond yield — Maastricht, monthly"}, {symbol="EUROSTAT:irt_lt_mcby_m?geo=DE&int_rt=MCBY", label="German 10y Bund", note="EU risk-free benchmark"}, {symbol="EUROSTAT:irt_lt_mcby_m?geo=FR&int_rt=MCBY", label="French 10y OAT", note="core-EU spread to Bund"}, {symbol="EUROSTAT:irt_lt_mcby_m?geo=IT&int_rt=MCBY", label="Italian 10y BTP", note="periphery — BTP/Bund spread = EU stress"},