add ECB Data Portal source; group-aware stale thresholds
ECB Statistical Data Warehouse joins as a 5th data source — open API, no key, daily euro-area yield curve data. Symbol format 'ECB:dataset/series_key', e.g. 'ECB:YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_10Y' for daily 10y AAA spot rate. Bonds tab adds ECB EZ 10y AAA + 2y AAA so there's at least some currently-fresh European sovereign data alongside the US Treasuries. Country-specific yields (Bund/OAT/BTP/Gilt/JGB) remain on Eurostat/FRED monthly mirrors — no free daily source exists for those. Stale threshold is now per-group instead of a flat 90 days. Daily-tape groups (bonds, rates, equity, etc.) flag stale after a week or three; monthly groups (economy, macro, valuation) stay at 60-90 days. The bonds tab will now correctly show 30-60 day-old country yields as stale next to the daily US/ECB ones. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
1edf9cad41
commit
4e7e4981e3
3 changed files with 118 additions and 6 deletions
|
|
@ -55,6 +55,25 @@ router = APIRouter(dependencies=[Depends(require_token)])
|
|||
JOB_NAMES = ("market_job", "news_job", "portfolio_job", "ai_log_job", "rollup_job")
|
||||
JOB_STALE_HOURS = 2.0 # job is "warn" if its last success was >2h ago
|
||||
|
||||
# Per-group expected freshness — bonds and intraday tape want daily data,
|
||||
# macro/economy/valuation are monthly/quarterly by nature. Older than this
|
||||
# many days from today → row gets a "stale" badge.
|
||||
_STALE_DAYS_BY_GROUP = {
|
||||
"bonds": 21,
|
||||
"rates": 7,
|
||||
"equity": 7,
|
||||
"mag7": 7,
|
||||
"commodities": 7,
|
||||
"fx": 7,
|
||||
"tech_ai": 7,
|
||||
"financials": 7,
|
||||
"bubble_watch":21,
|
||||
"valuation": 60,
|
||||
"macro": 60,
|
||||
"economy": 90,
|
||||
}
|
||||
_STALE_DAYS_DEFAULT = 90
|
||||
|
||||
|
||||
# --- Small helpers -----------------------------------------------------------
|
||||
|
||||
|
|
@ -158,17 +177,18 @@ async def indicators(
|
|||
.limit(1)
|
||||
)).scalar_one_or_none()
|
||||
|
||||
# Mark rows whose `as_of` is older than 90 days as stale so the UI
|
||||
# can dim them — some FRED international series are months/years
|
||||
# behind their primary source.
|
||||
# Mark rows whose `as_of` is older than the group-specific threshold.
|
||||
# Daily-tape groups (bonds, rates, equity, ...) flag stale earlier
|
||||
# than monthly groups (economy, macro, valuation).
|
||||
today = utcnow().date()
|
||||
threshold = _STALE_DAYS_BY_GROUP.get(group, _STALE_DAYS_DEFAULT)
|
||||
stale_symbols: set[str] = set()
|
||||
for r in rows:
|
||||
try:
|
||||
as_of_d = datetime.strptime(r.as_of, "%Y-%m-%d").date() if r.as_of else None
|
||||
except ValueError:
|
||||
as_of_d = None
|
||||
if as_of_d and (today - as_of_d).days > 90:
|
||||
if as_of_d and (today - as_of_d).days > threshold:
|
||||
stale_symbols.add(r.symbol)
|
||||
|
||||
return templates.TemplateResponse(
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ YAHOO_CHART = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
|
|||
FRED_API = "https://api.stlouisfed.org/fred/series/observations"
|
||||
EUROSTAT_API = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset}"
|
||||
ONS_API = "https://www.ons.gov.uk/{topic}/timeseries/{cdid}/{dataset}/data"
|
||||
ECB_API = "https://data-api.ecb.europa.eu/service/data/{dataset}/{series_key}"
|
||||
UA = {"User-Agent": "Mozilla/5.0 (cassandra) Python/httpx"}
|
||||
|
||||
|
||||
|
|
@ -424,6 +425,93 @@ async def fetch_ons(
|
|||
return Quote(symbol, "ons", label, note, None, None, None, error=str(e))
|
||||
|
||||
|
||||
# --- ECB Data Portal (no API key, daily euro-area data) ---------------------
|
||||
|
||||
|
||||
async def fetch_ecb(
|
||||
client: httpx.AsyncClient,
|
||||
symbol: str,
|
||||
label: str,
|
||||
note: str,
|
||||
anchor: str | None = None,
|
||||
) -> Quote:
|
||||
"""Fetch an ECB Statistical Data Warehouse series. `symbol` format:
|
||||
<dataset>/<series_key>
|
||||
e.g. 'YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_10Y' for 10y EZ AAA yield (daily).
|
||||
ECB returns SDMX-JSON; we extract the single observation series."""
|
||||
try:
|
||||
if "/" not in symbol:
|
||||
raise ValueError("ECB symbol must be dataset/series_key")
|
||||
dataset, series_key = symbol.split("/", 1)
|
||||
r = await client.get(
|
||||
ECB_API.format(dataset=dataset, series_key=series_key),
|
||||
params={"format": "jsondata"},
|
||||
headers={**UA, "Accept": "application/json"},
|
||||
timeout=20,
|
||||
)
|
||||
r.raise_for_status()
|
||||
data = r.json()
|
||||
|
||||
# SDMX-JSON: dataSets[0].series[key].observations is { "<idx>": [value, ...] }
|
||||
# where <idx> indexes into structure.dimensions.observation[0].values for time.
|
||||
ds = (data.get("dataSets") or [{}])[0]
|
||||
series_map = ds.get("series") or {}
|
||||
if not series_map:
|
||||
raise ValueError("no series in response")
|
||||
ser = next(iter(series_map.values()))
|
||||
obs = ser.get("observations") or {}
|
||||
times = [
|
||||
v["id"]
|
||||
for v in data["structure"]["dimensions"]["observation"][0]["values"]
|
||||
]
|
||||
|
||||
rows: list[tuple[str, float]] = []
|
||||
for idx_str in sorted(obs, key=int):
|
||||
v = obs[idx_str][0]
|
||||
if v is None:
|
||||
continue
|
||||
t = times[int(idx_str)]
|
||||
iso = _eurostat_time_to_iso(t) # reuse: ECB time codes match Eurostat
|
||||
try:
|
||||
rows.append((iso, float(v)))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
|
||||
if not rows:
|
||||
raise ValueError("no observations")
|
||||
last_date, last_val = rows[-1]
|
||||
|
||||
def _find_back(min_days: int) -> float | None:
|
||||
ref = datetime.strptime(last_date, "%Y-%m-%d").date()
|
||||
for d, v in reversed(rows[:-1]):
|
||||
if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days:
|
||||
return v
|
||||
return None
|
||||
|
||||
prev_val = rows[-2][1] if len(rows) >= 2 else None
|
||||
changes = {
|
||||
"1d": _pct(prev_val, last_val),
|
||||
"1m": _pct(_find_back(28), last_val),
|
||||
"1y": _pct(_find_back(360), last_val),
|
||||
}
|
||||
anchor_used: str | None = None
|
||||
if anchor:
|
||||
anchor_d = _parse_date(anchor).date()
|
||||
for d, v in reversed(rows):
|
||||
if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d:
|
||||
changes["anchor"] = _pct(v, last_val)
|
||||
anchor_used = d
|
||||
break
|
||||
|
||||
return Quote(
|
||||
symbol=symbol, source="ecb", label=label, note=note,
|
||||
price=last_val, currency=None, as_of=last_date,
|
||||
changes=changes, anchor_date=anchor_used,
|
||||
)
|
||||
except Exception as e:
|
||||
return Quote(symbol, "ecb", label, note, None, None, None, error=str(e))
|
||||
|
||||
|
||||
# --- Source registry ----------------------------------------------------------
|
||||
|
||||
FetcherFn = Callable[..., "Quote"]
|
||||
|
|
@ -432,6 +520,7 @@ SOURCES: dict[str, FetcherFn] = {
|
|||
"FRED": fetch_fred,
|
||||
"EUROSTAT": fetch_eurostat,
|
||||
"ONS": fetch_ons,
|
||||
"ECB": fetch_ecb,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -171,8 +171,11 @@ bonds = [
|
|||
{symbol="^FVX", label="US 5y Treasury yield", note="belly of the curve"},
|
||||
{symbol="^TNX", label="US 10y Treasury yield", note="benchmark long rate"},
|
||||
{symbol="^TYX", label="US 30y Treasury yield", note="long-end / term premium proxy"},
|
||||
# Eurozone — Maastricht convergence yields via Eurostat (monthly)
|
||||
{symbol="EUROSTAT:irt_lt_mcby_m?geo=EA&int_rt=MCBY", label="Eurozone 10y aggregate", note="EZ avg 10y govt bond yield"},
|
||||
# Eurozone — daily AAA yield curve via ECB Data Portal (most current)
|
||||
{symbol="ECB:YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_10Y", label="Eurozone 10y AAA (ECB, daily)", note="EZ AAA-rated 10y spot rate — ECB daily"},
|
||||
{symbol="ECB:YC/B.U2.EUR.4F.G_N_A.SV_C_YM.SR_2Y", label="Eurozone 2y AAA (ECB, daily)", note="EZ AAA short-end, daily"},
|
||||
# Eurozone country yields — Maastricht convergence series via Eurostat (monthly)
|
||||
{symbol="EUROSTAT:irt_lt_mcby_m?geo=EA&int_rt=MCBY", label="Eurozone 10y aggregate", note="EZ avg 10y govt bond yield — Maastricht, monthly"},
|
||||
{symbol="EUROSTAT:irt_lt_mcby_m?geo=DE&int_rt=MCBY", label="German 10y Bund", note="EU risk-free benchmark"},
|
||||
{symbol="EUROSTAT:irt_lt_mcby_m?geo=FR&int_rt=MCBY", label="French 10y OAT", note="core-EU spread to Bund"},
|
||||
{symbol="EUROSTAT:irt_lt_mcby_m?geo=IT&int_rt=MCBY", label="Italian 10y BTP", note="periphery — BTP/Bund spread = EU stress"},
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue