add Eurostat + UK ONS sources; valuation/bubble/economy/bonds groups; aggregate read; market-open header

Three new data sources hooked into the existing SOURCES registry. All
open APIs, no keys:

  - EUROSTAT: prefix EUROSTAT:dataset?dim=val&... — current EU bond
    yields (Bund/OAT/BTP/EZ) and Eurozone economic indicators that
    FRED's OECD-mirror series stopped updating in 2022-2023.
  - ONS: prefix ONS:topic/cdid/dataset — current UK CPI, unemployment,
    GDP, industrial production. Replaces the 5+ month-stale FRED
    LRHUTTTTGBM156S mirror.

New indicator groups in default.toml feed the strategic/fundamental
lens we converged on: valuation (CAPE/Buffett anchors), bubble_watch
(SKEW/VVIX/RSP vs SPY/HYG vs TLT/IPO/crypto), economy (multi-region,
ALL current-or-stale-flagged), bonds (UK/EU/US/JPN sovereign yields).

Indicator panel now opens with an AI "read" interpretation per group
(generated hourly at :07 UTC alongside an aggregate cross-group read
shown in the dashboard header). The aggregate is grounded by a markets
strip — NYSE/LSE/Frankfurt/Tokyo/HK/Shanghai with open/closed LEDs and
next-open countdown, computed locally from each exchange's tz.

Other UX bits: indicator-row tooltips populated from TOML notes;
rows whose last observation is >90 days old get a 'stale' chip;
ghost symbols (in DB but no longer in TOML) filtered out of the
panel; Eurostat/ONS symbols display as short codes rather than the
full API path.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Giorgio Gilestro 2026-05-15 23:07:42 +01:00
parent a10409c02b
commit 1edf9cad41
15 changed files with 1156 additions and 10 deletions

View file

@ -16,6 +16,8 @@ from app.config import get_settings
YAHOO_CHART = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
FRED_API = "https://api.stlouisfed.org/fred/series/observations"
EUROSTAT_API = "https://ec.europa.eu/eurostat/api/dissemination/statistics/1.0/data/{dataset}"
ONS_API = "https://www.ons.gov.uk/{topic}/timeseries/{cdid}/{dataset}/data"
UA = {"User-Agent": "Mozilla/5.0 (cassandra) Python/httpx"}
@ -212,10 +214,225 @@ async def fetch_fred(
return Quote(symbol, "fred", label, note, None, None, None, error=str(e))
# --- Eurostat (no API key needed) -------------------------------------------
def _eurostat_time_to_iso(t: str) -> str:
"""Convert Eurostat time codes into ISO-style dates so they sort and
compare correctly. Accepts YYYY-MM, YYYY-Qn, YYYY, and YYYY-MM-DD."""
t = t.strip()
if len(t) == 4 and t.isdigit(): # annual: "2026"
return f"{t}-01-01"
if len(t) == 6 and t[4] == "Q": # quarterly: "2026Q1"
q = int(t[5])
return f"{t[:4]}-{(q - 1) * 3 + 1:02d}-01"
if len(t) == 7 and t[4] == "-": # monthly: "2026-03"
return f"{t}-01"
if len(t) == 10: # daily: "2026-03-15"
return t
return t # fall through; caller may flag
async def fetch_eurostat(
client: httpx.AsyncClient,
symbol: str,
label: str,
note: str,
anchor: str | None = None,
) -> Quote:
"""Fetch a Eurostat time series. `symbol` format:
DATASET?dim1=val1&dim2=val2
e.g. 'irt_lt_mcby_m?geo=DE&int_rt=MCBY' for German 10y bond yield.
Eurostat's API is open (no key), uses JSON-stat 2.0."""
import urllib.parse
try:
if "?" in symbol:
dataset, query = symbol.split("?", 1)
params = dict(urllib.parse.parse_qsl(query))
else:
dataset, params = symbol, {}
params.setdefault("format", "JSON")
params.setdefault("lang", "EN")
r = await client.get(
EUROSTAT_API.format(dataset=dataset),
params=params, headers=UA, timeout=20,
)
r.raise_for_status()
data = r.json()
time_cat = data["dimension"]["time"]["category"]
# JSON-stat 2.0: {"index": {timecode: pos}, "label": {timecode: human}}
time_index = time_cat["index"]
values = data.get("value") or {}
# Build (iso_date, value) pairs, sorted ascending in time.
rows: list[tuple[str, float]] = []
for tcode, pos in sorted(time_index.items(), key=lambda kv: kv[1]):
raw = values.get(str(pos))
if raw is None:
continue
try:
rows.append((_eurostat_time_to_iso(tcode), float(raw)))
except (TypeError, ValueError):
continue
if not rows:
raise ValueError("no observations")
last_date, last_val = rows[-1]
def _find_back(min_days: int) -> float | None:
ref = datetime.strptime(last_date, "%Y-%m-%d").date()
for d, v in reversed(rows[:-1]):
if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days:
return v
return None
prev_val = rows[-2][1] if len(rows) >= 2 else None
changes = {
"1d": _pct(prev_val, last_val),
"1m": _pct(_find_back(28), last_val),
"1y": _pct(_find_back(360), last_val),
}
anchor_used: str | None = None
if anchor:
anchor_d = _parse_date(anchor).date()
for d, v in reversed(rows):
if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d:
changes["anchor"] = _pct(v, last_val)
anchor_used = d
break
return Quote(
symbol=symbol, source="eurostat", label=label, note=note,
price=last_val, currency=None, as_of=last_date,
changes=changes, anchor_date=anchor_used,
)
except Exception as e:
return Quote(symbol, "eurostat", label, note, None, None, None, error=str(e))
# --- UK ONS (Office for National Statistics, no API key needed) -------------
_ONS_MONTH = {
"JAN": 1, "FEB": 2, "MAR": 3, "APR": 4, "MAY": 5, "JUN": 6,
"JUL": 7, "AUG": 8, "SEP": 9, "OCT": 10, "NOV": 11, "DEC": 12,
}
def _ons_date_to_iso(s: str) -> str | None:
"""ONS date formats: monthly '2026 MAR', quarterly '2026 Q1', annual '2025'."""
s = s.strip().upper()
parts = s.split()
try:
if len(parts) == 1 and parts[0].isdigit():
return f"{parts[0]}-01-01"
if len(parts) == 2:
year = int(parts[0])
tag = parts[1]
if tag in _ONS_MONTH:
return f"{year:04d}-{_ONS_MONTH[tag]:02d}-01"
if tag.startswith("Q") and tag[1:].isdigit():
q = int(tag[1:])
return f"{year:04d}-{(q - 1) * 3 + 1:02d}-01"
except (ValueError, IndexError):
pass
return None
async def fetch_ons(
client: httpx.AsyncClient,
symbol: str,
label: str,
note: str,
anchor: str | None = None,
) -> Quote:
"""Fetch a UK ONS time series. `symbol` format:
<topic_path>/<cdid>/<dataset>
e.g. 'economy/inflationandpriceindices/d7g7/mm23' for UK CPI YoY.
ONS publishes via www.ons.gov.uk; no auth, JSON when Accept header set."""
try:
parts = symbol.split("/")
if len(parts) < 3:
raise ValueError("ONS symbol must be topic/cdid/dataset")
dataset = parts[-1]
cdid = parts[-2]
topic = "/".join(parts[:-2])
r = await client.get(
ONS_API.format(topic=topic, cdid=cdid, dataset=dataset),
headers={**UA, "Accept": "application/json"},
timeout=20,
)
r.raise_for_status()
data = r.json()
# Use the most granular series available: months > quarters > years.
for key in ("months", "quarters", "years"):
raw_seq = data.get(key) or []
if raw_seq:
break
if not raw_seq:
raise ValueError("no observations")
rows: list[tuple[str, float]] = []
for entry in raw_seq:
iso = _ons_date_to_iso(entry.get("date", ""))
v = entry.get("value")
if iso is None or v in (None, "", "."):
continue
try:
rows.append((iso, float(v)))
except (TypeError, ValueError):
continue
if not rows:
raise ValueError("no parseable observations")
last_date, last_val = rows[-1]
def _find_back(min_days: int) -> float | None:
ref = datetime.strptime(last_date, "%Y-%m-%d").date()
for d, v in reversed(rows[:-1]):
if (ref - datetime.strptime(d, "%Y-%m-%d").date()).days >= min_days:
return v
return None
prev_val = rows[-2][1] if len(rows) >= 2 else None
changes = {
"1d": _pct(prev_val, last_val),
"1m": _pct(_find_back(28), last_val),
"1y": _pct(_find_back(360), last_val),
}
anchor_used: str | None = None
if anchor:
anchor_d = _parse_date(anchor).date()
for d, v in reversed(rows):
if datetime.strptime(d, "%Y-%m-%d").date() <= anchor_d:
changes["anchor"] = _pct(v, last_val)
anchor_used = d
break
return Quote(
symbol=symbol, source="ons", label=label, note=note,
price=last_val, currency=None, as_of=last_date,
changes=changes, anchor_date=anchor_used,
)
except Exception as e:
return Quote(symbol, "ons", label, note, None, None, None, error=str(e))
# --- Source registry ----------------------------------------------------------
FetcherFn = Callable[..., "Quote"]
SOURCES: dict[str, FetcherFn] = {"yahoo": fetch_yahoo, "FRED": fetch_fred}
SOURCES: dict[str, FetcherFn] = {
"yahoo": fetch_yahoo,
"FRED": fetch_fred,
"EUROSTAT": fetch_eurostat,
"ONS": fetch_ons,
}
def parse_symbol(symbol: str) -> tuple[FetcherFn, str]:

84
app/services/markets.py Normal file
View file

@ -0,0 +1,84 @@
"""Market-open/close status for the dashboard header. Pure computation —
no API needed; the schedules are known constants. Holidays are NOT modelled
(would require a region-specific calendar); a closed Monday will still show
"open" if the time-of-day fits. Good enough for the strategic dashboard.
"""
from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, time, timedelta, timezone
from zoneinfo import ZoneInfo
@dataclass(frozen=True)
class Market:
code: str
name: str
tz: str # IANA zone (handles DST automatically)
open: time # local time
close: time # local time
# Mon=0 .. Sun=6. Markets observe Mon-Fri unless overridden.
_WORKWEEK = {0, 1, 2, 3, 4}
MARKETS: list[Market] = [
Market("NYSE", "NYSE", "America/New_York", time(9, 30), time(16, 0)),
Market("LSE", "LSE", "Europe/London", time(8, 0), time(16, 30)),
Market("XETRA", "Frankfurt","Europe/Berlin", time(9, 0), time(17, 30)),
Market("JPX", "Tokyo", "Asia/Tokyo", time(9, 0), time(15, 0)),
Market("HKEX", "Hong Kong","Asia/Hong_Kong", time(9, 30), time(16, 0)),
Market("SSE", "Shanghai", "Asia/Shanghai", time(9, 30), time(15, 0)),
]
def _next_open_at(m: Market, now_utc: datetime) -> datetime:
"""Earliest future open datetime (UTC) for this market, scanning ahead
up to 7 days for the next weekday."""
tz = ZoneInfo(m.tz)
local = now_utc.astimezone(tz)
candidate_date = local.date()
for _ in range(8): # today + 7 days
weekday = candidate_date.weekday()
if weekday in _WORKWEEK:
local_open = datetime.combine(candidate_date, m.open, tzinfo=tz)
if local_open > local:
return local_open.astimezone(timezone.utc)
candidate_date = candidate_date + timedelta(days=1)
return now_utc + timedelta(days=7) # fallback (shouldn't happen)
def _close_at(m: Market, now_utc: datetime) -> datetime:
"""Today's close in UTC (assumes we've already established it's open)."""
tz = ZoneInfo(m.tz)
local = now_utc.astimezone(tz)
return datetime.combine(local.date(), m.close, tzinfo=tz).astimezone(timezone.utc)
def status_for(m: Market, now_utc: datetime) -> dict:
tz = ZoneInfo(m.tz)
local = now_utc.astimezone(tz)
is_workday = local.weekday() in _WORKWEEK
in_session = is_workday and m.open <= local.time() < m.close
if in_session:
return {
"code": m.code,
"name": m.name,
"open": True,
"until": _close_at(m, now_utc),
"label": "open",
}
return {
"code": m.code,
"name": m.name,
"open": False,
"until": _next_open_at(m, now_utc),
"label": "closed",
}
def all_statuses(now_utc: datetime | None = None) -> list[dict]:
if now_utc is None:
now_utc = datetime.now(timezone.utc)
return [status_for(m, now_utc) for m in MARKETS]

View file

@ -20,7 +20,7 @@ OPENROUTER_URL = "https://openrouter.ai/api/v1/chat/completions"
# Bump when the composed prompt changes meaningfully. Stored on every
# StrategicLog row so historical logs can be linked to the prompt that produced
# them.
PROMPT_VERSION = 3
PROMPT_VERSION = 4
# --- Core: invariant across tone/analysis settings ----------------------------
@ -60,6 +60,28 @@ numbers in every paragraph. No section over ~150 words.
- End with a watch list: 3-5 specific items to track in the next week, \
each one sentence.
# Time-horizon discipline
- This is a STRATEGIC log, not a day-trader's read. Treat 1-day moves under \
2% as background noise; mention them only when they break or confirm a \
multi-week trend or are extreme outliers.
- Anchor every claim to multi-week (1m), multi-month (since-anchor), or \
multi-year (1y) changes not 1d. If the only thing happening is a 1d move, \
omit the paragraph.
- The watch list is for "structural tripwires over the next 1-3 months", not \
"things to watch tomorrow". Each watch item should name a level/threshold \
whose breach would change the regime, not a calendar-date event.
# Rational vs irrational framing
The reader's primary goal is to disconnect rational decisions from market \
irrationality. In every sector or theme paragraph, separately identify:
- The RATIONAL drivers: earnings, real-economy data, monetary policy, \
structural geopolitical shifts, valuation vs fundamentals.
- The IRRATIONAL drivers: positioning, narrative momentum, sentiment \
extremes, concentration, flow-driven moves, options gamma, credit complacency.
When the two diverge price moving on irrational drivers while fundamentals \
say otherwise, or vice versa flag the divergence explicitly. Those gaps \
are where the next regime change starts.
# Discipline
- No emojis, no marketing language, no "concerning" or "unprecedented" \
without a specific number behind it.
@ -68,7 +90,16 @@ without a specific number behind it.
predicted X and X did not happen". Both are useful; conflating them is not.
- Don't repeat the same point in different words across paragraphs.
- No buy/sell recommendations. Triggers are pre-set elsewhere; your job is \
to report whether reality is confirming, modifying, or refuting the thesis."""
to report whether reality is confirming, modifying, or refuting the thesis.
# System temperature (closing line, mandatory)
Close the log with a single sentence on a line of its own, formatted exactly:
System temperature: [cool|neutral|elevated|hot|extreme] [one clause naming the 2-3 specific divergences or readings that justify the label]
This is the line a reader who only sees the watch list scrolls down to. Make \
it earn its place: cite real signals (HY OAS, breadth, VIX, valuation, real \
yields), not vibes."""
# --- Tone: audience-shaping block --------------------------------------------
@ -141,6 +172,118 @@ question via the chat sidebar.
- Keep the same audience and analysis discipline established above."""
def build_summary_system_prompt(tone: str, analysis: str) -> str:
"""A lean, focused system prompt for the per-indicator-group hourly
summary. INTERPRETATION not description the reader has the table
next to this paragraph; they don't need numbers recited at them."""
tone_block = _TONE.get(tone.upper(), _TONE["INTERMEDIATE"])
analysis_block = _ANALYSIS.get(analysis.upper(), _ANALYSIS["SPECULATIVE"])
return f"""You write a TINY interpretation (≤60 words, 2-3 sentences) \
of ONE indicator group for a strategic markets dashboard.
# What this is for
The reader is looking at the table of numbers right next to your text. \
They can see the values. They CANNOT see the meaning. Your job is to \
**explain what the data means**, not to recite it. Each sentence should be \
a regime-level interpretation, a fundamental driver identification, or a \
cross-indicator implication not a description of moves.
# Hard constraints
- Plain prose, ONE paragraph. No markdown, no headers, no lists, no labels.
- Open IMMEDIATELY with substance. NEVER start with: "I need to", "I'll", \
"We need to", "We are asked", "Here's", "Let me", "Let's", "Sure", "Looking \
at", "Based on", "Summary:", "The data shows", "First", "To address". No \
meta-commentary at all.
- Cite at most 2-3 specific numbers and ONLY when they anchor an \
interpretation. Don't list moves; explain them.
- Multi-week / multi-month horizon. 1-day moves under 2% are noise skip.
- No buy/sell language. No predictions. No watch list. No TL;DR. No date \
header. No "system temperature" line that belongs to the full daily log.
{tone_block}
{analysis_block}
# Bad example — describes what happened
"S&P +5.2% 1m and Nasdaq +8.8% 1m diverge from FTSE -3.4% and Euro Stoxx \
-2.6%. The US-vs-rest gap is widening."
# Good example — interprets what it means
"The US-vs-rest equity gap is funded by AI-capex concentration in 7 names; \
the breadth-weighted RSP barely keeps pace with SPY, which is the classic \
late-cycle marker narrow leadership, not broad recovery. The 5% 1m gap \
between Nasdaq and FTSE is a narrative trade, not a fundamental one."
"""
def build_summary_user_prompt(group_name: str, quotes: list[dict]) -> str:
parts = [
f"# Group: {group_name}",
"Indicators (latest reading + 1d/1m/1y/since-anchor change):",
"```json",
json.dumps(quotes, indent=2, default=str)[:12000],
"```",
"\nWrite the 2-3 sentence read for this group now.",
]
return "\n".join(parts)
def build_aggregate_summary_system_prompt(tone: str, analysis: str) -> str:
"""System prompt for the cross-group aggregate read shown on the dashboard.
Wider lens than a per-group summary synthesise across all groups."""
tone_block = _TONE.get(tone.upper(), _TONE["INTERMEDIATE"])
analysis_block = _ANALYSIS.get(analysis.upper(), _ANALYSIS["SPECULATIVE"])
return f"""You write a single SHORT cross-asset INTERPRETATION (≤80 \
words, 2-4 sentences) for the dashboard header. The reader is glancing \
give them the meaning of the whole tape, not a recap.
# What this is for
The reader can see every indicator on the dashboard below this paragraph. \
Your job is NOT to summarise the moves. It is to explain what the moves, \
**taken together as a system**, mean: which regime is being signalled, \
which divergences are load-bearing, what fundamental story the cross-asset \
behaviour tells.
# Hard constraints
- Plain prose, ONE paragraph. No markdown, headers, lists, or labels.
- Open IMMEDIATELY with substance. NEVER start with: "I need to", "I'll", \
"We need to", "Here's", "Let me", "Looking at", "Based on", "Sure", "Summary:", \
"The data shows", "Across the board". No meta-commentary.
- Identify the single most important **cross-asset implication**: e.g. \
"rates and credit disagree", "equities outrun fundamentals", "geopolitical \
risk premium is in commodities but not vol". Cite no more than 3 specific \
numbers, and only as anchors for the interpretation.
- Multi-week / multi-month horizon. 1-day moves under 2% are noise.
- No buy/sell language. No predictions of specific levels.
{tone_block}
{analysis_block}
# Bad example — describes
"Equities are up, real yields are higher, HY OAS is tight, breadth is \
narrowing."
# Good example — interprets
"The tape is paying a rising real discount rate (US 10y real +15bp 1m) with \
conviction for AI growth, but credit refuses to confirm and breadth is \
narrowing that combination is what late-cycle looks like, not pre-crash. \
The risk is not the level but the convergence: if any one of credit, \
breadth, or vol turns, the others will follow fast."
"""
def build_aggregate_summary_user_prompt(quotes_by_group: dict[str, list[dict]]) -> str:
parts = [
"# All indicator groups (latest readings + change windows)",
"```json",
json.dumps(quotes_by_group, indent=2, default=str)[:20000],
"```",
"\nWrite the cross-asset aggregate read now.",
]
return "\n".join(parts)
def build_chat_system_prompt(
tone: str,
analysis: str,