Giorgio Gilestro
a10409c02b
Containerised macro-strategy dashboard: 4-panel web UI (indicators, portfolio, flash news, AI strategic log), MariaDB store, hourly ingestion jobs, OpenRouter-backed AI analysis. Ports the four prototype scripts in the parent dir (market_pulse, flash_news, trading212, strategic_log) into async services backed by a persistent DB and served via FastAPI + Jinja2 + HTMX. APScheduler runs as a separate compose service for crash-safety and easier restarts. Portfolio composition + position names come live from Trading 212; news per-ticker headlines reuse those names. Tone (NOVICE/INTERMEDIATE/ PRO) and analysis style (DRY/SPECULATIVE) are env-configurable and stored on each log row so historical entries show what produced them. Default model is deepseek/deepseek-v4-flash (overridable via env). Light/dark theme toggle, sans-serif for prose surfaces, monospace for data. Bearer-token auth, OpenRouter monthly cost cap, RSS feeds auto- disabled on consecutive failures. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
285 lines
9.3 KiB
Python
285 lines
9.3 KiB
Python
"""Market data fetchers — Yahoo Finance (no auth) and FRED (key required).
|
|
|
|
Ported from /home/gg/ownCloud/Family/Finances/Wealth/market_pulse.py.
|
|
Logic preserved verbatim where possible; HTTP switched to httpx.AsyncClient.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from typing import Callable
|
|
|
|
import httpx
|
|
|
|
from app.config import get_settings
|
|
|
|
|
|
YAHOO_CHART = "https://query1.finance.yahoo.com/v8/finance/chart/{symbol}"
|
|
FRED_API = "https://api.stlouisfed.org/fred/series/observations"
|
|
UA = {"User-Agent": "Mozilla/5.0 (cassandra) Python/httpx"}
|
|
|
|
|
|
# --- In-flight data shape (services return these; jobs persist to DB) --------
|
|
|
|
|
|
@dataclass
|
|
class Quote:
|
|
symbol: str
|
|
source: str
|
|
label: str
|
|
note: str
|
|
price: float | None
|
|
currency: str | None
|
|
as_of: str | None
|
|
changes: dict[str, float | None] = field(default_factory=dict)
|
|
price_base: float | None = None
|
|
currency_base: str | None = None
|
|
anchor_date: str | None = None
|
|
error: str | None = None
|
|
|
|
|
|
def _pct(old: float | None, new: float | None) -> float | None:
|
|
if old is None or new is None or old == 0:
|
|
return None
|
|
return (new - old) / old * 100.0
|
|
|
|
|
|
def _parse_date(s: str) -> datetime:
|
|
return datetime.strptime(s, "%Y-%m-%d")
|
|
|
|
|
|
def _yahoo_range_covering(anchor: str | None) -> str:
|
|
if not anchor:
|
|
return "1y"
|
|
days = (datetime.now(timezone.utc).date() - _parse_date(anchor).date()).days
|
|
if days <= 360:
|
|
return "1y"
|
|
if days <= 1800:
|
|
return "5y"
|
|
if days <= 3600:
|
|
return "10y"
|
|
return "max"
|
|
|
|
|
|
# --- Fetchers -----------------------------------------------------------------
|
|
|
|
|
|
async def fetch_yahoo(
|
|
client: httpx.AsyncClient,
|
|
symbol: str,
|
|
label: str,
|
|
note: str,
|
|
anchor: str | None = None,
|
|
) -> Quote:
|
|
"""Latest close + %1d / %1m / %1y / (optional) %anchor from Yahoo's chart endpoint."""
|
|
try:
|
|
r = await client.get(
|
|
YAHOO_CHART.format(symbol=symbol),
|
|
params={"interval": "1d", "range": _yahoo_range_covering(anchor),
|
|
"includePrePost": "false"},
|
|
headers=UA,
|
|
timeout=15,
|
|
)
|
|
r.raise_for_status()
|
|
result = r.json()["chart"]["result"]
|
|
if not result:
|
|
raise ValueError("empty result")
|
|
res = result[0]
|
|
meta = res["meta"]
|
|
price = meta.get("regularMarketPrice")
|
|
prev_session = meta.get("previousClose")
|
|
timestamps = res.get("timestamp") or []
|
|
closes = (res.get("indicators", {}).get("quote") or [{}])[0].get("close") or []
|
|
series = [(t, c) for t, c in zip(timestamps, closes) if c is not None]
|
|
if not series:
|
|
raise ValueError("no closes in series")
|
|
last_ts = series[-1][0]
|
|
if prev_session is None and len(series) >= 2:
|
|
prev_session = series[-2][1]
|
|
chg_1m = _pct(series[-22][1], price) if len(series) >= 22 else None
|
|
chg_1y = _pct(series[0][1], price) if len(series) >= 2 else None
|
|
changes: dict[str, float | None] = {
|
|
"1d": _pct(prev_session, price),
|
|
"1m": chg_1m,
|
|
"1y": chg_1y,
|
|
}
|
|
anchor_used: str | None = None
|
|
if anchor:
|
|
anchor_ts = int(_parse_date(anchor).replace(tzinfo=timezone.utc).timestamp())
|
|
anchor_close = next((c for t, c in series if t >= anchor_ts), None)
|
|
anchor_actual_ts = next((t for t, c in series if t >= anchor_ts), None)
|
|
changes["anchor"] = _pct(anchor_close, price)
|
|
if anchor_actual_ts:
|
|
anchor_used = datetime.fromtimestamp(
|
|
anchor_actual_ts, timezone.utc
|
|
).strftime("%Y-%m-%d")
|
|
return Quote(
|
|
symbol=symbol,
|
|
source="yahoo",
|
|
label=label,
|
|
note=note,
|
|
price=price,
|
|
currency=meta.get("currency"),
|
|
as_of=datetime.fromtimestamp(last_ts, timezone.utc).strftime("%Y-%m-%d"),
|
|
changes=changes,
|
|
anchor_date=anchor_used,
|
|
)
|
|
except Exception as e:
|
|
return Quote(symbol, "yahoo", label, note, None, None, None, error=str(e))
|
|
|
|
|
|
async def fetch_fred(
|
|
client: httpx.AsyncClient,
|
|
symbol: str,
|
|
label: str,
|
|
note: str,
|
|
anchor: str | None = None,
|
|
) -> Quote:
|
|
"""Latest value + 1d/1m/1y deltas from a FRED series. Requires FRED_API_KEY."""
|
|
api_key = get_settings().FRED_API_KEY
|
|
if not api_key:
|
|
return Quote(symbol, "fred", label, note, None, None, None,
|
|
error="FRED_API_KEY not set")
|
|
try:
|
|
r = await client.get(
|
|
FRED_API,
|
|
params={
|
|
"series_id": symbol,
|
|
"api_key": api_key,
|
|
"file_type": "json",
|
|
"sort_order": "desc",
|
|
"limit": 5000 if anchor else 800,
|
|
},
|
|
headers=UA,
|
|
timeout=20,
|
|
)
|
|
r.raise_for_status()
|
|
obs = r.json().get("observations", [])
|
|
data = [
|
|
(o["date"], float(o["value"]))
|
|
for o in obs if o.get("value") not in (".", "", None)
|
|
]
|
|
if not data:
|
|
raise ValueError("no observations")
|
|
last_date, last_val = data[0]
|
|
# CPI levels reported as YoY%.
|
|
if symbol in ("CPIAUCSL", "CPILFESL") and len(data) >= 13:
|
|
yoy = _pct(data[12][1], last_val)
|
|
changes: dict[str, float | None] = {"1d": None, "1m": None, "1y": None}
|
|
anchor_used: str | None = None
|
|
if anchor:
|
|
anchor_dt = _parse_date(anchor)
|
|
anchor_idx = next(
|
|
(i for i, (d, _) in enumerate(data) if _parse_date(d) <= anchor_dt),
|
|
None,
|
|
)
|
|
if anchor_idx is not None and anchor_idx + 12 < len(data):
|
|
yoy_then = _pct(data[anchor_idx + 12][1], data[anchor_idx][1])
|
|
changes["anchor"] = (yoy or 0) - (yoy_then or 0)
|
|
anchor_used = anchor
|
|
return Quote(symbol, "fred", label, note, yoy, "%", last_date,
|
|
changes=changes, anchor_date=anchor_used)
|
|
|
|
last_dt = _parse_date(last_date)
|
|
|
|
def find_back(min_days: int) -> float | None:
|
|
for d, v in data[1:]:
|
|
if (last_dt - _parse_date(d)).days >= min_days:
|
|
return v
|
|
return None
|
|
|
|
prev_val = data[1][1] if len(data) >= 2 else None
|
|
changes = {
|
|
"1d": _pct(prev_val, last_val),
|
|
"1m": _pct(find_back(28), last_val),
|
|
"1y": _pct(find_back(360), last_val),
|
|
}
|
|
anchor_used = None
|
|
if anchor:
|
|
anchor_dt = _parse_date(anchor)
|
|
anchor_obs = next(
|
|
((d, v) for d, v in data if _parse_date(d) <= anchor_dt), None
|
|
)
|
|
if anchor_obs:
|
|
changes["anchor"] = _pct(anchor_obs[1], last_val)
|
|
anchor_used = anchor_obs[0]
|
|
return Quote(
|
|
symbol=symbol, source="fred", label=label, note=note,
|
|
price=last_val, currency=None, as_of=last_date,
|
|
changes=changes, anchor_date=anchor_used,
|
|
)
|
|
except Exception as e:
|
|
return Quote(symbol, "fred", label, note, None, None, None, error=str(e))
|
|
|
|
|
|
# --- Source registry ----------------------------------------------------------
|
|
|
|
FetcherFn = Callable[..., "Quote"]
|
|
SOURCES: dict[str, FetcherFn] = {"yahoo": fetch_yahoo, "FRED": fetch_fred}
|
|
|
|
|
|
def parse_symbol(symbol: str) -> tuple[FetcherFn, str]:
|
|
if ":" in symbol:
|
|
prefix, _, ident = symbol.partition(":")
|
|
if prefix in SOURCES:
|
|
return SOURCES[prefix], ident
|
|
return SOURCES["yahoo"], symbol
|
|
|
|
|
|
async def fetch(
|
|
client: httpx.AsyncClient,
|
|
symbol: str,
|
|
label: str,
|
|
note: str,
|
|
anchor: str | None = None,
|
|
) -> Quote:
|
|
fn, ident = parse_symbol(symbol)
|
|
return await fn(client, ident, label, note, anchor)
|
|
|
|
|
|
# --- Currency normalisation ---------------------------------------------------
|
|
|
|
|
|
async def _get_fx_rate(
|
|
client: httpx.AsyncClient,
|
|
from_ccy: str,
|
|
to_ccy: str,
|
|
cache: dict[tuple[str, str], float | None],
|
|
) -> float | None:
|
|
if from_ccy == to_ccy:
|
|
return 1.0
|
|
if from_ccy == "GBp": # LSE pence
|
|
gbp = await _get_fx_rate(client, "GBP", to_ccy, cache)
|
|
return None if gbp is None else 0.01 * gbp
|
|
key = (from_ccy, to_ccy)
|
|
if key in cache:
|
|
return cache[key]
|
|
pair = f"{from_ccy}{to_ccy}=X"
|
|
try:
|
|
r = await client.get(
|
|
YAHOO_CHART.format(symbol=pair),
|
|
params={"interval": "1d", "range": "5d"},
|
|
headers=UA, timeout=10,
|
|
)
|
|
r.raise_for_status()
|
|
rate = r.json()["chart"]["result"][0]["meta"].get("regularMarketPrice")
|
|
cache[key] = rate
|
|
return rate
|
|
except Exception:
|
|
cache[key] = None
|
|
return None
|
|
|
|
|
|
async def normalise_to_base(
|
|
client: httpx.AsyncClient, quotes: list[Quote], base: str
|
|
) -> None:
|
|
cache: dict[tuple[str, str], float | None] = {}
|
|
base = base.upper()
|
|
for q in quotes:
|
|
if q.price is None or not q.currency:
|
|
continue
|
|
rate = await _get_fx_rate(client, q.currency, base, cache)
|
|
if rate is None:
|
|
continue
|
|
q.price_base = q.price * rate
|
|
q.currency_base = base
|