935 lines
41 KiB
Zig
935 lines
41 KiB
Zig
//! `zfin diagnose SYMBOL` - walk the whole candle chain for one symbol and say
|
|
//! where it breaks.
|
|
//!
|
|
//! The counterpart to `doctor`, and deliberately its opposite in both axes:
|
|
//! `doctor` checks the whole system and never touches the network; this checks
|
|
//! one symbol and queries every tier, including the provider.
|
|
//!
|
|
//! It exists because the question "is the provider broken for this symbol, or is
|
|
//! it us?" came up repeatedly and the only way to answer it was a hand-rolled
|
|
//! sequence of curl commands against Tiingo and the shared server. That answer
|
|
//! belongs in the tool.
|
|
//!
|
|
//! READ-ONLY. Nothing here writes to the cache, which is the property that makes
|
|
//! it safe to run while diagnosing: `Tiingo.fetchCandles` returns candles without
|
|
//! caching them, the server check is a plain GET, and local state comes from
|
|
//! `Store.readCandleMeta`.
|
|
//!
|
|
//! RATE LIMIT CAVEAT: the provider query uses a throwaway `Tiingo` client with
|
|
//! its own limiter, so that one request is invisible to the shared hourly budget
|
|
//! `DataService` accounts for. On a free tier close to its cap, `diagnose` can be
|
|
//! the request that trips it. Accepted to avoid widening `DataService`'s API with
|
|
//! a public provider accessor; revisit if this command ever fetches in bulk.
|
|
|
|
const std = @import("std");
|
|
const zfin = @import("../root.zig");
|
|
const cli = @import("common.zig");
|
|
const framework = @import("framework.zig");
|
|
const freshness = @import("../cache/freshness.zig");
|
|
const Tiingo = @import("../providers/tiingo.zig").Tiingo;
|
|
const http = @import("../net/http.zig");
|
|
const fmt = @import("../format.zig");
|
|
|
|
const Store = zfin.cache.Store;
|
|
const Date = zfin.Date;
|
|
|
|
pub const ParsedArgs = struct {
|
|
symbol: []const u8,
|
|
};
|
|
|
|
pub const meta: framework.Meta = .{
|
|
.name = "diagnose",
|
|
.group = .infra,
|
|
.synopsis = "Trace one symbol's candle data through cache, server and provider",
|
|
.help =
|
|
\\Usage: zfin diagnose SYMBOL
|
|
\\
|
|
\\Reports, in order:
|
|
\\ local newest cached bar, TTL state, provider, failure count
|
|
\\ peers newest bar held by other symbols of the same kind
|
|
\\ server what ZFIN_SERVER offers, whether it is ahead or behind, and
|
|
\\ whether it refreshes this symbol at all
|
|
\\ provider what the upstream provider actually has right now
|
|
\\
|
|
\\Then a verdict naming where the chain breaks, and what to do about it.
|
|
\\
|
|
\\Read-only: no cache writes, no fetches into the cache. Answers
|
|
\\"is the provider broken for this symbol, or is it us?".
|
|
\\
|
|
\\The provider query uses its own rate limiter, so it is not counted
|
|
\\against the shared hourly budget. One request per invocation.
|
|
\\
|
|
,
|
|
.uppercase_first_arg = true,
|
|
.user_errors = error{MissingSymbol},
|
|
};
|
|
|
|
pub fn parseArgs(ctx: *framework.RunCtx, cmd_args: []const []const u8) !ParsedArgs {
|
|
if (cmd_args.len < 1) {
|
|
cli.stderrPrint(ctx.io, "Error: 'diagnose' requires a symbol\n");
|
|
return error.MissingSymbol;
|
|
}
|
|
return .{ .symbol = cmd_args[0] };
|
|
}
|
|
|
|
// ── verdict ──────────────────────────────────────────────────
|
|
|
|
/// Everything the verdict is derived from. All optional, because any tier can be
|
|
/// absent: no local copy, a kind with no peers, no server configured, a provider
|
|
/// that returned nothing.
|
|
pub const Observed = struct {
|
|
local: ?Date = null,
|
|
/// True when the local copy's TTL has not yet lapsed, so nothing will fetch
|
|
/// it on the normal path regardless of how old the bar is.
|
|
local_fresh: bool = false,
|
|
peer: ?Date = null,
|
|
server: ?Date = null,
|
|
provider: ?Date = null,
|
|
/// Is the symbol in the set a normal run fetches (`fetchedSymbols`)? Null
|
|
/// when it could not be determined - no portfolio resolved - in which case
|
|
/// no conclusion is drawn from it.
|
|
tracked: ?bool = null,
|
|
/// Is it a projections benchmark symbol? Those are fetched on demand rather
|
|
/// than routinely, so "nothing tracks this" is true but misleading - the
|
|
/// remedy is `zfin projections`, not a config fix.
|
|
benchmark: bool = false,
|
|
/// Does the SHARED SERVER's refresh loop include this symbol? Null when the
|
|
/// server did not say: none configured, a build predating `/diagnostics`, or
|
|
/// a failed request. Only a definite `false` is ever concluded from.
|
|
server_tracked: ?bool = null,
|
|
};
|
|
|
|
pub const Verdict = enum {
|
|
/// Nothing cached locally at all.
|
|
not_cached,
|
|
/// Local matches the best evidence available.
|
|
current,
|
|
/// Fetched on demand rather than routinely: a benchmark symbol, refreshed
|
|
/// whenever `projections` runs. Not tracked, but not neglected either.
|
|
demand_fetched,
|
|
/// Nothing in the fetch set asks for this symbol, so no normal run will ever
|
|
/// update it however stale it gets. The most fundamental gate there is: it
|
|
/// outranks the TTL, because even a lapsed TTL is never consulted for a
|
|
/// symbol nobody requests.
|
|
not_tracked,
|
|
/// The provider has a newer bar and the local TTL is still in the future, so
|
|
/// the normal path will not look until it lapses.
|
|
held_by_ttl,
|
|
/// The provider has a newer bar, the TTL has lapsed, and the shared server is
|
|
/// also behind - so a normal run syncs the server's older copy and stops.
|
|
held_by_server,
|
|
/// As `held_by_server`, plus the server does not track the symbol at all - so
|
|
/// its copy is not merely behind, nothing there will ever move it. A local
|
|
/// force-refresh fixes today; only registering the symbol on the server stops
|
|
/// it recurring. Separated from `held_by_server` because the two have
|
|
/// different remedies and only one of them is durable.
|
|
server_untracked,
|
|
/// The provider has a newer bar and nothing is holding us back; a refresh
|
|
/// should simply work.
|
|
refreshable,
|
|
/// Peers have moved on but the provider has nothing newer for THIS symbol.
|
|
/// The "is the provider broken for this symbol?" question answered yes:
|
|
/// whatever is wrong is upstream or in the symbol itself, not in this cache.
|
|
provider_behind_peers,
|
|
/// No provider answer to compare against.
|
|
unknown,
|
|
|
|
pub fn summary(self: Verdict) []const u8 {
|
|
return switch (self) {
|
|
.not_cached => "nothing cached for this symbol",
|
|
.current => "up to date with everything available",
|
|
.demand_fetched => "a benchmark symbol, fetched on demand - `zfin projections` refreshes it when it runs",
|
|
.not_tracked => "nothing in the fetch set asks for this symbol, so no normal run will update it - however stale it gets",
|
|
.held_by_ttl => "the bar exists upstream, but the local TTL has not lapsed - nothing will fetch it until it does",
|
|
.held_by_server => "the bar exists upstream; the shared cache is behind too, so a normal run syncs its older copy and stops",
|
|
.server_untracked => "the shared cache serves this symbol but never refreshes it, so it will drift behind again after any local fix - register it there to make a fix stick",
|
|
.refreshable => "the bar exists upstream and nothing is holding it back",
|
|
.provider_behind_peers => "peers have a newer bar but the provider has nothing newer for this symbol - the gap is upstream, not in this cache",
|
|
.unknown => "no provider answer, so the chain cannot be traced past the cache",
|
|
};
|
|
}
|
|
|
|
/// What to run, when there is something. Split from the symbol so this stays
|
|
/// pure and testable; `takes_symbol` exists because not every remedy is
|
|
/// per-symbol - `projections` takes none, and printing `zfin projections AGG`
|
|
/// hands the operator a command that does not work.
|
|
pub const Action = struct {
|
|
subcommand: []const u8,
|
|
takes_symbol: bool,
|
|
};
|
|
|
|
pub fn action(self: Verdict) ?Action {
|
|
return switch (self) {
|
|
.not_tracked, .held_by_ttl, .held_by_server, .server_untracked, .refreshable => .{ .subcommand = "cache refresh", .takes_symbol = true },
|
|
.not_cached => .{ .subcommand = "quote", .takes_symbol = true },
|
|
.demand_fetched => .{ .subcommand = "projections", .takes_symbol = false },
|
|
.current, .provider_behind_peers, .unknown => null,
|
|
};
|
|
}
|
|
};
|
|
|
|
/// Classify what was observed.
|
|
///
|
|
/// Pure, and the only part of this command worth testing exhaustively - the rest
|
|
/// is I/O and formatting. Ordering matters: `held_by_ttl` is checked before
|
|
/// `held_by_server` because a future TTL stops the fetch earlier in `getCandles`
|
|
/// than a server sync does, so it is the more proximate cause.
|
|
pub fn classify(o: Observed) Verdict {
|
|
const local = o.local orelse return .not_cached;
|
|
const provider = o.provider orelse return .unknown;
|
|
if (!local.lessThan(provider)) {
|
|
// Nothing newer to fetch. Whether that is healthy depends entirely on
|
|
// the peers: if they have moved on and this symbol's provider data has
|
|
// not, the problem is upstream. If nobody has moved, everything is
|
|
// simply up to date - and calling that "the gap is upstream" would
|
|
// report a healthy symbol as a fault.
|
|
if (o.peer) |peer| {
|
|
if (local.lessThan(peer)) return .provider_behind_peers;
|
|
}
|
|
return .current;
|
|
}
|
|
// A benchmark outranks both: it is genuinely untracked, but saying so would
|
|
// send the operator hunting for a config fix when the answer is simply that
|
|
// `projections` fetches it on demand.
|
|
if (o.benchmark) return .demand_fetched;
|
|
// Untracked outranks the TTL: a lapsed TTL is never even consulted for a
|
|
// symbol no code path requests.
|
|
if (o.tracked) |t| {
|
|
if (!t) return .not_tracked;
|
|
}
|
|
if (o.local_fresh) return .held_by_ttl;
|
|
if (o.server) |srv| {
|
|
if (srv.lessThan(provider)) {
|
|
// The server being behind is the proximate cause either way. Whether
|
|
// it can EVER catch up is the part worth separating: a tracked symbol
|
|
// will move on the server's next pass, an untracked one never will.
|
|
if (o.server_tracked) |st| {
|
|
if (!st) return .server_untracked;
|
|
}
|
|
return .held_by_server;
|
|
}
|
|
}
|
|
// Deliberately NOT reporting an untracked-on-server symbol whose copy is
|
|
// current as a fault. A refresh works right now, which is what the verdict
|
|
// answers; the standing risk is carried by the `server` output line instead of
|
|
// being promoted into a blocker that does not exist yet.
|
|
return .refreshable;
|
|
}
|
|
|
|
// ── run ──────────────────────────────────────────────────────
|
|
|
|
pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
|
|
const io = ctx.io;
|
|
const out = ctx.out;
|
|
const symbol = parsed.symbol;
|
|
|
|
var arena_state = std.heap.ArenaAllocator.init(ctx.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
var obs: Observed = .{};
|
|
|
|
var store = Store.init(io, ctx.allocator, ctx.config.cache_dir);
|
|
|
|
// ── local ────────────────────────────────────────────────
|
|
if (store.readCandleMeta(symbol)) |m| {
|
|
obs.local = m.meta.last_date;
|
|
obs.local_fresh = store.isCandleMetaFresh(symbol);
|
|
try out.print("local newest {f}", .{m.meta.last_date});
|
|
try out.print(", {s}", .{if (obs.local_fresh) "TTL still in the future" else "TTL lapsed"});
|
|
try out.print(", {s}", .{@tagName(m.meta.provider)});
|
|
if (m.meta.fail_count > 0) try out.print(", {d} consecutive failures", .{m.meta.fail_count});
|
|
// A live Tiingo backoff means this symbol is being served by
|
|
// Yahoo on purpose (Tiingo 404'd it), not by accident. Surface
|
|
// it so a provider that looks "wrong" can be explained.
|
|
if (m.meta.tiingo_retry_after_s > ctx.now_s) {
|
|
try out.print(", Tiingo backoff until {f}", .{Date.fromEpoch(m.meta.tiingo_retry_after_s)});
|
|
} else if (m.meta.tiingo_retry_after_s != 0) {
|
|
try out.print(", Tiingo backoff lapsed (retries next refresh)", .{});
|
|
}
|
|
try out.print("\n", .{});
|
|
|
|
// Adjustment basis. The cache is append-only, so a distribution
|
|
// that goes ex after the last full fetch never marks down the
|
|
// bars behind it - total returns then read low by roughly the
|
|
// missed yield. Report it here because the symptom (a slightly
|
|
// low 1Y total return) is otherwise invisible.
|
|
if (freshness.newestCorporateAction(arena, &store, symbol)) |newest_action| {
|
|
if (freshness.adjustmentBasisStale(m.meta.adj_basis, m.meta.last_date, newest_action)) {
|
|
try out.print(
|
|
"adj basis {f} - STALE, {f} went ex behind it; total returns read low until restated\n",
|
|
.{ m.meta.adj_basis, newest_action },
|
|
);
|
|
} else {
|
|
try out.print(
|
|
"adj basis {f} - current through the newest corporate action ({f})\n",
|
|
.{ m.meta.adj_basis, newest_action },
|
|
);
|
|
}
|
|
} else {
|
|
try out.print("adj basis {f} - no dividends or splits cached, nothing to restate\n", .{m.meta.adj_basis});
|
|
}
|
|
} else {
|
|
try out.print("local nothing cached\n", .{});
|
|
}
|
|
|
|
// ── peers ────────────────────────────────────────────────
|
|
const kind = zfin.market.classify(symbol);
|
|
{
|
|
const keys = store.cacheKeys(arena) catch &.{};
|
|
var empty = std.StringHashMap(void).init(arena);
|
|
const entries = try freshness.collect(arena, &store, keys, &empty, &.{});
|
|
var newest: ?Date = null;
|
|
var counted: usize = 0;
|
|
for (entries) |e| {
|
|
if (e.kind != kind) continue;
|
|
if (std.mem.eql(u8, e.symbol, symbol)) continue;
|
|
const bar = e.last_date orelse continue;
|
|
counted += 1;
|
|
if (newest == null or newest.?.lessThan(bar)) newest = bar;
|
|
}
|
|
obs.peer = newest;
|
|
if (newest) |n| {
|
|
try out.print("peers {d} other {s} cached, newest {f}", .{ counted, @tagName(kind), n });
|
|
if (obs.local) |l| {
|
|
if (l.lessThan(n)) {
|
|
const days = @divTrunc(n.toEpoch() - l.toEpoch(), std.time.s_per_day);
|
|
try out.print(" - {s} is {d}d behind", .{ symbol, days });
|
|
}
|
|
}
|
|
try out.print("\n", .{});
|
|
} else {
|
|
try out.print("peers no other {s} cached, nothing to compare against\n", .{@tagName(kind)});
|
|
}
|
|
}
|
|
|
|
// ── tracked ──────────────────────────────────────────────
|
|
if (cli.loadPortfolio(ctx, ctx.today)) |loaded_pf| {
|
|
var l = loaded_pf;
|
|
defer l.deinit(ctx.allocator);
|
|
for (cli.demandFetchedSymbols(io, arena, l.anchor())) |b| {
|
|
if (std.mem.eql(u8, b, symbol)) obs.benchmark = true;
|
|
}
|
|
if (cli.trackedSymbols(ctx, arena, l.portfolio)) |set| {
|
|
obs.tracked = set.contains(symbol);
|
|
} else |_| {}
|
|
if (obs.benchmark) {
|
|
try out.print("tracked on demand - a projections benchmark symbol\n", .{});
|
|
} else if (obs.tracked) |t| {
|
|
try out.print("tracked {s}\n", .{if (t) "yes - a normal run fetches this symbol" else "NO - no normal run fetches this symbol"});
|
|
}
|
|
}
|
|
|
|
// ── server ───────────────────────────────────────────────
|
|
if (ctx.config.server_url) |base| {
|
|
switch (serverDiagnostics(io, arena, base, ctx.config.server_api_key, symbol)) {
|
|
.view => |v| {
|
|
obs.server = v.last_date;
|
|
obs.server_tracked = v.tracked;
|
|
if (v.last_date) |srv| {
|
|
try out.print("server offers {f}", .{srv});
|
|
try printRelativeToLocal(out, srv, obs.local);
|
|
} else {
|
|
try out.print("server nothing cached there", .{});
|
|
}
|
|
if (v.created) |c| {
|
|
try out.print(", written {f}", .{Date.fromEpoch(c)});
|
|
}
|
|
if (v.tracked) |t| {
|
|
try out.print(", tracked: {s}", .{if (t) "yes" else "NO"});
|
|
}
|
|
try out.print("\n", .{});
|
|
// The line the endpoint exists for. Said plainly, because "not
|
|
// tracked" is jargon for a state whose consequence is total.
|
|
if (v.tracked) |t| {
|
|
if (!t) try out.print(" the server will never refresh this symbol on its own\n", .{});
|
|
}
|
|
// Its OWN peer gap, labelled as the server's rather than folded
|
|
// into the local one - different corpora, and merging them would
|
|
// invent a number neither side reported.
|
|
if (v.days_behind) |db| {
|
|
if (db > 0) {
|
|
try out.print(" and is {d}d behind its own peers there", .{db});
|
|
// The original bug's signature in one line: stamped good,
|
|
// yet behind. Only worth saying when both hold - `fresh`
|
|
// on a current copy is just healthy.
|
|
if (v.fresh orelse false) {
|
|
try out.print(", with its TTL still unlapsed - so a normal run there will not look either", .{});
|
|
}
|
|
try out.print("\n", .{});
|
|
}
|
|
}
|
|
if (v.fail_count) |fc| {
|
|
if (fc > 0) try out.print(" {d} consecutive provider failure(s) recorded there\n", .{fc});
|
|
}
|
|
},
|
|
// Fall back to the pre-`/diagnostics` probe so this command keeps
|
|
// working against a server that has not been redeployed.
|
|
.unsupported => {
|
|
if (serverNewest(io, arena, base, ctx.config.server_api_key, symbol)) |srv| {
|
|
obs.server = srv;
|
|
try out.print("server offers {f}", .{srv});
|
|
try printRelativeToLocal(out, srv, obs.local);
|
|
try out.print(" (no /diagnostics - intent unknown)\n", .{});
|
|
} else {
|
|
try out.print("server no answer for this symbol\n", .{});
|
|
}
|
|
},
|
|
// Named, not collapsed: an auth rejection and a DNS failure send the
|
|
// operator to entirely different places.
|
|
.failed => |why| try out.print("server could not ask: {s}\n", .{why}),
|
|
}
|
|
} else {
|
|
try out.print("server ZFIN_SERVER not set\n", .{});
|
|
}
|
|
|
|
// ── provider ─────────────────────────────────────────────
|
|
if (ctx.config.tiingo_key) |key| {
|
|
// Throwaway client with its own limiter - see the caveat at the top of
|
|
// this file. One request.
|
|
var tg = Tiingo.init(io, arena, key, .{ .per_hour = ctx.config.tiingoHourlyLimit() });
|
|
defer tg.deinit();
|
|
const to = fmt.todayDate(io);
|
|
const from = to.addDays(-10);
|
|
if (tg.fetchCandles(arena, symbol, from, to)) |candles| {
|
|
if (candles.len == 0) {
|
|
try out.print("provider tiingo returned no bars for the last 10 days\n", .{});
|
|
} else {
|
|
var newest = candles[0].date;
|
|
for (candles[1..]) |c| {
|
|
if (newest.lessThan(c.date)) newest = c.date;
|
|
}
|
|
obs.provider = newest;
|
|
try out.print("provider tiingo newest {f} ({d} bars in the last 10 days)\n", .{ newest, candles.len });
|
|
}
|
|
} else |err| {
|
|
// Name the error. "provider failed" cannot distinguish a rate limit
|
|
// from a dead key from a symbol the provider does not carry, and
|
|
// those have completely different remedies.
|
|
try out.print("provider tiingo FAILED: {s}\n", .{@errorName(err)});
|
|
}
|
|
} else {
|
|
try out.print("provider TIINGO_API_KEY not set\n", .{});
|
|
}
|
|
|
|
// ── verdict ──────────────────────────────────────────────
|
|
const verdict = classify(obs);
|
|
try out.print("\nverdict {s}\n", .{verdict.summary()});
|
|
if (verdict.action()) |act| {
|
|
if (act.takes_symbol) {
|
|
try out.print(" try: zfin {s} {s}\n", .{ act.subcommand, symbol });
|
|
} else {
|
|
try out.print(" try: zfin {s}\n", .{act.subcommand});
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Newest bar the shared server reports for `symbol`, or null.
|
|
///
|
|
/// A plain GET of the server's `candles_meta`, parsed for `last_date`. Read-only
|
|
/// by construction: nothing is written to the local cache, which matters because
|
|
/// the normal sync path would overwrite local bytes with the server's.
|
|
/// The shared server's own account of a symbol, from `GET /:symbol/diagnostics`.
|
|
///
|
|
/// Every field optional: an older server has no such endpoint at all, and a
|
|
/// newer one may add fields this build does not know. Absent means "the server
|
|
/// did not say", never a default value that would be indistinguishable from one
|
|
/// it did say.
|
|
pub const ServerView = struct {
|
|
/// Does the SERVER's refresh loop include this symbol? This is the field the
|
|
/// whole endpoint exists for: a symbol the server serves but never refreshes
|
|
/// looks identical to a maintained one from the outside.
|
|
tracked: ?bool = null,
|
|
/// Is the server's own copy stamped fresh by its `#!expires=`?
|
|
fresh: ?bool = null,
|
|
fail_count: ?u32 = null,
|
|
/// Calendar days the server's copy is behind its same-kind peers ON THE
|
|
/// SERVER. Not comparable to the local peer gap - different corpus.
|
|
days_behind: ?i64 = null,
|
|
last_date: ?Date = null,
|
|
peer_date: ?Date = null,
|
|
/// Unix seconds when the server wrote its copy.
|
|
created: ?i64 = null,
|
|
};
|
|
|
|
/// Outcome of asking the server about a symbol. A tagged union rather than
|
|
/// `?ServerView` because "this server is too old to ask" and "the request broke"
|
|
/// lead to different output, and collapsing them would report an infrastructure
|
|
/// problem as a missing feature.
|
|
pub const ServerProbe = union(enum) {
|
|
view: ServerView,
|
|
/// The endpoint is absent: a server predating it. Fall back to `candles_meta`.
|
|
unsupported,
|
|
/// Anything else. Carries the error NAME so the output can say which, rather
|
|
/// than reporting a bare "unavailable" for a DNS failure and an auth
|
|
/// rejection alike.
|
|
failed: []const u8,
|
|
};
|
|
|
|
/// Parse a diagnostics body. Separated from the request so the shape can be
|
|
/// tested without a server.
|
|
///
|
|
/// Unknown fields are ignored and malformed ones are left null rather than
|
|
/// failing the parse: a newer server adding a field must not blind an older
|
|
/// client to the fields it does understand.
|
|
pub fn parseServerView(arena: std.mem.Allocator, body: []const u8) ?ServerView {
|
|
const parsed = std.json.parseFromSlice(std.json.Value, arena, body, .{}) catch return null;
|
|
const obj = switch (parsed.value) {
|
|
.object => |o| o,
|
|
else => return null,
|
|
};
|
|
var v = ServerView{};
|
|
if (obj.get("tracked")) |x| if (x == .bool) {
|
|
v.tracked = x.bool;
|
|
};
|
|
if (obj.get("fresh")) |x| if (x == .bool) {
|
|
v.fresh = x.bool;
|
|
};
|
|
if (obj.get("fail_count")) |x| if (x == .integer and x.integer >= 0) {
|
|
v.fail_count = @intCast(x.integer);
|
|
};
|
|
if (obj.get("days_behind")) |x| if (x == .integer) {
|
|
v.days_behind = x.integer;
|
|
};
|
|
if (obj.get("created")) |x| if (x == .integer) {
|
|
v.created = x.integer;
|
|
};
|
|
if (obj.get("last_date")) |x| if (x == .string) {
|
|
v.last_date = Date.parse(x.string) catch null;
|
|
};
|
|
if (obj.get("peer_date")) |x| if (x == .string) {
|
|
v.peer_date = Date.parse(x.string) catch null;
|
|
};
|
|
return v;
|
|
}
|
|
|
|
/// " - ahead of your local copy" and friends. Extracted because both the
|
|
/// `/diagnostics` path and the older `candles_meta` fallback print it, and two
|
|
/// copies would have drifted on the equal case.
|
|
fn printRelativeToLocal(out: *std.Io.Writer, srv: Date, local: ?Date) !void {
|
|
const l = local orelse return;
|
|
if (srv.lessThan(l)) {
|
|
try out.print(" - BEHIND your local copy", .{});
|
|
} else if (l.lessThan(srv)) {
|
|
try out.print(" - ahead of your local copy", .{});
|
|
} else {
|
|
try out.print(" - same as local", .{});
|
|
}
|
|
}
|
|
|
|
fn serverDiagnostics(
|
|
io: std.Io,
|
|
arena: std.mem.Allocator,
|
|
base: []const u8,
|
|
api_key: ?[]const u8,
|
|
symbol: []const u8,
|
|
) ServerProbe {
|
|
const url = std.fmt.allocPrint(arena, "{s}/{s}/diagnostics", .{ base, symbol }) catch |e|
|
|
return .{ .failed = @errorName(e) };
|
|
var client = http.Client.init(io, arena);
|
|
defer client.deinit();
|
|
var hdr: [1]std.http.Header = .{.{ .name = "", .value = "" }};
|
|
const extra: []const std.http.Header = if (api_key) |k| blk: {
|
|
hdr[0] = .{ .name = "X-API-Key", .value = k };
|
|
break :blk hdr[0..1];
|
|
} else &.{};
|
|
var resp = client.request(.GET, url, null, extra) catch |err| switch (err) {
|
|
// The one error that means "ask the old way" rather than "something is
|
|
// wrong". Every other status is a real failure and says so by name.
|
|
error.NotFound => return .unsupported,
|
|
else => return .{ .failed = @errorName(err) },
|
|
};
|
|
defer resp.deinit();
|
|
return if (parseServerView(arena, resp.body)) |v|
|
|
.{ .view = v }
|
|
else
|
|
.{ .failed = "MalformedBody" };
|
|
}
|
|
|
|
fn serverNewest(
|
|
io: std.Io,
|
|
arena: std.mem.Allocator,
|
|
base: []const u8,
|
|
api_key: ?[]const u8,
|
|
symbol: []const u8,
|
|
) ?Date {
|
|
const url = std.fmt.allocPrint(arena, "{s}/{s}/candles_meta", .{ base, symbol }) catch return null;
|
|
var client = http.Client.init(io, arena);
|
|
defer client.deinit();
|
|
var hdr: [1]std.http.Header = .{.{ .name = "", .value = "" }};
|
|
const extra: []const std.http.Header = if (api_key) |k| blk: {
|
|
hdr[0] = .{ .name = "X-API-Key", .value = k };
|
|
break :blk hdr[0..1];
|
|
} else &.{};
|
|
var resp = client.request(.GET, url, null, extra) catch return null;
|
|
defer resp.deinit();
|
|
return lastDateIn(resp.body);
|
|
}
|
|
|
|
/// `last_date::` value from an SRF body.
|
|
fn lastDateIn(body: []const u8) ?Date {
|
|
const key = "last_date::";
|
|
const idx = std.mem.indexOf(u8, body, key) orelse return null;
|
|
const rest = body[idx + key.len ..];
|
|
const end = std.mem.indexOfAny(u8, rest, ",\n\r") orelse rest.len;
|
|
return Date.parse(rest[0..end]) catch null;
|
|
}
|
|
|
|
// ── tests ────────────────────────────────────────────────────
|
|
|
|
const testing = std.testing;
|
|
|
|
fn d(y: i16, m: u8, day: u8) Date {
|
|
return Date.fromYmd(y, m, day);
|
|
}
|
|
|
|
test "classify: the observed failure - provider has it, TTL holds us back" {
|
|
// AMZN on Tue 2026-08-11: Tiingo had 08-10 for over a day, the local copy sat
|
|
// on 08-07, and the TTL ran to Tue 16:55 - so nothing would look until then.
|
|
// This is the state that prompted the command.
|
|
try testing.expectEqual(Verdict.held_by_ttl, classify(.{
|
|
.local = d(2026, 8, 7),
|
|
.local_fresh = true,
|
|
.peer = d(2026, 8, 10),
|
|
.server = d(2026, 8, 7),
|
|
.provider = d(2026, 8, 10),
|
|
}));
|
|
}
|
|
|
|
test "classify: TTL is reported before the server, being the earlier gate" {
|
|
// Both are true here. `getCandles` checks the TTL before it syncs, so the TTL
|
|
// is the more proximate cause and naming the server would send the operator
|
|
// to the wrong tier.
|
|
const v = classify(.{
|
|
.local = d(2026, 8, 7),
|
|
.local_fresh = true,
|
|
.server = d(2026, 8, 7),
|
|
.provider = d(2026, 8, 10),
|
|
});
|
|
try testing.expectEqual(Verdict.held_by_ttl, v);
|
|
}
|
|
|
|
test "classify: a lapsed TTL with a lagging server names the server" {
|
|
try testing.expectEqual(Verdict.held_by_server, classify(.{
|
|
.local = d(2026, 8, 7),
|
|
.local_fresh = false,
|
|
.server = d(2026, 8, 7),
|
|
.provider = d(2026, 8, 10),
|
|
}));
|
|
}
|
|
|
|
test "classify: nothing in the way means a refresh should just work" {
|
|
// No server configured.
|
|
try testing.expectEqual(Verdict.refreshable, classify(.{
|
|
.local = d(2026, 8, 7),
|
|
.local_fresh = false,
|
|
.provider = d(2026, 8, 10),
|
|
}));
|
|
// Server already current, so it is not the obstacle.
|
|
try testing.expectEqual(Verdict.refreshable, classify(.{
|
|
.local = d(2026, 8, 7),
|
|
.local_fresh = false,
|
|
.server = d(2026, 8, 10),
|
|
.provider = d(2026, 8, 10),
|
|
}));
|
|
}
|
|
|
|
test "classify: everything level is current, not a fault" {
|
|
// SPY on Tue 2026-08-11: local, peers, server and provider all on 08-10.
|
|
// The first draft called this "the gap is upstream", which reports a
|
|
// perfectly healthy symbol as a problem.
|
|
const v = classify(.{
|
|
.local = d(2026, 8, 10),
|
|
.local_fresh = true,
|
|
.peer = d(2026, 8, 10),
|
|
.server = d(2026, 8, 10),
|
|
.provider = d(2026, 8, 10),
|
|
});
|
|
try testing.expectEqual(Verdict.current, v);
|
|
try testing.expectEqual(@as(?Verdict.Action, null), v.action());
|
|
|
|
// Local somehow AHEAD of the provider, peers level - still nothing to do.
|
|
try testing.expectEqual(Verdict.current, classify(.{
|
|
.local = d(2026, 8, 11),
|
|
.peer = d(2026, 8, 11),
|
|
.provider = d(2026, 8, 10),
|
|
}));
|
|
}
|
|
|
|
test "classify: peers ahead with the provider stuck points upstream" {
|
|
// THE QUESTION THIS COMMAND EXISTS FOR, answered yes. Peers hold 08-11, the
|
|
// provider has nothing past 08-10 for this symbol, so no local action helps.
|
|
const v = classify(.{
|
|
.local = d(2026, 8, 10),
|
|
.local_fresh = false,
|
|
.peer = d(2026, 8, 11),
|
|
.provider = d(2026, 8, 10),
|
|
});
|
|
try testing.expectEqual(Verdict.provider_behind_peers, v);
|
|
try testing.expectEqual(@as(?Verdict.Action, null), v.action());
|
|
}
|
|
|
|
test "classify: absent tiers degrade to a named unknown, never a guess" {
|
|
// No local copy at all.
|
|
try testing.expectEqual(Verdict.not_cached, classify(.{ .provider = d(2026, 8, 10) }));
|
|
// No provider answer: the chain cannot be traced, and saying so beats
|
|
// inferring from peers.
|
|
try testing.expectEqual(Verdict.unknown, classify(.{
|
|
.local = d(2026, 8, 7),
|
|
.peer = d(2026, 8, 10),
|
|
}));
|
|
// Nothing at all.
|
|
try testing.expectEqual(Verdict.not_cached, classify(.{}));
|
|
}
|
|
|
|
test "classify: every verdict offering an action names a runnable command" {
|
|
// Enumerated from the type, NOT a hand-written list. The hand-written version
|
|
// of this claimed to cover "every verdict" and silently skipped
|
|
// `server_untracked` the moment it was added - a test that quietly stops
|
|
// covering new cases is worse than no test, because the green tick is read as
|
|
// coverage.
|
|
for (std.enums.values(Verdict)) |v| {
|
|
try testing.expect(v.summary().len > 0);
|
|
// A subcommand, not a full command line - the caller appends the symbol,
|
|
// so this must never already contain one.
|
|
if (v.action()) |act| {
|
|
try testing.expect(act.subcommand.len > 0);
|
|
try testing.expect(std.mem.indexOf(u8, act.subcommand, "zfin") == null);
|
|
try testing.expect(std.mem.indexOf(u8, act.subcommand, "<") == null);
|
|
}
|
|
}
|
|
}
|
|
|
|
test "lastDateIn: parses the meta shape, tolerates junk" {
|
|
const meta_body = "#!srfv1\n#!expires=1786481700\nlast_close:num:274.48,last_date::2026-08-07,provider::tiingo\n";
|
|
try testing.expect(lastDateIn(meta_body).?.eql(d(2026, 8, 7)));
|
|
try testing.expectEqual(@as(?Date, null), lastDateIn("#!srfv1\n"));
|
|
try testing.expectEqual(@as(?Date, null), lastDateIn("last_date::not-a-date\n"));
|
|
}
|
|
|
|
test "classify: an untracked symbol is named as such, ahead of the TTL" {
|
|
// THE CASE THAT PROMPTED THIS. A benchmark symbol whose shared-cache copy was
|
|
// CURRENT and whose provider had the bar - so the first version reported
|
|
// "nothing is holding it back" - and yet a normal run left it untouched,
|
|
// because no fetch path asks for benchmark symbols at all.
|
|
const v = classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = false,
|
|
.peer = d(2026, 8, 11),
|
|
.server = d(2026, 8, 11),
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = false,
|
|
});
|
|
try testing.expectEqual(Verdict.not_tracked, v);
|
|
|
|
// Outranks a fresh TTL: even once that lapses, nothing consults it.
|
|
try testing.expectEqual(Verdict.not_tracked, classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = true,
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = false,
|
|
}));
|
|
}
|
|
|
|
test "classify: tracked=null draws no conclusion" {
|
|
// No portfolio resolved, so trackedness is unknown. Guessing either way
|
|
// would be worse than falling through to the next observable cause.
|
|
try testing.expectEqual(Verdict.held_by_ttl, classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = true,
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = null,
|
|
}));
|
|
try testing.expectEqual(Verdict.refreshable, classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = false,
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = null,
|
|
}));
|
|
}
|
|
|
|
test "classify: being tracked does not mask the real cause" {
|
|
try testing.expectEqual(Verdict.held_by_ttl, classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = true,
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = true,
|
|
}));
|
|
try testing.expectEqual(Verdict.held_by_server, classify(.{
|
|
.local = d(2026, 6, 29),
|
|
.local_fresh = false,
|
|
.server = d(2026, 6, 29),
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = true,
|
|
}));
|
|
}
|
|
|
|
test "classify: a benchmark is demand-fetched, not untracked" {
|
|
// AGG. Genuinely absent from the routine fetch set, so `not_tracked` is
|
|
// literally true - and misleading, because it sends the operator hunting for
|
|
// a config fix when `zfin projections` refreshes it on demand. Outranks both
|
|
// the tracked check and the TTL for that reason.
|
|
const v = classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = false,
|
|
.peer = d(2026, 8, 11),
|
|
.server = d(2026, 8, 11),
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = false,
|
|
.benchmark = true,
|
|
});
|
|
try testing.expectEqual(Verdict.demand_fetched, v);
|
|
// No symbol: `zfin projections AGG` is not a command that works.
|
|
try testing.expectEqualStrings("projections", v.action().?.subcommand);
|
|
try testing.expect(!v.action().?.takes_symbol);
|
|
|
|
// Even with a fresh TTL, the demand-fetched fact is the useful one.
|
|
try testing.expectEqual(Verdict.demand_fetched, classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = true,
|
|
.provider = d(2026, 8, 11),
|
|
.benchmark = true,
|
|
}));
|
|
}
|
|
|
|
test "classify: a non-benchmark untracked symbol still reports not_tracked" {
|
|
// The flag must not swallow the genuine case.
|
|
try testing.expectEqual(Verdict.not_tracked, classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = false,
|
|
.provider = d(2026, 8, 11),
|
|
.tracked = false,
|
|
.benchmark = false,
|
|
}));
|
|
}
|
|
|
|
test "classify: a current benchmark is still just current" {
|
|
// Nothing newer anywhere, so the demand-fetched fact is not a finding.
|
|
try testing.expectEqual(Verdict.current, classify(.{
|
|
.local = d(2026, 8, 11),
|
|
.peer = d(2026, 8, 11),
|
|
.provider = d(2026, 8, 11),
|
|
.benchmark = true,
|
|
}));
|
|
}
|
|
|
|
test "parseServerView: the live production record for SPCX" {
|
|
// Verbatim from `GET /SPCX/diagnostics` against the deployed server. The
|
|
// combination is the one this endpoint was built to surface: served, behind,
|
|
// and refreshed by nothing.
|
|
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
|
defer arena.deinit();
|
|
const body =
|
|
\\{"symbol":"SPCX","tracked":false,"fresh":false,"fail_count":0,"days_behind":44,"last_date":"2026-06-29","peer_date":"2026-08-12","created":1782773387}
|
|
;
|
|
const v = parseServerView(arena.allocator(), body).?;
|
|
try testing.expectEqual(false, v.tracked.?);
|
|
try testing.expectEqual(false, v.fresh.?);
|
|
try testing.expectEqual(@as(u32, 0), v.fail_count.?);
|
|
try testing.expectEqual(@as(i64, 44), v.days_behind.?);
|
|
try testing.expect(v.last_date.?.eql(d(2026, 6, 29)));
|
|
try testing.expect(v.peer_date.?.eql(d(2026, 8, 12)));
|
|
try testing.expectEqual(@as(i64, 1782773387), v.created.?);
|
|
}
|
|
|
|
test "parseServerView: nulls stay null rather than becoming defaults" {
|
|
// An uncached symbol answers 200 with nulls. `tracked:false` and "the server
|
|
// did not say" must not both arrive as false - one is a finding, the other is
|
|
// an absence of information.
|
|
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
|
defer arena.deinit();
|
|
const body =
|
|
\\{"symbol":"NOSUCH","tracked":false,"fresh":false,"fail_count":0,"days_behind":0,"last_date":null,"peer_date":"2026-08-12","created":null}
|
|
;
|
|
const v = parseServerView(arena.allocator(), body).?;
|
|
try testing.expectEqual(@as(?Date, null), v.last_date);
|
|
try testing.expectEqual(@as(?i64, null), v.created);
|
|
try testing.expectEqual(false, v.tracked.?);
|
|
}
|
|
|
|
test "parseServerView: a newer server's extra fields do not blind an older client" {
|
|
// Forward compatibility is the whole reason fields are read individually
|
|
// rather than by struct coercion: an added field must not cost the client
|
|
// every field it does understand.
|
|
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
|
defer arena.deinit();
|
|
const body =
|
|
\\{"tracked":true,"unheard_of":{"nested":[1,2]},"days_behind":3,"expires":123}
|
|
;
|
|
const v = parseServerView(arena.allocator(), body).?;
|
|
try testing.expectEqual(true, v.tracked.?);
|
|
try testing.expectEqual(@as(i64, 3), v.days_behind.?);
|
|
}
|
|
|
|
test "parseServerView: junk yields null, and a wrong-typed field is skipped not fatal" {
|
|
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
|
defer arena.deinit();
|
|
const a = arena.allocator();
|
|
// Not JSON at all - e.g. an HTML error page from a proxy.
|
|
try testing.expectEqual(@as(?ServerView, null), parseServerView(a, "<html>502</html>"));
|
|
// Valid JSON, wrong shape.
|
|
try testing.expectEqual(@as(?ServerView, null), parseServerView(a, "[1,2,3]"));
|
|
// A field of the wrong type is dropped; the rest still parses. Better than
|
|
// failing the whole record over one bad value.
|
|
const v = parseServerView(a, "{\"tracked\":\"yes\",\"days_behind\":7}").?;
|
|
try testing.expectEqual(@as(?bool, null), v.tracked);
|
|
try testing.expectEqual(@as(i64, 7), v.days_behind.?);
|
|
// An unparseable date is null, not an error.
|
|
const v2 = parseServerView(a, "{\"last_date\":\"not-a-date\"}").?;
|
|
try testing.expectEqual(@as(?Date, null), v2.last_date);
|
|
}
|
|
|
|
test "classify: a server that never refreshes the symbol outranks plain held_by_server" {
|
|
// SPCX's real state. `held_by_server` is true but stops short of the part
|
|
// that matters: no pass on the server will ever move this copy, so a local
|
|
// refresh fixes today and nothing else.
|
|
const base = Observed{
|
|
.local = d(2026, 6, 29),
|
|
.local_fresh = false,
|
|
.peer = d(2026, 8, 12),
|
|
.server = d(2026, 6, 29),
|
|
.provider = d(2026, 8, 12),
|
|
.tracked = true,
|
|
};
|
|
var o = base;
|
|
o.server_tracked = false;
|
|
try testing.expectEqual(Verdict.server_untracked, classify(o));
|
|
|
|
// Tracked on the server: it will catch up on the next pass, so the weaker
|
|
// verdict is the correct one.
|
|
o.server_tracked = true;
|
|
try testing.expectEqual(Verdict.held_by_server, classify(o));
|
|
|
|
// The server did not say (older build, or no server): must not be read as
|
|
// untracked. Silence is not a finding.
|
|
o.server_tracked = null;
|
|
try testing.expectEqual(Verdict.held_by_server, classify(o));
|
|
}
|
|
|
|
test "classify: untracked on the server is not a blocker while its copy is current" {
|
|
// A refresh works right now - that is what the verdict answers. The standing
|
|
// risk that nothing will refresh it later belongs on the `server` output line,
|
|
// not promoted into a blocker that does not exist yet.
|
|
try testing.expectEqual(Verdict.refreshable, classify(.{
|
|
.local = d(2026, 8, 6),
|
|
.local_fresh = false,
|
|
.server = d(2026, 8, 12),
|
|
.provider = d(2026, 8, 12),
|
|
.tracked = true,
|
|
.server_tracked = false,
|
|
}));
|
|
}
|
|
|
|
test "classify: the local TTL still outranks an untracked server" {
|
|
// Ordering by proximate cause, matching `held_by_ttl` over `held_by_server`:
|
|
// an unlapsed TTL stops the fetch inside `getCandles`, before any sync is
|
|
// attempted, so the server's intent has not come into play yet.
|
|
try testing.expectEqual(Verdict.held_by_ttl, classify(.{
|
|
.local = d(2026, 6, 29),
|
|
.local_fresh = true,
|
|
.server = d(2026, 6, 29),
|
|
.provider = d(2026, 8, 12),
|
|
.tracked = true,
|
|
.server_tracked = false,
|
|
}));
|
|
}
|