add diagnostics route

build.zig.zon

@@ -14,8 +14,8 @@
             .hash = "httpz-0.0.0-PNVzrLjJCAD37S0CcrXpsjSqr86hVjK0rsALTDJ98AAJ",
         },
         .zfin = .{
-            .url = "git+https://git.lerch.org/lobo/zfin#937822f9b04847942da4f57e9dfbde8cdd39be86",
+            .url = "git+https://git.lerch.org/lobo/zfin#4a86ecb95e1b155b58c2cdc294d850309bab934d",
-            .hash = "zfin-0.0.0-J-B21hCbVgAPhcCoDDQ5BAPkwN_Vk8is57gshMJO56F4",
+            .hash = "zfin-0.0.0-J-B21tBcVwCRYeqB7LbX67Ik9a2y8olgk7OObrUtJ8YU",
         },
     },
 }

src/main.zig

@@ -536,6 +536,142 @@ const SrfKind = enum {
     tickers_companies,
 };
 
+/// What the server INTENDS for a symbol, as opposed to the data it happens to
+/// hold. Every other endpoint answers the second question; nothing answered the
+/// first, which is how a symbol the refresh loop never touches sat six weeks
+/// behind while being served to clients as though it were maintained.
+const SymbolDiagnostics = struct {
+    /// Will the refresh loop fetch this symbol? See `collectRefreshSymbols`.
+    tracked: bool,
+    /// Newest cached bar, or null when there is no candle meta at all.
+    last_date: ?zfin.Date,
+    /// When the cached copy was written (Unix seconds), null when uncached.
+    created: ?i64,
+    /// Consecutive transient provider failures on the primary provider.
+    fail_count: u8,
+    /// Is the cached copy stamped fresh by its own `#!expires=`? Reported rather
+    /// than the raw expiry because `fresh` alongside a non-zero `days_behind` is
+    /// precisely the pathology that started this: a copy stamped good until
+    /// tomorrow while sitting days behind its peers. The raw directive is still
+    /// on the wire via `/:symbol/candles_meta` for anyone who wants it.
+    fresh: bool,
+    /// Newest bar held by any same-kind peer in this cache, or null when there is
+    /// no peer to compare against.
+    peer_date: ?zfin.Date,
+    /// Calendar days behind `peer_date`; 0 when not behind or incomparable.
+    days_behind: i64,
+};
+
+/// Calendar days `last` sits behind `peer`, or 0 when it is not behind.
+///
+/// Computed here rather than read out of the sweep's findings. `scan` emits a
+/// Finding only for a TRACKED symbol (untracked ones divert to `orphans`), so
+/// reading `days_behind` from there returned 0 for every untracked symbol -
+/// exactly the class this endpoint exists to expose. An untracked symbol sitting
+/// 43 days behind, reported as `days_behind:0`, is the worst available answer:
+/// it reads as "current" for the one case nobody is watching.
+///
+/// Calendar days, truncated, to match `zfin.freshness.Finding.days_behind` - the
+/// magnitude an operator weighs against `max_normal_lag_days`.
+fn daysBehind(last: ?zfin.Date, peer: ?zfin.Date) i64 {
+    const l = last orelse return 0;
+    const p = peer orelse return 0;
+    if (!l.lessThan(p)) return 0;
+    return @divTrunc(p.toEpoch() - l.toEpoch(), std.time.s_per_day);
+}
+
+/// The peer reference date for `kind`, or null when the group cannot yield a
+/// comparison. `conclusive()` is the gate: with a single cached symbol of a kind
+/// there are no peers, and reporting that symbol's own date as its `peer_date`
+/// would manufacture agreement out of nothing.
+fn groupPeerDate(report: zfin.freshness.Report, kind: zfin.market.InstrumentKind) ?zfin.Date {
+    for (report.groups) |g| {
+        if (g.kind != kind) continue;
+        if (!g.conclusive()) return null;
+        return g.peer_date;
+    }
+    return null;
+}
+
+fn handleDiagnostics(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
+    const raw_symbol = req.param("symbol") orelse {
+        res.status = 400;
+        res.body = "Missing symbol";
+        return;
+    };
+    const arena = res.arena;
+    const symbol = try upperDupe(arena, raw_symbol);
+
+    var store = zfin.cache.Store.init(app.io, arena, app.config.cache_dir);
+
+    // An unreadable or unparseable portfolio leaves the set empty, which reports
+    // `tracked:false` - honest, because a refresh run reading the same file would
+    // fetch nothing either.
+    //
+    // NO `portfolio.deinit()` here, deliberately. The set's keys BORROW from the
+    // parsed lots, and every `contains` happens below this block, so freeing the
+    // portfolio dangles them - observed as all 25 tracked symbols reading as
+    // untracked. The request arena owns this memory and releases it with the
+    // response. This is the same trap `zfin`'s `trackedSymbols` documents: "the
+    // scoped-defer-frees version dangled its keys before the caller read them,
+    // which made every cached symbol look untracked."
+    var tracked = std.StringHashMap(void).init(arena);
+    const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf";
+    if (std.Io.Dir.cwd().readFileAlloc(app.io, portfolio_path, arena, .limited(10 * 1024 * 1024))) |data| {
+        if (zfin.cache.deserializePortfolio(arena, data)) |parsed| {
+            try collectRefreshSymbols(&tracked, parsed.lots);
+        } else |_| {}
+    } else |_| {}
+
+    // The peer sweep. `collect` + `scan` rather than a local "newest of this
+    // kind" loop: the definition of behind-its-peers lives in one place, and this
+    // endpoint exists to report that definition, not a second opinion on it.
+    const keys = store.cacheKeys(arena) catch &.{};
+    const entries = try zfin.freshness.collect(arena, &store, keys, &tracked, &.{});
+    // wall-clock required: peer freshness is judged against the market calendar.
+    const now_s = std.Io.Timestamp.now(app.io, .real).toSeconds();
+    const report = try zfin.freshness.scan(arena, entries, now_s);
+
+    const meta = store.readCandleMeta(symbol);
+    const last_date: ?zfin.Date = if (meta) |m| m.meta.last_date else null;
+    const peer_date = groupPeerDate(report, zfin.market.classify(symbol));
+    const d = SymbolDiagnostics{
+        .tracked = tracked.contains(symbol),
+        .last_date = last_date,
+        .created = if (meta) |m| m.created else null,
+        .fail_count = if (meta) |m| m.meta.fail_count else 0,
+        .fresh = store.isCandleMetaFresh(symbol),
+        .peer_date = peer_date,
+        .days_behind = daysBehind(last_date, peer_date),
+    };
+
+    var aw: std.Io.Writer.Allocating = .init(arena);
+    try aw.writer.print(
+        \\{{"symbol":"{s}","tracked":{},"fresh":{},"fail_count":{d},"days_behind":{d}
+    , .{ symbol, d.tracked, d.fresh, d.fail_count, d.days_behind });
+    // Null rather than a sentinel date for the absent cases: a client must be
+    // able to tell "no cached bar" from "a bar dated the epoch".
+    if (d.last_date) |ld| {
+        try aw.writer.print(",\"last_date\":\"{f}\"", .{ld});
+    } else {
+        try aw.writer.writeAll(",\"last_date\":null");
+    }
+    if (d.peer_date) |pd| {
+        try aw.writer.print(",\"peer_date\":\"{f}\"", .{pd});
+    } else {
+        try aw.writer.writeAll(",\"peer_date\":null");
+    }
+    if (d.created) |c| {
+        try aw.writer.print(",\"created\":{d}", .{c});
+    } else {
+        try aw.writer.writeAll(",\"created\":null");
+    }
+    try aw.writer.writeByte('}');
+
+    res.content_type = httpz.ContentType.JSON;
+    res.body = aw.written();
+}
+
 fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8, kind: SrfKind) !void {
     return handleSrfFileByKey(app, req, res, "symbol", filename, kind);
 }
@@ -951,6 +1087,22 @@ fn writeFileAtomic(io: std.Io, allocator: std.mem.Allocator, path: []const u8, b
 
 // ── Refresh command ──────────────────────────────────────────
 
+/// The symbols the refresh loop will fetch: `.stock` and `.watch` lots, keyed by
+/// `priceSymbol()` so a `ticker::` alias resolves the same way a fetch does.
+///
+/// Extracted rather than inlined because `/:symbol/diagnostics` reports a
+/// `tracked` flag, and a `tracked` that meant anything other than "refresh will
+/// fetch this" would be worse than not reporting it at all - an operator would
+/// read it as a promise the loop never made. One definition, two callers.
+fn collectRefreshSymbols(out: *std.StringHashMap(void), lots: []const zfin.Lot) !void {
+    for (lots) |lot| {
+        if (lot.security_type != .stock and lot.security_type != .watch) continue;
+        if (lot.symbol.len == 0) continue;
+        const sym = lot.priceSymbol();
+        if (!out.contains(sym)) try out.put(sym, {});
+    }
+}
+
 fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) !u8 {
     var config = zfin.Config.fromEnv(io, allocator, environ);
     defer config.deinit();
@@ -983,14 +1135,7 @@ fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process
 
     var symbols = std.StringHashMap(void).init(allocator);
     defer symbols.deinit();
-    for (portfolio.lots) |lot| {
+    try collectRefreshSymbols(&symbols, portfolio.lots);
-        if (lot.security_type != .stock and lot.security_type != .watch) continue;
-        if (lot.symbol.len == 0) continue;
-        const sym = lot.priceSymbol();
-        if (!symbols.contains(sym)) {
-            try symbols.put(sym, {});
-        }
-    }
 
     const stdout_file = std.Io.File.stdout();
     var buf: [4096]u8 = undefined;
@@ -1329,6 +1474,7 @@ pub fn main(init: std.process.Init) !u8 {
         router.get("/:symbol/quote", handleQuote, .{});
         router.get("/:symbol/candles", handleCandles, .{});
         router.get("/:symbol/candles_meta", handleCandlesMeta, .{});
+        router.get("/:symbol/diagnostics", handleDiagnostics, .{});
         router.get("/:symbol/dividends", handleDividends, .{});
         router.get("/:symbol/splits", handleSplits, .{});
         router.get("/:symbol/earnings", handleEarnings, .{});
@@ -1450,6 +1596,12 @@ test "pathIsPublic: gated surface (key required)" {
     try std.testing.expect(!pathIsPublic("/symbols"));
     try std.testing.expect(!pathIsPublic("/_edgar/tickers_funds"));
     try std.testing.expect(!pathIsPublic("/0000320193/entity_facts"));
+    // Diagnostics leaks the operator's tracked set and cache layout, so it must
+    // stay gated. It needs no entry in `pathIsPublic` to be gated - the default
+    // is closed - and this asserts the default rather than trusting it, because
+    // the cost of that assumption being wrong is silent disclosure.
+    try std.testing.expect(!pathIsPublic("/AAPL/diagnostics"));
+    try std.testing.expect(!pathIsPublic("/BRK.B/diagnostics"));
 }
 
 test "pathIsPublic: returns look-alikes do not slip through" {
@@ -1533,3 +1685,122 @@ test "printSymbolList: empty shows (none), non-empty joins with commas" {
     defer std.testing.allocator.free(b_out);
     try std.testing.expectEqualStrings("  lagging: NKE, AMZN\n", b_out);
 }
+
+test "collectRefreshSymbols: only .stock and .watch, keyed by priceSymbol" {
+    const a = std.testing.allocator;
+    var set = std.StringHashMap(void).init(a);
+    defer set.deinit();
+
+    const lots = [_]zfin.Lot{
+        .{ .symbol = "AAPL", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) },
+        .{ .symbol = "SPCX", .security_type = .watch, .shares = 0, .open_price = 0, .open_date = zfin.Date.fromYmd(2026, 1, 1) },
+        // Cash is not fetched, so it must not appear as tracked - reporting it
+        // would promise a refresh that never runs.
+        .{ .symbol = "USD", .security_type = .cash, .shares = 100, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) },
+        // A duplicate holding is one symbol, not two.
+        .{ .symbol = "AAPL", .security_type = .stock, .shares = 5, .open_price = 5, .open_date = zfin.Date.fromYmd(2026, 2, 1) },
+        // A CUSIP priced through a `ticker::` alias must register under the
+        // symbol a FETCH uses, not the one the statement shows - otherwise
+        // `tracked` says false for a symbol the loop refreshes every night.
+        .{ .symbol = "922908736", .ticker = "VTTHX", .security_type = .stock, .shares = 3, .open_price = 27, .open_date = zfin.Date.fromYmd(2026, 1, 1) },
+    };
+    try collectRefreshSymbols(&set, &lots);
+
+    try std.testing.expectEqual(@as(u32, 3), set.count());
+    try std.testing.expect(set.contains("AAPL"));
+    try std.testing.expect(set.contains("SPCX"));
+    try std.testing.expect(set.contains("VTTHX"));
+    try std.testing.expect(!set.contains("922908736"));
+    try std.testing.expect(!set.contains("USD"));
+}
+
+test "daysBehind: an UNTRACKED symbol behind its peers still reports the gap" {
+    // The regression this replaces: `days_behind` was read out of
+    // `zfin.freshness.Report.stale`/`far_behind`, and `scan` only emits findings
+    // for tracked symbols - untracked ones divert to `orphans`. So every
+    // untracked symbol reported 0, and an untracked symbol is precisely what this
+    // endpoint was built to expose. AGG's real numbers, observed against a live
+    // cache copy: five days behind, reported as current.
+    try std.testing.expectEqual(@as(i64, 5), daysBehind(
+        zfin.Date.fromYmd(2026, 8, 6),
+        zfin.Date.fromYmd(2026, 8, 11),
+    ));
+    // SPCX's real gap.
+    try std.testing.expectEqual(@as(i64, 43), daysBehind(
+        zfin.Date.fromYmd(2026, 6, 29),
+        zfin.Date.fromYmd(2026, 8, 11),
+    ));
+}
+
+test "daysBehind: not behind, or incomparable, is zero rather than negative" {
+    const d = zfin.Date.fromYmd(2026, 8, 11);
+    // Level with peers.
+    try std.testing.expectEqual(@as(i64, 0), daysBehind(d, d));
+    // AHEAD of peers - this symbol IS the peer maximum. Must not report a
+    // negative gap, which would sort as "most behind" in any worst-first list.
+    try std.testing.expectEqual(@as(i64, 0), daysBehind(d, zfin.Date.fromYmd(2026, 8, 1)));
+    // No cached bar, and no peer to compare against: unanswerable, not zero-ish.
+    // Callers distinguish these from "current" via the null `last_date`.
+    try std.testing.expectEqual(@as(i64, 0), daysBehind(null, d));
+    try std.testing.expectEqual(@as(i64, 0), daysBehind(d, null));
+}
+
+test "groupPeerDate: an inconclusive group has no peer date" {
+    const d = zfin.Date.fromYmd(2026, 8, 11);
+    // `dated = 1` is the symbol itself and nothing else. Returning its own date
+    // as `peer_date` would read as "agrees with its peers" when there are none.
+    const lonely = [_]zfin.freshness.GroupState{.{
+        .kind = .equity,
+        .peer_date = d,
+        .freshness = null,
+        .dated = 1,
+    }};
+    var report = zfin.freshness.Report{
+        .stale = &.{},
+        .far_behind = &.{},
+        .orphans = &.{},
+        .missing = &.{},
+        .groups = @constCast(lonely[0..]),
+    };
+    try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .equity));
+
+    // Two dated entries make a comparison possible.
+    const peers = [_]zfin.freshness.GroupState{.{
+        .kind = .equity,
+        .peer_date = d,
+        .freshness = null,
+        .dated = 2,
+    }};
+    report.groups = @constCast(peers[0..]);
+    try std.testing.expectEqual(@as(?zfin.Date, d), groupPeerDate(report, .equity));
+
+    // A kind with no group at all is not an error, just unanswerable.
+    try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .mutual_fund));
+}
+
+test "collectRefreshSymbols: keys stay readable while the source lots live" {
+    // Guards the bug this shipped with: the handler parsed the portfolio in an
+    // inner scope with a `defer portfolio.deinit()`, so by the time it asked
+    // `tracked.contains(symbol)` the keys pointed at released memory and all 25
+    // tracked symbols reported false. `zfin`'s own `trackedSymbols` carries a
+    // comment about the identical failure - "made every cached symbol look
+    // untracked" - which is what makes it worth a test rather than a comment.
+    //
+    // The invariant is ownership, not content: the set BORROWS from `lots`, so a
+    // lookup is only valid while `lots` is alive. Asserting a hit after the
+    // insert-scope has closed is the cheapest way to pin that.
+    const a = std.testing.allocator;
+    var set = std.StringHashMap(void).init(a);
+    defer set.deinit();
+
+    const lots = [_]zfin.Lot{
+        .{ .symbol = "AMZN", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) },
+    };
+    {
+        // A nested scope that ends before the lookup, mirroring the handler's
+        // shape. `lots` outlives it, so the keys remain valid.
+        try collectRefreshSymbols(&set, &lots);
+    }
+    try std.testing.expect(set.contains("AMZN"));
+    try std.testing.expectEqual(@as(u32, 1), set.count());
+}