nag when data is stale

TODO.md

@@ -62,43 +62,6 @@ lists others collapsed. Reconsider the interaction model — e.g. allow
 specifying an expiration date, showing all monthlies expanded by default,
 or filtering by strategy (covered calls, spreads).
 
-## Risk-free rate maintenance
-
-T-bill rates are hardcoded in `src/analytics/risk.zig` as a year-by-year table
-(source: FRED series DTB3). Each trailing period uses the average rate over its
-date range. The table includes update instructions as doc comments.
-
-**Action needed annually:** Update the current year's rate mid-year, finalize
-the prior year's rate in January. See the curl commands in the `tbill_rates`
-doc comment.
-
-## CLI/TUI code review (lower priority)
-
-No review has been done on these files. They are presentation-layer code
-and not part of the reusable library API.
-
-TUI:
-- `src/tui.zig`
-- `src/tui/chart.zig`
-- `src/tui/keybinds.zig`
-- `src/tui/theme.zig`
-
-Commands:
-- `src/commands/common.zig`
-- `src/commands/analysis.zig`
-- `src/commands/cache.zig`
-- `src/commands/divs.zig`
-- `src/commands/earnings.zig`
-- `src/commands/enrich.zig`
-- `src/commands/etf.zig`
-- `src/commands/history.zig`
-- `src/commands/lookup.zig`
-- `src/commands/options.zig`
-- `src/commands/perf.zig`
-- `src/commands/portfolio.zig`
-- `src/commands/quote.zig`
-- `src/commands/splits.zig`
-
 ## TUI: toggle to last symbol keybind
 
 Add a single-key toggle that flips between the current symbol and the
@@ -306,17 +269,29 @@ Until then the constant is the single knob.
 
 ## Torn SRF files from server sync (recurring bug)
 
-**Status:** First-layer fix done — 2026-05-02. `syncFromServer`
+**Status:** Multi-layer defense in place.
-(`src/service.zig`) now validates responses via
-`cache.Store.looksCompleteSrf` before `writeRaw`. Torn HTTP bodies
-(empty, missing `#!srfv1` header, or no trailing newline) are
-rejected with a warn-level log and NOT written to cache. Next fetch
-will try the provider fallback.
 
-**Remaining work (if it comes back):**
+- `syncFromServer` (`src/service.zig`) validates responses via
+  `cache.Store.looksCompleteSrf` before `writeRaw`. Torn HTTP bodies
+  (empty, missing `#!srfv1` header, or no trailing newline) are
+  rejected with a warn-level log and NOT written to cache.
+- HTTP responses are checked for an `ETag` sha256 header; on mismatch
+  we retry the request once before giving up and falling back to the
+  provider.
+- Read-path self-heal: on SRF parse failure during read, the cache
+  entry is invalidated so a subsequent refresh can repair without
+  user intervention.
+- Diagnostics: richer error capture around the sync path to pinpoint
+  where torn responses originate (HTTP transit has been the dominant
+  source so far).
 
-- HTTP-level Content-Length validation in `src/net/http.zig` to fail
+**Remaining work:**
-  closer to the source rather than at the cache write.
+
-- Read-path self-heal: on SRF parse failure during read, invalidate
+- Continue monitoring — if torn responses persist despite the etag
-  the cache entry so a subsequent refresh can repair without user
+  retry, investigate lower-level transport issues (proxy, keepalive,
-  intervention.
+  partial reads on the server side).
+
+(Content-Length validation was considered and rejected: once the
+server starts compressing response bodies, Content-Length reflects
+the compressed byte count, not the decoded payload, so it's not a
+reliable integrity check.)

src/analytics/risk.zig

@@ -30,7 +30,9 @@ pub const TrailingRisk = struct {
 
 /// Average annual 3-month T-bill rate by year (source: FRED series DTB3).
 /// Used to compute period-appropriate risk-free rates for Sharpe ratio.
-/// Update annually — last updated March 2026.
+/// Update annually — bump `tbill_rates_last_updated` below when you
+/// refresh the table. `src/data/staleness.zig` nags on stderr every
+/// invocation once it's past the annual due date (Jan 31).
 ///
 /// To update mid-year (e.g. refresh the current year's YTD average):
 ///   curl -s "https://fred.stlouisfed.org/graph/fredgraph.csv?id=DTB3&cosd=2026-01-01&coed=2026-12-31&fq=Daily" \
@@ -40,6 +42,11 @@ pub const TrailingRisk = struct {
 ///   curl -s "https://fred.stlouisfed.org/graph/fredgraph.csv?id=DTB3&cosd=2026-01-01&coed=2026-12-31&fq=Annual&fam=avg" \
 ///     | tail -1 | awk -F, '{printf "%.4f\n", $2/100}'
 ///   Then add a new entry for 2027 using the mid-year command above.
+///   Finally, bump `tbill_rates_last_updated` to today's date.
+///
+/// Registered with the staleness checker in `src/data/staleness.zig`.
+pub const tbill_rates_last_updated: Date = Date.fromYmd(2026, 3, 15);
+
 const tbill_rates = [_]struct { year: u16, rate: f64 }{
     .{ .year = 2015, .rate = 0.0005 },
     .{ .year = 2016, .rate = 0.0032 },

src/data/shiller.zig

@@ -8,12 +8,20 @@
 ///   3. File → Save As → CSV (ie_data.csv)
 ///   4. Replace src/data/ie_data.csv with the new file
 ///   5. Rebuild — build/gen_shiller.zig regenerates the data automatically
+///   6. Bump `ie_data_last_updated` below to today's date.
 ///
 /// All returns are nominal, expressed as decimals (0.12 = 12%).
 const std = @import("std");
+const Date = @import("../models/date.zig").Date;
 const generated = @import("shiller_generated");
 pub const ShillerYear = @import("shiller_year").ShillerYear;
 
+/// Last time `ie_data.csv` was refreshed. Bump this whenever you
+/// replace the CSV — drives the annual staleness nag in
+/// `src/data/staleness.zig` (nags on stderr from April 1 each year
+/// until refreshed).
+pub const ie_data_last_updated: Date = Date.fromYmd(2026, 4, 27);
+
 /// Annual returns from the Shiller dataset, generated at build time from ie_data.csv.
 pub const annual_returns: []const ShillerYear = &generated.data;
 

src/data/staleness.zig

@@ -0,0 +1,299 @@
+//! Staleness nags for hand-maintained data sources.
+//!
+//! Some datasets in this codebase require periodic human maintenance
+//! (e.g. appending the latest year's T-bill rate, replacing Shiller's
+//! `ie_data.csv`). Each such source declares a `pub const <name>_last_updated: Date`
+//! constant which a human bumps when the refresh happens.
+//!
+//! This module registers those sources and prints a warning to stderr
+//! on every `zfin` invocation once the annual refresh window opens
+//! and the data is still stale. The refresh window is expressed as a
+//! single `(month, day)` per year — the earliest date by which fresh
+//! upstream data is expected to be available — not as a rolling
+//! day-count.
+//!
+//! ### Nagging semantics
+//!
+//! Given today's date and an entry's `(due_month, due_day, last_updated)`:
+//!
+//! 1. Compute `this_years_due = Date.fromYmd(today.year(), due_month, due_day)`.
+//! 2. If `today < this_years_due` — not yet nag season, silent.
+//! 3. If `last_updated >= this_years_due` — already refreshed this cycle, silent.
+//! 4. Otherwise — nag.
+//!
+//! The nag keeps firing every invocation until the human bumps
+//! `last_updated` past `this_years_due`.
+//!
+//! ### Adding a new source
+//!
+//! 1. Add `pub const <name>_last_updated: Date = Date.fromYmd(YYYY, MM, DD);`
+//!    to the module that owns the data.
+//! 2. Append a new `StaleEntry` to `entries` below.
+//! 3. Document the refresh procedure in the owning module's doc
+//!    comment (or a `TODO` at the top).
+
+const std = @import("std");
+const Date = @import("../models/date.zig").Date;
+const risk = @import("../analytics/risk.zig");
+const shiller = @import("shiller.zig");
+
+/// A hand-maintained data source that nags once a year if it hasn't
+/// been refreshed by its annual `(due_month, due_day)`.
+pub const StaleEntry = struct {
+    /// Human-readable name surfaced in the warning.
+    name: []const u8,
+    /// The last time a human refreshed this data. Bumped by hand.
+    last_updated: Date,
+    /// Calendar month (1-12) by which fresh upstream data is expected.
+    due_month: u8,
+    /// Calendar day (1-31) within `due_month`.
+    due_day: u8,
+    /// Source file to point the user at for refresh instructions.
+    source_file: []const u8,
+};
+
+/// Registry of hand-maintained data sources in this codebase. Passed
+/// to `check` from `main.zig`; tests construct their own slices.
+pub const entries = [_]StaleEntry{
+    .{
+        .name = "T-bill risk-free rate table",
+        .last_updated = risk.tbill_rates_last_updated,
+        .due_month = 1,
+        .due_day = 31,
+        .source_file = "src/analytics/risk.zig",
+    },
+    .{
+        .name = "Shiller annual returns (ie_data.csv)",
+        .last_updated = shiller.ie_data_last_updated,
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/data/shiller.zig",
+    },
+};
+
+/// Write a warning line for each entry in `entries` that is overdue
+/// for refresh as of `today`. Writes nothing when every entry is
+/// fresh. Entries are processed in order; multiple warnings are
+/// separated by a blank line.
+pub fn check(
+    writer: *std.Io.Writer,
+    today: Date,
+    list: []const StaleEntry,
+) !void {
+    var wrote_any = false;
+    for (list) |entry| {
+        const this_years_due = Date.fromYmd(
+            today.year(),
+            entry.due_month,
+            entry.due_day,
+        );
+        // Not yet nag season.
+        if (today.lessThan(this_years_due)) continue;
+        // Already refreshed this cycle.
+        if (!entry.last_updated.lessThan(this_years_due)) continue;
+
+        if (wrote_any) try writer.writeAll("\n");
+        wrote_any = true;
+
+        var due_buf: [10]u8 = undefined;
+        var upd_buf: [10]u8 = undefined;
+        const due_str = this_years_due.format(&due_buf);
+        const upd_str = entry.last_updated.format(&upd_buf);
+
+        try writer.print(
+            "warning: {s} is overdue for refresh.\n" ++
+                "  Expected by {s}; last updated {s}.\n" ++
+                "  See {s} for refresh instructions.\n",
+            .{ entry.name, due_str, upd_str, entry.source_file },
+        );
+    }
+}
+
+// ── Tests ────────────────────────────────────────────────────
+
+test "silent before due date" {
+    var buf: [1024]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{.{
+        .name = "Test data",
+        .last_updated = Date.fromYmd(2025, 3, 15),
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/test.zig",
+    }};
+    try check(&w, Date.fromYmd(2026, 3, 31), &list);
+    try std.testing.expectEqualStrings("", w.buffered());
+}
+
+test "nag on due date when stale" {
+    var buf: [1024]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{.{
+        .name = "Test data",
+        .last_updated = Date.fromYmd(2025, 3, 15),
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/test.zig",
+    }};
+    try check(&w, Date.fromYmd(2026, 4, 1), &list);
+    const expected =
+        "warning: Test data is overdue for refresh.\n" ++
+        "  Expected by 2026-04-01; last updated 2025-03-15.\n" ++
+        "  See src/test.zig for refresh instructions.\n";
+    try std.testing.expectEqualStrings(expected, w.buffered());
+}
+
+test "nag after due date when stale" {
+    var buf: [1024]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{.{
+        .name = "Test data",
+        .last_updated = Date.fromYmd(2025, 3, 15),
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/test.zig",
+    }};
+    try check(&w, Date.fromYmd(2026, 6, 15), &list);
+    const expected =
+        "warning: Test data is overdue for refresh.\n" ++
+        "  Expected by 2026-04-01; last updated 2025-03-15.\n" ++
+        "  See src/test.zig for refresh instructions.\n";
+    try std.testing.expectEqualStrings(expected, w.buffered());
+}
+
+test "silent when already refreshed this cycle" {
+    var buf: [1024]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{.{
+        .name = "Test data",
+        .last_updated = Date.fromYmd(2026, 4, 2),
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/test.zig",
+    }};
+    try check(&w, Date.fromYmd(2026, 6, 15), &list);
+    try std.testing.expectEqualStrings("", w.buffered());
+}
+
+test "silent when last_updated equals this year's due date" {
+    var buf: [1024]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{.{
+        .name = "Test data",
+        .last_updated = Date.fromYmd(2026, 4, 1),
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/test.zig",
+    }};
+    try check(&w, Date.fromYmd(2026, 4, 1), &list);
+    try std.testing.expectEqualStrings("", w.buffered());
+}
+
+test "two-cycle stale still fires" {
+    var buf: [1024]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{.{
+        .name = "Test data",
+        .last_updated = Date.fromYmd(2024, 2, 1),
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/test.zig",
+    }};
+    try check(&w, Date.fromYmd(2026, 6, 15), &list);
+    const expected =
+        "warning: Test data is overdue for refresh.\n" ++
+        "  Expected by 2026-04-01; last updated 2024-02-01.\n" ++
+        "  See src/test.zig for refresh instructions.\n";
+    try std.testing.expectEqualStrings(expected, w.buffered());
+}
+
+test "empty list is a no-op" {
+    var buf: [16]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    try check(&w, Date.fromYmd(2030, 12, 31), &[_]StaleEntry{});
+    try std.testing.expectEqualStrings("", w.buffered());
+}
+
+test "multiple stale entries separated by blank line" {
+    var buf: [2048]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{
+        .{
+            .name = "Alpha",
+            .last_updated = Date.fromYmd(2024, 12, 1),
+            .due_month = 1,
+            .due_day = 31,
+            .source_file = "src/alpha.zig",
+        },
+        .{
+            .name = "Beta",
+            .last_updated = Date.fromYmd(2025, 3, 15),
+            .due_month = 4,
+            .due_day = 1,
+            .source_file = "src/beta.zig",
+        },
+    };
+    try check(&w, Date.fromYmd(2026, 6, 15), &list);
+    const expected =
+        "warning: Alpha is overdue for refresh.\n" ++
+        "  Expected by 2026-01-31; last updated 2024-12-01.\n" ++
+        "  See src/alpha.zig for refresh instructions.\n" ++
+        "\n" ++
+        "warning: Beta is overdue for refresh.\n" ++
+        "  Expected by 2026-04-01; last updated 2025-03-15.\n" ++
+        "  See src/beta.zig for refresh instructions.\n";
+    try std.testing.expectEqualStrings(expected, w.buffered());
+}
+
+test "fresh entry does not emit blank line before later stale entry" {
+    var buf: [2048]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{
+        .{
+            .name = "Alpha",
+            .last_updated = Date.fromYmd(2026, 2, 1), // fresh
+            .due_month = 1,
+            .due_day = 31,
+            .source_file = "src/alpha.zig",
+        },
+        .{
+            .name = "Beta",
+            .last_updated = Date.fromYmd(2025, 3, 15), // stale
+            .due_month = 4,
+            .due_day = 1,
+            .source_file = "src/beta.zig",
+        },
+    };
+    try check(&w, Date.fromYmd(2026, 6, 15), &list);
+    const expected =
+        "warning: Beta is overdue for refresh.\n" ++
+        "  Expected by 2026-04-01; last updated 2025-03-15.\n" ++
+        "  See src/beta.zig for refresh instructions.\n";
+    try std.testing.expectEqualStrings(expected, w.buffered());
+}
+
+test "silent when today is one day before due" {
+    var buf: [256]u8 = undefined;
+    var w = std.Io.Writer.fixed(&buf);
+    const list = [_]StaleEntry{.{
+        .name = "Test data",
+        .last_updated = Date.fromYmd(2020, 1, 1),
+        .due_month = 4,
+        .due_day = 1,
+        .source_file = "src/test.zig",
+    }};
+    try check(&w, Date.fromYmd(2026, 3, 31), &list);
+    try std.testing.expectEqualStrings("", w.buffered());
+}
+
+test "real registry compiles and is non-empty" {
+    // Guard that the registry stays wired up; doesn't assert any
+    // particular nag behavior (real dates drift over time).
+    try std.testing.expect(entries.len >= 2);
+    for (entries) |e| {
+        try std.testing.expect(e.name.len > 0);
+        try std.testing.expect(e.source_file.len > 0);
+        try std.testing.expect(e.due_month >= 1 and e.due_month <= 12);
+        try std.testing.expect(e.due_day >= 1 and e.due_day <= 31);
+    }
+}

src/main.zig

@@ -273,6 +273,21 @@ fn runCli() !u8 {
         return 1;
     }
 
+    // Nag on stderr when hand-maintained data sources are overdue for
+    // refresh (T-bill rates, Shiller ie_data.csv). See
+    // src/data/staleness.zig for the registry and rules. Runs here —
+    // after globals parse, before command dispatch — so the warning
+    // lands above command output on every CLI and TUI invocation.
+    {
+        const staleness = @import("data/staleness.zig");
+        const Date = @import("models/date.zig").Date;
+        var stale_buf: [2048]u8 = undefined;
+        var stale_writer = std.fs.File.stderr().writer(&stale_buf);
+        const today = Date.fromEpoch(std.time.timestamp());
+        staleness.check(&stale_writer.interface, today, &staleness.entries) catch {};
+        stale_writer.interface.flush() catch {};
+    }
+
     const color = @import("format.zig").shouldUseColor(globals.no_color);
 
     const command = args[globals.cursor];