ensure that new candles do not end up with an assumption of successful refresh

src/service.zig

@@ -630,8 +630,8 @@ pub const DataService = struct {
             // wall-clock required: stamp the candle-meta freshness boundary
             // (market-aware next post-close / NAV-availability time).
             const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
-            const expires = market.nextCandleExpiry(now_s, market.classify(symbol));
-            s.cacheCandles(symbol, triple.candles, .tiingo, 0, expires);
+            const kind = market.classify(symbol);
+            s.cacheCandles(symbol, triple.candles, .tiingo, 0, expiryAfterFetch(now_s, kind, triple.candles));
         }
         // Dividends and splits use the supplement write path: Tiingo's
         // view merges into existing (typically Polygon-sourced) records
@@ -965,8 +965,11 @@ pub const DataService = struct {
                         if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
                             return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
                     } else {
-                        // Append new candles to existing file + update meta, reset fail_count
-                        s.appendCandles(symbol, new_candles, result.provider, 0, expires);
+                        // Append new candles to existing file + update meta, reset fail_count.
+                        // TTL via `expiryAfterFetch`, NOT the precomputed
+                        // next-boundary `expires`: getting a bar back does not
+                        // mean getting the RIGHT bar back.
+                        s.appendCandles(symbol, new_candles, result.provider, 0, expiryAfterFetch(now_s, kind, new_candles));
                         if (s.read(self.allocator, Candle, symbol, null, .any)) |r| {
                             self.allocator.free(new_candles);
                             return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
@@ -3233,6 +3236,40 @@ pub const DataService = struct {
         return best;
     }
 
+    /// TTL to stamp after a fetch that DID return candles.
+    ///
+    /// A successful fetch is not necessarily a caught-up one. An incremental
+    /// request asks for everything after the last cached bar, so it can return
+    /// the bar missed in a PREVIOUS session while the current session's bar is
+    /// still unposted - a success by every measure the fetch itself can see.
+    ///
+    /// Stamping the next-boundary TTL there writes the session off after a
+    /// partial catch-up, and for a symbol whose provider posts later than
+    /// `provider_lag_grace_s` that never converges: each session it recovers the
+    /// previous session's bar and is immediately a session behind again.
+    /// Observed on AMZN and NKE, which sat exactly one session behind for days
+    /// while the bar had been sitting at the provider the whole time -
+    ///
+    ///   Fri 18:36  zero bars, 99min past grace  -> next boundary (Mon 16:55)
+    ///   Mon 17:00  returned FRIDAY's bar        -> next boundary (Tue 16:55)  <- here
+    ///   Tue 16:55  returns MONDAY's bar         -> next boundary (Wed 16:55)
+    ///
+    /// `staleCandleExpiry` asks the question the zero-bar branch already asks -
+    /// is the newest bar we now hold the one that should be available? - and
+    /// returns the next boundary when it is, so a genuinely caught-up fetch
+    /// behaves exactly as before.
+    ///
+    /// Takes the maximum rather than the last element: provider ordering is not
+    /// something this decision should depend on.
+    fn expiryAfterFetch(now_s: i64, kind: market.InstrumentKind, candles: []const Candle) i64 {
+        if (candles.len == 0) return market.nextCandleExpiry(now_s, kind);
+        var newest = candles[0].date;
+        for (candles[1..]) |c| {
+            if (newest.lessThan(c.date)) newest = c.date;
+        }
+        return market.staleCandleExpiry(now_s, kind, newest);
+    }
+
     fn syncCandlesFromServer(self: *DataService, symbol: []const u8) bool {
         const daily = self.syncFromServer(symbol, .candles_daily);
         const meta = self.syncFromServer(symbol, .candles_meta);
@@ -5047,3 +5084,57 @@ test "serverBarRegression: blocks a regression and reports both dates" {
     try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "NOLOCAL", older));
     try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", "#!srfv1\n"));
 }
+
+test "expiryAfterFetch: a partial catch-up keeps retrying instead of writing off the session" {
+    // THE REGRESSION THIS GUARDS, with Monday's real numbers. AMZN was fetched
+    // Mon 2026-08-10 at 17:00 ET holding Thursday's bar, and the incremental
+    // request returned FRIDAY's. One new candle - a success - so the old code
+    // stamped the next-boundary TTL and stopped looking until Tue 16:55, while
+    // Monday's bar showed up at the provider minutes later. Repeat daily and the
+    // symbol is permanently one session behind.
+    const kind = market.InstrumentKind.equity;
+    // Mon 2026-08-10 17:00 ET, five minutes past the 16:55 equity target.
+    const mon_1700 = Date.fromYmd(2026, 8, 10).toEpoch() + 21 * std.time.s_per_hour;
+
+    var friday_only = [_]Candle{.{ .date = Date.fromYmd(2026, 8, 7), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 }};
+    const partial = DataService.expiryAfterFetch(mon_1700, kind, &friday_only);
+    try std.testing.expectEqual(mon_1700 + market.short_retry_s, partial);
+    try std.testing.expect(partial != market.nextCandleExpiry(mon_1700, kind));
+
+    // And the case that must NOT change: a fetch that actually caught up gets
+    // the full next-boundary TTL exactly as before.
+    var through_monday = [_]Candle{
+        .{ .date = Date.fromYmd(2026, 8, 7), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 },
+        .{ .date = Date.fromYmd(2026, 8, 10), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 },
+    };
+    try std.testing.expectEqual(
+        market.nextCandleExpiry(mon_1700, kind),
+        DataService.expiryAfterFetch(mon_1700, kind, &through_monday),
+    );
+}
+
+test "expiryAfterFetch: takes the maximum, not the last element" {
+    // Provider ordering is not something this decision should depend on.
+    const kind = market.InstrumentKind.equity;
+    const mon_1700 = Date.fromYmd(2026, 8, 10).toEpoch() + 21 * std.time.s_per_hour;
+    var descending = [_]Candle{
+        .{ .date = Date.fromYmd(2026, 8, 10), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 },
+        .{ .date = Date.fromYmd(2026, 8, 7), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 },
+    };
+    // Newest is 08-10 even though it is first, so this is caught up.
+    try std.testing.expectEqual(
+        market.nextCandleExpiry(mon_1700, kind),
+        DataService.expiryAfterFetch(mon_1700, kind, &descending),
+    );
+}
+
+test "expiryAfterFetch: an empty slice falls back to the next boundary" {
+    // Both call sites are guarded by a non-empty check; this keeps the helper
+    // safe if a third one is ever added.
+    const kind = market.InstrumentKind.equity;
+    const mon_1700 = Date.fromYmd(2026, 8, 10).toEpoch() + 21 * std.time.s_per_hour;
+    try std.testing.expectEqual(
+        market.nextCandleExpiry(mon_1700, kind),
+        DataService.expiryAfterFetch(mon_1700, kind, &.{}),
+    );
+}