diff --git a/src/service.zig b/src/service.zig index 82c28f5..f324d2c 100644 --- a/src/service.zig +++ b/src/service.zig @@ -630,8 +630,8 @@ pub const DataService = struct { // wall-clock required: stamp the candle-meta freshness boundary // (market-aware next post-close / NAV-availability time). const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds(); - const expires = market.nextCandleExpiry(now_s, market.classify(symbol)); - s.cacheCandles(symbol, triple.candles, .tiingo, 0, expires); + const kind = market.classify(symbol); + s.cacheCandles(symbol, triple.candles, .tiingo, 0, expiryAfterFetch(now_s, kind, triple.candles)); } // Dividends and splits use the supplement write path: Tiingo's // view merges into existing (typically Polygon-sourced) records @@ -965,8 +965,11 @@ pub const DataService = struct { if (s.read(self.allocator, Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } else { - // Append new candles to existing file + update meta, reset fail_count - s.appendCandles(symbol, new_candles, result.provider, 0, expires); + // Append new candles to existing file + update meta, reset fail_count. + // TTL via `expiryAfterFetch`, NOT the precomputed + // next-boundary `expires`: getting a bar back does not + // mean getting the RIGHT bar back. + s.appendCandles(symbol, new_candles, result.provider, 0, expiryAfterFetch(now_s, kind, new_candles)); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| { self.allocator.free(new_candles); return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; @@ -3233,6 +3236,40 @@ pub const DataService = struct { return best; } + /// TTL to stamp after a fetch that DID return candles. + /// + /// A successful fetch is not necessarily a caught-up one. An incremental + /// request asks for everything after the last cached bar, so it can return + /// the bar missed in a PREVIOUS session while the current session's bar is + /// still unposted - a success by every measure the fetch itself can see. + /// + /// Stamping the next-boundary TTL there writes the session off after a + /// partial catch-up, and for a symbol whose provider posts later than + /// `provider_lag_grace_s` that never converges: each session it recovers the + /// previous session's bar and is immediately a session behind again. + /// Observed on AMZN and NKE, which sat exactly one session behind for days + /// while the bar had been sitting at the provider the whole time - + /// + /// Fri 18:36 zero bars, 99min past grace -> next boundary (Mon 16:55) + /// Mon 17:00 returned FRIDAY's bar -> next boundary (Tue 16:55) <- here + /// Tue 16:55 returns MONDAY's bar -> next boundary (Wed 16:55) + /// + /// `staleCandleExpiry` asks the question the zero-bar branch already asks - + /// is the newest bar we now hold the one that should be available? - and + /// returns the next boundary when it is, so a genuinely caught-up fetch + /// behaves exactly as before. + /// + /// Takes the maximum rather than the last element: provider ordering is not + /// something this decision should depend on. + fn expiryAfterFetch(now_s: i64, kind: market.InstrumentKind, candles: []const Candle) i64 { + if (candles.len == 0) return market.nextCandleExpiry(now_s, kind); + var newest = candles[0].date; + for (candles[1..]) |c| { + if (newest.lessThan(c.date)) newest = c.date; + } + return market.staleCandleExpiry(now_s, kind, newest); + } + fn syncCandlesFromServer(self: *DataService, symbol: []const u8) bool { const daily = self.syncFromServer(symbol, .candles_daily); const meta = self.syncFromServer(symbol, .candles_meta); @@ -5047,3 +5084,57 @@ test "serverBarRegression: blocks a regression and reports both dates" { try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "NOLOCAL", older)); try std.testing.expectEqual(@as(?@TypeOf(reg), null), DataService.serverBarRegression(&s, "AAPL", "#!srfv1\n")); } + +test "expiryAfterFetch: a partial catch-up keeps retrying instead of writing off the session" { + // THE REGRESSION THIS GUARDS, with Monday's real numbers. AMZN was fetched + // Mon 2026-08-10 at 17:00 ET holding Thursday's bar, and the incremental + // request returned FRIDAY's. One new candle - a success - so the old code + // stamped the next-boundary TTL and stopped looking until Tue 16:55, while + // Monday's bar showed up at the provider minutes later. Repeat daily and the + // symbol is permanently one session behind. + const kind = market.InstrumentKind.equity; + // Mon 2026-08-10 17:00 ET, five minutes past the 16:55 equity target. + const mon_1700 = Date.fromYmd(2026, 8, 10).toEpoch() + 21 * std.time.s_per_hour; + + var friday_only = [_]Candle{.{ .date = Date.fromYmd(2026, 8, 7), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 }}; + const partial = DataService.expiryAfterFetch(mon_1700, kind, &friday_only); + try std.testing.expectEqual(mon_1700 + market.short_retry_s, partial); + try std.testing.expect(partial != market.nextCandleExpiry(mon_1700, kind)); + + // And the case that must NOT change: a fetch that actually caught up gets + // the full next-boundary TTL exactly as before. + var through_monday = [_]Candle{ + .{ .date = Date.fromYmd(2026, 8, 7), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 }, + .{ .date = Date.fromYmd(2026, 8, 10), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 }, + }; + try std.testing.expectEqual( + market.nextCandleExpiry(mon_1700, kind), + DataService.expiryAfterFetch(mon_1700, kind, &through_monday), + ); +} + +test "expiryAfterFetch: takes the maximum, not the last element" { + // Provider ordering is not something this decision should depend on. + const kind = market.InstrumentKind.equity; + const mon_1700 = Date.fromYmd(2026, 8, 10).toEpoch() + 21 * std.time.s_per_hour; + var descending = [_]Candle{ + .{ .date = Date.fromYmd(2026, 8, 10), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 }, + .{ .date = Date.fromYmd(2026, 8, 7), .open = 1, .high = 1, .low = 1, .close = 1, .adj_close = 1, .volume = 1 }, + }; + // Newest is 08-10 even though it is first, so this is caught up. + try std.testing.expectEqual( + market.nextCandleExpiry(mon_1700, kind), + DataService.expiryAfterFetch(mon_1700, kind, &descending), + ); +} + +test "expiryAfterFetch: an empty slice falls back to the next boundary" { + // Both call sites are guarded by a non-empty check; this keeps the helper + // safe if a third one is ever added. + const kind = market.InstrumentKind.equity; + const mon_1700 = Date.fromYmd(2026, 8, 10).toEpoch() + 21 * std.time.s_per_hour; + try std.testing.expectEqual( + market.nextCandleExpiry(mon_1700, kind), + DataService.expiryAfterFetch(mon_1700, kind, &.{}), + ); +}