more measured fallback and logging

src/analytics/performance.zig

@@ -62,14 +62,14 @@ fn totalReturnFromAdjCloseSnap(candles: []const Candle, from: Date, to: Date, st
 /// Compute total return with manual dividend reinvestment.
 /// Uses raw close prices and dividend records independently.
 /// Candles and dividends must be sorted by date ascending.
-/// `from` snaps forward, `to` snaps backward.
+/// `from` snaps backward (last trading day on/before), `to` snaps backward.
 pub fn totalReturnWithDividends(
     candles: []const Candle,
     dividends: []const Dividend,
     from: Date,
     to: Date,
 ) ?PerformanceResult {
-    return totalReturnWithDividendsSnap(candles, dividends, from, to, .forward);
+    return totalReturnWithDividendsSnap(candles, dividends, from, to, .backward);
 }
 
 /// Same as totalReturnWithDividends but both dates snap backward.
@@ -105,7 +105,7 @@ fn totalReturnWithDividendsSnap(
 
     for (dividends) |div| {
         if (div.ex_date.lessThan(start.date)) continue;
-        if (end.date.lessThan(div.ex_date)) break;
+        if (end.date.lessThan(div.ex_date)) continue;
 
         // Find close price on or near the ex-date.
         // For stable-NAV funds, dividends before candle history use $1.
@@ -619,3 +619,48 @@ test "withFallback -- both null stays null" {
     try std.testing.expect(merged.one_year == null);
     try std.testing.expect(merged.ten_year == null);
 }
+
+test "splits-only adj_close -- dividend reinvestment preferred" {
+    // Simulates a TwelveData-like provider where adj_close only accounts for splits.
+    // Stock pays $2/quarter dividend, price stays at ~$100.
+    // adj_close shows no return (splits-only), but dividend reinvestment should
+    // show the correct total return from the distributions.
+    const candles = [_]Candle{
+        // adj_close == close here (no splits), so adj_close return ≈ 0%
+        makeCandle(Date.fromYmd(2025, 1, 2), 100),
+        makeCandle(Date.fromYmd(2025, 3, 15), 100),
+        makeCandle(Date.fromYmd(2025, 6, 15), 100),
+        makeCandle(Date.fromYmd(2025, 9, 15), 100),
+        makeCandle(Date.fromYmd(2025, 12, 31), 100),
+    };
+    const divs = [_]Dividend{
+        .{ .ex_date = Date.fromYmd(2025, 3, 15), .amount = 2.0 },
+        .{ .ex_date = Date.fromYmd(2025, 6, 15), .amount = 2.0 },
+        .{ .ex_date = Date.fromYmd(2025, 9, 15), .amount = 2.0 },
+        .{ .ex_date = Date.fromYmd(2025, 12, 15), .amount = 2.0 },
+    };
+
+    // adj_close return should be ~0% (price flat, no dividend adjustment)
+    const adj_result = totalReturnFromAdjClose(&candles, Date.fromYmd(2025, 1, 2), Date.fromYmd(2025, 12, 31));
+    try std.testing.expect(adj_result != null);
+    try std.testing.expectApproxEqAbs(@as(f64, 0.0), adj_result.?.total_return, 0.01);
+
+    // Dividend reinvestment: 3 of 4 dividends are reinvested (Dec 15 is >10 days
+    // from any candle so it's skipped by snap tolerance). ~6.12% total return.
+    const div_result = totalReturnWithDividends(&candles, &divs, Date.fromYmd(2025, 1, 2), Date.fromYmd(2025, 12, 31));
+    try std.testing.expect(div_result != null);
+    try std.testing.expect(div_result.?.total_return > 0.05);
+
+    // When adj_close is splits-only, dividend reinvestment should be primary.
+    // Wrapping in TrailingReturns to test withFallback:
+    const adj_tr: TrailingReturns = .{ .one_year = adj_result };
+    const div_tr: TrailingReturns = .{ .one_year = div_result };
+
+    // withFallback(div, adj) keeps div_result where available
+    const total = withFallback(div_tr, adj_tr);
+    try std.testing.expect(total.one_year.?.total_return > 0.05);
+
+    // Reversed order (adj_close primary) would give ~0% — wrong for splits-only
+    const wrong = withFallback(adj_tr, div_tr);
+    try std.testing.expectApproxEqAbs(@as(f64, 0.0), wrong.one_year.?.total_return, 0.01);
+}

src/cache/store.zig

@@ -215,7 +215,7 @@ pub const Store = struct {
 
     /// Write a full set of candles to cache (no expiry — historical facts don't expire).
     /// Also updates candle metadata.
-    pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle) void {
+    pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8) void {
         if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
             defer self.allocator.free(srf_data);
             self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
@@ -227,14 +227,14 @@ pub const Store = struct {
 
         if (candles.len > 0) {
             const last = candles[candles.len - 1];
-            self.updateCandleMeta(symbol, last.close, last.date);
+            self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count);
         }
     }
 
     /// Append new candle records to the existing cache file.
     /// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
     /// Also updates candle metadata.
-    pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle) void {
+    pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8) void {
         if (new_candles.len == 0) return;
 
         if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
@@ -263,21 +263,17 @@ pub const Store = struct {
         }
 
         const last = new_candles[new_candles.len - 1];
-        self.updateCandleMeta(symbol, last.close, last.date);
+        self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count);
     }
 
-    /// Write (or refresh) candle metadata without touching the candle data file.
-    pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date) void {
-        self.updateCandleMetaWithProvider(symbol, last_close, last_date, .twelvedata);
-    }
-
-    /// Write candle metadata with a specific provider source.
-    pub fn updateCandleMetaWithProvider(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider) void {
+    /// Write (or refresh) candle metadata with a specific provider source.
+    pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8) void {
         const expires = std.time.timestamp() + Ttl.candles_latest;
         const meta = CandleMeta{
             .last_close = last_close,
             .last_date = last_date,
             .provider = provider,
+            .fail_count = fail_count,
         };
         if (serializeCandleMeta(self.allocator, meta, .{ .expires = expires })) |meta_data| {
             defer self.allocator.free(meta_data);
@@ -398,8 +394,12 @@ pub const Store = struct {
         last_close: f64,
         last_date: Date,
         /// Which provider sourced the candle data. Used during incremental refresh
-        /// to go directly to the right provider instead of trying TwelveData first.
+        /// to go directly to the right provider instead of trying Tiingo first.
         provider: CandleProvider = .tiingo,
+        /// Consecutive transient failure count for the primary provider (Tiingo).
+        /// Incremented on ServerError; reset to 0 on success. When >= 3, the
+        /// symbol is degraded to a fallback provider until Tiingo recovers.
+        fail_count: u8 = 0,
     };
 
     pub const CandleProvider = enum {

src/service.zig

@@ -39,6 +39,11 @@ pub const DataError = error{
     CacheError,
     ParseError,
     OutOfMemory,
+    /// Transient provider failure (server error, connection issue).
+    /// Caller should stop and retry later.
+    TransientError,
+    /// Provider auth failure (bad API key). Entire refresh should stop.
+    AuthError,
 };
 
 /// Re-exported provider types needed by commands via DataService.
@@ -263,85 +268,108 @@ pub const DataService = struct {
 
     // ── Public data methods ──────────────────────────────────────
 
-    /// Fetch candles from providers with fallback logic.
-    /// Tries the provider recorded in meta (if any), then Tiingo (primary), then TwelveData, then Yahoo.
-    /// Returns the candles and which provider succeeded.
+    /// Fetch candles from providers with error classification.
+    ///
+    /// Error handling:
+    ///   - ServerError/RateLimited/RequestFailed from Tiingo → TransientError (stop refresh, retry later)
+    ///   - NotFound/ParseError/InvalidResponse from Tiingo → try Yahoo (symbol-level issue)
+    ///   - Unauthorized → TransientError (config problem, stop refresh)
+    ///
+    /// The `preferred` param controls incremental fetch consistency: use the same
+    /// provider that sourced the existing cache data.
     fn fetchCandlesFromProviders(
         self: *DataService,
         symbol: []const u8,
         from: Date,
         to: Date,
         preferred: cache.Store.CandleProvider,
-    ) !struct { candles: []Candle, provider: cache.Store.CandleProvider } {
-        // If preferred is Yahoo, try it first
+    ) (DataError || error{NotFound})!struct { candles: []Candle, provider: cache.Store.CandleProvider } {
+        // If preferred is Yahoo (degraded symbol), try Yahoo first
         if (preferred == .yahoo) {
             if (self.getProvider(Yahoo)) |yh| {
                 if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
                     log.debug("{s}: candles from Yahoo (preferred)", .{symbol});
                     return .{ .candles = candles, .provider = .yahoo };
-                } else |_| {}
-            } else |_| {}
-        }
-
-        // If preferred is TwelveData, try it before Tiingo
-        if (preferred == .twelvedata) {
-            if (self.getProvider(TwelveData)) |td| {
-                if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
-                    log.debug("{s}: candles from TwelveData (preferred)", .{symbol});
-                    return .{ .candles = candles, .provider = .twelvedata };
                 } else |err| {
-                    if (err == error.RateLimited) {
-                        self.rateLimitBackoff();
-                        if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
-                            log.debug("{s}: candles from TwelveData (preferred, after rate limit retry)", .{symbol});
-                            return .{ .candles = candles, .provider = .twelvedata };
-                        } else |_| {}
-                    }
+                    log.warn("{s}: Yahoo (preferred) failed: {s}", .{ symbol, @errorName(err) });
                 }
             } else |_| {}
         }
 
-        // Primary: Tiingo (1000 req/day, no per-minute limit)
+        // Primary: Tiingo
         if (self.getProvider(Tiingo)) |tg| {
             if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
                 log.debug("{s}: candles from Tiingo", .{symbol});
                 return .{ .candles = candles, .provider = .tiingo };
-            } else |_| {}
-        } else |_| {}
-
-        // Fallback: TwelveData (if not already tried as preferred)
-        if (preferred != .twelvedata) {
-            if (self.getProvider(TwelveData)) |td| {
-                if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
-                    log.debug("{s}: candles from TwelveData (fallback)", .{symbol});
-                    return .{ .candles = candles, .provider = .twelvedata };
             } else |err| {
-                    if (err == error.RateLimited) {
-                        self.rateLimitBackoff();
-                        if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
-                            log.debug("{s}: candles from TwelveData (fallback, after rate limit retry)", .{symbol});
-                            return .{ .candles = candles, .provider = .twelvedata };
-                        } else |_| {}
-                    }
-                }
-            } else |_| {}
+                log.warn("{s}: Tiingo failed: {s}", .{ symbol, @errorName(err) });
+
+                if (err == error.Unauthorized) {
+                    log.err("{s}: Tiingo auth failed — check TIINGO_API_KEY", .{symbol});
+                    return DataError.AuthError;
                 }
 
-        // Last resort: Yahoo (if not already tried as preferred)
+                if (err == error.RateLimited) {
+                    // Rate limited: back off and retry — this is expected, not a failure
+                    log.info("{s}: Tiingo rate limited, backing off", .{symbol});
+                    self.rateLimitBackoff();
+                    if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
+                        log.debug("{s}: candles from Tiingo (after rate limit backoff)", .{symbol});
+                        return .{ .candles = candles, .provider = .tiingo };
+                    } else |retry_err| {
+                        log.warn("{s}: Tiingo retry after backoff failed: {s}", .{ symbol, @errorName(retry_err) });
+                        if (retry_err == error.RateLimited) {
+                            // Still rate limited after backoff — one more try
+                            self.rateLimitBackoff();
+                            if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
+                                log.debug("{s}: candles from Tiingo (after second backoff)", .{symbol});
+                                return .{ .candles = candles, .provider = .tiingo };
+                            } else |_| {}
+                        }
+                        // Exhausted rate limit retries — treat as transient
+                        return DataError.TransientError;
+                    }
+                }
+
+                if (isTransientError(err)) {
+                    // Server error or connection failure — stop, don't fall back
+                    return DataError.TransientError;
+                }
+
+                // NotFound, ParseError, InvalidResponse — symbol-level issue, try Yahoo
+                log.info("{s}: Tiingo does not have this symbol, trying Yahoo", .{symbol});
+            }
+        } else |_| {
+            log.warn("{s}: Tiingo provider not available (no API key?)", .{symbol});
+        }
+
+        // Fallback: Yahoo (symbol not on Tiingo)
         if (preferred != .yahoo) {
             if (self.getProvider(Yahoo)) |yh| {
                 if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
-                    log.debug("{s}: candles from Yahoo (fallback)", .{symbol});
+                    log.info("{s}: candles from Yahoo (Tiingo fallback)", .{symbol});
                     return .{ .candles = candles, .provider = .yahoo };
-                } else |_| {}
-            } else |_| {}
+                } else |err| {
+                    log.warn("{s}: Yahoo fallback also failed: {s}", .{ symbol, @errorName(err) });
+                }
+            } else |_| {
+                log.warn("{s}: Yahoo provider not available", .{symbol});
+            }
         }
 
-        return error.FetchFailed;
+        return DataError.FetchFailed;
+    }
+
+    /// Classify whether a provider error is transient (provider is down).
+    /// ServerError = HTTP 5xx, RequestFailed = connection/network failure.
+    /// Note: RateLimited and Unauthorized are handled separately.
+    fn isTransientError(err: anyerror) bool {
+        return err == error.ServerError or
+            err == error.RequestFailed;
     }
 
     /// Fetch daily candles for a symbol (10+ years for trailing returns).
-    /// Checks cache first; fetches from Tiingo (primary), TwelveData, or Yahoo if stale/missing.
+    /// Checks cache first; fetches from Tiingo (primary) or Yahoo (fallback) if stale/missing.
     /// Uses incremental updates: when the cache is stale, only fetches
     /// candles newer than the last cached date rather than re-fetching
     /// the entire history.
@@ -353,13 +381,17 @@ pub const DataService = struct {
         const meta_result = s.readCandleMeta(symbol);
         if (meta_result) |mr| {
             const m = mr.meta;
-            if (s.isCandleMetaFresh(symbol)) {
+
+            // If cached data is from TwelveData (deprecated for candles due to
+            // unreliable adj_close), skip cache and fall through to full re-fetch.
+            if (m.provider == .twelvedata) {
+                log.debug("{s}: cached candles from TwelveData — forcing full re-fetch", .{symbol});
+            } else if (s.isCandleMetaFresh(symbol)) {
                 // Fresh — deserialize candles and return
                 log.debug("{s}: candles fresh in local cache", .{symbol});
                 if (s.read(Candle, symbol, null, .any)) |r|
                     return .{ .data = r.data, .source = .cached, .timestamp = mr.created };
-            }
-
+            } else {
                 // Stale — try server sync before incremental fetch
                 if (self.syncCandlesFromServer(symbol)) {
                     if (s.isCandleMetaFresh(symbol)) {
@@ -368,7 +400,6 @@ pub const DataService = struct {
                             return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
                     }
                     log.debug("{s}: candles synced from server but stale, falling through to incremental fetch", .{symbol});
-                // Server data also stale — fall through to incremental fetch
                 }
 
                 // Stale — try incremental update using last_date from meta
@@ -376,13 +407,27 @@ pub const DataService = struct {
 
                 // If last cached date is today or later, just refresh the TTL (meta only)
                 if (!fetch_from.lessThan(today)) {
-                s.updateCandleMeta(symbol, m.last_close, m.last_date);
+                    s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count);
                     if (s.read(Candle, symbol, null, .any)) |r|
                         return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
                 } else {
                     // Incremental fetch from day after last cached candle
-                const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch {
-                    // All providers failed — return stale data
+                    const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch |err| {
+                        if (err == DataError.TransientError) {
+                            // Increment fail_count for this symbol
+                            const new_fail_count = m.fail_count +| 1; // saturating add
+                            log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count });
+                            s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count);
+
+                            // If degraded (fail_count >= 3), return stale data rather than failing
+                            if (new_fail_count >= 3) {
+                                log.warn("{s}: degraded after {d} consecutive failures, returning stale data", .{ symbol, new_fail_count });
+                                if (s.read(Candle, symbol, null, .any)) |r|
+                                    return .{ .data = r.data, .source = .cached, .timestamp = mr.created };
+                            }
+                            return DataError.TransientError;
+                        }
+                        // Non-transient failure — return stale data if available
                         if (s.read(Candle, symbol, null, .any)) |r|
                             return .{ .data = r.data, .source = .cached, .timestamp = mr.created };
                         return DataError.FetchFailed;
@@ -390,24 +435,23 @@ pub const DataService = struct {
                     const new_candles = result.candles;
 
                     if (new_candles.len == 0) {
-                    // No new candles (weekend/holiday) — refresh TTL only (meta rewrite)
+                        // No new candles (weekend/holiday) — refresh TTL, reset fail_count
                         self.allocator.free(new_candles);
-                    s.updateCandleMetaWithProvider(symbol, m.last_close, m.last_date, result.provider);
+                        s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0);
                         if (s.read(Candle, symbol, null, .any)) |r|
                             return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
                     } else {
-                    // Append new candles to existing file + update meta
-                    s.appendCandles(symbol, new_candles);
-                    // Load the full (now-updated) file for the caller
+                        // Append new candles to existing file + update meta, reset fail_count
+                        s.appendCandles(symbol, new_candles, result.provider, 0);
                         if (s.read(Candle, symbol, null, .any)) |r| {
                             self.allocator.free(new_candles);
                             return .{ .data = r.data, .source = .fetched, .timestamp = std.time.timestamp() };
                         }
-                    // Append failed or file unreadable — just return new candles
                         return .{ .data = new_candles, .source = .fetched, .timestamp = std.time.timestamp() };
                     }
                 }
             }
+        }
 
         // No usable cache — try server sync first
         if (self.syncCandlesFromServer(symbol)) {
@@ -417,23 +461,27 @@ pub const DataService = struct {
                     return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
             }
             log.debug("{s}: candles synced from server but stale, falling through to full fetch", .{symbol});
-            // Server data also stale — fall through to full fetch
         }
 
         // No usable cache — full fetch (~10 years, plus buffer for leap years)
         log.debug("{s}: fetching full candle history from provider", .{symbol});
         const from = today.addDays(-3700);
 
-        const result = self.fetchCandlesFromProviders(symbol, from, today, .tiingo) catch {
+        const result = self.fetchCandlesFromProviders(symbol, from, today, .tiingo) catch |err| {
+            if (err == DataError.TransientError) {
+                // On a fresh fetch, increment fail_count if we have meta
+                if (meta_result) |mr| {
+                    const new_fail_count = mr.meta.fail_count +| 1;
+                    s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count);
+                }
+                return DataError.TransientError;
+            }
             s.writeNegative(symbol, .candles_daily);
             return DataError.FetchFailed;
         };
 
         if (result.candles.len > 0) {
-            s.cacheCandles(symbol, result.candles);
-            // Record which provider sourced this data
-            const last = result.candles[result.candles.len - 1];
-            s.updateCandleMetaWithProvider(symbol, last.close, last.date, result.provider);
+            s.cacheCandles(symbol, result.candles, result.provider, 0); // reset fail_count on success
         }
 
         return .{ .data = result.candles, .source = .fetched, .timestamp = std.time.timestamp() };
@@ -596,10 +644,9 @@ pub const DataService = struct {
 
         if (self.getDividends(symbol)) |div_result| {
             divs = div_result.data;
-            // adj_close returns are the primary total return source (accounts for
-            // splits + dividends). Dividend-reinvestment is only used as a fallback
-            // for periods where adj_close returns null (e.g. candle history too short
-            // for stable-NAV funds like money markets).
+            // adj_close is the primary total return source (accounts for splits +
+            // dividends). Dividend-reinvestment only fills gaps where adj_close
+            // returns null (e.g. stable-NAV funds with short candle history).
             const asof_div = performance.trailingReturnsWithDividends(c, div_result.data);
             asof_total = performance.withFallback(asof_price, asof_div);
             const me_div = performance.trailingReturnsMonthEndWithDividends(c, div_result.data, today);