From c65a034bf0a8ac35ff8350de22cbf59925791431 Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Wed, 4 Nov 2026 09:04:35 -0800 Subject: [PATCH] serve stale data for up to 2h if upstream fails --- src/cache/Cache.zig | 93 +++++++++++++++++++++--- src/cache/Lru.zig | 42 ++++++++++- src/http/handler.zig | 149 ++++++++++++++++++++++++++++++++++++++- src/main.zig | 2 + src/weather/MetNo.zig | 8 +++ src/weather/Mock.zig | 3 + src/weather/Provider.zig | 80 ++++++++++++++++++++- 7 files changed, 361 insertions(+), 16 deletions(-) diff --git a/src/cache/Cache.zig b/src/cache/Cache.zig index 176b758..8d63a26 100644 --- a/src/cache/Cache.zig +++ b/src/cache/Cache.zig @@ -16,6 +16,18 @@ cache_dir: ?[]const u8, pub const Config = struct { max_entries: usize = 1_000, cache_dir: ?[]const u8, + /// How long entries survive past their TTL, in memory and on disk, so + /// `getStale` can serve them while the upstream is unavailable. `get` is + /// unaffected: it only ever returns fresh entries. + stale_retention_s: u64 = 0, +}; + +/// An entry returned by `getStale`. `expires` lets the caller say how old the +/// data it is serving is. +pub const Entry = struct { + value: []const u8, + /// Expiry time, in milliseconds since the epoch. In the past for stale data. + expires: i64, }; pub fn init(allocator: std.mem.Allocator, io: std.Io, config: Config) !*Cache { @@ -34,6 +46,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, config: Config) !*Cache { .cache_dir = if (config.cache_dir) |d| try allocator.dupe(u8, d) else null, }; + cache.lru.retention_ms = @intCast(config.stale_retention_s * std.time.ms_per_s); cache.lru.setEvictionCallback(cache, evictCallback); // Clean up expired files and populate L1 cache from L2 @@ -51,9 +64,26 @@ fn evictCallback(ctx: *anyopaque, key: []const u8) void { self.deleteFile(key); } +/// Returns the value only if it has not expired. pub fn get(self: *Cache, key: []const u8) ?[]const u8 { - // Check L1 (memory) first - if (self.lru.get(key)) |value| return value; + const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds(); + const entry = self.lookup(key, now) orelse return null; + if (now > entry.expires) return null; + return entry.value; +} + +/// Returns the value whether fresh or expired-but-retained. Meant as a +/// fallback when refreshing an expired entry has failed. +pub fn getStale(self: *Cache, key: []const u8) ?Entry { + const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds(); + return self.lookup(key, now); +} + +fn lookup(self: *Cache, key: []const u8, now_ms: i64) ?Entry { + // Check L1 (memory) first. A retained-but-expired L1 entry is the newest + // copy there is, so it also short-circuits the disk read. + if (self.lru.lookup(key, now_ms)) |item| + return .{ .value = item.value, .expires = item.expires }; // Check L2 (disk) const cached = self.loadFromFile(key) catch return null; @@ -62,7 +92,8 @@ pub fn get(self: *Cache, key: []const u8) ?[]const u8 { // L2 exists - promote to L1 self.lru.put(key, cached.value, cached.expires) catch return null; - return self.lru.get(key); + const item = self.lru.lookup(key, now_ms) orelse return null; + return .{ .value = item.value, .expires = item.expires }; } pub fn put(self: *Cache, key: []const u8, value: []const u8, ttl_seconds: u64) !void { @@ -107,8 +138,8 @@ const CacheEntry = struct { }; /// Loads cached value from file (path calculated by the key). An error will be thrown -/// if the file access fails OR if the data has expired. -/// If the data has expired, the file will be deleted +/// if the file access fails OR if the data is past its retention window, in +/// which case the file is deleted. Expired-but-retained entries are returned. fn loadFromFile(self: *Cache, key: []const u8) !CacheEntry { if (self.cache_dir == null) return error.NoCacheDir; @@ -122,8 +153,8 @@ fn loadFromFile(self: *Cache, key: []const u8) !CacheEntry { } /// Loads cached value from file (using a path). An error will be thrown -/// if the file access fails OR if the data has expired. -/// If the data has expired, the file will be deleted +/// if the file access fails OR if the data is past its retention window, in +/// which case the file is deleted. Expired-but-retained entries are returned. fn loadFromFilePath(self: *Cache, file_path: []const u8) !CacheEntry { const file = try std.Io.Dir.cwd().openFile(self.io, file_path, .{}); defer file.close(self.io); @@ -135,10 +166,10 @@ fn loadFromFilePath(self: *Cache, file_path: []const u8) !CacheEntry { const cached = try deserialize(self.allocator, reader); errdefer cached.deinit(self.allocator); - // Check if expired + // Check if past retention. Merely expired entries are kept: they are the + // fallback `getStale` serves while the upstream is down. const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds(); - if (cached.expires <= now) { - // We're expired, delete expired file + if (cached.expires + self.lru.retention_ms <= now) { self.deleteFile(cached.key); return error.Expired; } @@ -373,3 +404,45 @@ test "L1/L2 cache flow" { // Now it should be in L1 try std.testing.expectEqualStrings("value1", cache.lru.get("key1").?); } + +test "stale entries are retained for getStale but never returned by get" { + const allocator = std.testing.allocator; + + var tmp_dir = std.testing.tmpDir(.{}); + defer tmp_dir.cleanup(); + + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const cache_dir_len = try tmp_dir.dir.realPath(std.testing.io, &path_buf); + const cache_dir = path_buf[0..cache_dir_len]; + + const cache = try Cache.init(allocator, std.testing.io, .{ + .max_entries = 10, + .cache_dir = cache_dir, + .stale_retention_s = 3600, + }); + defer cache.deinit(); + + const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds(); + + // Expired a minute ago, still inside the hour of retention. Written only to + // L2 so the disk path is what gets exercised, as it is after a restart. + try cache.saveToFile("recent", "old-but-usable", now - 60_000); + try std.testing.expect(cache.get("recent") == null); + const stale = cache.getStale("recent").?; + try std.testing.expectEqualStrings("old-but-usable", stale.value); + try std.testing.expectEqual(now - 60_000, stale.expires); + // Now promoted to L1, and still not fresh from there either + try std.testing.expect(cache.get("recent") == null); + try std.testing.expectEqualStrings("old-but-usable", cache.getStale("recent").?.value); + + // Expired two hours ago: past retention, unusable even as stale, and the + // file is cleaned up. + try cache.saveToFile("ancient", "too-old", now - 2 * 3600 * 1000); + try std.testing.expect(cache.getStale("ancient") == null); + try std.testing.expectError(error.FileNotFound, cache.loadFromFile("ancient")); + + // Fresh entries come back from both + try cache.put("fresh", "new", 900); + try std.testing.expectEqualStrings("new", cache.get("fresh").?); + try std.testing.expectEqualStrings("new", cache.getStale("fresh").?.value); +} diff --git a/src/cache/Lru.zig b/src/cache/Lru.zig index 473d57f..b057f75 100644 --- a/src/cache/Lru.zig +++ b/src/cache/Lru.zig @@ -7,6 +7,10 @@ allocator: std.mem.Allocator, io: std.Io, map: std.StringHashMap(Entry), max_entries: usize, +/// How long an entry is kept after it expires, so it can still be served as a +/// fallback when the upstream is down. `get` never returns such an entry; only +/// `lookup` does. Zero drops entries as soon as they expire. +retention_ms: i64 = 0, evict_fn: ?*const fn (ctx: *anyopaque, key: []const u8) void = null, evict_ctx: ?*anyopaque = null, @@ -31,16 +35,25 @@ pub fn setEvictionCallback(self: *Lru, ctx: *anyopaque, callback: *const fn (ctx } pub fn get(self: *Lru, key: []const u8) ?[]const u8 { + const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds(); + const item = self.lookup(key, now) orelse return null; + if (now > item.expires) return null; + return item.value; +} + +/// Returns the entry whether fresh or expired-but-retained, leaving the +/// freshness decision to the caller. Entries past the retention window are +/// dropped. +pub fn lookup(self: *Lru, key: []const u8, now_ms: i64) ?Iterator.Item { var entry = self.map.getPtr(key) orelse return null; - const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds(); - if (now > entry.expires) { + if (now_ms > entry.expires + self.retention_ms) { self.remove(key); return null; } entry.access_count += 1; - return entry.value; + return .{ .key = key, .value = entry.value, .expires = entry.expires }; } pub fn put(self: *Lru, key: []const u8, value: []const u8, expires: i64) !void { @@ -154,4 +167,27 @@ test "LRU expired entry returns null" { // Get should return null for expired entry const result = lru.get("key1"); try std.testing.expect(result == null); + // And with no retention it is gone entirely + try std.testing.expect(lru.lookup("key1", now) == null); +} + +test "LRU retains expired entries for lookup within the retention window" { + var lru = try Lru.init(std.testing.allocator, std.testing.io, 10); + defer lru.deinit(); + lru.retention_ms = 60_000; + + const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds(); + try lru.put("recent", "v1", now - 1000); + try lru.put("ancient", "v2", now - 120_000); + + // Expired, so never served as fresh... + try std.testing.expect(lru.get("recent") == null); + // ...but still available to a caller willing to take stale data + const item = lru.lookup("recent", now).?; + try std.testing.expectEqualStrings("v1", item.value); + try std.testing.expectEqual(now - 1000, item.expires); + + // Past the window it is dropped + try std.testing.expect(lru.lookup("ancient", now) == null); + try std.testing.expect(!lru.map.contains("ancient")); } diff --git a/src/http/handler.zig b/src/http/handler.zig index 989b4e7..7c9032b 100644 --- a/src/http/handler.zig +++ b/src/http/handler.zig @@ -83,6 +83,18 @@ pub fn handleWeather( res.body = help.translation_page; return; } + // Any other `:` name is an endpoint we do not have, not a place + res.status = 404; + res.content_type = .TEXT; + res.body = "Unknown endpoint\n"; + return; + } + if (isJunkLocation(location)) { + log.debug("rejecting junk location {s}", .{location}); + res.status = 404; + res.content_type = .TEXT; + res.body = "Not found\n"; + return; } try handleWeatherInternal(opts, req, res, location, client_ip); @@ -117,6 +129,11 @@ fn handleWeatherInternal( res.body = "Location not found (location query resolution failure)\n"; return; }, + error.GeocodingFailed => { + log.warn("geocoding unavailable for query {s}", .{location_query}); + upstreamUnavailable(res, "Location lookup is temporarily unavailable"); + return; + }, else => return err, } }; @@ -130,7 +147,14 @@ fn handleWeatherInternal( res.body = "Location not found (provider fetch failure)\n"; return; }, - else => return err, + error.OutOfMemory => return err, + // Anything else is the upstream failing (status, network, TLS, or + // a malformed response) with no usable cached data to fall back on + else => { + log.warn("weather unavailable for {d:.4},{d:.4}: {}", .{ location.coords.latitude, location.coords.longitude, err }); + upstreamUnavailable(res, "Weather data is temporarily unavailable"); + return; + }, } }; defer weather.deinit(); @@ -184,6 +208,45 @@ fn handleWeatherInternal( try renderWeatherData(res.writer(), weather, params, render_options, now_unix_s); } +/// A dependency we cannot work without is down. That is a 503 the client can +/// retry, not a 500 implying we are broken. +fn upstreamUnavailable(res: *httpz.Response, comptime message: []const u8) void { + res.status = 503; + res.content_type = .TEXT; + res.header("Retry-After", "60"); + res.body = message ++ ", please try again shortly\n"; +} + +/// File extensions that only ever show up in requests from scanners probing +/// for files. No place name ends in one, and letting them through costs a +/// Nominatim lookup each time (failures are not cached) plus possibly a +/// weather fetch. +const junk_extensions = [_][]const u8{ + "7z", "asp", "aspx", "bak", "bz2", "cfg", "cgi", "conf", + "config", "css", "env", "gif", "gz", "htm", "html", "ico", + "ini", "jpeg", "jpg", "js", "json", "jsp", "log", "php", + "png", "rar", "sql", "svg", "tar", "tgz", "txt", "webp", + "xml", "yaml", "yml", "zip", +}; + +/// Whether a location is a probe for something other than weather. Runs after +/// the endpoints we do serve (favicon.ico, robots.txt, ...) are handled. +fn isJunkLocation(location: []const u8) bool { + // `@domain` lookups legitimately end in what looks like an extension + if (location.len == 0 or location[0] == '@') return false; + const name = if (location[0] == '~') location[1..] else location; + if (name.len == 0) return true; + // Dotfiles: .env, .git, .aws, ... + if (name[0] == '.') return true; + // WordPress probes: wp-login.php is caught below, wp-admin is not + if (std.ascii.startsWithIgnoreCase(name, "wp-")) return true; + const dot = std.mem.lastIndexOfScalar(u8, name, '.') orelse return false; + const extension = name[dot + 1 ..]; + for (junk_extensions) |junk| + if (std.ascii.eqlIgnoreCase(extension, junk)) return true; + return false; +} + fn renderWeatherData( writer: *std.Io.Writer, weather: types.WeatherData, @@ -624,3 +687,87 @@ test "handler: an explicit unit request still overrides the pin's country" { try ht.expectStatus(200); try ht.expectBody("☀️ +20°C\n"); } + +test "isJunkLocation: rejects scanner probes" { + const junk = [_][]const u8{ + "sitemap.xml.gz", "wp-login.php", "wp-admin", "WP-Admin", ".env", + ".git", "config.json", "backup.sql", "index.HTML", "apple-touch-icon.png", + "ads.txt", "~", "~shell.php", + }; + for (junk) |location| { + std.testing.expect(isJunkLocation(location)) catch |err| { + std.debug.print("expected junk: {s}\n", .{location}); + return err; + }; + } +} + +test "isJunkLocation: lets real locations through" { + const real = [_][]const u8{ + "London", "St. Louis", "St.Louis", "Washington.DC", "48.85,2.35", + "50.39.107.54", "2001:db8::1", "@github.com", "@wttr.in", "~Eiffel Tower", + "Mt. Hood", + "Москва", + "PDX", "", + }; + for (real) |location| { + std.testing.expect(!isJunkLocation(location)) catch |err| { + std.debug.print("expected real: {s}\n", .{location}); + return err; + }; + } +} + +test "handler: junk paths and unknown endpoints are 404 without a lookup" { + const allocator = std.testing.allocator; + const MockHarness = @import("Server.zig").MockHarness; + + var harness = try MockHarness.init(allocator); + defer harness.deinit(); + // Any lookup reaching the provider would fail loudly with a 503 instead + harness.mock.fail_with = error.WeatherApiFailed; + + const cases = [_][]const u8{ "sitemap.xml.gz", ":", ":bogus", ".env", "wp-admin" }; + for (cases) |location| { + var ht = httpz.testing.init(.{}); + defer ht.deinit(); + + const url = try std.fmt.allocPrint(allocator, "/{s}", .{location}); + defer allocator.free(url); + ht.url(url); + ht.param("location", location); + + try handleWeather(&harness.opts, ht.req, ht.res, "127.0.0.1"); + try ht.expectStatus(404); + } +} + +test "handler: upstream weather failure with nothing cached is a retryable 503" { + const allocator = std.testing.allocator; + const MockHarness = @import("Server.zig").MockHarness; + + var harness = try MockHarness.init(allocator); + defer harness.deinit(); + harness.mock.fail_with = error.WeatherApiFailed; + + // The harness cache directory is shared across tests and runs, so other + // tests' responses for these coordinates would be served from it. An empty + // in-memory cache guarantees there is nothing to fall back on. + const Cache = @import("../cache/Cache.zig"); + const empty_cache = try Cache.init(allocator, std.testing.io, .{ + .cache_dir = null, + .stale_retention_s = WeatherProvider.stale_retention_s, + }); + defer empty_cache.deinit(); + harness.opts.provider = harness.mock.provider(empty_cache); + var ht = httpz.testing.init(.{}); + defer ht.deinit(); + + ht.url("/73.158.64.1?format=1"); + ht.param("location", "73.158.64.1"); + + try handleWeather(&harness.opts, ht.req, ht.res, "73.158.64.1"); + + try ht.expectStatus(503); + try ht.expectHeader("Retry-After", "60"); +} diff --git a/src/main.zig b/src/main.zig index 8ea251d..ca0dd58 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2,6 +2,7 @@ const std = @import("std"); const Config = @import("Config.zig"); const Cache = @import("cache/Cache.zig"); const MetNo = @import("weather/MetNo.zig"); +const WeatherProvider = @import("weather/Provider.zig"); const Server = @import("http/Server.zig"); const RateLimiter = @import("http/RateLimiter.zig"); const GeoIp = @import("location/GeoIp.zig"); @@ -104,6 +105,7 @@ fn serve( const cache = try Cache.init(allocator, io, .{ .max_entries = cfg.cache_size, .cache_dir = cfg.cache_dir, + .stale_retention_s = WeatherProvider.stale_retention_s, }); defer cache.deinit(); diff --git a/src/weather/MetNo.zig b/src/weather/MetNo.zig index 59fc59f..947b1cc 100644 --- a/src/weather/MetNo.zig +++ b/src/weather/MetNo.zig @@ -147,6 +147,14 @@ fn fetchRaw(ptr: *anyopaque, allocator: std.mem.Allocator, coords: Coordinates) }); if (result.status != .ok) { + const body = response_buf[0..writer.end]; + std.log.warn("met.no returned HTTP {d} ({s}) for {s} with User-Agent '{s}': {s}", .{ + @intFromEnum(result.status), + result.status.phrase() orelse "unknown", + url, + user_agent, + body[0..@min(body.len, 512)], + }); return error.WeatherApiFailed; } diff --git a/src/weather/Mock.zig b/src/weather/Mock.zig index a38b172..1fa6996 100644 --- a/src/weather/Mock.zig +++ b/src/weather/Mock.zig @@ -9,6 +9,8 @@ const Mock = @This(); allocator: std.mem.Allocator, responses: std.StringHashMap([]const u8), parse_fn: ?*const fn (ptr: *anyopaque, allocator: std.mem.Allocator, raw: []const u8) anyerror!types.WeatherData = null, +/// When set, every fetch fails with this error, simulating an upstream outage. +fail_with: ?anyerror = null, pub fn init(allocator: std.mem.Allocator) !Mock { return Mock{ @@ -37,6 +39,7 @@ pub fn addResponse(self: *Mock, coords: Coordinates, raw_json: []const u8) !void fn fetchRaw(ptr: *anyopaque, allocator: std.mem.Allocator, coords: Coordinates) ![]const u8 { const self: *Mock = @ptrCast(@alignCast(ptr)); + if (self.fail_with) |err| return err; const key = try std.fmt.allocPrint(allocator, "{d:.4},{d:.4}", .{ coords.latitude, coords.longitude }); defer allocator.free(key); diff --git a/src/weather/Provider.zig b/src/weather/Provider.zig index 744b6f8..c8f309b 100644 --- a/src/weather/Provider.zig +++ b/src/weather/Provider.zig @@ -11,6 +11,14 @@ const Cache = @import("../cache/Cache.zig"); const WeatherProvider = @This(); +const log = std.log.scoped(.weather); + +/// How long past its TTL a cached response may still be served when the +/// upstream fails. With the 16-33 minute TTL below, the oldest data ever served +/// is about two hours old; beyond that "current" conditions are fiction. +/// Configure the weather cache with this as `Cache.Config.stale_retention_s`. +pub const stale_retention_s: u64 = 90 * 60; + ptr: *anyopaque, vtable: *const VTable, cache: *Cache, @@ -30,8 +38,20 @@ pub fn fetch(self: WeatherProvider, allocator: std.mem.Allocator, coords: Coordi if (self.cache.get(cache_key)) |cached_raw| return self.vtable.parse(self.ptr, allocator, cached_raw); - // Cache miss - fetch raw data from provider - const raw = try self.vtable.fetchRaw(self.ptr, allocator, coords); + // Cache miss - fetch raw data from provider, falling back to an expired + // entry if the upstream is down + const raw = self.vtable.fetchRaw(self.ptr, allocator, coords) catch |err| { + if (self.cache.getStale(cache_key)) |stale| { + const now_ms = std.Io.Timestamp.now(self.cache.io, .real).toMilliseconds(); + log.warn("weather fetch for {s} failed ({}); serving cached data that expired {d}s ago", .{ + cache_key, + err, + @divTrunc(now_ms - stale.expires, std.time.ms_per_s), + }); + return self.vtable.parse(self.ptr, allocator, stale.value); + } + return err; + }; defer allocator.free(raw); // TTL: 1000-2000 seconds (16-33 minutes) to avoid thundering herd. @@ -79,3 +99,59 @@ test "provider fetch" { try std.testing.expect(weather1.forecast.len > 0); try std.testing.expect(weather2.forecast.len > 0); } + +test "provider serves expired cached data when the upstream fails" { + const Mock = @import("Mock.zig"); + const MetNo = @import("MetNo.zig"); + + const cache = try Cache.init(std.testing.allocator, std.testing.io, .{ + .max_entries = 10, + .cache_dir = null, + .stale_retention_s = stale_retention_s, + }); + defer cache.deinit(); + + var mock = try Mock.init(std.testing.allocator); + defer mock.deinit(); + mock.parse_fn = MetNo.parse; + mock.fail_with = error.WeatherApiFailed; + + const provider = mock.provider(cache); + const coords = Coordinates{ .latitude = 33.4484, .longitude = -112.0740 }; + + // Nothing cached: the upstream error surfaces unchanged + try std.testing.expectError(error.WeatherApiFailed, provider.fetch(std.testing.allocator, coords)); + + // A response that expired ten minutes ago is better than nothing + const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds(); + try cache.lru.put("33.4484,-112.0740", @embedFile("../tests/metno_test_data.json"), now - 10 * 60 * 1000); + + const weather = try provider.fetch(std.testing.allocator, coords); + defer weather.deinit(); + try std.testing.expect(weather.forecast.len > 0); +} + +test "provider does not serve data past the stale retention window" { + const Mock = @import("Mock.zig"); + const MetNo = @import("MetNo.zig"); + + const cache = try Cache.init(std.testing.allocator, std.testing.io, .{ + .max_entries = 10, + .cache_dir = null, + .stale_retention_s = stale_retention_s, + }); + defer cache.deinit(); + + var mock = try Mock.init(std.testing.allocator); + defer mock.deinit(); + mock.parse_fn = MetNo.parse; + mock.fail_with = error.WeatherApiFailed; + + const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds(); + const too_old = now - @as(i64, @intCast(stale_retention_s * std.time.ms_per_s)) - 1000; + try cache.lru.put("33.4484,-112.0740", @embedFile("../tests/metno_test_data.json"), too_old); + + const provider = mock.provider(cache); + const coords = Coordinates{ .latitude = 33.4484, .longitude = -112.0740 }; + try std.testing.expectError(error.WeatherApiFailed, provider.fetch(std.testing.allocator, coords)); +}