serve stale data for up to 2h if upstream fails

src/cache/Cache.zig

@@ -16,6 +16,18 @@ cache_dir: ?[]const u8,
 pub const Config = struct {
     max_entries: usize = 1_000,
     cache_dir: ?[]const u8,
+    /// How long entries survive past their TTL, in memory and on disk, so
+    /// `getStale` can serve them while the upstream is unavailable. `get` is
+    /// unaffected: it only ever returns fresh entries.
+    stale_retention_s: u64 = 0,
+};
+
+/// An entry returned by `getStale`. `expires` lets the caller say how old the
+/// data it is serving is.
+pub const Entry = struct {
+    value: []const u8,
+    /// Expiry time, in milliseconds since the epoch. In the past for stale data.
+    expires: i64,
 };
 
 pub fn init(allocator: std.mem.Allocator, io: std.Io, config: Config) !*Cache {
@@ -34,6 +46,7 @@ pub fn init(allocator: std.mem.Allocator, io: std.Io, config: Config) !*Cache {
         .cache_dir = if (config.cache_dir) |d| try allocator.dupe(u8, d) else null,
     };
 
+    cache.lru.retention_ms = @intCast(config.stale_retention_s * std.time.ms_per_s);
     cache.lru.setEvictionCallback(cache, evictCallback);
 
     // Clean up expired files and populate L1 cache from L2
@@ -51,9 +64,26 @@ fn evictCallback(ctx: *anyopaque, key: []const u8) void {
     self.deleteFile(key);
 }
 
+/// Returns the value only if it has not expired.
 pub fn get(self: *Cache, key: []const u8) ?[]const u8 {
-    // Check L1 (memory) first
-    if (self.lru.get(key)) |value| return value;
+    const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds();
+    const entry = self.lookup(key, now) orelse return null;
+    if (now > entry.expires) return null;
+    return entry.value;
+}
+
+/// Returns the value whether fresh or expired-but-retained. Meant as a
+/// fallback when refreshing an expired entry has failed.
+pub fn getStale(self: *Cache, key: []const u8) ?Entry {
+    const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds();
+    return self.lookup(key, now);
+}
+
+fn lookup(self: *Cache, key: []const u8, now_ms: i64) ?Entry {
+    // Check L1 (memory) first. A retained-but-expired L1 entry is the newest
+    // copy there is, so it also short-circuits the disk read.
+    if (self.lru.lookup(key, now_ms)) |item|
+        return .{ .value = item.value, .expires = item.expires };
 
     // Check L2 (disk)
     const cached = self.loadFromFile(key) catch return null;
@@ -62,7 +92,8 @@ pub fn get(self: *Cache, key: []const u8) ?[]const u8 {
     // L2 exists - promote to L1
     self.lru.put(key, cached.value, cached.expires) catch return null;
 
-    return self.lru.get(key);
+    const item = self.lru.lookup(key, now_ms) orelse return null;
+    return .{ .value = item.value, .expires = item.expires };
 }
 
 pub fn put(self: *Cache, key: []const u8, value: []const u8, ttl_seconds: u64) !void {
@@ -107,8 +138,8 @@ const CacheEntry = struct {
 };
 
 /// Loads cached value from file (path calculated by the key). An error will be thrown
-/// if the file access fails OR if the data has expired.
-/// If the data has expired, the file will be deleted
+/// if the file access fails OR if the data is past its retention window, in
+/// which case the file is deleted. Expired-but-retained entries are returned.
 fn loadFromFile(self: *Cache, key: []const u8) !CacheEntry {
     if (self.cache_dir == null) return error.NoCacheDir;
 
@@ -122,8 +153,8 @@ fn loadFromFile(self: *Cache, key: []const u8) !CacheEntry {
 }
 
 /// Loads cached value from file (using a path). An error will be thrown
-/// if the file access fails OR if the data has expired.
-/// If the data has expired, the file will be deleted
+/// if the file access fails OR if the data is past its retention window, in
+/// which case the file is deleted. Expired-but-retained entries are returned.
 fn loadFromFilePath(self: *Cache, file_path: []const u8) !CacheEntry {
     const file = try std.Io.Dir.cwd().openFile(self.io, file_path, .{});
     defer file.close(self.io);
@@ -135,10 +166,10 @@ fn loadFromFilePath(self: *Cache, file_path: []const u8) !CacheEntry {
     const cached = try deserialize(self.allocator, reader);
     errdefer cached.deinit(self.allocator);
 
-    // Check if expired
+    // Check if past retention. Merely expired entries are kept: they are the
+    // fallback `getStale` serves while the upstream is down.
     const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds();
-    if (cached.expires <= now) {
-        // We're expired, delete expired file
+    if (cached.expires + self.lru.retention_ms <= now) {
         self.deleteFile(cached.key);
         return error.Expired;
     }
@@ -373,3 +404,45 @@ test "L1/L2 cache flow" {
     // Now it should be in L1
     try std.testing.expectEqualStrings("value1", cache.lru.get("key1").?);
 }
+
+test "stale entries are retained for getStale but never returned by get" {
+    const allocator = std.testing.allocator;
+
+    var tmp_dir = std.testing.tmpDir(.{});
+    defer tmp_dir.cleanup();
+
+    var path_buf: [std.fs.max_path_bytes]u8 = undefined;
+    const cache_dir_len = try tmp_dir.dir.realPath(std.testing.io, &path_buf);
+    const cache_dir = path_buf[0..cache_dir_len];
+
+    const cache = try Cache.init(allocator, std.testing.io, .{
+        .max_entries = 10,
+        .cache_dir = cache_dir,
+        .stale_retention_s = 3600,
+    });
+    defer cache.deinit();
+
+    const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds();
+
+    // Expired a minute ago, still inside the hour of retention. Written only to
+    // L2 so the disk path is what gets exercised, as it is after a restart.
+    try cache.saveToFile("recent", "old-but-usable", now - 60_000);
+    try std.testing.expect(cache.get("recent") == null);
+    const stale = cache.getStale("recent").?;
+    try std.testing.expectEqualStrings("old-but-usable", stale.value);
+    try std.testing.expectEqual(now - 60_000, stale.expires);
+    // Now promoted to L1, and still not fresh from there either
+    try std.testing.expect(cache.get("recent") == null);
+    try std.testing.expectEqualStrings("old-but-usable", cache.getStale("recent").?.value);
+
+    // Expired two hours ago: past retention, unusable even as stale, and the
+    // file is cleaned up.
+    try cache.saveToFile("ancient", "too-old", now - 2 * 3600 * 1000);
+    try std.testing.expect(cache.getStale("ancient") == null);
+    try std.testing.expectError(error.FileNotFound, cache.loadFromFile("ancient"));
+
+    // Fresh entries come back from both
+    try cache.put("fresh", "new", 900);
+    try std.testing.expectEqualStrings("new", cache.get("fresh").?);
+    try std.testing.expectEqualStrings("new", cache.getStale("fresh").?.value);
+}

src/cache/Lru.zig

@@ -7,6 +7,10 @@ allocator: std.mem.Allocator,
 io: std.Io,
 map: std.StringHashMap(Entry),
 max_entries: usize,
+/// How long an entry is kept after it expires, so it can still be served as a
+/// fallback when the upstream is down. `get` never returns such an entry; only
+/// `lookup` does. Zero drops entries as soon as they expire.
+retention_ms: i64 = 0,
 evict_fn: ?*const fn (ctx: *anyopaque, key: []const u8) void = null,
 evict_ctx: ?*anyopaque = null,
 
@@ -31,16 +35,25 @@ pub fn setEvictionCallback(self: *Lru, ctx: *anyopaque, callback: *const fn (ctx
 }
 
 pub fn get(self: *Lru, key: []const u8) ?[]const u8 {
+    const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds();
+    const item = self.lookup(key, now) orelse return null;
+    if (now > item.expires) return null;
+    return item.value;
+}
+
+/// Returns the entry whether fresh or expired-but-retained, leaving the
+/// freshness decision to the caller. Entries past the retention window are
+/// dropped.
+pub fn lookup(self: *Lru, key: []const u8, now_ms: i64) ?Iterator.Item {
     var entry = self.map.getPtr(key) orelse return null;
 
-    const now = std.Io.Timestamp.now(self.io, .real).toMilliseconds();
-    if (now > entry.expires) {
+    if (now_ms > entry.expires + self.retention_ms) {
         self.remove(key);
         return null;
     }
 
     entry.access_count += 1;
-    return entry.value;
+    return .{ .key = key, .value = entry.value, .expires = entry.expires };
 }
 
 pub fn put(self: *Lru, key: []const u8, value: []const u8, expires: i64) !void {
@@ -154,4 +167,27 @@ test "LRU expired entry returns null" {
     // Get should return null for expired entry
     const result = lru.get("key1");
     try std.testing.expect(result == null);
+    // And with no retention it is gone entirely
+    try std.testing.expect(lru.lookup("key1", now) == null);
+}
+
+test "LRU retains expired entries for lookup within the retention window" {
+    var lru = try Lru.init(std.testing.allocator, std.testing.io, 10);
+    defer lru.deinit();
+    lru.retention_ms = 60_000;
+
+    const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds();
+    try lru.put("recent", "v1", now - 1000);
+    try lru.put("ancient", "v2", now - 120_000);
+
+    // Expired, so never served as fresh...
+    try std.testing.expect(lru.get("recent") == null);
+    // ...but still available to a caller willing to take stale data
+    const item = lru.lookup("recent", now).?;
+    try std.testing.expectEqualStrings("v1", item.value);
+    try std.testing.expectEqual(now - 1000, item.expires);
+
+    // Past the window it is dropped
+    try std.testing.expect(lru.lookup("ancient", now) == null);
+    try std.testing.expect(!lru.map.contains("ancient"));
 }

src/http/handler.zig

@@ -83,6 +83,18 @@ pub fn handleWeather(
             res.body = help.translation_page;
             return;
         }
+        // Any other `:` name is an endpoint we do not have, not a place
+        res.status = 404;
+        res.content_type = .TEXT;
+        res.body = "Unknown endpoint\n";
+        return;
+    }
+    if (isJunkLocation(location)) {
+        log.debug("rejecting junk location {s}", .{location});
+        res.status = 404;
+        res.content_type = .TEXT;
+        res.body = "Not found\n";
+        return;
     }
 
     try handleWeatherInternal(opts, req, res, location, client_ip);
@@ -117,6 +129,11 @@ fn handleWeatherInternal(
                 res.body = "Location not found (location query resolution failure)\n";
                 return;
             },
+            error.GeocodingFailed => {
+                log.warn("geocoding unavailable for query {s}", .{location_query});
+                upstreamUnavailable(res, "Location lookup is temporarily unavailable");
+                return;
+            },
             else => return err,
         }
     };
@@ -130,7 +147,14 @@ fn handleWeatherInternal(
                 res.body = "Location not found (provider fetch failure)\n";
                 return;
             },
-            else => return err,
+            error.OutOfMemory => return err,
+            // Anything else is the upstream failing (status, network, TLS, or
+            // a malformed response) with no usable cached data to fall back on
+            else => {
+                log.warn("weather unavailable for {d:.4},{d:.4}: {}", .{ location.coords.latitude, location.coords.longitude, err });
+                upstreamUnavailable(res, "Weather data is temporarily unavailable");
+                return;
+            },
         }
     };
     defer weather.deinit();
@@ -184,6 +208,45 @@ fn handleWeatherInternal(
     try renderWeatherData(res.writer(), weather, params, render_options, now_unix_s);
 }
 
+/// A dependency we cannot work without is down. That is a 503 the client can
+/// retry, not a 500 implying we are broken.
+fn upstreamUnavailable(res: *httpz.Response, comptime message: []const u8) void {
+    res.status = 503;
+    res.content_type = .TEXT;
+    res.header("Retry-After", "60");
+    res.body = message ++ ", please try again shortly\n";
+}
+
+/// File extensions that only ever show up in requests from scanners probing
+/// for files. No place name ends in one, and letting them through costs a
+/// Nominatim lookup each time (failures are not cached) plus possibly a
+/// weather fetch.
+const junk_extensions = [_][]const u8{
+    "7z",     "asp",  "aspx", "bak", "bz2",  "cfg", "cgi",  "conf",
+    "config", "css",  "env",  "gif", "gz",   "htm", "html", "ico",
+    "ini",    "jpeg", "jpg",  "js",  "json", "jsp", "log",  "php",
+    "png",    "rar",  "sql",  "svg", "tar",  "tgz", "txt",  "webp",
+    "xml",    "yaml", "yml",  "zip",
+};
+
+/// Whether a location is a probe for something other than weather. Runs after
+/// the endpoints we do serve (favicon.ico, robots.txt, ...) are handled.
+fn isJunkLocation(location: []const u8) bool {
+    // `@domain` lookups legitimately end in what looks like an extension
+    if (location.len == 0 or location[0] == '@') return false;
+    const name = if (location[0] == '~') location[1..] else location;
+    if (name.len == 0) return true;
+    // Dotfiles: .env, .git, .aws, ...
+    if (name[0] == '.') return true;
+    // WordPress probes: wp-login.php is caught below, wp-admin is not
+    if (std.ascii.startsWithIgnoreCase(name, "wp-")) return true;
+    const dot = std.mem.lastIndexOfScalar(u8, name, '.') orelse return false;
+    const extension = name[dot + 1 ..];
+    for (junk_extensions) |junk|
+        if (std.ascii.eqlIgnoreCase(extension, junk)) return true;
+    return false;
+}
+
 fn renderWeatherData(
     writer: *std.Io.Writer,
     weather: types.WeatherData,
@@ -624,3 +687,87 @@ test "handler: an explicit unit request still overrides the pin's country" {
     try ht.expectStatus(200);
     try ht.expectBody("☀️ +20°C\n");
 }
+
+test "isJunkLocation: rejects scanner probes" {
+    const junk = [_][]const u8{
+        "sitemap.xml.gz", "wp-login.php", "wp-admin",   "WP-Admin",   ".env",
+        ".git",           "config.json",  "backup.sql", "index.HTML", "apple-touch-icon.png",
+        "ads.txt",        "~",            "~shell.php",
+    };
+    for (junk) |location| {
+        std.testing.expect(isJunkLocation(location)) catch |err| {
+            std.debug.print("expected junk: {s}\n", .{location});
+            return err;
+        };
+    }
+}
+
+test "isJunkLocation: lets real locations through" {
+    const real = [_][]const u8{
+        "London",       "St. Louis",   "St.Louis",    "Washington.DC", "48.85,2.35",
+        "50.39.107.54", "2001:db8::1", "@github.com", "@wttr.in",      "~Eiffel Tower",
+        "Mt. Hood",
+        "Москва",
+        "PDX",          "",
+    };
+    for (real) |location| {
+        std.testing.expect(!isJunkLocation(location)) catch |err| {
+            std.debug.print("expected real: {s}\n", .{location});
+            return err;
+        };
+    }
+}
+
+test "handler: junk paths and unknown endpoints are 404 without a lookup" {
+    const allocator = std.testing.allocator;
+    const MockHarness = @import("Server.zig").MockHarness;
+
+    var harness = try MockHarness.init(allocator);
+    defer harness.deinit();
+    // Any lookup reaching the provider would fail loudly with a 503 instead
+    harness.mock.fail_with = error.WeatherApiFailed;
+
+    const cases = [_][]const u8{ "sitemap.xml.gz", ":", ":bogus", ".env", "wp-admin" };
+    for (cases) |location| {
+        var ht = httpz.testing.init(.{});
+        defer ht.deinit();
+
+        const url = try std.fmt.allocPrint(allocator, "/{s}", .{location});
+        defer allocator.free(url);
+        ht.url(url);
+        ht.param("location", location);
+
+        try handleWeather(&harness.opts, ht.req, ht.res, "127.0.0.1");
+        try ht.expectStatus(404);
+    }
+}
+
+test "handler: upstream weather failure with nothing cached is a retryable 503" {
+    const allocator = std.testing.allocator;
+    const MockHarness = @import("Server.zig").MockHarness;
+
+    var harness = try MockHarness.init(allocator);
+    defer harness.deinit();
+    harness.mock.fail_with = error.WeatherApiFailed;
+
+    // The harness cache directory is shared across tests and runs, so other
+    // tests' responses for these coordinates would be served from it. An empty
+    // in-memory cache guarantees there is nothing to fall back on.
+    const Cache = @import("../cache/Cache.zig");
+    const empty_cache = try Cache.init(allocator, std.testing.io, .{
+        .cache_dir = null,
+        .stale_retention_s = WeatherProvider.stale_retention_s,
+    });
+    defer empty_cache.deinit();
+    harness.opts.provider = harness.mock.provider(empty_cache);
+    var ht = httpz.testing.init(.{});
+    defer ht.deinit();
+
+    ht.url("/73.158.64.1?format=1");
+    ht.param("location", "73.158.64.1");
+
+    try handleWeather(&harness.opts, ht.req, ht.res, "73.158.64.1");
+
+    try ht.expectStatus(503);
+    try ht.expectHeader("Retry-After", "60");
+}

src/main.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 const Config = @import("Config.zig");
 const Cache = @import("cache/Cache.zig");
 const MetNo = @import("weather/MetNo.zig");
+const WeatherProvider = @import("weather/Provider.zig");
 const Server = @import("http/Server.zig");
 const RateLimiter = @import("http/RateLimiter.zig");
 const GeoIp = @import("location/GeoIp.zig");
@@ -104,6 +105,7 @@ fn serve(
     const cache = try Cache.init(allocator, io, .{
         .max_entries = cfg.cache_size,
         .cache_dir = cfg.cache_dir,
+        .stale_retention_s = WeatherProvider.stale_retention_s,
     });
     defer cache.deinit();
 

src/weather/MetNo.zig

@@ -147,6 +147,14 @@ fn fetchRaw(ptr: *anyopaque, allocator: std.mem.Allocator, coords: Coordinates)
     });
 
     if (result.status != .ok) {
+        const body = response_buf[0..writer.end];
+        std.log.warn("met.no returned HTTP {d} ({s}) for {s} with User-Agent '{s}': {s}", .{
+            @intFromEnum(result.status),
+            result.status.phrase() orelse "unknown",
+            url,
+            user_agent,
+            body[0..@min(body.len, 512)],
+        });
         return error.WeatherApiFailed;
     }
 

src/weather/Mock.zig

@@ -9,6 +9,8 @@ const Mock = @This();
 allocator: std.mem.Allocator,
 responses: std.StringHashMap([]const u8),
 parse_fn: ?*const fn (ptr: *anyopaque, allocator: std.mem.Allocator, raw: []const u8) anyerror!types.WeatherData = null,
+/// When set, every fetch fails with this error, simulating an upstream outage.
+fail_with: ?anyerror = null,
 
 pub fn init(allocator: std.mem.Allocator) !Mock {
     return Mock{
@@ -37,6 +39,7 @@ pub fn addResponse(self: *Mock, coords: Coordinates, raw_json: []const u8) !void
 
 fn fetchRaw(ptr: *anyopaque, allocator: std.mem.Allocator, coords: Coordinates) ![]const u8 {
     const self: *Mock = @ptrCast(@alignCast(ptr));
+    if (self.fail_with) |err| return err;
     const key = try std.fmt.allocPrint(allocator, "{d:.4},{d:.4}", .{ coords.latitude, coords.longitude });
     defer allocator.free(key);
 

src/weather/Provider.zig

@@ -11,6 +11,14 @@ const Cache = @import("../cache/Cache.zig");
 
 const WeatherProvider = @This();
 
+const log = std.log.scoped(.weather);
+
+/// How long past its TTL a cached response may still be served when the
+/// upstream fails. With the 16-33 minute TTL below, the oldest data ever served
+/// is about two hours old; beyond that "current" conditions are fiction.
+/// Configure the weather cache with this as `Cache.Config.stale_retention_s`.
+pub const stale_retention_s: u64 = 90 * 60;
+
 ptr: *anyopaque,
 vtable: *const VTable,
 cache: *Cache,
@@ -30,8 +38,20 @@ pub fn fetch(self: WeatherProvider, allocator: std.mem.Allocator, coords: Coordi
     if (self.cache.get(cache_key)) |cached_raw|
         return self.vtable.parse(self.ptr, allocator, cached_raw);
 
-    // Cache miss - fetch raw data from provider
-    const raw = try self.vtable.fetchRaw(self.ptr, allocator, coords);
+    // Cache miss - fetch raw data from provider, falling back to an expired
+    // entry if the upstream is down
+    const raw = self.vtable.fetchRaw(self.ptr, allocator, coords) catch |err| {
+        if (self.cache.getStale(cache_key)) |stale| {
+            const now_ms = std.Io.Timestamp.now(self.cache.io, .real).toMilliseconds();
+            log.warn("weather fetch for {s} failed ({}); serving cached data that expired {d}s ago", .{
+                cache_key,
+                err,
+                @divTrunc(now_ms - stale.expires, std.time.ms_per_s),
+            });
+            return self.vtable.parse(self.ptr, allocator, stale.value);
+        }
+        return err;
+    };
     defer allocator.free(raw);
 
     // TTL: 1000-2000 seconds (16-33 minutes) to avoid thundering herd.
@@ -79,3 +99,59 @@ test "provider fetch" {
     try std.testing.expect(weather1.forecast.len > 0);
     try std.testing.expect(weather2.forecast.len > 0);
 }
+
+test "provider serves expired cached data when the upstream fails" {
+    const Mock = @import("Mock.zig");
+    const MetNo = @import("MetNo.zig");
+
+    const cache = try Cache.init(std.testing.allocator, std.testing.io, .{
+        .max_entries = 10,
+        .cache_dir = null,
+        .stale_retention_s = stale_retention_s,
+    });
+    defer cache.deinit();
+
+    var mock = try Mock.init(std.testing.allocator);
+    defer mock.deinit();
+    mock.parse_fn = MetNo.parse;
+    mock.fail_with = error.WeatherApiFailed;
+
+    const provider = mock.provider(cache);
+    const coords = Coordinates{ .latitude = 33.4484, .longitude = -112.0740 };
+
+    // Nothing cached: the upstream error surfaces unchanged
+    try std.testing.expectError(error.WeatherApiFailed, provider.fetch(std.testing.allocator, coords));
+
+    // A response that expired ten minutes ago is better than nothing
+    const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds();
+    try cache.lru.put("33.4484,-112.0740", @embedFile("../tests/metno_test_data.json"), now - 10 * 60 * 1000);
+
+    const weather = try provider.fetch(std.testing.allocator, coords);
+    defer weather.deinit();
+    try std.testing.expect(weather.forecast.len > 0);
+}
+
+test "provider does not serve data past the stale retention window" {
+    const Mock = @import("Mock.zig");
+    const MetNo = @import("MetNo.zig");
+
+    const cache = try Cache.init(std.testing.allocator, std.testing.io, .{
+        .max_entries = 10,
+        .cache_dir = null,
+        .stale_retention_s = stale_retention_s,
+    });
+    defer cache.deinit();
+
+    var mock = try Mock.init(std.testing.allocator);
+    defer mock.deinit();
+    mock.parse_fn = MetNo.parse;
+    mock.fail_with = error.WeatherApiFailed;
+
+    const now = std.Io.Timestamp.now(std.testing.io, .real).toMilliseconds();
+    const too_old = now - @as(i64, @intCast(stale_retention_s * std.time.ms_per_s)) - 1000;
+    try cache.lru.put("33.4484,-112.0740", @embedFile("../tests/metno_test_data.json"), too_old);
+
+    const provider = mock.provider(cache);
+    const coords = Coordinates{ .latitude = 33.4484, .longitude = -112.0740 };
+    try std.testing.expectError(error.WeatherApiFailed, provider.fetch(std.testing.allocator, coords));
+}