From f58fc54eb25534583234abf4d898068f1936ffcb Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Thu, 13 Aug 2026 08:54:55 -0700 Subject: [PATCH] add (local) explain command to provide insights on the resolution chain --- src/cli/explain.zig | 482 ++++++++++++++++++++++++++++++++ src/cli/pin.zig | 10 +- src/location/GeoIp.zig | 618 +++++++++++++++++++++++++++++++++++++++-- src/location/Pins.zig | 9 + src/main.zig | 3 + 5 files changed, 1100 insertions(+), 22 deletions(-) create mode 100644 src/cli/explain.zig diff --git a/src/cli/explain.zig b/src/cli/explain.zig new file mode 100644 index 0000000..91e6bb9 --- /dev/null +++ b/src/cli/explain.zig @@ -0,0 +1,482 @@ +const std = @import("std"); +const Config = @import("../Config.zig"); +const GeoIp = @import("../location/GeoIp.zig"); +const GeoLite2 = @import("../location/GeoLite2.zig"); + +/// `wttr explain
`: reports how an address resolves, stage by stage. +/// +/// Deliberately a command rather than an HTTP endpoint. The report reveals the +/// pin table, which records where particular networks are, and `--online` spends +/// the provider's quota and appends to a cache that is never evicted. Gating all +/// of that behind shell access on the server makes it safe by construction, with +/// no credential to configure, no endpoint to rate limit, and no way for a +/// spoofed header to reach it. +pub fn runExplain( + allocator: std.mem.Allocator, + io: std.Io, + cfg: Config, + args: []const []const u8, +) !u8 { + var err_buf: [512]u8 = undefined; + var err_out = std.Io.File.stderr().writer(io, &err_buf); + const ew = &err_out.interface; + + var address: ?[]const u8 = null; + var online = false; + + for (args) |arg| { + if (std.mem.eql(u8, arg, "--online")) { + online = true; + } else if (std.mem.startsWith(u8, arg, "-")) { + try ew.print("unknown option {s}\n", .{arg}); + try ew.flush(); + return 2; + } else if (address == null) { + address = arg; + } else { + try ew.writeAll("usage: wttr explain
[--online]\n"); + try ew.flush(); + return 2; + } + } + + const ip = address orelse { + try ew.writeAll("usage: wttr explain
[--online]\n"); + try ew.flush(); + return 2; + }; + + var geoip = GeoIp.init(allocator, io, cfg.geolite_path, cfg) catch |e| { + // Not downloaded here on purpose: a diagnostic should not pull tens of + // megabytes as a side effect. If the server runs, the file is present. + try ew.print( + \\could not open the GeoLite2 database at {s}: {t} + \\the server downloads this on first start; run it once, or set WTTR_GEOLITE_PATH + \\ + , .{ cfg.geolite_path, e }); + try ew.flush(); + return 1; + }; + defer geoip.deinit(); + + const explanation = try geoip.explain(ip, .{ .online = online }); + defer explanation.deinit(); + + var out_buf: [4096]u8 = undefined; + var out = std.Io.File.stdout().writer(io, &out_buf); + const w = &out.interface; + try report(w, io, cfg, ip, online, explanation); + try w.flush(); + + // Non-zero when nothing resolved, so this is usable in a script without + // parsing the report. + return if (explanation.result == null) 1 else 0; +} + +fn report( + w: *std.Io.Writer, + io: std.Io, + cfg: Config, + ip: []const u8, + online: bool, + e: GeoIp.Explanation, +) !void { + try w.print("address {s}\n", .{ip}); + if (!e.parsed) { + // Stated plainly because it is the one case where no stage runs: nothing + // is handed to a provider unless it parses as an address first. + try w.writeAll(" not an IP address; no source was consulted\n"); + return; + } + + try w.writeAll("\npins "); + try w.print("{s}", .{cfg.pins_file}); + try printAge(w, io, cfg.pins_file); + try w.print("\n {d} pin(s) loaded\n", .{e.pins_loaded}); + if (e.pin) |p| { + try w.print(" MATCH {s} -> {s}\n", .{ p.cidr, p.location.name }); + try w.print(" {d:.4},{d:.4} country {s}\n", .{ + p.location.coords.latitude, + p.location.coords.longitude, + isoText(p.location.iso_country), + }); + } else { + try w.writeAll(" no pin matches this address\n"); + } + + try w.writeAll("\ngeolite2 "); + try w.print("{s}", .{cfg.geolite_path}); + try printAge(w, io, cfg.geolite_path); + try w.writeAll("\n"); + switch (e.database) { + .absent => try w.writeAll(" no entry for this address\n"), + .unusable => try w.writeAll(" entry found, but it has no usable coordinates\n"), + .accepted, .rejected => |hit| { + try w.print(" {s}\n", .{hit.location.name}); + try w.print(" {d:.4},{d:.4} country {s}\n", .{ + hit.location.coords.latitude, + hit.location.coords.longitude, + isoText(hit.location.iso_country), + }); + if (hit.accuracy_radius) |radius| { + // The number that explains a confidently wrong answer. A small + // radius on a wrong city is exactly the case pins exist for, + // because no confidence test and no provider can correct it. + try w.print(" accuracy radius {d} km ({s} threshold {d} km)\n", .{ + radius, + if (e.database == .rejected) "REJECTED, over" else "accepted, within", + GeoIp.max_accuracy_radius_km, + }); + } else { + try w.writeAll(" no accuracy radius recorded (accepted)\n"); + } + }, + } + + // The provider's answer is shown whether or not it was used. When the + // database answers confidently and wrongly, "what would the other source + // have said" is the question, and only a side-by-side answers it. + try w.print("\nfallback {s}", .{e.fallback_provider}); + if (!e.fallbackWasUsed()) try w.writeAll(" (not used, an earlier stage answered)"); + try w.writeAll("\n"); + switch (e.fallback) { + .not_cached => try w.writeAll(" not cached; re-run with --online to ask the provider\n"), + .failed => try w.writeAll(" queried, no usable answer\n"), + .cached, .fetched => |loc| { + try w.print(" {s}\n", .{loc.name}); + try w.print(" {d:.4},{d:.4} country {s}\n", .{ + loc.coords.latitude, + loc.coords.longitude, + isoText(loc.iso_country), + }); + if (e.fallback == .fetched) + try w.writeAll(" fetched just now, and cached permanently\n"); + }, + } + // Last, as a footnote: the path matters when diagnosing a wrong cache + // directory, but it is not what the reader came for. + try w.print(" cache {s}", .{e.fallback_cache_path}); + try printAge(w, io, e.fallback_cache_path); + try w.writeAll("\n"); + + try w.writeAll("\nresult "); + if (e.result) |r| { + try w.print("{s}\n", .{r.location.name}); + try w.print(" {d:.4},{d:.4}\n", .{ + r.location.coords.latitude, + r.location.coords.longitude, + }); + try w.print(" from {s}\n", .{r.source.label()}); + } else { + try w.writeAll("no source resolved this address\n"); + if (!online and e.fallback == .not_cached) + try w.writeAll(" (the provider was not queried; --online would try it)\n"); + } + + try w.print(" units {s}", .{ + if (e.country) |iso| if (std.mem.eql(u8, &iso, "US")) "imperial" else "metric" else "metric", + }); + if (e.country) |iso| { + try w.print(" (country {s})\n", .{iso[0..]}); + } else { + try w.writeAll(" (no country known for this address)\n"); + } + + // Said every time rather than only on a mismatch, because there is nothing + // here that can detect one: a running server holds its own mapped database + // and its own pins, and this process cannot see either. + try w.writeAll( + \\ + \\note this re-runs the chain against the files above. A running + \\ server may differ if it has not reloaded them (SIGHUP is + \\ sent by `wttr pin`, and the database is remapped on refresh). + \\ + ); +} + +/// Renders the value of an optional country code for display. +fn isoText(iso: ?[2]u8) []const u8 { + if (iso) |*code| return code[0..]; + return "unknown"; +} + +/// Appends "(updated N ago)" for a path, or notes that it is missing. +/// +/// The age is what lets an operator judge whether the file this command read is +/// the one a long-running server actually has mapped. +/// +/// Stats directly rather than going through `GeoLite2.ageInSeconds`, which logs +/// any failure as a missing *GeoLite2 database*; that would misreport an absent +/// pins file, and an absent pins file is the normal state. +fn printAge(w: *std.Io.Writer, io: std.Io, path: []const u8) !void { + const stat = std.Io.Dir.cwd().statFile(io, path, .{}) catch { + try w.writeAll(" (absent)"); + return; + }; + const age = GeoLite2.ageOf(std.Io.Timestamp.now(io, .real), stat.mtime); + if (age < std.time.s_per_hour) { + try w.print(" (updated {d}m ago)", .{@divFloor(age, std.time.s_per_min)}); + } else if (age < std.time.s_per_day) { + try w.print(" (updated {d}h ago)", .{@divFloor(age, std.time.s_per_hour)}); + } else { + try w.print(" (updated {d}d ago)", .{@divFloor(age, std.time.s_per_day)}); + } +} + +test "explain: rejects an argument that is not an address" { + const allocator = std.testing.allocator; + var buf: [4096]u8 = undefined; + var w = std.Io.Writer.fixed(&buf); + + const cfg = try Config.loadForTest(allocator); + defer cfg.deinit(allocator); + + // Reported rather than passed on. The provider URLs interpolate the argument, + // so "not an address" has to stop before any stage runs. + const e: GeoIp.Explanation = .{ + .allocator = allocator, + .parsed = false, + .pins_loaded = 0, + .pin = null, + .database = .absent, + .fallback = .not_cached, + .fallback_provider = "ipwho.is", + .fallback_cache_path = "/tmp/none", + .result = null, + .country = null, + }; + + try report(&w, std.testing.io, cfg, "not-an-address", false, e); + + const output = buf[0..w.end]; + try std.testing.expect(std.mem.indexOf(u8, output, "not an IP address") != null); + // None of the stage headings should appear: there is nothing to say about them. + try std.testing.expect(std.mem.indexOf(u8, output, "geolite2") == null); +} + +test "explain: names the accuracy radius and shows the provider it displaced" { + const allocator = std.testing.allocator; + var buf: [4096]u8 = undefined; + var w = std.Io.Writer.fixed(&buf); + + const cfg = try Config.loadForTest(allocator); + defer cfg.deinit(allocator); + + // The case this command exists for: a small radius on the wrong city, which + // passes the threshold and so prevents the provider from being asked. Both + // answers are shown, because seeing them side by side is what reveals that + // the database is the thing that is wrong. + const database_answer: @import("../location/resolver.zig").Location = .{ + .allocator = allocator, + .name = "Portland, Oregon, United States", + .coords = .{ .latitude = 45.5136, .longitude = -122.5946 }, + .iso_country = .{ 'U', 'S' }, + }; + + const e: GeoIp.Explanation = .{ + .allocator = allocator, + .parsed = true, + .pins_loaded = 0, + .pin = null, + .database = .{ .accepted = .{ + .location = database_answer, + .accuracy_radius = 20, + } }, + .fallback = .{ .cached = .{ + .allocator = allocator, + .name = "Portland, Washington, United States", + .coords = .{ .latitude = 47.2255, .longitude = -122.4112 }, + .iso_country = .{ 'U', 'S' }, + } }, + .fallback_provider = "ipwho.is", + .fallback_cache_path = "/tmp/none", + .result = .{ .location = database_answer, .source = .database }, + .country = .{ 'U', 'S' }, + }; + + try report(&w, std.testing.io, cfg, "70.102.70.100", false, e); + + const output = buf[0..w.end]; + try std.testing.expect(std.mem.indexOf(u8, output, "accuracy radius 20 km") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "accepted, within threshold 200 km") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "from geolite2") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "units imperial (country US)") != null); + + // The provider's answer is present and marked as not the one in force. + try std.testing.expect(std.mem.indexOf(u8, output, "ipwho.is (not used, an earlier stage answered)") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "Portland, Washington, United States") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "47.2255,-122.4112 country US") != null); +} + +test "explain: a rejected entry is labelled as over the threshold" { + const allocator = std.testing.allocator; + var buf: [4096]u8 = undefined; + var w = std.Io.Writer.fixed(&buf); + + const cfg = try Config.loadForTest(allocator); + defer cfg.deinit(allocator); + + const e: GeoIp.Explanation = .{ + .allocator = allocator, + .parsed = true, + .pins_loaded = 0, + .pin = null, + .database = .{ .rejected = .{ + .location = .{ + .allocator = allocator, + .name = "Somewhere Vague", + .coords = .{ .latitude = 1, .longitude = 2 }, + .iso_country = null, + }, + .accuracy_radius = 500, + } }, + .fallback = .not_cached, + .fallback_provider = "ipwho.is", + .fallback_cache_path = "/tmp/none", + .result = null, + .country = null, + }; + + try report(&w, std.testing.io, cfg, "203.0.113.7", false, e); + + const output = buf[0..w.end]; + try std.testing.expect(std.mem.indexOf(u8, output, "REJECTED, over threshold 200 km") != null); + // The operator has to be told the provider is reachable but was not asked, + // otherwise "no source resolved this" looks like a dead end. + try std.testing.expect(std.mem.indexOf(u8, output, "--online") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "no source resolved this address") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "no country known") != null); +} + +test "explain: a pin is reported as the match, with the database it overrides" { + const allocator = std.testing.allocator; + var buf: [4096]u8 = undefined; + var w = std.Io.Writer.fixed(&buf); + + const cfg = try Config.loadForTest(allocator); + defer cfg.deinit(allocator); + + const pin_location: @import("../location/resolver.zig").Location = .{ + .allocator = allocator, + .name = "Seattle, Washington, United States", + .coords = .{ .latitude = 47.6, .longitude = -122.3 }, + .iso_country = .{ 'U', 'S' }, + }; + + const e: GeoIp.Explanation = .{ + .allocator = allocator, + .parsed = true, + .pins_loaded = 1, + .pin = .{ .cidr = "70.102.70.100/32", .location = pin_location }, + // Still reported, because an operator checking a pin needs to see what it + // is overriding. + .database = .{ .accepted = .{ + .location = .{ + .allocator = allocator, + .name = "Portland, Oregon, United States", + .coords = .{ .latitude = 45.5136, .longitude = -122.5946 }, + .iso_country = .{ 'U', 'S' }, + }, + .accuracy_radius = 20, + } }, + .fallback = .not_cached, + .fallback_provider = "ipwho.is", + .fallback_cache_path = "/tmp/none", + .result = .{ .location = pin_location, .source = .pin }, + .country = .{ 'U', 'S' }, + }; + + try report(&w, std.testing.io, cfg, "70.102.70.100", false, e); + + const output = buf[0..w.end]; + try std.testing.expect(std.mem.indexOf(u8, output, "MATCH 70.102.70.100/32") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "1 pin(s) loaded") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "Portland, Oregon") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "from pin") != null); +} + +test "explain: a cached provider answer says so, and a fetched one says it was stored" { + const allocator = std.testing.allocator; + const cfg = try Config.loadForTest(allocator); + defer cfg.deinit(allocator); + + const loc: @import("../location/resolver.zig").Location = .{ + .allocator = allocator, + .name = "Tacoma, Washington, United States", + .coords = .{ .latitude = 47.2255, .longitude = -122.4111 }, + .iso_country = .{ 'U', 'S' }, + }; + + const base: GeoIp.Explanation = .{ + .allocator = allocator, + .parsed = true, + .pins_loaded = 0, + .pin = null, + .database = .absent, + .fallback = .not_cached, + .fallback_provider = "ipwho.is", + .fallback_cache_path = "/tmp/none", + .result = null, + .country = .{ 'U', 'S' }, + }; + + { + var buf: [4096]u8 = undefined; + var w = std.Io.Writer.fixed(&buf); + var cached = base; + cached.fallback = .{ .cached = loc }; + cached.result = .{ .location = loc, .source = .fallback_cache }; + try report(&w, std.testing.io, cfg, "198.51.100.7", false, cached); + const output = buf[0..w.end]; + // Used, so it must NOT be labelled as displaced. + try std.testing.expect(std.mem.indexOf(u8, output, "not used") == null); + try std.testing.expect(std.mem.indexOf(u8, output, "from fallback (cached)") != null); + // Nothing was fetched, so nothing may claim to have been stored. + try std.testing.expect(std.mem.indexOf(u8, output, "cached permanently") == null); + } + + { + // The report has to admit the side effect: this cache is append-only and + // never evicted, so a fetch is a permanent change. + var buf: [4096]u8 = undefined; + var w = std.Io.Writer.fixed(&buf); + var fetched = base; + fetched.fallback = .{ .fetched = loc }; + fetched.result = .{ .location = loc, .source = .fallback_online }; + try report(&w, std.testing.io, cfg, "198.51.100.7", true, fetched); + const output = buf[0..w.end]; + try std.testing.expect(std.mem.indexOf(u8, output, "cached permanently") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "from fallback (fetched)") != null); + } +} + +test "explain: reports the pins file even when there are none" { + const allocator = std.testing.allocator; + var buf: [4096]u8 = undefined; + var w = std.Io.Writer.fixed(&buf); + + const cfg = try Config.loadForTest(allocator); + defer cfg.deinit(allocator); + + const e: GeoIp.Explanation = .{ + .allocator = allocator, + .parsed = true, + .pins_loaded = 0, + .pin = null, + .database = .absent, + .fallback = .not_cached, + .fallback_provider = "ipwho.is", + .fallback_cache_path = "/tmp/none", + .result = null, + .country = null, + }; + + try report(&w, std.testing.io, cfg, "203.0.113.7", false, e); + + const output = buf[0..w.end]; + // Naming the path matters more than the count: the usual mistake is running + // this against a different cache directory than the server uses. + try std.testing.expect(std.mem.indexOf(u8, output, cfg.pins_file) != null); + try std.testing.expect(std.mem.indexOf(u8, output, "0 pin(s) loaded") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "no pin matches") != null); +} diff --git a/src/cli/pin.zig b/src/cli/pin.zig index 6729b93..91cca4f 100644 --- a/src/cli/pin.zig +++ b/src/cli/pin.zig @@ -33,21 +33,29 @@ pub fn printUsage(io: std.Io, exe: []const u8) !u8 { \\ pin Override the location for an IP or range \\ unpin Remove an override \\ pins List overrides + \\ explain
Show how an address resolves, stage by stage \\ help Show this message \\ \\Pins take precedence over the GeoLite2 database, which is the point: \\they exist for addresses the database resolves confidently and wrongly. + \\`explain` is how you tell that case apart from the rest; it names the + \\source that answered and the accuracy radius the database claimed. \\ \\Examples: \\ {s} pin 12.94.132.0/24 San Francisco \\ {s} pin 12.94.132.170 Portland, Oregon \\ {s} pin 2001:db8::/32 London \\ {s} unpin 12.94.132.0/24 + \\ {s} explain 12.94.132.170 \\ \\A bare address is treated as a single host (/32 or /128). When ranges \\overlap, the longest prefix wins, so a host pin beats a range pin. \\ - , .{ base, base, base, base, base }); + \\`explain` does not query the online provider unless given --online, + \\because that spends the provider's quota and writes to a cache that is + \\never evicted. + \\ + , .{ base, base, base, base, base, base }); try w.flush(); return 0; } diff --git a/src/location/GeoIp.zig b/src/location/GeoIp.zig index e9e48a2..d359b91 100644 --- a/src/location/GeoIp.zig +++ b/src/location/GeoIp.zig @@ -54,6 +54,31 @@ const FallbackClient = union(enum) { }; } + /// Full cached answer for an address, without contacting the provider. + fn cachedLookup(self: FallbackClient, ip: []const u8) ?Location { + const packed_ip = Ip2location.packIp(ip) orelse return null; + return switch (self) { + .ip2location => |client| client.cache.get(packed_ip.key), + .ipwhois => |client| client.cache.get(packed_ip.key), + }; + } + + /// Human-readable provider name, for diagnostics and logs. + fn name(self: FallbackClient) []const u8 { + return switch (self) { + .ip2location => "ip2location.io", + .ipwhois => "ipwho.is", + }; + } + + /// Where this provider's permanent cache lives. + fn cachePath(self: FallbackClient) []const u8 { + return switch (self) { + .ip2location => |client| client.cache.path, + .ipwhois => |client| client.cache.path, + }; + } + fn deinit(self: FallbackClient, allocator: std.mem.Allocator) void { switch (self) { .ip2location => |client| { @@ -205,10 +230,31 @@ pub fn reloadPins(self: *GeoIP) !void { log.info("reloaded {d} IP pin(s) from {s}", .{ self.pins.entries.items.len, self.pins_path }); } +/// Where a resolution came from. +/// +/// Reported by `explain` and named in the per-resolution log line, because +/// "which source answered" is the question an operator actually has when a +/// client is placed in the wrong city. +pub const Source = enum { + pin, + database, + fallback_cache, + fallback_online, + + pub fn label(self: Source) []const u8 { + return switch (self) { + .pin => "pin", + .database => "geolite2", + .fallback_cache => "fallback (cached)", + .fallback_online => "fallback (fetched)", + }; + } +}; + pub fn lookup(self: *GeoIP, ip: []const u8) ?Location { - // Try MaxMind first. The shared lock has to cover `extractCoordinates` as - // well as the lookup itself: `MMDB_lookup_result_s.entry` holds a pointer - // back to the `MMDB_s`, which `extractCoordinates` dereferences. + // The shared lock has to cover `readEntry` as well as the lookup itself: + // `MMDB_lookup_result_s.entry` holds a pointer back to the `MMDB_s`, which + // `readEntry` dereferences. const from_db: ?Location = blk: { self.lock.lockSharedUncancelable(self.io); defer self.lock.unlockShared(self.io); @@ -225,7 +271,24 @@ pub fn lookup(self: *GeoIP, ip: []const u8) ?Location { log.debug("lookup geoip db for ip {s}. Found: {}", .{ ip, result.found_entry }); if (!result.found_entry) break :blk null; - break :blk self.extractCoordinates(ip, result); + + const hit = self.readEntry(ip, result) orelse break :blk null; + + // Reject low-confidence entries so the online provider gets a chance. + // `readEntry` reports the radius rather than acting on it, so that + // `explain` can describe this decision without repeating it. + if (hit.accuracy_radius) |radius| { + if (radius > max_accuracy_radius_km) { + log.info( + "GeoLite2 accuracy_radius for ip {s} is {d} km (>{d} km threshold), falling back to the online provider", + .{ ip, radius, max_accuracy_radius_km }, + ); + hit.location.deinit(); + break :blk null; + } + } + + break :blk hit.location; }; if (from_db) |coords| return coords; @@ -249,6 +312,227 @@ fn lookupInternal(mmdb: *c.MMDB_s, ip: []const u8) !c.MMDB_lookup_result_s { return result; } +/// A full account of how an address resolves, for `wttr explain`. +/// +/// Reports every stage rather than just the winner, because the interesting +/// failures are the ones where an early stage answered confidently and wrongly +/// and later stages were therefore never consulted. +pub const Explanation = struct { + allocator: std.mem.Allocator, + + /// False when the argument was not an address at all. Everything else is + /// skipped in that case: nothing should reach a provider with a value that is + /// not an address. + parsed: bool, + /// How many pins are loaded, so "no pin matched" reads differently from + /// "there are no pins". + pins_loaded: usize, + pin: ?Pinned, + database: Database, + fallback: Fallback, + /// The provider that would be consulted, named even when it is not reached. + fallback_provider: []const u8, + fallback_cache_path: []const u8, + + /// The answer `lookup` would return, and where it came from. + result: ?Result, + /// The country `lookupCountry` would return, which decides units. + country: ?[2]u8, + + pub const Pinned = struct { + /// Owned text of the matching range. + cidr: []const u8, + location: Location, + }; + + pub const Result = struct { + location: Location, + source: Source, + }; + + pub const Database = union(enum) { + /// No entry for this address. + absent, + /// An entry whose coordinates could not be read. + unusable, + /// An entry within the confidence threshold, so `lookup` uses it. + accepted: DbHit, + /// An entry too coarse to trust, so `lookup` falls through to the + /// provider. This is the case the threshold exists for. + rejected: DbHit, + + pub fn hit(self: Database) ?DbHit { + return switch (self) { + .accepted, .rejected => |h| h, + .absent, .unusable => null, + }; + } + }; + + pub const Fallback = union(enum) { + /// Answered from the permanent cache. + cached: Location, + /// Fetched over the network during this run. + fetched: Location, + /// Not cached, and no online lookup was asked for. + not_cached, + /// An online lookup was asked for and did not produce an answer. + failed, + + pub fn location(self: Fallback) ?Location { + return switch (self) { + .cached, .fetched => |loc| loc, + .not_cached, .failed => null, + }; + } + }; + + /// Whether the provider's answer is the one `lookup` would return. + /// + /// Derived rather than stored: the report needs to show what the provider + /// says even when an earlier stage won, so "do we have an answer" and "is it + /// the answer" are separate questions. + pub fn fallbackWasUsed(self: Explanation) bool { + const r = self.result orelse return false; + return r.source == .fallback_cache or r.source == .fallback_online; + } + + pub fn deinit(self: Explanation) void { + if (self.pin) |p| { + self.allocator.free(p.cidr); + p.location.deinit(); + } + if (self.database.hit()) |h| h.location.deinit(); + if (self.fallback.location()) |loc| loc.deinit(); + // `result` borrows whichever stage won rather than owning a copy, so it + // is deliberately not freed here. + } +}; + +pub const ExplainOptions = struct { + /// Whether a cache miss may be resolved over the network. + /// + /// Off by default, and that default matters: the fallback cache is permanent + /// and append-only, so a diagnostic that fetched by default would grow the + /// cache and spend the provider's quota every time it ran. + online: bool = false, +}; + +/// Explains how `ip` resolves, walking every stage. +/// +/// This re-runs the chain rather than replaying a past request. The result +/// matches what a running server would answer as long as that server's pins and +/// mapped database are the same as the ones on disk, which is why the caller +/// reports both files. +pub fn explain(self: *GeoIP, ip: []const u8, opts: ExplainOptions) !Explanation { + var out: Explanation = .{ + .allocator = self.allocator, + .parsed = Ip2location.packIp(ip) != null, + .pins_loaded = 0, + .pin = null, + .database = .absent, + // Overwritten below for any address that parses. Left as "not cached" so + // an address that never reaches the provider reads as untried rather than + // as a miss the provider reported. + .fallback = .not_cached, + .fallback_provider = self.fallback_client.name(), + .fallback_cache_path = self.fallback_client.cachePath(), + .result = null, + .country = null, + }; + // Reads whatever has been filled in so far, so a failure partway through + // does not leak the stages that already succeeded. + errdefer out.deinit(); + + { + self.lock.lockSharedUncancelable(self.io); + defer self.lock.unlockShared(self.io); + + out.pins_loaded = self.pins.entries.items.len; + + if (out.parsed) { + if (self.pins.matchFor(ip)) |entry| { + var buf: [64]u8 = undefined; + const text = try Pins.formatCidr( + .{ .family = entry.family, .network = entry.network, .prefix_len = entry.prefix_len }, + &buf, + ); + // Both copies are owned, because `entry` points into the pin set + // and is only valid while the lock is held. + const cidr = try self.allocator.dupe(u8, text); + errdefer self.allocator.free(cidr); + const name = try self.allocator.dupe(u8, entry.name); + + out.pin = .{ + .cidr = cidr, + .location = .{ + .allocator = self.allocator, + .name = name, + .coords = entry.coords, + .iso_country = entry.iso_country, + }, + }; + } + + // The database is read even when a pin matched. An operator debugging + // a pin needs to see what it is overriding. + if (lookupInternal(self.mmdb, ip)) |result| { + if (!result.found_entry) { + out.database = .absent; + } else if (self.readEntry(ip, result)) |hit| { + const too_coarse = if (hit.accuracy_radius) |r| r > max_accuracy_radius_km else false; + out.database = if (too_coarse) .{ .rejected = hit } else .{ .accepted = hit }; + } else { + out.database = .unusable; + } + } else |_| { + out.database = .absent; + } + } + } + + if (!out.parsed) return out; + + // The provider is consulted even when an earlier stage already answered. + // "What would the other source have said" is the question worth asking when + // the database answers confidently and wrongly, and comparing the two is the + // whole reason to run this. It does not affect `result`: precedence below + // still prefers the pin and the database. + // + // Reading the cache is free. Going to the network is not, which is why it + // stays behind `--online` regardless of whether the answer is needed. + if (self.fallback_client.cachedLookup(ip)) |cached| { + out.fallback = .{ .cached = cached }; + } else if (opts.online) { + if (self.fallback_client.lookup(ip)) |fetched| { + out.fallback = .{ .fetched = fetched }; + } else { + out.fallback = .failed; + } + } else { + out.fallback = .not_cached; + } + + // Precedence, restated here against the collected stages so the report and + // `resolve` cannot disagree about who wins. The drift test below asserts + // this matches `lookup` for real addresses. + if (out.pin) |p| { + out.result = .{ .location = p.location, .source = .pin }; + } else if (out.database.hit()) |h| { + if (out.database == .accepted) out.result = .{ .location = h.location, .source = .database }; + } + if (out.result == null) { + switch (out.fallback) { + .cached => |loc| out.result = .{ .location = loc, .source = .fallback_cache }, + .fetched => |loc| out.result = .{ .location = loc, .source = .fallback_online }, + else => {}, + } + } + + out.country = self.lookupCountry(ip); + return out; +} + pub fn isUSIp(self: *GeoIP, ip: []const u8) bool { const iso = self.lookupCountry(ip) orelse return false; return std.mem.eql(u8, &iso, "US"); @@ -314,25 +598,36 @@ fn readIso(result: c.MMDB_lookup_result_s) ?[2]u8 { /// /// Note this only catches entries GeoLite2 admits are vague. An entry can be /// precise *and* wrong, which is what pins exist for. -const max_accuracy_radius_km = 200; +pub const max_accuracy_radius_km = 200; -fn extractCoordinates(self: *GeoIP, ip: []const u8, result: c.MMDB_lookup_result_s) ?Location { +/// A database entry, read but not yet judged. +pub const DbHit = struct { + location: Location, + /// MaxMind's own confidence signal, in km. Null when the entry carries none. + accuracy_radius: ?u16, +}; + +/// Reads an entry into a `Location`, without applying the confidence gate. +/// +/// The gate is the caller's business: `resolve` rejects a coarse entry so the +/// online provider gets a chance, while `explain` reports the same decision +/// rather than acting on it. Keeping the reading here means one place knows the +/// database's field layout. +/// +/// Must be called with the shared lock held: the returned strings are read +/// through `result.entry`, which points into the mapped database. +fn readEntry(self: *GeoIP, ip: []const u8, result: c.MMDB_lookup_result_s) ?DbHit { if (!result.found_entry) return null; var entry_copy = result.entry; - // Check accuracy_radius first -- reject low-confidence entries so we fall - // back to the configured online provider instead. // SAFETY: accuracy_data set by MMDB_get_value var accuracy_data: c.MMDB_entry_data_s = undefined; const acc_status = c.MMDB_get_value(&entry_copy, &accuracy_data, "location", "accuracy_radius", @as([*c]const u8, null)); - if (acc_status == c.MMDB_SUCCESS and accuracy_data.has_data) { - const radius = accuracy_data.unnamed_0.uint16; - if (radius > max_accuracy_radius_km) { - log.info("GeoLite2 accuracy_radius for ip {s} is {d} km (>{d} km threshold), falling back to the online provider", .{ ip, radius, max_accuracy_radius_km }); - return null; - } - } + const accuracy_radius: ?u16 = if (acc_status == c.MMDB_SUCCESS and accuracy_data.has_data) + accuracy_data.unnamed_0.uint16 + else + null; entry_copy = result.entry; @@ -398,13 +693,16 @@ fn extractCoordinates(self: *GeoIP, ip: []const u8, result: c.MMDB_lookup_result const final_name = Location.buildDisplayName(self.allocator, city, subdivision, country, ip); return .{ - .allocator = self.allocator, - .name = final_name, - .coords = .{ - .latitude = latitude, - .longitude = longitude, + .location = .{ + .allocator = self.allocator, + .name = final_name, + .coords = .{ + .latitude = latitude, + .longitude = longitude, + }, + .iso_country = iso_country, }, - .iso_country = iso_country, + .accuracy_radius = accuracy_radius, }; } @@ -822,3 +1120,281 @@ test "lookupCountry: null when no source knows the address" { try std.testing.expect(geoip.lookupCountry("not-an-address") == null); try std.testing.expect(!geoip.isUSIp("not-an-address")); } + +test "explain agrees with lookup for every source" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_len = try tmp.dir.realPath(io, &path_buf); + const dir = path_buf[0..dir_len]; + + var config = try testConfigIn(allocator, io, dir); + defer config.deinit(allocator); + + // `explain` walks the chain separately from `resolve` so it can report the + // stages `resolve` short-circuits past. That means the two can drift, and a + // diagnostic that disagrees with the code it describes is worse than none. + // Cover one address per source. + { + var pins: Pins = .init(allocator); + defer pins.deinit(); + try pins.put(try Pins.parseCidr("203.0.113.0/24"), "Pinned Place", .{ .latitude = 1.5, .longitude = 2.5 }, Location.isoFrom("US")); + try pins.save(io, config.pins_file); + } + + // Seeded so the fallback-cache source is exercised without a network call. + const cached_ip = "198.51.100.7"; + { + var cache = try Ip2location.Cache.init(allocator, io, config.ipwhois_cache_file); + defer cache.deinit(); + const key = Ip2location.packIp(cached_ip).?; + try cache.put(key.key, key.family, .{ + .allocator = allocator, + .name = "Cached Place", + .coords = .{ .latitude = 3.5, .longitude = 4.5 }, + .iso_country = Location.isoFrom("US"), + }); + } + + var geoip = GeoIP.init(allocator, io, config.geolite_path, config) catch + return error.SkipZigTest; + defer geoip.deinit(); + + const cases = [_][]const u8{ + "203.0.113.7", // pin + "73.158.64.1", // database + cached_ip, // fallback cache + // Null agreement is covered by an address that parses as nothing, rather + // than by one that is merely absent: for an absent address `lookup` is + // supposed to reach the provider, which a unit test cannot let it do. + // The absent case is asserted against `explain` alone below. + "not-an-address", + }; + + for (cases) |ip| { + const explanation = try geoip.explain(ip, .{}); + defer explanation.deinit(); + + const direct = geoip.lookup(ip); + defer if (direct) |d| d.deinit(); + + if (direct) |expected| { + const got = explanation.result orelse { + std.debug.print("explain found nothing for {s}, lookup returned {s}\n", .{ ip, expected.name }); + return error.TestUnexpectedResult; + }; + try std.testing.expectEqualStrings(expected.name, got.location.name); + try std.testing.expectEqual(expected.coords.latitude, got.location.coords.latitude); + try std.testing.expectEqual(expected.coords.longitude, got.location.coords.longitude); + try std.testing.expectEqual(expected.iso_country, got.location.iso_country); + } else { + if (explanation.result) |got| { + std.debug.print("explain found {s} for {s}, lookup returned nothing\n", .{ got.location.name, ip }); + return error.TestUnexpectedResult; + } + } + + // And the units it reports have to be the ones the handler would apply. + try std.testing.expectEqual(geoip.lookupCountry(ip), explanation.country); + } +} + +test "explain: reports the source each address resolved through" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_len = try tmp.dir.realPath(io, &path_buf); + const dir = path_buf[0..dir_len]; + + var config = try testConfigIn(allocator, io, dir); + defer config.deinit(allocator); + + { + var pins: Pins = .init(allocator); + defer pins.deinit(); + try pins.put(try Pins.parseCidr("203.0.113.0/24"), "Pinned Place", .{ .latitude = 1.5, .longitude = 2.5 }, Location.isoFrom("US")); + try pins.save(io, config.pins_file); + } + + var geoip = GeoIP.init(allocator, io, config.geolite_path, config) catch + return error.SkipZigTest; + defer geoip.deinit(); + + { + const e = try geoip.explain("203.0.113.7", .{}); + defer e.deinit(); + try std.testing.expectEqual(Source.pin, e.result.?.source); + try std.testing.expectEqualStrings("203.0.113.0/24", e.pin.?.cidr); + try std.testing.expectEqual(@as(usize, 1), e.pins_loaded); + // A pin does not stop the database from being reported: an operator + // checking a pin needs to see what it overrides. Here there is nothing. + try std.testing.expectEqual(Explanation.Database.absent, std.meta.activeTag(e.database)); + } + + { + const e = try geoip.explain("73.158.64.1", .{}); + defer e.deinit(); + try std.testing.expectEqual(Source.database, e.result.?.source); + try std.testing.expect(e.pin == null); + // The radius is the whole point of the report, so it must be populated. + const hit = e.database.hit() orelse return error.TestUnexpectedResult; + try std.testing.expect(hit.accuracy_radius != null); + // The provider is still reported for comparison, but must not be the + // answer, and must not have been fetched without --online. + try std.testing.expect(!e.fallbackWasUsed()); + try std.testing.expect(std.meta.activeTag(e.fallback) != .fetched); + } + + { + // Absent everywhere. Must report the miss rather than reaching the network. + const e = try geoip.explain("192.0.2.99", .{}); + defer e.deinit(); + try std.testing.expect(e.result == null); + try std.testing.expectEqual(Explanation.Database.absent, std.meta.activeTag(e.database)); + try std.testing.expectEqual(Explanation.Fallback.not_cached, std.meta.activeTag(e.fallback)); + } + + { + // Not an address: no stage may run, because the argument would otherwise + // be interpolated into a provider URL. + const e = try geoip.explain("not-an-address", .{}); + defer e.deinit(); + try std.testing.expect(!e.parsed); + try std.testing.expect(e.result == null); + try std.testing.expect(e.pin == null); + // Not even the cache is consulted: nothing that is not an address should + // reach a provider or its cache key. + try std.testing.expectEqual(Explanation.Fallback.not_cached, std.meta.activeTag(e.fallback)); + } +} + +test "explain: a pin wins over an accepted database entry" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_len = try tmp.dir.realPath(io, &path_buf); + const dir = path_buf[0..dir_len]; + + var config = try testConfigIn(allocator, io, dir); + defer config.deinit(allocator); + + // The original case: GeoLite2 answers this confidently, so the report has to + // show both the pin that won and the entry it displaced. + { + var pins: Pins = .init(allocator); + defer pins.deinit(); + try pins.put(try Pins.parseCidr("12.94.132.0/24"), "Seattle, Washington, United States", .{ .latitude = 47.6, .longitude = -122.3 }, Location.isoFrom("US")); + try pins.save(io, config.pins_file); + } + + var geoip = GeoIP.init(allocator, io, config.geolite_path, config) catch + return error.SkipZigTest; + defer geoip.deinit(); + + const e = try geoip.explain("12.94.132.170", .{}); + defer e.deinit(); + + try std.testing.expectEqual(Source.pin, e.result.?.source); + try std.testing.expectEqualStrings("Seattle, Washington, United States", e.result.?.location.name); + + const hit = e.database.hit() orelse return error.SkipZigTest; + // Different answer, still reported, and reported as one the database was + // confident about. That combination is what tells an operator the pin is + // load-bearing rather than redundant. + try std.testing.expect(!std.mem.eql(u8, hit.location.name, "Seattle, Washington, United States")); + try std.testing.expectEqual(Explanation.Database.accepted, std.meta.activeTag(e.database)); +} + +test "explain: reports the provider's answer alongside the database that displaced it" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_len = try tmp.dir.realPath(io, &path_buf); + const dir = path_buf[0..dir_len]; + + var config = try testConfigIn(allocator, io, dir); + defer config.deinit(allocator); + + // An address the database answers, with a *different* provider answer seeded + // in the cache. `resolve` never reaches the provider for this address, but the + // report has to show what it would have said: when the database is confidently + // wrong, a side-by-side is what makes that visible. + const ip = "73.158.64.1"; + { + var cache = try Ip2location.Cache.init(allocator, io, config.ipwhois_cache_file); + defer cache.deinit(); + const key = Ip2location.packIp(ip).?; + try cache.put(key.key, key.family, .{ + .allocator = allocator, + .name = "Provider Answer, Elsewhere", + .coords = .{ .latitude = 11.25, .longitude = 22.5 }, + .iso_country = Location.isoFrom("GB"), + }); + } + + var geoip = GeoIP.init(allocator, io, config.geolite_path, config) catch + return error.SkipZigTest; + defer geoip.deinit(); + + const e = try geoip.explain(ip, .{}); + defer e.deinit(); + + // The database still wins, and is still what the server would serve. + const db = e.database.hit() orelse return error.SkipZigTest; + try std.testing.expectEqual(Source.database, e.result.?.source); + try std.testing.expectEqualStrings(db.location.name, e.result.?.location.name); + + // And the provider's answer is reported, flagged as not the one in force. + try std.testing.expect(!e.fallbackWasUsed()); + const provider = e.fallback.location() orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("Provider Answer, Elsewhere", provider.name); + try std.testing.expectEqual(@as(f64, 11.25), provider.coords.latitude); + try std.testing.expectEqual(Explanation.Fallback.cached, std.meta.activeTag(e.fallback)); + + // Reporting it must not have changed which units the handler applies: the + // provider cache is only consulted for units when nothing earlier answered. + const iso = e.country orelse return error.SkipZigTest; + try std.testing.expectEqualStrings("US", &iso); +} + +test "explain: does not reach the network for an unneeded provider answer" { + const allocator = std.testing.allocator; + const io = std.testing.io; + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_len = try tmp.dir.realPath(io, &path_buf); + const dir = path_buf[0..dir_len]; + + var config = try testConfigIn(allocator, io, dir); + defer config.deinit(allocator); + + var geoip = GeoIP.init(allocator, io, config.geolite_path, config) catch + return error.SkipZigTest; + defer geoip.deinit(); + + // Now that the provider is reported even when unused, the default must still + // never fetch. Otherwise every run of this command would spend quota and + // append to a cache that is never evicted. + const e = try geoip.explain("73.158.64.1", .{}); + defer e.deinit(); + + try std.testing.expectEqual(Explanation.Fallback.not_cached, std.meta.activeTag(e.fallback)); + // A fetch would also have been recorded in the cache file. + var cache = try Ip2location.Cache.init(allocator, io, config.ipwhois_cache_file); + defer cache.deinit(); + try std.testing.expectEqual(@as(usize, 0), cache.entries.count()); +} diff --git a/src/location/Pins.zig b/src/location/Pins.zig index 53cc5ac..c8204b3 100644 --- a/src/location/Pins.zig +++ b/src/location/Pins.zig @@ -137,6 +137,15 @@ fn bestMatch(self: *const Pins, ip_str: []const u8) ?*const Entry { return best; } +/// The matching entry itself, for callers that need more than a `Location`. +/// +/// `explain` reports the range that matched, which the `Location` does not carry. +/// Borrowed rather than copied, so the caller must hold whatever lock guards the +/// pin set for as long as it uses the result. +pub fn matchFor(self: *const Pins, ip_str: []const u8) ?*const Entry { + return self.bestMatch(ip_str); +} + /// Longest-prefix match for `ip_str`, or null when no pin applies. pub fn lookup(self: *const Pins, allocator: std.mem.Allocator, ip_str: []const u8) ?Location { const entry = self.bestMatch(ip_str) orelse return null; diff --git a/src/main.zig b/src/main.zig index 14140cc..8ea251d 100644 --- a/src/main.zig +++ b/src/main.zig @@ -12,6 +12,7 @@ const GeoLite2 = @import("location/GeoLite2.zig"); const Refresher = @import("location/Refresher.zig"); const Signals = @import("Signals.zig"); const pin_cmd = @import("cli/pin.zig"); +const explain_cmd = @import("cli/explain.zig"); const version = @import("build_options").version; /// Zig 0.16 entry point: the runtime supplies allocators, the `Io` @@ -40,6 +41,7 @@ pub fn main(init: std.process.Init) !u8 { if (std.mem.eql(u8, cmd, "pin")) return pin_cmd.runPin(allocator, io, cfg, args[2..]); if (std.mem.eql(u8, cmd, "unpin")) return pin_cmd.runUnpin(allocator, io, cfg, args[2..]); if (std.mem.eql(u8, cmd, "pins")) return pin_cmd.runList(allocator, io, cfg); + if (std.mem.eql(u8, cmd, "explain")) return explain_cmd.runExplain(allocator, io, cfg, args[2..]); if (std.mem.eql(u8, cmd, "help") or std.mem.eql(u8, cmd, "--help") or std.mem.eql(u8, cmd, "-h")) return pin_cmd.printUsage(io, args[0]); @@ -193,4 +195,5 @@ test { _ = @import("location/Pins.zig"); _ = @import("Signals.zig"); _ = @import("cli/pin.zig"); + _ = @import("cli/explain.zig"); }