add .tmp/ to gitignore
This commit is contained in:
parent
a5a9d0d7d7
commit
6bc5b5a209
2 changed files with 88 additions and 20 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -2,5 +2,6 @@
|
||||||
.zig-cache/
|
.zig-cache/
|
||||||
zig-out/
|
zig-out/
|
||||||
coverage/
|
coverage/
|
||||||
|
.tmp/
|
||||||
# Zig 0.16 stores fetched packages in-tree rather than the global cache
|
# Zig 0.16 stores fetched packages in-tree rather than the global cache
|
||||||
zig-pkg/
|
zig-pkg/
|
||||||
|
|
|
||||||
|
|
@ -1,13 +1,62 @@
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
const log = std.log.scoped(.geolite2);
|
const log = std.log.scoped(.geolite2);
|
||||||
|
|
||||||
pub fn ensureDatabase(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !void {
|
/// Ensures a GeoLite2 database exists at `path`, downloading it when missing
|
||||||
std.Io.Dir.cwd().access(io, path, .{}) catch {
|
/// and refreshing it once it is older than `max_age_seconds`.
|
||||||
log.info("GeoLite2 database not found at {s}, will download", .{path});
|
///
|
||||||
try downloadDatabase(allocator, io, path);
|
/// Refresh matters: the upstream release is rebuilt continuously and IP
|
||||||
log.info("GeoLite2 database downloaded successfully", .{});
|
/// allocations move between cities, so a stale database silently resolves
|
||||||
|
/// clients to the wrong place. Before this, the database was only fetched when
|
||||||
|
/// absent, so a deployment could -- and did -- run for months on the copy it
|
||||||
|
/// happened to download first.
|
||||||
|
///
|
||||||
|
/// A missing database is fatal (there is nothing to fall back on). A failed
|
||||||
|
/// *refresh* is not: an out-of-date database still answers lookups, so a
|
||||||
|
/// network blip must not stop the service from starting.
|
||||||
|
pub fn ensureDatabase(
|
||||||
|
allocator: std.mem.Allocator,
|
||||||
|
io: std.Io,
|
||||||
|
path: []const u8,
|
||||||
|
max_age_seconds: u64,
|
||||||
|
) !void {
|
||||||
|
const stat = std.Io.Dir.cwd().statFile(io, path, .{}) catch |err| switch (err) {
|
||||||
|
error.FileNotFound => {
|
||||||
|
log.info("GeoLite2 database not found at {s}, will download", .{path});
|
||||||
|
try downloadDatabase(allocator, io, path);
|
||||||
|
log.info("GeoLite2 database downloaded successfully", .{});
|
||||||
|
return;
|
||||||
|
},
|
||||||
|
else => return err,
|
||||||
|
};
|
||||||
|
|
||||||
|
const age_seconds = ageInSeconds(io, stat.mtime);
|
||||||
|
if (age_seconds < max_age_seconds) {
|
||||||
|
log.info("GeoLite2 database is {d} day(s) old, no refresh needed", .{@divFloor(age_seconds, std.time.s_per_day)});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
log.info(
|
||||||
|
"GeoLite2 database is {d} day(s) old (limit {d}), refreshing",
|
||||||
|
.{ @divFloor(age_seconds, std.time.s_per_day), @divFloor(max_age_seconds, std.time.s_per_day) },
|
||||||
|
);
|
||||||
|
downloadDatabase(allocator, io, path) catch |err| {
|
||||||
|
// Keep serving from the stale database rather than refusing to start.
|
||||||
|
log.warn(
|
||||||
|
"GeoLite2 refresh failed ({t}); continuing with the existing database at {s}",
|
||||||
|
.{ err, path },
|
||||||
|
);
|
||||||
return;
|
return;
|
||||||
};
|
};
|
||||||
|
log.info("GeoLite2 database refreshed successfully", .{});
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Age of `mtime` in seconds, clamped at 0 so a clock skew into the future
|
||||||
|
/// cannot read as an enormous age and trigger a pointless download.
|
||||||
|
fn ageInSeconds(io: std.Io, mtime: std.Io.Timestamp) u64 {
|
||||||
|
const now_ns = std.Io.Timestamp.now(io, .real).nanoseconds;
|
||||||
|
const age_ns = now_ns - mtime.nanoseconds;
|
||||||
|
if (age_ns <= 0) return 0;
|
||||||
|
return @intCast(@divFloor(age_ns, std.time.ns_per_s));
|
||||||
}
|
}
|
||||||
|
|
||||||
fn downloadDatabase(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !void {
|
fn downloadDatabase(allocator: std.mem.Allocator, io: std.Io, path: []const u8) !void {
|
||||||
|
|
@ -18,27 +67,45 @@ fn downloadDatabase(allocator: std.mem.Allocator, io: std.Io, path: []const u8)
|
||||||
defer client.deinit();
|
defer client.deinit();
|
||||||
|
|
||||||
const uri = try std.Uri.parse(latest_url);
|
const uri = try std.Uri.parse(latest_url);
|
||||||
const response_buf = try allocator.alloc(u8, 64 * 1024 * 1024);
|
|
||||||
defer allocator.free(response_buf);
|
|
||||||
|
|
||||||
var writer = std.Io.Writer.fixed(response_buf);
|
|
||||||
const result = try client.fetch(.{
|
|
||||||
.location = .{ .uri = uri },
|
|
||||||
.method = .GET,
|
|
||||||
.response_writer = &writer,
|
|
||||||
});
|
|
||||||
|
|
||||||
if (result.status != .ok) return error.DownloadFailed;
|
|
||||||
|
|
||||||
// Ensure directory exists
|
// Ensure directory exists
|
||||||
if (std.fs.path.dirname(path)) |dir| {
|
if (std.fs.path.dirname(path)) |dir| {
|
||||||
try std.Io.Dir.cwd().createDirPath(io, dir);
|
try std.Io.Dir.cwd().createDirPath(io, dir);
|
||||||
}
|
}
|
||||||
|
|
||||||
try std.Io.Dir.cwd().writeFile(io, .{
|
// Download to a sibling temp file and rename into place. The rename is
|
||||||
.sub_path = path,
|
// atomic, so a download that dies partway cannot leave a truncated
|
||||||
.data = response_buf[0..writer.end],
|
// database behind, and a database already mmap'd by this process keeps
|
||||||
});
|
// pointing at the old inode until it is reopened.
|
||||||
|
const tmp_path = try std.fmt.allocPrint(allocator, "{s}.download", .{path});
|
||||||
|
defer allocator.free(tmp_path);
|
||||||
|
|
||||||
|
{
|
||||||
|
const file = try std.Io.Dir.cwd().createFile(io, tmp_path, .{});
|
||||||
|
// The temp file is only useful if everything below succeeds.
|
||||||
|
errdefer std.Io.Dir.cwd().deleteFile(io, tmp_path) catch |err|
|
||||||
|
log.warn("could not remove partial download {s}: {t}", .{ tmp_path, err });
|
||||||
|
defer file.close(io);
|
||||||
|
|
||||||
|
// Stream the response straight to disk. Buffering the whole database in
|
||||||
|
// memory needed an allocation sized by guesswork, and the guess (64 MiB)
|
||||||
|
// was already within ~1 MB of the real database size.
|
||||||
|
var buf: [64 * 1024]u8 = undefined;
|
||||||
|
var file_writer = file.writer(io, &buf);
|
||||||
|
const result = try client.fetch(.{
|
||||||
|
.location = .{ .uri = uri },
|
||||||
|
.method = .GET,
|
||||||
|
.response_writer = &file_writer.interface,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (result.status != .ok) {
|
||||||
|
log.err("GeoLite2 download returned HTTP {d}", .{@intFromEnum(result.status)});
|
||||||
|
return error.DownloadFailed;
|
||||||
|
}
|
||||||
|
try file_writer.interface.flush();
|
||||||
|
}
|
||||||
|
|
||||||
|
try std.Io.Dir.cwd().rename(tmp_path, .cwd(), path, io);
|
||||||
}
|
}
|
||||||
|
|
||||||
fn getLatestReleaseUrl(allocator: std.mem.Allocator, io: std.Io) ![]const u8 {
|
fn getLatestReleaseUrl(allocator: std.mem.Allocator, io: std.Io) ![]const u8 {
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue