extract brokerage discovery and export for downstream
This commit is contained in:
parent
a3d235bd39
commit
5e22e1dccc
3 changed files with 316 additions and 256 deletions
|
|
@ -10,5 +10,6 @@ pub const types = @import("brokerage/types.zig");
|
|||
pub const schwab = @import("brokerage/schwab.zig");
|
||||
pub const fidelity = @import("brokerage/fidelity.zig");
|
||||
pub const wells_fargo = @import("brokerage/wells_fargo.zig");
|
||||
pub const discover = @import("brokerage/discover.zig");
|
||||
|
||||
pub const BrokeragePosition = types.BrokeragePosition;
|
||||
|
|
|
|||
307
src/brokerage/discover.zig
Normal file
307
src/brokerage/discover.zig
Normal file
|
|
@ -0,0 +1,307 @@
|
|||
//! Find and identify brokerage export files on disk.
|
||||
//!
|
||||
//! Split from the audit hygiene command so every consumer shares one
|
||||
//! implementation. Detection is the part worth centralising: it keys on each
|
||||
//! file's own self-identifying marker, and the near-miss cases below are not
|
||||
//! obvious enough to re-derive safely (a Schwab CSV holding a Fidelity fund
|
||||
//! names "FIDELITY" in a ticker description, and Fidelity has already
|
||||
//! re-cased its column header once).
|
||||
//!
|
||||
//! Ownership: `brokerFiles` returns a slice the caller frees, and each
|
||||
//! `DiscoveredFile.path` is separately allocated and must also be freed.
|
||||
//! Detection reads each candidate and discards the bytes, so a caller that
|
||||
//! needs the contents reads the file again - cheap at export sizes, and it
|
||||
//! keeps this module free of a buffer-lifetime contract.
|
||||
|
||||
const std = @import("std");
|
||||
|
||||
/// Size ceiling for non-CSV candidates. A CSV is exempt because a positions
|
||||
/// export legitimately gets large; anything else this big is not an export.
|
||||
const max_size_non_csv = 512 * 1024;
|
||||
|
||||
/// Type of a discovered brokerage file.
|
||||
pub const BrokerFileKind = enum {
|
||||
fidelity_csv,
|
||||
schwab_csv,
|
||||
schwab_summary,
|
||||
};
|
||||
|
||||
/// A discovered brokerage file ready for reconciliation.
|
||||
pub const DiscoveredFile = struct {
|
||||
path: []const u8,
|
||||
kind: BrokerFileKind,
|
||||
dir_label: []const u8, // e.g. "audit/" or "$ZFIN_AUDIT_FILES"
|
||||
};
|
||||
|
||||
/// Detect the brokerage type from file contents.
|
||||
///
|
||||
/// Detection answers only "which brokerage produced this file?" - it is
|
||||
/// deliberately separate from "can we parse it?", which each parser
|
||||
/// enforces via its own header/column validation. Keying detection on a
|
||||
/// stable self-identifying marker (rather than the column header) means a
|
||||
/// format tweak surfaces as a specific parse error, not a misroute.
|
||||
pub fn detectBrokerFileKind(data: []const u8) ?BrokerFileKind {
|
||||
// Strip optional UTF-8 BOM
|
||||
const content = if (data.len >= 3 and data[0] == 0xEF and data[1] == 0xBB and data[2] == 0xBF)
|
||||
data[3..]
|
||||
else
|
||||
data;
|
||||
|
||||
// Fidelity CSV: identified by its legal-disclaimer footer, which names
|
||||
// the "Fidelity Brokerage Services LLC" legal entity. This is the file's
|
||||
// true self-identification and is stable boilerplate - unlike the column
|
||||
// header, which Fidelity has already re-cased (Title Case -> sentence
|
||||
// case) and could change again. Keying on the legal-entity phrase also
|
||||
// avoids false positives that bare "Fidelity" would cause: a *Schwab* CSV
|
||||
// holding a Fidelity fund shows "FIDELITY ..." as a ticker description,
|
||||
// but never the legal-entity name. Whether the columns are still
|
||||
// parseable is a separate concern, enforced by the parser's header check.
|
||||
if (std.mem.indexOf(u8, content, "Fidelity Brokerage Services LLC") != null)
|
||||
return .fidelity_csv;
|
||||
|
||||
// Schwab per-account CSV: starts with a quoted title line like "Positions for ..."
|
||||
if (std.mem.startsWith(u8, content, "\"Positions for")) return .schwab_csv;
|
||||
|
||||
// Schwab summary: the "Account number ending in" anchor is exactly what
|
||||
// parseSummary keys on to find account blocks, so detection matches what
|
||||
// the parser can actually handle. There is intentionally no looser
|
||||
// fallback (e.g. account-type labels + "$"): a file lacking this anchor
|
||||
// cannot be parsed as a summary, so classifying it as one would only
|
||||
// produce a guaranteed parse failure - and would misclassify unrelated
|
||||
// CSVs (Fidelity's legal footer contains "Brokerage", every export has "$").
|
||||
if (std.mem.indexOf(u8, content, "Account number ending in") != null) return .schwab_summary;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Discover brokerage files in a directory. Filters by recency (< 24h)
|
||||
/// and applies size limits for non-CSV files.
|
||||
pub fn brokerFiles(
|
||||
io: std.Io,
|
||||
allocator: std.mem.Allocator,
|
||||
dir_path: []const u8,
|
||||
dir_label: []const u8,
|
||||
now_s: i64,
|
||||
/// Skip files older than this. Callers differ - see the hygiene
|
||||
/// command's constant for the reasoning.
|
||||
max_age_hours: i64,
|
||||
) ![]DiscoveredFile {
|
||||
var results = std.ArrayList(DiscoveredFile).empty;
|
||||
defer results.deinit(allocator);
|
||||
|
||||
var dir = std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }) catch return try results.toOwnedSlice(allocator);
|
||||
defer dir.close(io);
|
||||
|
||||
const max_age_s: i128 = @as(i128, max_age_hours) * 3600;
|
||||
|
||||
var it = dir.iterate();
|
||||
while (try it.next(io)) |entry| {
|
||||
if (entry.kind != .file) continue;
|
||||
|
||||
// Check file modification time
|
||||
const stat = dir.statFile(io, entry.name, .{}) catch continue;
|
||||
const mtime_s: i128 = @divFloor(stat.mtime.nanoseconds, std.time.ns_per_s);
|
||||
const age_s = now_s - mtime_s;
|
||||
if (age_s > max_age_s) continue;
|
||||
|
||||
// Check if it's a CSV (no size limit) or non-CSV (size limit applies)
|
||||
const is_csv = std.mem.endsWith(u8, entry.name, ".csv") or std.mem.endsWith(u8, entry.name, ".CSV");
|
||||
if (!is_csv and stat.size > max_size_non_csv) continue;
|
||||
|
||||
// Read and detect content type
|
||||
const data = dir.readFileAlloc(io, entry.name, allocator, .limited(10 * 1024 * 1024)) catch continue;
|
||||
defer allocator.free(data);
|
||||
|
||||
const kind = detectBrokerFileKind(data) orelse continue;
|
||||
const full_path = std.fs.path.join(allocator, &.{ dir_path, entry.name }) catch continue;
|
||||
try results.append(allocator, .{
|
||||
.path = full_path,
|
||||
.kind = kind,
|
||||
.dir_label = dir_label,
|
||||
});
|
||||
}
|
||||
|
||||
return results.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: fidelity csv identified by legal footer" {
|
||||
// Detection keys on the self-identifying legal-entity footer, not the
|
||||
// column header (which Fidelity has already re-cased). A realistic
|
||||
// export: header, a data row, blank line, then the disclaimer footer.
|
||||
const data =
|
||||
"Account number,Account name,Symbol,Description,Quantity,Last price,Last price change,Current value,Today's gain/loss dollar,Today's gain/loss percent,Total gain/loss dollar,Total gain/loss percent,Percent of account,Cost basis total,Average cost basis,Type\n" ++
|
||||
"Z123,Individual - TOD,AAPL,APPLE INC,100,$150.00,-$2.00,$15000.00,,,,,99.88%,$10000.00,$100.00,Margin,\n" ++
|
||||
"\n" ++
|
||||
"\"Brokerage services are provided by Fidelity Brokerage Services LLC (FBS), 900 Salem Street, Smithfield, RI 02917. ...\"\n";
|
||||
try std.testing.expectEqual(BrokerFileKind.fidelity_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
test "detectBrokerFileKind: fidelity detected by footer even with an unrecognized header" {
|
||||
// The footer is the identity signal, so a future header change (or a
|
||||
// leading BOM, or no header at all) does not defeat detection. Whether
|
||||
// the columns are parseable is a separate concern enforced by the
|
||||
// parser's header validation.
|
||||
const data =
|
||||
"\xEF\xBB\xBFsome future header layout we do not recognize\n" ++
|
||||
"...data...\n\n" ++
|
||||
"\"... Both are Fidelity Brokerage Services LLC companies and members SIPC ...\"\n";
|
||||
try std.testing.expectEqual(BrokerFileKind.fidelity_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
test "detectBrokerFileKind: a Fidelity fund inside a Schwab CSV is not mis-detected as Fidelity" {
|
||||
// Bare "Fidelity" would false-match here (FIDELITY STOCK SELECTOR is a
|
||||
// holding), but the legal-entity phrase never appears as a ticker
|
||||
// description - so this is correctly detected as a Schwab CSV.
|
||||
const data =
|
||||
"\"Positions for account Sample IRA ...1234 as of 10:00 AM ET, 2026/06/27\"\n" ++
|
||||
"\n" ++
|
||||
"\"Symbol\",\"Description\",\"Price\"\n" ++
|
||||
"\"FDSCX\",\"FIDELITY STOCK SELECTOR SMALL CAP\",\"51.65\"\n";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
test "detectBrokerFileKind: schwab csv" {
|
||||
const schwab_header = "\"Positions for account Roth IRA ...1234 as of\"";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_csv, detectBrokerFileKind(schwab_header).?);
|
||||
}
|
||||
test "detectBrokerFileKind: schwab summary" {
|
||||
const schwab_summary_data = "Brokerage ...1234\nAccount number ending in 1234\n$500,000.00";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_summary, detectBrokerFileKind(schwab_summary_data).?);
|
||||
}
|
||||
test "detectBrokerFileKind: unknown file" {
|
||||
const random_data = "This is just some random text that doesn't match any pattern";
|
||||
try std.testing.expect(detectBrokerFileKind(random_data) == null);
|
||||
}
|
||||
test "detectBrokerFileKind: schwab csv with Positions header" {
|
||||
const data = "\"Positions for account Brokerage ...1234 as of 11:31 AM ET, 2026/04/25\"\n\nSymbol,Description,Quantity";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
test "detectBrokerFileKind: summary-shaped text without the anchor is not detected" {
|
||||
// Regression guard against re-introducing the old "account-type label +
|
||||
// $" fallback. This blob looks summary-ish (account-type words, dollar
|
||||
// amounts) but lacks the "Account number ending in" anchor that
|
||||
// parseSummary requires - so classifying it as a summary would only
|
||||
// yield a guaranteed NoAccountsFound. Detection must return null.
|
||||
const data = "Roth IRA ...1234\nSome text\n$50,000.00\n";
|
||||
try std.testing.expect(detectBrokerFileKind(data) == null);
|
||||
}
|
||||
|
||||
test "brokerFiles: finds files in temp directory" {
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
// Create a temp directory with test files
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
// Write a fidelity CSV (identified by its legal-entity footer)
|
||||
tmp.dir.writeFile(io, .{
|
||||
.sub_path = "fidelity.csv",
|
||||
.data = "Account number,Account name,Symbol,Description,Quantity,Last price,Current value\nZ123,Test,AAPL,Apple,100,200,20000\n\n\"Brokerage services are provided by Fidelity Brokerage Services LLC (FBS), 900 Salem Street, Smithfield, RI 02917. ...\"\n",
|
||||
}) catch unreachable;
|
||||
|
||||
// Write a schwab summary (non-CSV)
|
||||
tmp.dir.writeFile(io, .{
|
||||
.sub_path = "schwab.txt",
|
||||
.data = "Brokerage ...1234\nAccount number ending in 1234\n$500,000.00\n",
|
||||
}) catch unreachable;
|
||||
|
||||
// Write a random non-matching file
|
||||
tmp.dir.writeFile(io, .{
|
||||
.sub_path = "notes.txt",
|
||||
.data = "Just some random notes",
|
||||
}) catch unreachable;
|
||||
|
||||
// Get the temp dir path
|
||||
const tmp_path = tmp.dir.realPathFileAlloc(io, ".", allocator) catch unreachable;
|
||||
defer allocator.free(tmp_path);
|
||||
|
||||
// wall-clock required: test writes real files and verifies they're
|
||||
// treated as fresh. A fixed synthetic `now_s` would drift relative
|
||||
// to the file mtime and produce flaky results.
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const files = try brokerFiles(io, allocator, tmp_path, "test/", now_s, 24);
|
||||
defer {
|
||||
for (files) |f| allocator.free(f.path);
|
||||
allocator.free(files);
|
||||
}
|
||||
|
||||
// Should find fidelity CSV and schwab summary, but not notes.txt
|
||||
try std.testing.expectEqual(@as(usize, 2), files.len);
|
||||
|
||||
var found_fidelity = false;
|
||||
var found_schwab = false;
|
||||
for (files) |f| {
|
||||
switch (f.kind) {
|
||||
.fidelity_csv => found_fidelity = true,
|
||||
.schwab_summary => found_schwab = true,
|
||||
else => {},
|
||||
}
|
||||
}
|
||||
try std.testing.expect(found_fidelity);
|
||||
try std.testing.expect(found_schwab);
|
||||
}
|
||||
|
||||
test "brokerFiles: empty directory returns empty" {
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
const tmp_path = tmp.dir.realPathFileAlloc(io, ".", allocator) catch unreachable;
|
||||
defer allocator.free(tmp_path);
|
||||
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const files = try brokerFiles(io, allocator, tmp_path, "test/", now_s, 24);
|
||||
defer allocator.free(files);
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 0), files.len);
|
||||
}
|
||||
|
||||
test "brokerFiles: nonexistent directory returns empty" {
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const files = try brokerFiles(io, allocator, "/nonexistent/path/audit", "test/", now_s, 24);
|
||||
defer allocator.free(files);
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 0), files.len);
|
||||
}
|
||||
|
||||
test "brokerFiles: max_age_hours is the caller's to choose" {
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
tmp.dir.writeFile(io, .{
|
||||
.sub_path = "export.csv",
|
||||
.data = "Account number,Symbol\nZ1,AAPL\n\n\"... Fidelity Brokerage Services LLC ...\"\n",
|
||||
}) catch unreachable;
|
||||
const tmp_path = tmp.dir.realPathFileAlloc(io, ".", allocator) catch unreachable;
|
||||
defer allocator.free(tmp_path);
|
||||
|
||||
// Advancing `now` rather than back-dating the file: mtime is the thing
|
||||
// under test, so moving the clock instead keeps this independent of
|
||||
// whether the filesystem honours a mtime write.
|
||||
const real_now = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const looks_48h_old = real_now + 48 * 3600;
|
||||
|
||||
// The window the hygiene report uses would drop it.
|
||||
const tight = try brokerFiles(io, allocator, tmp_path, "test/", looks_48h_old, 24);
|
||||
defer {
|
||||
for (tight) |f| allocator.free(f.path);
|
||||
allocator.free(tight);
|
||||
}
|
||||
try std.testing.expectEqual(@as(usize, 0), tight.len);
|
||||
|
||||
// A caller willing to look further back gets it, which is the whole
|
||||
// reason this is a parameter: a reconciler would rather see a stale
|
||||
// export and say so than silently find nothing.
|
||||
const loose = try brokerFiles(io, allocator, tmp_path, "test/", looks_48h_old, 24 * 7);
|
||||
defer {
|
||||
for (loose) |f| allocator.free(f.path);
|
||||
allocator.free(loose);
|
||||
}
|
||||
try std.testing.expectEqual(@as(usize, 1), loose.len);
|
||||
try std.testing.expectEqual(BrokerFileKind.fidelity_csv, loose[0].kind);
|
||||
}
|
||||
|
|
@ -28,14 +28,18 @@ const test_git = @import("../../testutil/git.zig");
|
|||
const common = @import("common.zig");
|
||||
const fidelity = @import("../../analytics/reconcile/fidelity.zig");
|
||||
const schwab = @import("schwab.zig");
|
||||
const discover = zfin.brokerage.discover;
|
||||
const fmt = cli.fmt;
|
||||
|
||||
// ── Hygiene check (flagless audit) ──────────────────────────
|
||||
|
||||
/// Constants for hygiene check behavior. Kept as named constants for
|
||||
/// easy future tuning.
|
||||
/// How recent a brokerage export must be for the hygiene check to pick it
|
||||
/// up. Passed to `discover.brokerFiles` rather than living inside it, because
|
||||
/// callers disagree: a hygiene report wants only today's downloads, while a
|
||||
/// reconciler is better served by seeing a stale file and saying so.
|
||||
const audit_file_max_age_hours = 24;
|
||||
const audit_file_max_size_non_csv = 512 * 1024; // 512KB, for non-CSV files only
|
||||
pub const default_stale_days: u32 = 3;
|
||||
const stale_warning_multiplier: u32 = 2; // yellow -> red at 2× threshold
|
||||
/// Safety cap on how many portfolio commits the account-cadence walk
|
||||
|
|
@ -73,108 +77,6 @@ const harvested_stale_days: u32 = 90;
|
|||
/// `AccountTaxEntry.tax_mix_date`.
|
||||
const tax_mix_stale_days: u32 = 90;
|
||||
|
||||
/// Type of a discovered brokerage file.
|
||||
const BrokerFileKind = enum {
|
||||
fidelity_csv,
|
||||
schwab_csv,
|
||||
schwab_summary,
|
||||
};
|
||||
|
||||
/// A discovered brokerage file ready for reconciliation.
|
||||
const DiscoveredFile = struct {
|
||||
path: []const u8,
|
||||
kind: BrokerFileKind,
|
||||
dir_label: []const u8, // e.g. "audit/" or "$ZFIN_AUDIT_FILES"
|
||||
};
|
||||
|
||||
/// Detect the brokerage type from file contents.
|
||||
///
|
||||
/// Detection answers only "which brokerage produced this file?" - it is
|
||||
/// deliberately separate from "can we parse it?", which each parser
|
||||
/// enforces via its own header/column validation. Keying detection on a
|
||||
/// stable self-identifying marker (rather than the column header) means a
|
||||
/// format tweak surfaces as a specific parse error, not a misroute.
|
||||
fn detectBrokerFileKind(data: []const u8) ?BrokerFileKind {
|
||||
// Strip optional UTF-8 BOM
|
||||
const content = if (data.len >= 3 and data[0] == 0xEF and data[1] == 0xBB and data[2] == 0xBF)
|
||||
data[3..]
|
||||
else
|
||||
data;
|
||||
|
||||
// Fidelity CSV: identified by its legal-disclaimer footer, which names
|
||||
// the "Fidelity Brokerage Services LLC" legal entity. This is the file's
|
||||
// true self-identification and is stable boilerplate - unlike the column
|
||||
// header, which Fidelity has already re-cased (Title Case -> sentence
|
||||
// case) and could change again. Keying on the legal-entity phrase also
|
||||
// avoids false positives that bare "Fidelity" would cause: a *Schwab* CSV
|
||||
// holding a Fidelity fund shows "FIDELITY ..." as a ticker description,
|
||||
// but never the legal-entity name. Whether the columns are still
|
||||
// parseable is a separate concern, enforced by the parser's header check.
|
||||
if (std.mem.indexOf(u8, content, "Fidelity Brokerage Services LLC") != null)
|
||||
return .fidelity_csv;
|
||||
|
||||
// Schwab per-account CSV: starts with a quoted title line like "Positions for ..."
|
||||
if (std.mem.startsWith(u8, content, "\"Positions for")) return .schwab_csv;
|
||||
|
||||
// Schwab summary: the "Account number ending in" anchor is exactly what
|
||||
// parseSummary keys on to find account blocks, so detection matches what
|
||||
// the parser can actually handle. There is intentionally no looser
|
||||
// fallback (e.g. account-type labels + "$"): a file lacking this anchor
|
||||
// cannot be parsed as a summary, so classifying it as one would only
|
||||
// produce a guaranteed parse failure - and would misclassify unrelated
|
||||
// CSVs (Fidelity's legal footer contains "Brokerage", every export has "$").
|
||||
if (std.mem.indexOf(u8, content, "Account number ending in") != null) return .schwab_summary;
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// Discover brokerage files in a directory. Filters by recency (< 24h)
|
||||
/// and applies size limits for non-CSV files.
|
||||
fn discoverBrokerFiles(
|
||||
io: std.Io,
|
||||
allocator: std.mem.Allocator,
|
||||
dir_path: []const u8,
|
||||
dir_label: []const u8,
|
||||
now_s: i64,
|
||||
) ![]DiscoveredFile {
|
||||
var results = std.ArrayList(DiscoveredFile).empty;
|
||||
defer results.deinit(allocator);
|
||||
|
||||
var dir = std.Io.Dir.cwd().openDir(io, dir_path, .{ .iterate = true }) catch return try results.toOwnedSlice(allocator);
|
||||
defer dir.close(io);
|
||||
|
||||
const max_age_s: i128 = audit_file_max_age_hours * 3600;
|
||||
|
||||
var it = dir.iterate();
|
||||
while (try it.next(io)) |entry| {
|
||||
if (entry.kind != .file) continue;
|
||||
|
||||
// Check file modification time
|
||||
const stat = dir.statFile(io, entry.name, .{}) catch continue;
|
||||
const mtime_s: i128 = @divFloor(stat.mtime.nanoseconds, std.time.ns_per_s);
|
||||
const age_s = now_s - mtime_s;
|
||||
if (age_s > max_age_s) continue;
|
||||
|
||||
// Check if it's a CSV (no size limit) or non-CSV (size limit applies)
|
||||
const is_csv = std.mem.endsWith(u8, entry.name, ".csv") or std.mem.endsWith(u8, entry.name, ".CSV");
|
||||
if (!is_csv and stat.size > audit_file_max_size_non_csv) continue;
|
||||
|
||||
// Read and detect content type
|
||||
const data = dir.readFileAlloc(io, entry.name, allocator, .limited(10 * 1024 * 1024)) catch continue;
|
||||
defer allocator.free(data);
|
||||
|
||||
const kind = detectBrokerFileKind(data) orelse continue;
|
||||
const full_path = std.fs.path.join(allocator, &.{ dir_path, entry.name }) catch continue;
|
||||
try results.append(allocator, .{
|
||||
.path = full_path,
|
||||
.kind = kind,
|
||||
.dir_label = dir_label,
|
||||
});
|
||||
}
|
||||
|
||||
return results.toOwnedSlice(allocator);
|
||||
}
|
||||
|
||||
/// Compute which accounts have been modified between two parsed portfolios.
|
||||
/// Returns a set of account names that have any lot-level differences.
|
||||
/// Compares by serializing each lot to a canonical string per account,
|
||||
|
|
@ -1134,7 +1036,7 @@ pub fn runHygieneCheck(
|
|||
// Resolve audit directories
|
||||
const portfolio_dir = std.fs.path.dirnamePosix(portfolio_path) orelse ".";
|
||||
|
||||
var all_files = std.ArrayList(DiscoveredFile).empty;
|
||||
var all_files = std.ArrayList(discover.DiscoveredFile).empty;
|
||||
defer {
|
||||
for (all_files.items) |f| allocator.free(f.path);
|
||||
all_files.deinit(allocator);
|
||||
|
|
@ -1143,7 +1045,7 @@ pub fn runHygieneCheck(
|
|||
// Check $ZFIN_AUDIT_FILES first
|
||||
const env_audit_dir = if (svc.config.environ_map) |em| em.get("ZFIN_AUDIT_FILES") else null;
|
||||
if (env_audit_dir) |edir| {
|
||||
const env_files = try discoverBrokerFiles(io, allocator, edir, "$ZFIN_AUDIT_FILES", now_s);
|
||||
const env_files = try discover.brokerFiles(io, allocator, edir, "$ZFIN_AUDIT_FILES", now_s, audit_file_max_age_hours);
|
||||
defer allocator.free(env_files);
|
||||
for (env_files) |f| try all_files.append(allocator, f);
|
||||
}
|
||||
|
|
@ -1153,7 +1055,7 @@ pub fn runHygieneCheck(
|
|||
defer if (default_audit_dir) |d| allocator.free(d);
|
||||
|
||||
if (default_audit_dir) |adir| {
|
||||
const dir_files = try discoverBrokerFiles(io, allocator, adir, "audit/", now_s);
|
||||
const dir_files = try discover.brokerFiles(io, allocator, adir, "audit/", now_s, audit_file_max_age_hours);
|
||||
defer allocator.free(dir_files);
|
||||
for (dir_files) |f| try all_files.append(allocator, f);
|
||||
}
|
||||
|
|
@ -1410,57 +1312,6 @@ test "accumulatePresent: result strings are owned copies (survive source free)"
|
|||
try std.testing.expectEqualStrings("9012", dst.items[0]);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: fidelity csv identified by legal footer" {
|
||||
// Detection keys on the self-identifying legal-entity footer, not the
|
||||
// column header (which Fidelity has already re-cased). A realistic
|
||||
// export: header, a data row, blank line, then the disclaimer footer.
|
||||
const data =
|
||||
"Account number,Account name,Symbol,Description,Quantity,Last price,Last price change,Current value,Today's gain/loss dollar,Today's gain/loss percent,Total gain/loss dollar,Total gain/loss percent,Percent of account,Cost basis total,Average cost basis,Type\n" ++
|
||||
"Z123,Individual - TOD,AAPL,APPLE INC,100,$150.00,-$2.00,$15000.00,,,,,99.88%,$10000.00,$100.00,Margin,\n" ++
|
||||
"\n" ++
|
||||
"\"Brokerage services are provided by Fidelity Brokerage Services LLC (FBS), 900 Salem Street, Smithfield, RI 02917. ...\"\n";
|
||||
try std.testing.expectEqual(BrokerFileKind.fidelity_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: fidelity detected by footer even with an unrecognized header" {
|
||||
// The footer is the identity signal, so a future header change (or a
|
||||
// leading BOM, or no header at all) does not defeat detection. Whether
|
||||
// the columns are parseable is a separate concern enforced by the
|
||||
// parser's header validation.
|
||||
const data =
|
||||
"\xEF\xBB\xBFsome future header layout we do not recognize\n" ++
|
||||
"...data...\n\n" ++
|
||||
"\"... Both are Fidelity Brokerage Services LLC companies and members SIPC ...\"\n";
|
||||
try std.testing.expectEqual(BrokerFileKind.fidelity_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: a Fidelity fund inside a Schwab CSV is not mis-detected as Fidelity" {
|
||||
// Bare "Fidelity" would false-match here (FIDELITY STOCK SELECTOR is a
|
||||
// holding), but the legal-entity phrase never appears as a ticker
|
||||
// description - so this is correctly detected as a Schwab CSV.
|
||||
const data =
|
||||
"\"Positions for account Sample IRA ...1234 as of 10:00 AM ET, 2026/06/27\"\n" ++
|
||||
"\n" ++
|
||||
"\"Symbol\",\"Description\",\"Price\"\n" ++
|
||||
"\"FDSCX\",\"FIDELITY STOCK SELECTOR SMALL CAP\",\"51.65\"\n";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: schwab csv" {
|
||||
const schwab_header = "\"Positions for account Roth IRA ...1234 as of\"";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_csv, detectBrokerFileKind(schwab_header).?);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: schwab summary" {
|
||||
const schwab_summary_data = "Brokerage ...1234\nAccount number ending in 1234\n$500,000.00";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_summary, detectBrokerFileKind(schwab_summary_data).?);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: unknown file" {
|
||||
const random_data = "This is just some random text that doesn't match any pattern";
|
||||
try std.testing.expect(detectBrokerFileKind(random_data) == null);
|
||||
}
|
||||
|
||||
test "stalenessColor: within threshold" {
|
||||
try std.testing.expectEqual(cli.CLR_MUTED, stalenessColor(2, 3));
|
||||
try std.testing.expectEqual(cli.CLR_MUTED, stalenessColor(3, 3));
|
||||
|
|
@ -2131,21 +1982,6 @@ test "findPriceDateMismatches: CD price change is ignored" {
|
|||
try std.testing.expectEqual(@as(usize, 0), m.items.len);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: schwab csv with Positions header" {
|
||||
const data = "\"Positions for account Brokerage ...1234 as of 11:31 AM ET, 2026/04/25\"\n\nSymbol,Description,Quantity";
|
||||
try std.testing.expectEqual(BrokerFileKind.schwab_csv, detectBrokerFileKind(data).?);
|
||||
}
|
||||
|
||||
test "detectBrokerFileKind: summary-shaped text without the anchor is not detected" {
|
||||
// Regression guard against re-introducing the old "account-type label +
|
||||
// $" fallback. This blob looks summary-ish (account-type words, dollar
|
||||
// amounts) but lacks the "Account number ending in" anchor that
|
||||
// parseSummary requires - so classifying it as a summary would only
|
||||
// yield a guaranteed NoAccountsFound. Detection must return null.
|
||||
const data = "Roth IRA ...1234\nSome text\n$50,000.00\n";
|
||||
try std.testing.expect(detectBrokerFileKind(data) == null);
|
||||
}
|
||||
|
||||
test "UpdateCadence label" {
|
||||
try std.testing.expectEqualStrings("weekly", analysis.UpdateCadence.weekly.label());
|
||||
try std.testing.expectEqualStrings("monthly", analysis.UpdateCadence.monthly.label());
|
||||
|
|
@ -2153,90 +1989,6 @@ test "UpdateCadence label" {
|
|||
try std.testing.expectEqualStrings("none", analysis.UpdateCadence.none.label());
|
||||
}
|
||||
|
||||
test "discoverBrokerFiles: finds files in temp directory" {
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
// Create a temp directory with test files
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
// Write a fidelity CSV (identified by its legal-entity footer)
|
||||
tmp.dir.writeFile(io, .{
|
||||
.sub_path = "fidelity.csv",
|
||||
.data = "Account number,Account name,Symbol,Description,Quantity,Last price,Current value\nZ123,Test,AAPL,Apple,100,200,20000\n\n\"Brokerage services are provided by Fidelity Brokerage Services LLC (FBS), 900 Salem Street, Smithfield, RI 02917. ...\"\n",
|
||||
}) catch unreachable;
|
||||
|
||||
// Write a schwab summary (non-CSV)
|
||||
tmp.dir.writeFile(io, .{
|
||||
.sub_path = "schwab.txt",
|
||||
.data = "Brokerage ...1234\nAccount number ending in 1234\n$500,000.00\n",
|
||||
}) catch unreachable;
|
||||
|
||||
// Write a random non-matching file
|
||||
tmp.dir.writeFile(io, .{
|
||||
.sub_path = "notes.txt",
|
||||
.data = "Just some random notes",
|
||||
}) catch unreachable;
|
||||
|
||||
// Get the temp dir path
|
||||
const tmp_path = tmp.dir.realPathFileAlloc(io, ".", allocator) catch unreachable;
|
||||
defer allocator.free(tmp_path);
|
||||
|
||||
// wall-clock required: test writes real files and verifies they're
|
||||
// treated as fresh. A fixed synthetic `now_s` would drift relative
|
||||
// to the file mtime and produce flaky results.
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const files = try discoverBrokerFiles(io, allocator, tmp_path, "test/", now_s);
|
||||
defer {
|
||||
for (files) |f| allocator.free(f.path);
|
||||
allocator.free(files);
|
||||
}
|
||||
|
||||
// Should find fidelity CSV and schwab summary, but not notes.txt
|
||||
try std.testing.expectEqual(@as(usize, 2), files.len);
|
||||
|
||||
var found_fidelity = false;
|
||||
var found_schwab = false;
|
||||
for (files) |f| {
|
||||
switch (f.kind) {
|
||||
.fidelity_csv => found_fidelity = true,
|
||||
.schwab_summary => found_schwab = true,
|
||||
else => {},
|
||||
}
|
||||
}
|
||||
try std.testing.expect(found_fidelity);
|
||||
try std.testing.expect(found_schwab);
|
||||
}
|
||||
|
||||
test "discoverBrokerFiles: empty directory returns empty" {
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
|
||||
const tmp_path = tmp.dir.realPathFileAlloc(io, ".", allocator) catch unreachable;
|
||||
defer allocator.free(tmp_path);
|
||||
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const files = try discoverBrokerFiles(io, allocator, tmp_path, "test/", now_s);
|
||||
defer allocator.free(files);
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 0), files.len);
|
||||
}
|
||||
|
||||
test "discoverBrokerFiles: nonexistent directory returns empty" {
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const files = try discoverBrokerFiles(io, allocator, "/nonexistent/path/audit", "test/", now_s);
|
||||
defer allocator.free(files);
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 0), files.len);
|
||||
}
|
||||
|
||||
test "printLargeLotWarning: cash destination emits dest_lot::cash template" {
|
||||
var buf: [1024]u8 = undefined;
|
||||
var writer = std.Io.Writer.fixed(&buf);
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue