harden schwab csv parsing in manner similar to fidelity
All checks were successful
Generic zig build / build (push) Successful in 5m38s
Generic zig build / publish-macos (push) Successful in 14s
Generic zig build / deploy (push) Successful in 22s

This commit is contained in:
Emil Lerch 2026-07-22 10:03:02 -07:00
parent d59c2da58b
commit dd4d68280f
Signed by: lobo
GPG key ID: A7B62D657EF764F8

View file

@ -25,9 +25,11 @@
//! 4. Rows with symbol "Cash & Cash Investments" are treated as cash.
//! The row with symbol "Positions Total" is skipped.
//!
//! 5. Hardcodes the expected column layout. If Schwab changes the CSV
//! format, this parser will break. The header row is not validated
//! beyond being skipped.
//! 5. Hardcodes the expected column layout via fixed indices. The header
//! row is validated (see `validateHeader`) to confirm the columns the
//! parser reads are where it expects them, so a Schwab reorder/rename of
//! a read-column fails loudly as `error.UnexpectedHeader` instead of
//! silently mis-reading a value.
//!
//! ## Schwab summary - limitations
//!
@ -75,6 +77,47 @@ const Col = struct {
const asset_type = 16;
};
/// Distinctive prefix for each column `parseCsv` reads, at its expected
/// index, matched case-insensitively. `Price` is matched exactly (not by
/// prefix) because its bare label is a prefix of the "Price Chng ..."
/// columns, so a startsWith could not tell them apart on a reorder.
///
/// Only the columns the parser actually reads are validated: Schwab
/// reordering, renaming, or adding a column we ignore is tolerated, while a
/// change that would shift a value we depend on is rejected with
/// `error.UnexpectedHeader` instead of silently mis-reading. Schwab labels
/// use a "Short (Long)" form ("Qty (Quantity)"); older exports and the test
/// fixtures use the short form ("Qty") - a prefix match accepts both. Keep
/// in sync with `Col`.
const ExpectedColumn = struct { idx: usize, prefix: []const u8, exact: bool = false };
const expected_header = [_]ExpectedColumn{
.{ .idx = Col.symbol, .prefix = "Symbol" },
.{ .idx = Col.price, .prefix = "Price", .exact = true },
.{ .idx = Col.quantity, .prefix = "Qty" },
.{ .idx = Col.market_value, .prefix = "Mkt Val" },
.{ .idx = Col.cost_basis, .prefix = "Cost Basis" },
.{ .idx = Col.asset_type, .prefix = "Asset Type" },
};
/// Validate that the header carries the columns `parseCsv` indexes into,
/// at their expected positions (case-insensitive). Reuses `splitCsvLine`
/// so the surrounding quotes are stripped the same way as data rows. A
/// missing/renamed/reordered read-column - or too few columns - fails with
/// `error.UnexpectedHeader`; trailing or unused-column changes are
/// tolerated. See `expected_header`.
fn validateHeader(header: []const u8) error{UnexpectedHeader}!void {
var cols: [expected_columns][]const u8 = undefined;
const n = splitCsvLine(header, &cols);
if (n < expected_columns) return error.UnexpectedHeader;
inline for (expected_header) |c| {
const ok = if (c.exact)
std.ascii.eqlIgnoreCase(cols[c.idx], c.prefix)
else
std.ascii.startsWithIgnoreCase(cols[c.idx], c.prefix);
if (!ok) return error.UnexpectedHeader;
}
}
/// Split a Schwab CSV line on commas, stripping surrounding quotes from each field.
/// Returns the number of columns parsed. Fields are slices into the input line.
fn splitCsvLine(line: []const u8, cols: *[expected_columns][]const u8) usize {
@ -147,8 +190,12 @@ pub fn parseCsv(allocator: std.mem.Allocator, data: []const u8) !CsvResult {
// Line 2: blank (skip)
_ = lines.next();
// Line 3: header row (skip)
_ = lines.next();
// Line 3: header row - validate the columns we read are where we expect,
// so a Schwab format change surfaces as UnexpectedHeader (and, in the
// flagless audit, a "could not parse" warning) rather than silently
// reading a shifted column.
const header_line = lines.next() orelse return error.UnexpectedHeader;
try validateHeader(std.mem.trimEnd(u8, header_line, &.{ '\r', ' ' }));
// Data rows
while (lines.next()) |line| {
@ -346,6 +393,65 @@ test "parseCsv basic" {
try std.testing.expect(parsed.positions[1].quantity == null);
}
test "parseCsv accepts the real Short (Long) header labels" {
// Real Schwab exports label columns "Qty (Quantity)", "Mkt Val (Market
// Value)", etc. (the fixtures above use the short form). The prefix-token
// header validation must accept both forms.
const csv =
"\"Positions for account Sample IRA ...1234 as of 10:00 AM ET, 2026/06/27\"\n" ++
"\n" ++
"\"Symbol\",\"Description\",\"Price Chng $ (Price Change $)\",\"Price Chng % (Price Change %)\",\"Price\",\"Qty (Quantity)\",\"Day Chng $ (Day Change $)\",\"Day Chng % (Day Change %)\",\"Mkt Val (Market Value)\",\"Cost Basis\",\"Gain $ (Gain/Loss $)\",\"Gain % (Gain/Loss %)\",\"Ratings\",\"Reinvest?\",\"Reinvest Capital Gains?\",\"% of Acct (% of Account)\",\"Asset Type\",\n" ++
"\"AMZN\",\"AMAZON.COM INC\",\"5.55\",\"2.38%\",\"239.20\",\"100\",\"$8.00\",\"2.38%\",\"$23,920.00\",\"$10,000.00\",\"$13,920.00\",\"139%\",\"C\",\"No\",\"N/A\",\"41.54%\",\"Equity\",\n";
const allocator = std.testing.allocator;
const parsed = try parseCsv(allocator, csv);
defer allocator.free(parsed.positions);
try std.testing.expectEqual(@as(usize, 1), parsed.positions.len);
try std.testing.expectEqualStrings("AMZN", parsed.positions[0].symbol);
try std.testing.expectApproxEqAbs(@as(f64, 100), parsed.positions[0].quantity.?, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 23920.00), parsed.positions[0].current_value.?, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 10000.00), parsed.positions[0].cost_basis.?, 0.01);
}
test "parseCsv rejects a header whose read-columns are reordered" {
// Symbol (idx 0) and Price (idx 4) are swapped. Positional parsing would
// read a price where a symbol belongs, so validateHeader must reject it.
const csv =
"\"Positions for account Sample IRA ...1234 as of 10:00 AM ET, 2026/06/27\"\n" ++
"\n" ++
"\"Price\",\"Description\",\"Price Chng $\",\"Price Chng %\",\"Symbol\",\"Qty\",\"Day Chng $\",\"Day Chng %\",\"Mkt Val\",\"Cost Basis\",\"Gain $\",\"Gain %\",\"Ratings\",\"Reinvest?\",\"Reinvest Capital Gains?\",\"% of Acct\",\"Asset Type\",\n" ++
"\"239.20\",\"AMAZON.COM INC\",\"5.55\",\"2.38%\",\"AMZN\",\"100\",\"$8.00\",\"2.38%\",\"$23,920.00\",\"$10,000.00\",\"$13,920.00\",\"139%\",\"C\",\"No\",\"N/A\",\"41.54%\",\"Equity\",\n";
const allocator = std.testing.allocator;
try std.testing.expectError(error.UnexpectedHeader, parseCsv(allocator, csv));
}
test "parseCsv tolerates reordering of columns it does not read" {
// The two Price-change columns (idx 2, 3) are swapped. Neither is read,
// so validation passes and the read-columns still resolve correctly.
const csv =
"\"Positions for account Sample IRA ...1234 as of 10:00 AM ET, 2026/06/27\"\n" ++
"\n" ++
"\"Symbol\",\"Description\",\"Price Chng % (Price Change %)\",\"Price Chng $ (Price Change $)\",\"Price\",\"Qty (Quantity)\",\"Day Chng $ (Day Change $)\",\"Day Chng % (Day Change %)\",\"Mkt Val (Market Value)\",\"Cost Basis\",\"Gain $ (Gain/Loss $)\",\"Gain % (Gain/Loss %)\",\"Ratings\",\"Reinvest?\",\"Reinvest Capital Gains?\",\"% of Acct (% of Account)\",\"Asset Type\",\n" ++
"\"AMZN\",\"AMAZON.COM INC\",\"2.38%\",\"5.55\",\"239.20\",\"100\",\"$8.00\",\"2.38%\",\"$23,920.00\",\"$10,000.00\",\"$13,920.00\",\"139%\",\"C\",\"No\",\"N/A\",\"41.54%\",\"Equity\",\n";
const allocator = std.testing.allocator;
const parsed = try parseCsv(allocator, csv);
defer allocator.free(parsed.positions);
try std.testing.expectEqual(@as(usize, 1), parsed.positions.len);
try std.testing.expectEqualStrings("AMZN", parsed.positions[0].symbol);
try std.testing.expectApproxEqAbs(@as(f64, 100), parsed.positions[0].quantity.?, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 23920.00), parsed.positions[0].current_value.?, 0.01);
}
test "parseCsv rejects a header with too few columns" {
const csv =
"\"Positions for account Sample IRA ...1234 as of 10:00 AM ET, 2026/06/27\"\n" ++
"\n" ++
"\"Symbol\",\"Description\",\"Price\"\n";
const allocator = std.testing.allocator;
try std.testing.expectError(error.UnexpectedHeader, parseCsv(allocator, csv));
}
test "parseSummary basic" {
const data =
\\Sample Roth