//! EDGAR provider - SEC's electronic filing system as a data source.
//!
//! ## What this provider does
//!
//! Given a stock or fund symbol, EDGAR can answer:
//!
//!   * "What's this fund made of?" - the latest portfolio holdings,
//!     sector breakdown, and net assets, parsed from the fund's most
//!     recent NPORT-P filing.
//!   * "How many shares does this company have outstanding?" - read
//!     from XBRL-tagged fields on the company's most recent 10-K /
//!     10-Q / 40-F cover page. Combined with a price quote (from
//!     elsewhere) this gives market cap.
//!   * "Where in EDGAR does this symbol live?" - symbol -> CIK
//!     lookup via SEC's two ticker-map indexes.
//!
//! ## Workflow when a caller asks about one symbol
//!
//! Symbols don't carry CIKs, so the first step is always a
//! ticker-map lookup. From there the path forks:
//!
//!   AAPL (operating company)
//!     1. Look up "AAPL" in the company ticker map -> CIK 320193.
//!     2. Fetch the submissions feed for CIK 320193 -> entityType
//!        "operating", no NPORT-P. Classify as `not_a_fund`.
//!     3. (Optional) fetch shares-outstanding from the XBRL
//!        companyconcept endpoint for use in market cap math.
//!
//!   VTI (mutual-fund-trust ETF)
//!     1. Look up "VTI" in the mutual-fund ticker map -> CIK 36405,
//!        seriesId S000002848.
//!     2. Run the EDGAR full-text search for that seriesId, filtered
//!        to NPORT-P. Get the URL of the most recent filing.
//!     3. Download the NPORT-P primary_doc.xml. Parse profile,
//!        sectors, holdings.
//!
//!   SPY (unit-investment-trust ETF)
//!     1. Not in mutual-fund ticker map. Look up "SPY" in the
//!        company ticker map -> CIK 884394.
//!     2. Fetch the submissions feed -> entityType "other", has a
//!        NPORT-P at trust-CIK level (UITs don't have a seriesId).
//!     3. Download that NPORT-P. Parse like a fund.
//!
//!   GLD (commodity trust)
//!     1. Not in mutual-fund ticker map. Look up "GLD" in the
//!        company ticker map -> CIK 1222333.
//!     2. Submissions feed -> entityType "operating", SIC describes a
//!        commodity trust. No NPORT-P. Return profile-only metrics
//!        (the trust exists but has no portfolio to disclose).
//!
//! ## Glossary
//!
//!   CIK         Central Index Key. SEC's primary identifier for a
//!               filer. 10-digit zero-padded number; we normalize to
//!               that shape at the boundary so all callers can
//!               assume it.
//!   NPORT-P     Form NPORT-P (public). Quarterly portfolio
//!               disclosure filed by registered investment companies
//!               (mutual funds, most ETFs). Contains every position,
//!               aggregated valuation, and asset/issuer classifiers.
//!   10-K        Annual report filed by US-domiciled operating
//!               companies. Cover page carries shares-outstanding.
//!   10-Q        Quarterly equivalent of 10-K.
//!   40-F        Annual report filed by Canadian companies that
//!               participate in the SEC's MJDS regime. Same XBRL
//!               cover-page fields as 10-K - the dei taxonomy
//!               handles both. Barrick Mining, Shopify, etc.
//!   20-F        Annual report filed by other foreign private
//!               issuers (BP, Toyota, Sony, ...). Covers the same
//!               financial-statement ground as 10-K but the SEC
//!               doesn't require dei-tagged shares-outstanding here,
//!               so the XBRL companyconcept endpoint returns 404 for
//!               many of them. Caller treats this as "shares unknown."
//!   XBRL        Structured-data tagging for SEC filings. Makes
//!               specific fields (revenue, shares outstanding, etc.)
//!               machine-readable across forms.
//!   dei         Document and Entity Information - XBRL taxonomy for
//!               cover-page metadata (entity name, registrant info,
//!               shares outstanding). Cross-form, cross-jurisdiction.
//!   us-gaap     XBRL taxonomy for US GAAP financial concepts.
//!               Carries fallback shares-outstanding for dual-class
//!               issuers (GOOGL, META) that don't tag the dei field.
//!   UIT         Unit Investment Trust. A specific kind of fund
//!               structure (SPY, GLD, IVV, ...) that files NPORT-P
//!               at the trust-CIK level rather than under a
//!               series-of-trust seriesId like mutual funds do.
//!   SIC         Standard Industrial Classification. Four-digit
//!               industry code on the submissions feed; we use it to
//!               distinguish commodity trusts (SIC 6221) from
//!               operating companies (most other codes).
//!
//! ## SEC endpoints used
//!
//!   1. https://www.sec.gov/files/company_tickers_mf.json
//!      Mutual fund and ETF ticker map: (ticker -> CIK, seriesId,
//!      classId). One file, ~3 MB.
//!
//!   2. https://www.sec.gov/files/company_tickers.json
//!      Stocks and unit-investment-trust ETFs: (ticker -> CIK,
//!      title). One file, ~5 MB.
//!
//!   3. https://efts.sec.gov/LATEST/search-index?q=<seriesId>&forms=NPORT-P
//!      Full-text search for NPORT-P filings referencing
//!      `seriesId`. Necessary because the submissions feed only
//!      lists at trust-CIK level - a trust hosting hundreds of
//!      series would otherwise force us to download every NPORT-P
//!      to find the one we want.
//!
//!   4. https://data.sec.gov/submissions/CIK{cik:0>10}.json
//!      Per-CIK submissions feed. Carries entityType,
//!      sicDescription, ticker list, and the most-recent NPORT-P URL
//!      for UIT-style ETFs that lack a seriesId.
//!
//!   5. https://www.sec.gov/Archives/edgar/data/<CIK>/<ACC>/primary_doc.xml
//!      The actual NPORT-P document. XML, ~50-100 MB depending on
//!      fund size.
//!
//!   6. https://data.sec.gov/api/xbrl/companyconcept/CIK{cik:0>10}/{taxonomy}/{Concept}.json
//!      XBRL companyconcept endpoint. Used for shares-outstanding
//!      via `dei:EntityCommonStockSharesOutstanding` (single-class
//!      issuers) with fallback to `us-gaap:CommonStockSharesOutstanding`
//!      (dual-class issuers like GOOGL, META).
//!
//! ## Politeness
//!
//! SEC requires a descriptive User-Agent + From: header on every
//! request, populated from `Config.user_email` (env
//! `ZFIN_USER_EMAIL`). The provider takes the email as a non-null
//! constructor argument; callers must surface a clear error if the
//! env var is missing rather than letting requests go out
//! un-identified. SEC's documented ceiling is 10 req/s per IP; we
//! throttle at 8 req/s via a `RateLimiter`, leaving a 20% margin
//! against timing jitter and retry bursts. A per-symbol loop over a
//! typical portfolio reaches this ceiling quickly without it.
//!
//! ## Caching
//!
//! `Edgar` carries no cache state of its own. Every method does HTTP
//! + parse and returns a typed result; the `DataService` layer
//! writes the parsed results to the user-facing cache files
//! (`classification.srf`, `etf_metrics.srf`, `entity_facts.srf`)
//! and reads them back on subsequent calls.
//!
//! Ticker maps (`company_tickers*.json`) are the one upstream
//! document we cache through `Store` - typed
//! `[]MutualFundTickerEntry` / `[]CompanyTickerEntry` slices under
//! a synthetic `_edgar` key - because they're refreshed at SEC's
//! daily cadence rather than per symbol. Everything else gets
//! parsed into typed records and
//! written to the user-facing per-symbol or per-CIK cache files.

const std = @import("std");
const http = @import("../net/http.zig");
const RateLimiter = @import("../net/RateLimiter.zig");
const fmt = @import("../format.zig");
const xml = @import("xml.zig");

const tickers_funds_url = "https://www.sec.gov/files/company_tickers_mf.json";
const tickers_companies_url = "https://www.sec.gov/files/company_tickers.json";
const search_url_prefix = "https://efts.sec.gov/LATEST/search-index?";

// ── Edgar provider state ─────────────────────────────────────────
//
// File-as-struct: the file's top-level fields and methods together
// form the `Edgar` provider. Callers do
// `const Edgar = @import("providers/Edgar.zig");` followed by
// `var ed = Edgar.init(...);` and `ed.fetchMutualFundTickerMap(...)`
// etc.

client: http.Client,
/// Contact email for the User-Agent + From headers SEC requires on
/// every request. Sourced from `Config.user_email`. Required, not
/// optional: callers must surface a clear missing-config error
/// before constructing this provider rather than letting requests
/// go out un-identified.
user_email: []const u8,
/// Token-bucket throttle keeping us under SEC's 10 req/s ceiling.
/// Sized at 8 req/s to leave a 20% margin against timing jitter and
/// any retry bursts. Per-symbol fetch loops over a portfolio reach
/// this ceiling quickly without it.
rate_limiter: RateLimiter,
allocator: std.mem.Allocator,

const Edgar = @This();

pub fn init(io: std.Io, allocator: std.mem.Allocator, user_email: []const u8) Edgar {
    return .{
        .client = http.Client.init(io, allocator),
        .user_email = user_email,
        .rate_limiter = RateLimiter.init(io, 8, std.time.ns_per_s),
        .allocator = allocator,
    };
}

pub fn deinit(self: *Edgar) void {
    self.client.deinit();
}

/// GET wrapper that attaches the User-Agent + From headers SEC
/// requires on every request and acquires a rate-limit token before
/// issuing the call.
fn httpGet(self: *Edgar, url: []const u8) !http.Response {
    self.rate_limiter.acquire();

    var ua_buf: [256]u8 = undefined;
    const ua = std.fmt.bufPrint(&ua_buf, "zfin/0.1 ({s})", .{self.user_email}) catch return error.UserEmailTooLong;

    const headers = [_]std.http.Header{
        .{ .name = "User-Agent", .value = ua },
        .{ .name = "From", .value = self.user_email },
    };

    return self.client.request(.GET, url, null, &headers);
}

/// Fetch and parse SEC's mutual-fund/ETF ticker map
/// (`company_tickers_mf.json`). Returns an owned slice of
/// `MutualFundTickerEntry`. Caller manages the slice lifetime
/// (caching is the DataService's job); typical usage is to hand
/// the slice to `TickerMap(MutualFundTickerEntry).fromEntries`
/// which takes ownership.
pub fn fetchMutualFundTickerMap(self: *Edgar, allocator: std.mem.Allocator) ![]MutualFundTickerEntry {
    var resp = try self.httpGet(tickers_funds_url);
    defer resp.deinit();
    return parseTickerMap(allocator, resp.body);
}

/// Fetch and parse SEC's stocks-and-UITs ticker map
/// (`company_tickers.json`). Despite the filename, this file covers
/// operating companies AND unit investment trust ETFs (SPY, GLD,
/// IVV) - anything that doesn't file under a series-of-trust shape.
/// Returns an owned slice of `CompanyTickerEntry`.
pub fn fetchCompanyTickerMap(self: *Edgar, allocator: std.mem.Allocator) ![]CompanyTickerEntry {
    var resp = try self.httpGet(tickers_companies_url);
    defer resp.deinit();
    return parseStockTickerMap(allocator, resp.body);
}

/// Find the most recent NPORT-P filing for `series_id`. Returns null
/// if no filing exists. Caller owns the returned URL.
pub fn findLatestNportP(
    self: *Edgar,
    allocator: std.mem.Allocator,
    series_id: []const u8,
) !?[]u8 {
    const url = try std.fmt.allocPrint(
        allocator,
        "{s}q=%22{s}%22&forms=NPORT-P",
        .{ search_url_prefix, series_id },
    );
    defer allocator.free(url);
    var resp = try self.httpGet(url);
    defer resp.deinit();

    return parseLatestNportPFromSearch(allocator, resp.body);
}

/// Find the most recent NPORT-P filing for a CIK. Used for UIT-style
/// ETFs (SPY, etc.) that file at the trust-CIK level rather than a
/// series. Returns null if the CIK has no NPORT-P.
pub fn findLatestNportPByCik(
    self: *Edgar,
    allocator: std.mem.Allocator,
    cik: []const u8,
) !?[]u8 {
    const sub = try self.fetchSubmissionsFeed(allocator, cik);
    return sub.latest_nport_p_url;
}

/// Fetch and parse the per-CIK submissions feed.
pub fn fetchSubmissionsFeed(
    self: *Edgar,
    allocator: std.mem.Allocator,
    cik: []const u8,
) !SubmissionsSummary {
    const url = try std.fmt.allocPrint(
        allocator,
        "https://data.sec.gov/submissions/CIK{s:0>10}.json",
        .{cik},
    );
    defer allocator.free(url);
    var resp = try self.httpGet(url);
    defer resp.deinit();

    return parseSubmissionsFeed(allocator, resp.body, cik);
}

/// Fetch the most recent shares-outstanding value for a CIK. Returns
/// null on 404 (e.g. 20-F-only filers). Cascades through
/// `dei:EntityCommonStockSharesOutstanding` (single-class) then
/// `us-gaap:CommonStockSharesOutstanding` (dual-class fallback,
/// e.g. GOOGL, META).
pub fn fetchSharesOutstanding(
    self: *Edgar,
    allocator: std.mem.Allocator,
    cik: []const u8,
) !?SharesOutstanding {
    if (try self.fetchSharesConcept(allocator, cik, "dei", "EntityCommonStockSharesOutstanding")) |so| {
        return so;
    }
    return try self.fetchSharesConcept(allocator, cik, "us-gaap", "CommonStockSharesOutstanding");
}

fn fetchSharesConcept(
    self: *Edgar,
    allocator: std.mem.Allocator,
    cik: []const u8,
    taxonomy: []const u8,
    concept: []const u8,
) !?SharesOutstanding {
    const url = try std.fmt.allocPrint(
        allocator,
        "https://data.sec.gov/api/xbrl/companyconcept/CIK{s:0>10}/{s}/{s}.json",
        .{ cik, taxonomy, concept },
    );
    defer allocator.free(url);
    var resp = self.httpGet(url) catch |err| {
        if (err == error.NotFound) return null;
        return err;
    };
    defer resp.deinit();

    return parseSharesOutstanding(allocator, resp.body);
}

/// Fetch and parse N-PORT-P metrics for one ETF/MF ticker. The
/// return value describes what was found (full holdings,
/// profile-only, or not-a-fund). `top_n_holdings` caps holdings
/// emitted (sorted by pctVal descending).
pub fn fetchEtfMetrics(
    self: *Edgar,
    io: std.Io,
    allocator: std.mem.Allocator,
    mf_ticker_map: *const TickerMap(MutualFundTickerEntry),
    stock_ticker_map: *const TickerMap(CompanyTickerEntry),
    symbol: []const u8,
    top_n_holdings: usize,
) !EtfMetricsResult {
    // MF/ETF map first - authoritative for symbols filed under a
    // series. Series-keyed full-text search; CIK fallback would
    // yield arbitrary other series under the same trust.
    if (mf_ticker_map.get(symbol)) |entry| {
        const filing_url = (try self.findLatestNportP(allocator, entry.series_id.?)) orelse {
            return .not_a_fund;
        };
        defer allocator.free(filing_url);
        const m = try self.fetchAndParseNportP(
            io,
            allocator,
            entry.toGeneric(),
            filing_url,
            symbol,
            top_n_holdings,
        );
        return .{ .full = m };
    }

    // Stock map: probe the submissions feed (one extra HTTP per
    // unique CIK) to classify the entity. Branches:
    //   - fund_shaped  + has NPORT-P -> full holdings (SPY)
    //   - fund_shaped  + no NPORT-P  -> profile-only (SLVO ETN issuer)
    //   - trust_shaped               -> profile-only (GLD commodity)
    //   - operating                  -> not-a-fund (AAPL, MSFT)
    if (stock_ticker_map.get(symbol)) |entry| {
        var sub = try self.fetchSubmissionsFeed(allocator, entry.cik);
        defer sub.deinit(allocator);

        const class = classifyByEntityType(&sub);
        switch (class) {
            .operating => return .not_a_fund,
            .fund_shaped => {
                if (sub.latest_nport_p_url) |url| {
                    const m = try self.fetchAndParseNportP(
                        io,
                        allocator,
                        entry.toGeneric(),
                        url,
                        symbol,
                        top_n_holdings,
                    );
                    return .{ .full = m };
                }
                const profile = try buildProfileOnlyMetrics(io, allocator, entry.toGeneric(), &sub, symbol);
                return .{ .profile_only = profile };
            },
            .trust_shaped => {
                // Skip the NPORT-P probe - by definition these
                // don't file one. Saves an HTTP roundtrip.
                const profile = try buildProfileOnlyMetrics(io, allocator, entry.toGeneric(), &sub, symbol);
                return .{ .profile_only = profile };
            },
        }
    }

    return .not_in_edgar;
}

/// Download and parse a NPORT-P primary_doc.xml at `filing_url`.
/// Used by both the MF and UIT paths in `fetchEtfMetrics`. The
/// parsed `EtfMetrics` is the cacheable artifact; the XML bytes are
/// discarded after parsing - no provider-internal XML cache, so
/// re-fetches always re-download.
///
/// `entry` is a `GenericTickerEntry`; callers from either ticker
/// map (`MutualFundTickerEntry` / `CompanyTickerEntry`) collapse
/// to this shape via `.toGeneric()`.
fn fetchAndParseNportP(
    self: *Edgar,
    io: std.Io,
    allocator: std.mem.Allocator,
    entry: GenericTickerEntry,
    filing_url: []const u8,
    symbol: []const u8,
    top_n_holdings: usize,
) !EtfMetrics {
    var resp = try self.httpGet(filing_url);
    defer resp.deinit();

    return parseNportP(
        io,
        allocator,
        resp.body,
        symbol,
        entry,
        top_n_holdings,
    );
}

// ── Free types and helpers (no `self`) ───────────────────────────

pub const SectorWeight = struct {
    code: []const u8, // owned; raw NPORT-P code, e.g. "EC/CORP"
    description: []const u8, // owned; human-readable, e.g. "Equity / Corporate"
    pct_of_portfolio: f64,
};

pub const Holding = struct {
    name: []const u8, // owned
    ticker: ?[]const u8 = null, // owned; present for some equity holdings
    cusip: ?[]const u8 = null, // owned
    lei: ?[]const u8 = null, // owned; ISO 17442 Legal Entity Identifier
    country: ?[]const u8 = null, // owned; ISO-3166 alpha-2 from <invCountry>
    pct_of_portfolio: f64,
};

pub const EtfMetrics = struct {
    symbol: []const u8, // owned
    series_name: ?[]const u8 = null, // owned
    cik: []const u8, // owned
    /// Null for unit-investment-trust ETFs (SPY, etc.) that file
    /// NPORT-P at the trust-CIK level without a series identifier.
    series_id: ?[]const u8 = null, // owned
    net_assets: ?f64 = null,
    period_end: ?[]const u8 = null, // owned
    as_of: []const u8, // owned (date scraper ran)
    holdings: []Holding, // owned
    sectors: []SectorWeight, // owned

    pub fn deinit(self: EtfMetrics, allocator: std.mem.Allocator) void {
        allocator.free(self.symbol);
        if (self.series_name) |s| allocator.free(s);
        allocator.free(self.cik);
        if (self.series_id) |s| allocator.free(s);
        if (self.period_end) |s| allocator.free(s);
        allocator.free(self.as_of);
        for (self.holdings) |*h| {
            allocator.free(h.name);
            if (h.ticker) |t| allocator.free(t);
            if (h.cusip) |c| allocator.free(c);
            if (h.lei) |l| allocator.free(l);
            if (h.country) |c| allocator.free(c);
        }
        allocator.free(self.holdings);
        for (self.sectors) |*s| {
            allocator.free(s.code);
            allocator.free(s.description);
        }
        allocator.free(self.sectors);
    }
};

/// SRF-emit shape for the SEC's `company_tickers_mf.json` document.
/// One row per (symbol, series, class). The cache file lives under
/// `<cache_dir>/_edgar/tickers_funds.srf`. `MutualFundTickerEntry`
/// and `CompanyTickerEntry` are structurally identical but exist as
/// distinct types because `Store.dataTypeFor(T)` keys on Zig type;
/// distinct types map to distinct on-disk caches.
pub const MutualFundTickerEntry = TickerEntry(MutualFund);
const MutualFund = struct {};
fn TickerEntry(comptime T: type) type {
    return struct {
        /// Ticker symbol - e.g. "VTI", "AGG". The hashmap key when the
        /// caller builds a `TickerMap` for fast lookup.
        symbol: []const u8,
        /// Filer CIK, zero-padded to 10 digits.
        cik: []const u8,
        /// SEC series identifier (e.g. "S000002848"). Always present
        /// for entries from `company_tickers_mf.json`.
        series_id: ?[]const u8 = null,
        /// SEC class identifier (e.g. "C000007808"). Always present
        /// for entries from `company_tickers_mf.json`.
        class_id: ?[]const u8 = null,
        /// Trust / company name. Not populated by the MF ticker map
        /// (it carries no human-readable name); kept here for shape
        /// parity with `CompanyTickerEntry`.
        title: ?[]const u8 = null,

        const Child = T;
        const Self = @This();

        pub fn deinit(self: Self, allocator: std.mem.Allocator) void {
            allocator.free(self.symbol);
            allocator.free(self.cik);
            if (self.series_id) |s| allocator.free(s);
            if (self.class_id) |s| allocator.free(s);
            if (self.title) |s| allocator.free(s);
        }

        pub fn freeSlice(allocator: std.mem.Allocator, entries: []const Self) void {
            for (entries) |e| e.deinit(allocator);
            allocator.free(entries);
        }
        pub fn toGeneric(self: Self) GenericTickerEntry {
            // SAFETY: setting all fields in for loop
            var ge: GenericTickerEntry = undefined;
            inline for (@typeInfo(Self).@"struct".fields) |f|
                @field(ge, f.name) = @field(self, f.name);
            return ge;
        }
    };
}

pub const CompanyTickerEntry = TickerEntry(CompanyTicker);
const CompanyTicker = struct {};
const GenericTickerEntry = TickerEntry(Ticker);
const Ticker = struct {};

/// Fast-lookup wrapper around a slice of ticker entries. Built by
/// `fromEntries` after a fetch or cache read; takes ownership of
/// the slice. The hashmap stores `*const EntryT` pointers into the
/// owned slice - no string duping. `deinit` frees each entry's
/// owned strings, the slice itself, and the hashmap structure.
///
/// Generic over `EntryT` (currently `MutualFundTickerEntry` or
/// `CompanyTickerEntry`) so the same logic serves both ticker
/// maps. Callers instantiate as
/// `Edgar.TickerMap(MutualFundTickerEntry)` etc.
pub fn TickerMap(comptime T: type) type {
    return struct {
        entries: []T,
        map: std.StringHashMap(*const T),
        allocator: std.mem.Allocator,

        const Self = @This();

        /// Build a TickerMap from a slice of entries. Takes
        /// ownership of `entries` - caller must NOT free the
        /// slice; `deinit` will. The hashmap stores pointers into
        /// the owned slice keyed by `entry.symbol`.
        pub fn fromEntries(allocator: std.mem.Allocator, entries: []T) !Self {
            var map: std.StringHashMap(*const T) = .init(allocator);
            errdefer map.deinit();
            try map.ensureTotalCapacity(@intCast(entries.len));
            for (entries) |*e| {
                // First-wins on duplicate symbols. Same rule the
                // old hashmap-during-parse code applied.
                const gop = map.getOrPutAssumeCapacity(e.symbol);
                if (!gop.found_existing) gop.value_ptr.* = e;
            }
            return .{
                .entries = entries,
                .map = map,
                .allocator = allocator,
            };
        }

        pub fn get(self: Self, symbol: []const u8) ?*const T {
            return self.map.get(symbol);
        }

        pub fn deinit(self: *Self) void {
            self.map.deinit();
            T.freeSlice(self.allocator, self.entries);
        }
    };
}

/// Parse the SEC's `company_tickers_mf.json` shape into a slice of
/// `MutualFundTickerEntry`. Caller owns the slice - free via
/// `MutualFundTickerEntry.freeSlice` (or hand to `TickerMap`).
pub fn parseTickerMap(allocator: std.mem.Allocator, json_bytes: []const u8) ![]MutualFundTickerEntry {
    var out: std.ArrayList(MutualFundTickerEntry) = .empty;
    errdefer {
        for (out.items) |e| e.deinit(allocator);
        out.deinit(allocator);
    }

    const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json_bytes, .{});
    defer parsed.deinit();

    const root = switch (parsed.value) {
        .object => |o| o,
        else => return error.InvalidTickerMap,
    };
    const data_array = switch (root.get("data") orelse return error.InvalidTickerMap) {
        .array => |a| a.items,
        else => return error.InvalidTickerMap,
    };

    // Track first-seen symbols so duplicates skip cleanly without
    // also constructing a hashmap that we'd just throw away.
    var seen: std.StringHashMap(void) = .init(allocator);
    defer seen.deinit();

    for (data_array) |row| {
        const fields = switch (row) {
            .array => |a| a.items,
            else => continue,
        };
        if (fields.len < 4) continue;

        const cik_n = switch (fields[0]) {
            .integer => |n| n,
            else => continue,
        };
        const series_id = switch (fields[1]) {
            .string => |s| s,
            else => continue,
        };
        const class_id = switch (fields[2]) {
            .string => |s| s,
            else => continue,
        };
        const symbol = switch (fields[3]) {
            .string => |s| s,
            else => continue,
        };

        if (seen.contains(symbol)) continue;

        // CIKs are normalized to 10-digit zero-padded strings at
        // every boundary. Wikidata's P5531 uses this convention, so
        // downstream merge logic can join on the same key shape.
        // EDGAR ticker-map JSON delivers them as bare integers, so
        // we pad here. Cast to u64 first because signed `{d:0>10}`
        // reserves a slot for the sign character and produces
        // "0000+36405".
        const cik_str = try std.fmt.allocPrint(allocator, "{d:0>10}", .{@as(u64, @intCast(cik_n))});
        errdefer allocator.free(cik_str);
        const symbol_owned = try allocator.dupe(u8, symbol);
        errdefer allocator.free(symbol_owned);
        const series_owned = try allocator.dupe(u8, series_id);
        errdefer allocator.free(series_owned);
        const class_owned = try allocator.dupe(u8, class_id);
        errdefer allocator.free(class_owned);

        try seen.put(symbol_owned, {});

        try out.append(allocator, .{
            .symbol = symbol_owned,
            .cik = cik_str,
            .series_id = series_owned,
            .class_id = class_owned,
            .title = null,
        });
    }
    return out.toOwnedSlice(allocator);
}

/// Parser for the stocks-and-UITs `company_tickers.json` shape, which
/// is keyed by integer-string indices rather than the array-of-arrays
/// shape used by `company_tickers_mf.json`. Each entry has
/// `cik_str`, `ticker`, `title`. Returns a slice of
/// `CompanyTickerEntry`; caller owns via `CompanyTickerEntry.freeSlice`
/// (or hands to `TickerMap`).
pub fn parseStockTickerMap(allocator: std.mem.Allocator, json_bytes: []const u8) ![]CompanyTickerEntry {
    var out: std.ArrayList(CompanyTickerEntry) = .empty;
    errdefer {
        for (out.items) |e| e.deinit(allocator);
        out.deinit(allocator);
    }

    const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json_bytes, .{});
    defer parsed.deinit();

    const root = switch (parsed.value) {
        .object => |o| o,
        else => return error.InvalidTickerMap,
    };

    var seen: std.StringHashMap(void) = .init(allocator);
    defer seen.deinit();

    var it = root.iterator();
    while (it.next()) |entry| {
        const obj = switch (entry.value_ptr.*) {
            .object => |o| o,
            else => continue,
        };
        const cik_n = switch (obj.get("cik_str") orelse continue) {
            .integer => |n| n,
            else => continue,
        };
        const symbol = switch (obj.get("ticker") orelse continue) {
            .string => |s| s,
            else => continue,
        };
        const title = if (obj.get("title")) |v| switch (v) {
            .string => |s| s,
            else => null,
        } else null;

        if (seen.contains(symbol)) continue;

        const cik_str = try std.fmt.allocPrint(allocator, "{d:0>10}", .{@as(u64, @intCast(cik_n))});
        errdefer allocator.free(cik_str);
        const symbol_owned = try allocator.dupe(u8, symbol);
        errdefer allocator.free(symbol_owned);
        const title_owned = if (title) |t| try allocator.dupe(u8, t) else null;
        errdefer if (title_owned) |t| allocator.free(t);

        try seen.put(symbol_owned, {});

        try out.append(allocator, .{
            .symbol = symbol_owned,
            .cik = cik_str,
            .series_id = null,
            .class_id = null,
            .title = title_owned,
        });
    }
    return out.toOwnedSlice(allocator);
}

/// Lightweight summary of a CIK's `submissions/CIK*.json` feed.
/// Pulls out the four fields callers need (entity_name, entity_type,
/// sic_description, latest_nport_p_url) so they can branch without
/// re-parsing the full JSON. All owned strings allocated by the
/// caller's allocator; caller must free via `deinit`.
pub const SubmissionsSummary = struct {
    entity_name: ?[]const u8 = null,
    entity_type: ?[]const u8 = null,
    sic_description: ?[]const u8 = null,
    /// URL to the most-recent NPORT-P primary_doc.xml, if any.
    latest_nport_p_url: ?[]const u8 = null,

    pub fn deinit(self: SubmissionsSummary, allocator: std.mem.Allocator) void {
        if (self.entity_name) |s| allocator.free(s);
        if (self.entity_type) |s| allocator.free(s);
        if (self.sic_description) |s| allocator.free(s);
        if (self.latest_nport_p_url) |s| allocator.free(s);
    }
};

fn parseSubmissionsFeed(
    allocator: std.mem.Allocator,
    json_bytes: []const u8,
    cik: []const u8,
) !SubmissionsSummary {
    var out: SubmissionsSummary = .{};
    errdefer out.deinit(allocator);

    const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json_bytes, .{});
    defer parsed.deinit();

    const root = switch (parsed.value) {
        .object => |o| o,
        else => return out,
    };
    if (root.get("name")) |v| switch (v) {
        .string => |s| out.entity_name = try allocator.dupe(u8, s),
        else => {},
    };
    if (root.get("entityType")) |v| switch (v) {
        .string => |s| out.entity_type = try allocator.dupe(u8, s),
        else => {},
    };
    if (root.get("sicDescription")) |v| switch (v) {
        .string => |s| if (s.len > 0) {
            out.sic_description = try allocator.dupe(u8, s);
        },
        else => {},
    };

    out.latest_nport_p_url = try findNportPUrlInSubmissions(allocator, root, cik);
    return out;
}

/// Shares-outstanding from EDGAR XBRL companyconcept endpoint.
/// Sourced from the `dei:EntityCommonStockSharesOutstanding` concept,
/// which the SEC's Document and Entity Information taxonomy mandates
/// on the cover page of 10-K, 10-Q, 40-F, and similar forms.
///
/// The dei concept is preferred over `us-gaap:CommonStockSharesOutstanding`
/// because it covers Canadian 40-F filers (e.g. Barrick Mining) that
/// don't file under us-gaap. EU 20-F filers (e.g. BP) are still NOT
/// covered - they use pure ifrs-full without dei tagging - so callers
/// must tolerate `null` returns.
///
/// `value` is the share count from the most recent reporting period.
/// `period_end` is the `end` date that count was reported as-of, in
/// `YYYY-MM-DD` form. `form` is the SEC form name (`10-K`, `10-Q`,
/// `40-F`, etc.) that supplied the number, useful for staleness
/// reasoning ("a 10-Q is 3 months stale, a 40-F is 12 months stale").
pub const SharesOutstanding = struct {
    value: u64,
    period_end: []const u8, // owned
    form: []const u8, // owned

    pub fn deinit(self: SharesOutstanding, allocator: std.mem.Allocator) void {
        allocator.free(self.period_end);
        allocator.free(self.form);
    }
};

/// Per-symbol shares-outstanding record, ready for SRF emission. Joins
/// the bare `SharesOutstanding` fetch result (CIK-level) with caller-
/// supplied `symbol` and `as_of` so each output row carries the full
/// provenance needed by downstream merge logic.
///
/// The `source` field has no default - provenance is always emitted
/// (per the project's source-pure invariant: every row in a shared
/// classification file must self-identify which source produced it).
pub const SharesRecord = struct {
    symbol: []const u8, // owned
    shares_outstanding: u64,
    period_end: []const u8, // owned, YYYY-MM-DD
    form: ?[]const u8 = null, // owned (e.g. "10-Q", "40-F")
    cik: []const u8, // owned
    as_of: []const u8, // owned (date scraper ran)
    source: []const u8, // no default

    pub fn deinit(self: SharesRecord, allocator: std.mem.Allocator) void {
        allocator.free(self.symbol);
        allocator.free(self.period_end);
        if (self.form) |f| allocator.free(f);
        allocator.free(self.cik);
        allocator.free(self.as_of);
        allocator.free(self.source);
    }
};

/// Tagged union of XBRL-derived per-entity facts. Stored in the
/// per-CIK `entity_facts.srf` cache file. Currently only carries
/// `shares_outstanding`; new variants (revenue, net income, EPS,
/// etc.) get added here as new methods on `Edgar` extract them.
/// SRF's default `type` discriminator is what we want, so no
/// `srf_tag_field` override is declared.
pub const EntityFactRecord = union(enum) {
    shares_outstanding: SharesRecord,

    pub fn deinit(self: EntityFactRecord, allocator: std.mem.Allocator) void {
        switch (self) {
            .shares_outstanding => |r| r.deinit(allocator),
        }
    }

    /// Free a slice of records, calling deinit on each element first.
    pub fn freeSlice(allocator: std.mem.Allocator, recs: []const EntityFactRecord) void {
        for (recs) |r| r.deinit(allocator);
        allocator.free(recs);
    }
};

/// SRF-emit shape for the `profile` variant of an ETF metrics record.
/// One per fund. Disjoint from the internal `EtfMetrics` struct, which
/// holds the whole fund's data (profile + N sectors + M holdings) in
/// nested arrays for parsing convenience.
pub const EtfProfileRecord = struct {
    symbol: []const u8, // owned
    series_name: ?[]const u8 = null, // owned
    cik: []const u8, // owned
    series_id: ?[]const u8 = null, // owned
    net_assets: ?f64 = null,
    period_end: ?[]const u8 = null, // owned, YYYY-MM-DD
    as_of: []const u8, // owned
    source: []const u8, // no default

    pub fn deinit(self: EtfProfileRecord, allocator: std.mem.Allocator) void {
        allocator.free(self.symbol);
        if (self.series_name) |s| allocator.free(s);
        allocator.free(self.cik);
        if (self.series_id) |s| allocator.free(s);
        if (self.period_end) |s| allocator.free(s);
        allocator.free(self.as_of);
        allocator.free(self.source);
    }
};

/// SRF-emit shape for the `sector` variant. One per (assetCat,
/// issuerCat) pair within a fund. The `code` field is the raw
/// NPORT-P abbreviation; `description` is the human-readable
/// translation per `sectorDescription`.
pub const EtfSectorRecord = struct {
    symbol: []const u8, // owned
    code: []const u8, // owned, e.g. "EC/CORP"
    description: []const u8, // owned, e.g. "Equity / Corporate"
    pct_of_portfolio: f64,
    as_of: []const u8, // owned
    source: []const u8, // no default

    pub fn deinit(self: EtfSectorRecord, allocator: std.mem.Allocator) void {
        allocator.free(self.symbol);
        allocator.free(self.code);
        allocator.free(self.description);
        allocator.free(self.as_of);
        allocator.free(self.source);
    }
};

/// SRF-emit shape for the `holding` variant. One per top-N holding
/// retained from NPORT-P. Carries the full identifier inventory so
/// downstream display can prefer ticker > cusip > lei without
/// refetching.
pub const EtfHoldingRecord = struct {
    symbol: []const u8, // owned; the FUND's symbol
    name: []const u8, // owned; holding's company / instrument name
    ticker: ?[]const u8 = null, // owned
    cusip: ?[]const u8 = null, // owned
    lei: ?[]const u8 = null, // owned
    country: ?[]const u8 = null, // owned, ISO-3166 alpha-2
    pct_of_portfolio: f64,
    as_of: []const u8, // owned
    source: []const u8, // no default

    pub fn deinit(self: EtfHoldingRecord, allocator: std.mem.Allocator) void {
        allocator.free(self.symbol);
        allocator.free(self.name);
        if (self.ticker) |s| allocator.free(s);
        if (self.cusip) |s| allocator.free(s);
        if (self.lei) |s| allocator.free(s);
        if (self.country) |s| allocator.free(s);
        allocator.free(self.as_of);
        allocator.free(self.source);
    }
};

/// Tagged union covering all three rows of `etf_metrics.srf`. SRF's
/// default `type` discriminator is what we want, so no `srf_tag_field`
/// override is declared. Builders (in `main.zig`) construct the slice
/// by appending one `.profile` then N `.sector` then M `.holding`
/// variants per fund.
pub const EtfMetricRecord = union(enum) {
    profile: EtfProfileRecord,
    sector: EtfSectorRecord,
    holding: EtfHoldingRecord,

    pub fn deinit(self: EtfMetricRecord, allocator: std.mem.Allocator) void {
        switch (self) {
            .profile => |r| r.deinit(allocator),
            .sector => |r| r.deinit(allocator),
            .holding => |r| r.deinit(allocator),
        }
    }

    /// Free a slice of records, calling deinit on each element first.
    pub fn freeSlice(allocator: std.mem.Allocator, recs: []const EtfMetricRecord) void {
        for (recs) |r| r.deinit(allocator);
        allocator.free(recs);
    }
};

/// Decompose one fund's internal `EtfMetrics` struct into the SRF-
/// emit-shaped union slice. Appends one `.profile` variant then N
/// `.sector` variants then M `.holding` variants to `out`. All
/// strings on the resulting union values are freshly duped so the
/// caller can deinit `metrics` independently. Caller owns the
/// appended union values and must deinit them.
pub fn appendEtfMetricRecords(
    allocator: std.mem.Allocator,
    out: *std.ArrayList(EtfMetricRecord),
    metrics: EtfMetrics,
) !void {
    try out.append(allocator, .{ .profile = .{
        .symbol = try allocator.dupe(u8, metrics.symbol),
        .series_name = if (metrics.series_name) |s| try allocator.dupe(u8, s) else null,
        .cik = try allocator.dupe(u8, metrics.cik),
        .series_id = if (metrics.series_id) |s| try allocator.dupe(u8, s) else null,
        .net_assets = metrics.net_assets,
        .period_end = if (metrics.period_end) |s| try allocator.dupe(u8, s) else null,
        .as_of = try allocator.dupe(u8, metrics.as_of),
        .source = try allocator.dupe(u8, "edgar"),
    } });
    for (metrics.sectors) |s| {
        try out.append(allocator, .{ .sector = .{
            .symbol = try allocator.dupe(u8, metrics.symbol),
            .code = try allocator.dupe(u8, s.code),
            .description = try allocator.dupe(u8, s.description),
            .pct_of_portfolio = s.pct_of_portfolio,
            .as_of = try allocator.dupe(u8, metrics.as_of),
            .source = try allocator.dupe(u8, "edgar"),
        } });
    }
    for (metrics.holdings) |h| {
        try out.append(allocator, .{ .holding = .{
            .symbol = try allocator.dupe(u8, metrics.symbol),
            .name = try allocator.dupe(u8, h.name),
            .ticker = if (h.ticker) |t| try allocator.dupe(u8, t) else null,
            .cusip = if (h.cusip) |c| try allocator.dupe(u8, c) else null,
            .lei = if (h.lei) |l| try allocator.dupe(u8, l) else null,
            .country = if (h.country) |c| try allocator.dupe(u8, c) else null,
            .pct_of_portfolio = h.pct_of_portfolio,
            .as_of = try allocator.dupe(u8, metrics.as_of),
            .source = try allocator.dupe(u8, "edgar"),
        } });
    }
}

fn parseSharesOutstanding(
    allocator: std.mem.Allocator,
    json_bytes: []const u8,
) !?SharesOutstanding {
    const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json_bytes, .{});
    defer parsed.deinit();

    const root = switch (parsed.value) {
        .object => |o| o,
        else => return null,
    };
    const units = switch (root.get("units") orelse return null) {
        .object => |o| o,
        else => return null,
    };
    // The unit key is "shares". Defensive: take the first units
    // entry whose array has at least one row.
    var rows: []std.json.Value = &.{};
    var unit_it = units.iterator();
    while (unit_it.next()) |entry| {
        switch (entry.value_ptr.*) {
            .array => |a| if (a.items.len > 0) {
                rows = a.items;
                break;
            },
            else => continue,
        }
    }
    if (rows.len == 0) return null;

    // Pick the row with the latest `end` date. EDGAR usually returns
    // them in chronological order but don't rely on that.
    var best_idx: usize = 0;
    var best_end: []const u8 = "";
    for (rows, 0..) |row, i| {
        const obj = switch (row) {
            .object => |o| o,
            else => continue,
        };
        const end = switch (obj.get("end") orelse continue) {
            .string => |s| s,
            else => continue,
        };
        if (std.mem.order(u8, end, best_end) == .gt) {
            best_end = end;
            best_idx = i;
        }
    }
    if (best_end.len == 0) return null;

    const obj = switch (rows[best_idx]) {
        .object => |o| o,
        else => return null,
    };
    const val_node = obj.get("val") orelse return null;
    const val: u64 = switch (val_node) {
        .integer => |n| if (n < 0) return null else @intCast(n),
        .float => |f| if (f < 0) return null else @intFromFloat(f),
        else => return null,
    };
    const form_str: []const u8 = switch (obj.get("form") orelse .null) {
        .string => |s| s,
        else => "",
    };

    return .{
        .value = val,
        .period_end = try allocator.dupe(u8, best_end),
        .form = try allocator.dupe(u8, form_str),
    };
}

fn findNportPUrlInSubmissions(
    allocator: std.mem.Allocator,
    root: std.json.ObjectMap,
    cik: []const u8,
) !?[]u8 {
    const filings = switch (root.get("filings") orelse return null) {
        .object => |o| o,
        else => return null,
    };
    const recent = switch (filings.get("recent") orelse return null) {
        .object => |o| o,
        else => return null,
    };
    const forms = switch (recent.get("form") orelse return null) {
        .array => |a| a.items,
        else => return null,
    };
    const accessions = switch (recent.get("accessionNumber") orelse return null) {
        .array => |a| a.items,
        else => return null,
    };
    const dates = switch (recent.get("filingDate") orelse return null) {
        .array => |a| a.items,
        else => return null,
    };

    var best_idx: ?usize = null;
    var best_date: []const u8 = "";
    for (forms, 0..) |form, i| {
        const fname = switch (form) {
            .string => |s| s,
            else => continue,
        };
        if (!std.mem.eql(u8, fname, "NPORT-P")) continue;
        if (i >= dates.len) continue;
        const fd = switch (dates[i]) {
            .string => |s| s,
            else => continue,
        };
        if (std.mem.order(u8, fd, best_date) == .gt) {
            best_date = fd;
            best_idx = i;
        }
    }
    const idx = best_idx orelse return null;

    const acc = switch (accessions[idx]) {
        .string => |s| s,
        else => return null,
    };

    var cik_no_zeros = cik;
    while (cik_no_zeros.len > 1 and cik_no_zeros[0] == '0') cik_no_zeros = cik_no_zeros[1..];

    var adsh_buf: std.ArrayList(u8) = .empty;
    defer adsh_buf.deinit(allocator);
    for (acc) |c| if (c != '-') try adsh_buf.append(allocator, c);

    return try std.fmt.allocPrint(
        allocator,
        "https://www.sec.gov/Archives/edgar/data/{s}/{s}/primary_doc.xml",
        .{ cik_no_zeros, adsh_buf.items },
    );
}

/// Extract the most-recent filing URL from an EDGAR full-text search
/// response. Used by `findLatestNportP` (series-keyed search). Lifted
/// out so the same parser can be reused if we add more search calls.
fn parseLatestNportPFromSearch(allocator: std.mem.Allocator, json_bytes: []const u8) !?[]u8 {
    const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json_bytes, .{});
    defer parsed.deinit();

    const root = switch (parsed.value) {
        .object => |o| o,
        else => return null,
    };
    const hits_obj = switch (root.get("hits") orelse return null) {
        .object => |o| o,
        else => return null,
    };
    const hits_arr = switch (hits_obj.get("hits") orelse return null) {
        .array => |a| a.items,
        else => return null,
    };
    if (hits_arr.len == 0) return null;

    var best_idx: usize = 0;
    var best_date: []const u8 = "";
    for (hits_arr, 0..) |hit, i| {
        const hit_obj = switch (hit) {
            .object => |o| o,
            else => continue,
        };
        const src = switch (hit_obj.get("_source") orelse continue) {
            .object => |o| o,
            else => continue,
        };
        const fd = switch (src.get("file_date") orelse continue) {
            .string => |s| s,
            else => continue,
        };
        if (std.mem.order(u8, fd, best_date) == .gt) {
            best_date = fd;
            best_idx = i;
        }
    }

    const best = switch (hits_arr[best_idx]) {
        .object => |o| o,
        else => return null,
    };
    const src = switch (best.get("_source") orelse return null) {
        .object => |o| o,
        else => return null,
    };
    const adsh = switch (src.get("adsh") orelse return null) {
        .string => |s| s,
        else => return null,
    };
    const ciks_arr = switch (src.get("ciks") orelse return null) {
        .array => |a| a.items,
        else => return null,
    };
    if (ciks_arr.len == 0) return null;
    const cik_padded = switch (ciks_arr[0]) {
        .string => |s| s,
        else => return null,
    };

    var cik_no_zeros = cik_padded;
    while (cik_no_zeros.len > 1 and cik_no_zeros[0] == '0') cik_no_zeros = cik_no_zeros[1..];

    var adsh_buf = std.ArrayList(u8).empty;
    defer adsh_buf.deinit(allocator);
    for (adsh) |c| if (c != '-') try adsh_buf.append(allocator, c);

    return try std.fmt.allocPrint(
        allocator,
        "https://www.sec.gov/Archives/edgar/data/{s}/{s}/primary_doc.xml",
        .{ cik_no_zeros, adsh_buf.items },
    );
}

/// Classify a CIK based on its submissions-feed metadata. Decides
/// whether the symbol is a registered fund (probe NPORT-P), a
/// trust/ETN-style instrument (profile-only), or a plain operating
/// company (skip).
///
/// Decision rules - kept in one place because they're load-bearing
/// for what `EtfMetricsResult` variant `fetchEtfMetrics` returns.
/// Rules are based on observation across ~100 real symbols:
///
///   1. Has NPORT-P filing                    -> fund_shaped.
///      The presence of a NPORT-P is the unambiguous signal that
///      the entity is a registered investment company. Catches all
///      ETFs and mutual funds regardless of entityType / SIC.
///
///   2. entityType == "other" AND SIC indicates
///      a securities issuer or commodity dealer -> trust_shaped.
///      Catches ETN issuers (Credit Suisse AG -> SLVO), commodity
///      brokers (some smaller commodity trusts), without a NPORT-P.
///      Does NOT catch foreign issuers like BP/Barrick (entityType
///      "other" but SIC is industry-specific, not securities-related).
///
///   3. entityType == "operating" AND SIC contains
///      "Commodity"                           -> trust_shaped.
///      Catches commodity grantor trusts (GLD, SLV, IAU, GBTC).
///      `entityType` is "operating" for these despite their
///      trust-like nature - SEC classifies them as commodity-
///      contracts brokers because they hold physical commodities.
///
///   4. otherwise                             -> operating.
///      Plain operating companies (AAPL, NFLX, BRK.B, BP, etc.).
///      No fund records emitted; Wikidata covers their classification.
///
/// Note: REITs (e.g. Realty Income, O) are `operating` + SIC
/// "Real Estate Investment Trusts". They are operating companies
/// that distribute rental income, not registered investment
/// companies. They get bucketed under `operating` - Wikidata is
/// the right source for them.
fn classifyByEntityType(sub: *const SubmissionsSummary) enum {
    fund_shaped,
    trust_shaped,
    operating,
} {
    // Rule 1: NPORT-P presence is the strongest fund signal.
    if (sub.latest_nport_p_url != null) return .fund_shaped;

    const et = sub.entity_type orelse return .operating;
    const sic_opt = sub.sic_description;

    // Rule 2: securities issuers (ETN sponsor banks).
    if (std.mem.eql(u8, et, "other")) {
        if (sic_opt) |sic| {
            const securities_hints = [_][]const u8{
                "Security Brokers", // "Security Brokers, Dealers..."
                "Commodity Contracts",
                "Investment Trust", // explicit, not "Real Estate Investment Trusts"
            };
            for (securities_hints) |h| {
                if (std.mem.indexOf(u8, sic, h) != null) return .trust_shaped;
            }
        }
        return .operating;
    }

    // Rule 3: commodity grantor trusts classified as "operating".
    if (std.mem.eql(u8, et, "operating")) {
        if (sic_opt) |sic| {
            if (std.mem.indexOf(u8, sic, "Commodity") != null) {
                return .trust_shaped;
            }
        }
    }
    return .operating;
}

test "classifyByEntityType buckets real-world entities" {
    const T = std.testing;

    // SPY: NPORT-P present -> fund_shaped (regardless of other fields).
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "other");
        s.latest_nport_p_url = try T.allocator.dupe(u8, "https://example/primary_doc.xml");
        try T.expectEqual(.fund_shaped, classifyByEntityType(&s));
    }
    // SLVO/GLDI/USOI issuer (Credit Suisse AG): no NPORT-P, "other"
    // entityType, SIC = "Security Brokers..." -> trust_shaped.
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "other");
        s.sic_description = try T.allocator.dupe(u8, "Security Brokers, Dealers & Flotation Companies");
        try T.expectEqual(.trust_shaped, classifyByEntityType(&s));
    }
    // BP plc: foreign issuer, "other" entityType, SIC = industry.
    // Should be `operating`, not trust_shaped.
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "other");
        s.sic_description = try T.allocator.dupe(u8, "Petroleum Refining");
        try T.expectEqual(.operating, classifyByEntityType(&s));
    }
    // Barrick: same shape as BP.
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "other");
        s.sic_description = try T.allocator.dupe(u8, "Gold and Silver Ores");
        try T.expectEqual(.operating, classifyByEntityType(&s));
    }
    // GLD: "operating" entityType but SIC is commodity-contracts.
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "operating");
        s.sic_description = try T.allocator.dupe(u8, "Commodity Contracts Brokers & Dealers");
        try T.expectEqual(.trust_shaped, classifyByEntityType(&s));
    }
    // AAPL.
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "operating");
        s.sic_description = try T.allocator.dupe(u8, "Electronic Computers");
        try T.expectEqual(.operating, classifyByEntityType(&s));
    }
    // NFLX.
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "operating");
        s.sic_description = try T.allocator.dupe(u8, "Services-Video Tape Rental");
        try T.expectEqual(.operating, classifyByEntityType(&s));
    }
    // Realty Income (O): REIT, operating company.
    {
        var s: SubmissionsSummary = .{};
        defer s.deinit(T.allocator);
        s.entity_type = try T.allocator.dupe(u8, "operating");
        s.sic_description = try T.allocator.dupe(u8, "Real Estate Investment Trusts");
        try T.expectEqual(.operating, classifyByEntityType(&s));
    }
}

/// Result kind for `fetchEtfMetrics`. The caller - see `main.zig` -
/// distinguishes a full holdings record from a profile-only record so
/// it can log the right thing and produce accurate coverage stats.
pub const EtfMetricsResult = union(enum) {
    /// Full NPORT-P parse with holdings + sectors.
    full: EtfMetrics,
    /// Submissions-feed metadata only. Used for unit-investment trusts
    /// that file 10-K instead of NPORT-P (commodity trusts like GLD,
    /// some grantor trusts).
    profile_only: EtfMetrics,
    /// Symbol is in the stock-ticker map but is a plain operating
    /// company (AAPL, MSFT, ...). Not a fund. Caller should skip.
    not_a_fund: void,
    /// Symbol isn't in either ticker map. Caller should skip.
    not_in_edgar: void,
};

/// Construct an EtfMetrics record from submissions-feed metadata
/// alone, with no holdings or sectors. Used for trust entities (e.g.
/// commodity trusts) that lack a NPORT-P filing but for which we
/// still want to surface name + CIK in `etf_metrics.srf`.
///
/// Generic over the entry type (`MutualFundTickerEntry` or
/// `CompanyTickerEntry`).
fn buildProfileOnlyMetrics(
    io: std.Io,
    allocator: std.mem.Allocator,
    entry: GenericTickerEntry,
    sub: *const SubmissionsSummary,
    symbol: []const u8,
) !EtfMetrics {
    var as_of_buf: [10]u8 = undefined;
    const today_date = fmt.todayDate(io);
    const as_of = try std.fmt.bufPrint(&as_of_buf, "{f}", .{today_date});

    // Prefer the submissions-feed name (canonical, "SPDR GOLD TRUST")
    // over the company_tickers.json title (less authoritative).
    const name_src: ?[]const u8 = sub.entity_name orelse entry.title;
    const series_name: ?[]u8 = if (name_src) |n| try allocator.dupe(u8, n) else null;
    errdefer if (series_name) |s| allocator.free(s);

    return .{
        .symbol = try allocator.dupe(u8, symbol),
        .series_name = series_name,
        .cik = try allocator.dupe(u8, entry.cik),
        .series_id = null,
        .net_assets = null,
        .period_end = null,
        .as_of = try allocator.dupe(u8, as_of),
        .holdings = &.{},
        .sectors = &.{},
    };
}

/// Parse N-PORT-P bytes into an EtfMetrics struct. Heavy XML - we use
/// the vendored `xml.zig` DOM parser.
///
/// `entry` is a `GenericTickerEntry` - `MutualFundTickerEntry`
/// and `CompanyTickerEntry` callers use `.toGeneric()` to collapse
/// to this shape, so `parseNportP` doesn't need to know which
/// on-disk cache the entry came from.
fn parseNportP(
    io: std.Io,
    allocator: std.mem.Allocator,
    xml_bytes: []const u8,
    symbol: []const u8,
    entry: GenericTickerEntry,
    top_n_holdings: usize,
) !EtfMetrics {
    var as_of_buf: [10]u8 = undefined;
    const today_date = fmt.todayDate(io);
    const as_of = try std.fmt.bufPrint(&as_of_buf, "{f}", .{today_date});

    var doc = try xml.parse(allocator, xml_bytes);
    defer doc.deinit();
    const root = doc.root;

    // Walk: edgarSubmission > formData > genInfo and fundInfo.
    const form_data = (try root.findChildByTag("formData")) orelse return error.MissingFormData;
    const gen_info = try form_data.findChildByTag("genInfo");
    const fund_info = try form_data.findChildByTag("fundInfo");
    const invst_or_secs = try form_data.findChildByTag("invstOrSecs");

    var series_name: ?[]const u8 = null;
    var period_end: ?[]const u8 = null;
    if (gen_info) |gi| {
        if (try gi.findChildByTag("seriesName")) |e| {
            if (e.children.items.len > 0) {
                if (e.children.items[0] == .CharData) {
                    const sn = e.children.items[0].CharData;
                    // Single-series trusts (SPY, IVV, ...) write
                    // "N/A" here - drop it so we fall through to the
                    // ticker-map title below.
                    if (!std.mem.eql(u8, sn, "N/A") and sn.len > 0) {
                        series_name = try allocator.dupe(u8, sn);
                    }
                }
            }
        }
    }
    // Fall back to the ticker-map title (e.g. "SPDR S&P 500 ETF Trust"
    // for SPY) if NPORT-P didn't supply a useful series name. The
    // title comes from `company_tickers.json` for stock-map entries.
    if (series_name == null) {
        if (entry.title) |t| {
            series_name = try allocator.dupe(u8, t);
        }
    }
    if (gen_info) |gi| {
        if (try gi.findChildByTag("repPdEnd")) |e| {
            if (e.children.items.len > 0) {
                if (e.children.items[0] == .CharData) {
                    period_end = try allocator.dupe(u8, e.children.items[0].CharData);
                }
            }
        }
    }

    var net_assets: ?f64 = null;
    if (fund_info) |fi| {
        if (try fi.findChildByTag("netAssets")) |e| {
            if (e.children.items.len > 0) {
                if (e.children.items[0] == .CharData) {
                    net_assets = std.fmt.parseFloat(f64, e.children.items[0].CharData) catch null;
                }
            }
        }
    }

    // Holdings + sector breakdown.
    var holdings_list: std.ArrayList(Holding) = .empty;
    errdefer {
        for (holdings_list.items) |h| {
            allocator.free(h.name);
            if (h.ticker) |t| allocator.free(t);
            if (h.cusip) |c| allocator.free(c);
            if (h.lei) |l| allocator.free(l);
            if (h.country) |c| allocator.free(c);
        }
        holdings_list.deinit(allocator);
    }

    // Sector aggregation: assetCat × issuerCat -> cumulative weight
    var sector_map: std.StringHashMap(f64) = .init(allocator);
    defer {
        var it = sector_map.iterator();
        while (it.next()) |entry2| allocator.free(entry2.key_ptr.*);
        sector_map.deinit();
    }

    if (invst_or_secs) |secs| {
        for (secs.children.items) |child| {
            if (child != .Element) continue;
            const sec = child.Element;
            if (!std.mem.eql(u8, sec.tag, "invstOrSec")) continue;

            const name_text = elementText(sec, "name") orelse continue;
            const pct_text = elementText(sec, "pctVal") orelse continue;
            const pct = std.fmt.parseFloat(f64, pct_text) catch continue;

            try holdings_list.append(allocator, .{
                .name = try allocator.dupe(u8, name_text),
                .ticker = if (elementAttrValue(sec, "identifiers", "ticker")) |t| try allocator.dupe(u8, t) else null,
                .cusip = if (elementText(sec, "cusip")) |c| try allocator.dupe(u8, c) else null,
                .lei = if (elementTextOptional(sec, "lei")) |l| try allocator.dupe(u8, l) else null,
                .country = if (elementTextOptional(sec, "invCountry")) |c| try allocator.dupe(u8, c) else null,
                .pct_of_portfolio = pct,
            });

            // Aggregate by (assetCat, issuerCat).
            const asset_cat = elementText(sec, "assetCat") orelse "?";
            const issuer_cat = elementText(sec, "issuerCat") orelse "?";
            const key = try std.fmt.allocPrint(allocator, "{s}/{s}", .{ asset_cat, issuer_cat });
            const gop = try sector_map.getOrPut(key);
            if (gop.found_existing) {
                allocator.free(key);
                gop.value_ptr.* += pct;
            } else {
                gop.value_ptr.* = pct;
            }
        }
    }

    // Top N holdings by pct_of_portfolio.
    const all_holdings = try holdings_list.toOwnedSlice(allocator);
    std.mem.sort(Holding, all_holdings, {}, struct {
        fn gt(_: void, a: Holding, b: Holding) bool {
            return a.pct_of_portfolio > b.pct_of_portfolio;
        }
    }.gt);

    const keep = @min(all_holdings.len, top_n_holdings);
    const top = try allocator.alloc(Holding, keep);
    for (all_holdings[0..keep], 0..) |h, i| top[i] = h;
    // Free the rest.
    for (all_holdings[keep..]) |h| {
        allocator.free(h.name);
        if (h.ticker) |t| allocator.free(t);
        if (h.cusip) |c| allocator.free(c);
        if (h.lei) |l| allocator.free(l);
        if (h.country) |c| allocator.free(c);
    }
    allocator.free(all_holdings);

    // Sector list.
    var sectors_list: std.ArrayList(SectorWeight) = .empty;
    errdefer sectors_list.deinit(allocator);
    var s_it = sector_map.iterator();
    while (s_it.next()) |s_entry| {
        const code = s_entry.key_ptr.*;
        try sectors_list.append(allocator, .{
            .code = try allocator.dupe(u8, code),
            .description = try allocator.dupe(u8, sectorDescription(code)),
            .pct_of_portfolio = s_entry.value_ptr.*,
        });
    }
    const sectors = try sectors_list.toOwnedSlice(allocator);
    std.mem.sort(SectorWeight, sectors, {}, struct {
        fn gt(_: void, a: SectorWeight, b: SectorWeight) bool {
            return a.pct_of_portfolio > b.pct_of_portfolio;
        }
    }.gt);

    return .{
        .symbol = try allocator.dupe(u8, symbol),
        .series_name = series_name,
        .cik = try allocator.dupe(u8, entry.cik),
        .series_id = if (entry.series_id) |sid| try allocator.dupe(u8, sid) else null,
        .net_assets = net_assets,
        .period_end = period_end,
        .as_of = try allocator.dupe(u8, as_of),
        .holdings = top,
        .sectors = sectors,
    };
}

/// Translation table for NPORT-P `assetCat/issuerCat` codes. The
/// values are the SEC's own form-instruction abbreviations; the
/// descriptions are condensed-but-accurate human readings used to
/// populate `SectorWeight.description`.
///
/// Coverage targets the codes observed across a representative
/// real-world portfolio (~32 distinct codes seen across stock /
/// bond / blended ETFs). Unrecognized codes round-trip raw (the
/// lookup falls back to the code itself) so unknowns surface for
/// table extension rather than silently corrupting downstream
/// classification.
///
/// AssetCat values per SEC form instructions:
///   EC       Equity (common)        DE       Derivative
///   EP       Equity Preferred       DFE      Derivative - Foreign Exchange
///   DBT      Debt                   DIR      Direct Investment in Real Property
///   ABS-MBS  Asset-Backed Mortgage  DCR      Direct Credit Risk
///   ABS-O    Asset-Backed Other     LON      Loan
///   ABS-CBDO Asset-Backed CBO/CDO   STIV     Short-Term Investment Vehicle
///   RA       Repurchase Agreement   ?        Other / Unknown
///
/// IssuerCat values per SEC form instructions:
///   CORP   Corporate              MUN    Municipal
///   UST    US Treasury            NUSS   Non-US Sovereign
///   USGA   US Government Agency   RF     Registered Fund
///   USGSE  US Government-Sponsored Enterprise
///   PF     Private Fund           ?      Other / Unknown
const sector_code_descriptions = [_]struct {
    code: []const u8,
    description: []const u8,
}{
    // Equity (common)
    .{ .code = "EC/CORP", .description = "Equity / Corporate" },
    .{ .code = "EC/RF", .description = "Equity / Registered Fund" },
    .{ .code = "EC/NUSS", .description = "Equity / Non-US Sovereign" },
    .{ .code = "EC/?", .description = "Equity / Other" },

    // Equity preferred
    .{ .code = "EP/CORP", .description = "Equity Preferred / Corporate" },
    .{ .code = "EP/NUSS", .description = "Equity Preferred / Non-US Sovereign" },

    // Debt
    .{ .code = "DBT/UST", .description = "Debt / US Treasury" },
    .{ .code = "DBT/USGA", .description = "Debt / US Gov Agency" },
    .{ .code = "DBT/USGSE", .description = "Debt / US GSE" },
    .{ .code = "DBT/CORP", .description = "Debt / Corporate" },
    .{ .code = "DBT/MUN", .description = "Debt / Municipal" },
    .{ .code = "DBT/NUSS", .description = "Debt / Non-US Sovereign" },
    .{ .code = "DBT/?", .description = "Debt / Other" },

    // Asset-backed
    .{ .code = "ABS-MBS/USGSE", .description = "Asset-Backed / US GSE Mortgage" },
    .{ .code = "ABS-MBS/USGA", .description = "Asset-Backed / US Gov Agency Mortgage" },
    .{ .code = "ABS-MBS/CORP", .description = "Asset-Backed / Corporate Mortgage" },
    .{ .code = "ABS-O/CORP", .description = "Asset-Backed Other / Corporate" },
    .{ .code = "ABS-CBDO/CORP", .description = "Asset-Backed CBO/CDO / Corporate" },

    // Repurchase agreements
    .{ .code = "RA/CORP", .description = "Repurchase Agreement / Corporate" },
    .{ .code = "RA/?", .description = "Repurchase Agreement / Other" },

    // Loans
    .{ .code = "LON/CORP", .description = "Loan / Corporate" },

    // Short-term investment vehicles
    .{ .code = "STIV/CORP", .description = "Short-Term Investment Vehicle / Corporate" },
    .{ .code = "STIV/RF", .description = "Short-Term Investment Vehicle / Registered Fund" },
    .{ .code = "STIV/PF", .description = "Short-Term Investment Vehicle / Private Fund" },

    // Derivatives
    .{ .code = "DE/CORP", .description = "Derivative / Corporate" },
    .{ .code = "DE/?", .description = "Derivative / Other" },
    .{ .code = "DFE/CORP", .description = "Derivative-FX / Corporate" },
    .{ .code = "DFE/?", .description = "Derivative-FX / Other" },

    // Direct investment / direct credit risk
    .{ .code = "DIR/?", .description = "Direct Real Property / Other" },
    .{ .code = "DCR/?", .description = "Direct Credit Risk / Other" },

    // Catch-all unknowns. We translate "?/X" to a more readable
    // shape but preserve the structure (issuer is known even if
    // asset class isn't).
    .{ .code = "?/CORP", .description = "Other / Corporate" },
    .{ .code = "?/?", .description = "Other / Other" },
};

/// Look up an NPORT-P sector code's human-readable description. For
/// unknown codes returns the code itself, so the caller can always
/// `dupe` the result without conditional handling.
pub fn sectorDescription(code: []const u8) []const u8 {
    for (sector_code_descriptions) |entry| {
        if (std.mem.eql(u8, entry.code, code)) return entry.description;
    }
    return code;
}

fn elementText(parent: *xml.Element, tag: []const u8) ?[]const u8 {
    const child = (parent.findChildByTag(tag) catch return null) orelse return null;
    if (child.children.items.len == 0) return null;
    if (child.children.items[0] != .CharData) return null;
    return child.children.items[0].CharData;
}

/// Read the `value` attribute of a child element identified by `tag`,
/// optionally nested inside `parent_tag` (use `null` for direct
/// children). Returns null when either path step fails. Used to pull
/// `<identifiers><ticker value="AGX"/></identifiers>` out of NPORT-P
/// holding records, where the ticker is encoded as an attribute on a
/// nested element rather than as text content.
fn elementAttrValue(parent: *xml.Element, parent_tag: ?[]const u8, tag: []const u8) ?[]const u8 {
    const container: *xml.Element = if (parent_tag) |pt|
        (parent.findChildByTag(pt) catch return null) orelse return null
    else
        parent;
    const child = (container.findChildByTag(tag) catch return null) orelse return null;
    return child.getAttribute("value");
}

/// Like `elementText` but treats NPORT-P's `"N/A"` sentinel and the
/// empty string as absent. NPORT-P uses literal `"N/A"` for missing
/// LEIs on issuers without one, and empty `<invCountry/>` for
/// holdings whose country can't be determined; both should round-trip
/// as null in Zig.
fn elementTextOptional(parent: *xml.Element, tag: []const u8) ?[]const u8 {
    const text = elementText(parent, tag) orelse return null;
    if (text.len == 0) return null;
    if (std.mem.eql(u8, text, "N/A")) return null;
    return text;
}

test "parseTickerMap parses fixture rows" {
    const fixture =
        \\{"fields":["cik","seriesId","classId","symbol"],"data":[
        \\  [36405,"S000002848","C000007808","VTI"],
        \\  [1100663,"S000004362","C000012092","AGG"]
        \\]}
    ;
    const allocator = std.testing.allocator;
    const entries = try parseTickerMap(allocator, fixture);
    var map = try TickerMap(MutualFundTickerEntry).fromEntries(allocator, entries);
    defer map.deinit();

    const vti = map.get("VTI") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("VTI", vti.symbol);
    try std.testing.expectEqualStrings("0000036405", vti.cik);
    try std.testing.expectEqualStrings("S000002848", vti.series_id orelse return error.TestFailed);

    const agg = map.get("AGG") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("0001100663", agg.cik);
}

test "parseStockTickerMap parses fixture" {
    const fixture =
        \\{
        \\  "0":{"cik_str":78462,"ticker":"SPY","title":"SPDR S&P 500 ETF Trust"},
        \\  "1":{"cik_str":1222333,"ticker":"GLD","title":"SPDR GOLD TRUST"}
        \\}
    ;
    const allocator = std.testing.allocator;
    const entries = try parseStockTickerMap(allocator, fixture);
    var map = try TickerMap(CompanyTickerEntry).fromEntries(allocator, entries);
    defer map.deinit();

    const spy = map.get("SPY") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("SPY", spy.symbol);
    try std.testing.expectEqualStrings("0000078462", spy.cik);
    try std.testing.expect(spy.series_id == null);
    try std.testing.expectEqualStrings("SPDR S&P 500 ETF Trust", spy.title orelse return error.TestFailed);

    const gld = map.get("GLD") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("0001222333", gld.cik);
}

test "sectorDescription translates known codes and round-trips unknown" {
    // Known codes get translated.
    try std.testing.expectEqualStrings("Equity / Corporate", sectorDescription("EC/CORP"));
    try std.testing.expectEqualStrings("Debt / US Treasury", sectorDescription("DBT/UST"));
    try std.testing.expectEqualStrings("Asset-Backed / US GSE Mortgage", sectorDescription("ABS-MBS/USGSE"));

    // Codes added to cover real-world NPORT-P output.
    try std.testing.expectEqualStrings("Debt / Municipal", sectorDescription("DBT/MUN"));
    try std.testing.expectEqualStrings("Short-Term Investment Vehicle / Registered Fund", sectorDescription("STIV/RF"));
    try std.testing.expectEqualStrings("Repurchase Agreement / Corporate", sectorDescription("RA/CORP"));
    try std.testing.expectEqualStrings("Other / Other", sectorDescription("?/?"));

    // Unknown codes round-trip raw so future additions surface for
    // table extension rather than getting silently mistranslated.
    try std.testing.expectEqualStrings("MADE/UPCODE", sectorDescription("MADE/UPCODE"));
    try std.testing.expectEqualStrings("", sectorDescription(""));
}

// ── buildProfileOnlyMetrics ────────────────────────────────────

test "buildProfileOnlyMetrics: prefers submissions entity_name over entry.title" {
    const allocator = std.testing.allocator;
    const sub = SubmissionsSummary{
        .entity_name = "SPDR GOLD TRUST",
    };
    const entry = GenericTickerEntry{
        .symbol = "GLD",
        .cik = "0001222333",
        .title = "less authoritative title",
    };
    var metrics = try buildProfileOnlyMetrics(std.testing.io, allocator, entry, &sub, "GLD");
    defer metrics.deinit(allocator);

    try std.testing.expectEqualStrings("SPDR GOLD TRUST", metrics.series_name orelse return error.SeriesNameMissing);
    try std.testing.expectEqualStrings("GLD", metrics.symbol);
    try std.testing.expectEqualStrings("0001222333", metrics.cik);
    try std.testing.expect(metrics.series_id == null);
    try std.testing.expect(metrics.net_assets == null);
    try std.testing.expect(metrics.period_end == null);
    try std.testing.expectEqual(@as(usize, 0), metrics.holdings.len);
    try std.testing.expectEqual(@as(usize, 0), metrics.sectors.len);
}

test "buildProfileOnlyMetrics: falls back to entry.title when submissions has no name" {
    const allocator = std.testing.allocator;
    const sub = SubmissionsSummary{}; // entity_name = null
    const entry = GenericTickerEntry{
        .symbol = "FOO",
        .cik = "0",
        .title = "Foo Corp",
    };
    var metrics = try buildProfileOnlyMetrics(std.testing.io, allocator, entry, &sub, "FOO");
    defer metrics.deinit(allocator);
    try std.testing.expectEqualStrings("Foo Corp", metrics.series_name orelse return error.SeriesNameMissing);
}

test "buildProfileOnlyMetrics: both name sources null -> series_name is null" {
    const allocator = std.testing.allocator;
    const sub = SubmissionsSummary{};
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try buildProfileOnlyMetrics(std.testing.io, allocator, entry, &sub, "X");
    defer metrics.deinit(allocator);
    try std.testing.expect(metrics.series_name == null);
}

test "buildProfileOnlyMetrics: as_of is a 10-char YYYY-MM-DD string" {
    const allocator = std.testing.allocator;
    const sub = SubmissionsSummary{};
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try buildProfileOnlyMetrics(std.testing.io, allocator, entry, &sub, "X");
    defer metrics.deinit(allocator);
    try std.testing.expectEqual(@as(usize, 10), metrics.as_of.len);
    try std.testing.expectEqual(@as(u8, '-'), metrics.as_of[4]);
    try std.testing.expectEqual(@as(u8, '-'), metrics.as_of[7]);
}

test "parseNportP holdings: ticker/lei/country populated when present" {
    const allocator = std.testing.allocator;
    // Minimal NPORT-P fixture covering the holding-identifier shapes.
    // Two holdings: first has all identifiers, second is bare-bones
    // with the "N/A" LEI sentinel and an empty <invCountry/>.
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>Argan Inc</name>
        \\        <lei>529900E4KZWBV9KGBS83</lei>
        \\        <cusip>04010E109</cusip>
        \\        <identifiers>
        \\          <isin value="US04010E1091"/>
        \\          <ticker value="AGX"/>
        \\        </identifiers>
        \\        <pctVal>4.89</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\        <invCountry>US</invCountry>
        \\      </invstOrSec>
        \\      <invstOrSec>
        \\        <name>Mystery Bond</name>
        \\        <lei>N/A</lei>
        \\        <cusip>000000000</cusip>
        \\        <pctVal>0.50</pctVal>
        \\        <assetCat>DBT</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\        <invCountry></invCountry>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{
        .symbol = "TEST",
        .cik = "0000000000",
    };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "TEST", entry, 10);
    defer metrics.deinit(allocator);

    try std.testing.expectEqual(@as(usize, 2), metrics.holdings.len);

    // Holdings are sorted by pct descending - Argan first.
    const argan = metrics.holdings[0];
    try std.testing.expectEqualStrings("Argan Inc", argan.name);
    try std.testing.expectEqualStrings("AGX", argan.ticker orelse return error.TickerMissing);
    try std.testing.expectEqualStrings("04010E109", argan.cusip orelse return error.CusipMissing);
    try std.testing.expectEqualStrings("529900E4KZWBV9KGBS83", argan.lei orelse return error.LeiMissing);
    try std.testing.expectEqualStrings("US", argan.country orelse return error.CountryMissing);

    // Mystery Bond: no <ticker>, "N/A" lei, empty <invCountry/>.
    const mystery = metrics.holdings[1];
    try std.testing.expectEqualStrings("Mystery Bond", mystery.name);
    try std.testing.expect(mystery.ticker == null);
    try std.testing.expect(mystery.lei == null);
    try std.testing.expect(mystery.country == null);
    try std.testing.expectEqualStrings("000000000", mystery.cusip orelse return error.CusipMissing);
}

test "parseNportP: <genInfo><seriesName> populated -> series_name from XML" {
    // Multi-series trusts (Vanguard family, Fidelity family) put
    // the canonical series name in <genInfo><seriesName>. We
    // should pick that up over the entry.title fallback.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <genInfo>
        \\      <seriesName>Vanguard Total Bond Market Index Fund</seriesName>
        \\      <repPdEnd>2025-09-30</repPdEnd>
        \\    </genInfo>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>US Treasury 4.5% 2030</name>
        \\        <pctVal>5.0</pctVal>
        \\        <assetCat>DBT</assetCat>
        \\        <issuerCat>UST</issuerCat>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{
        .symbol = "VBTLX",
        .cik = "0000000000",
        .title = "Should Be Ignored When XML Has Series Name",
    };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "VBTLX", entry, 10);
    defer metrics.deinit(allocator);

    try std.testing.expectEqualStrings(
        "Vanguard Total Bond Market Index Fund",
        metrics.series_name orelse return error.SeriesNameMissing,
    );
    try std.testing.expectEqualStrings(
        "2025-09-30",
        metrics.period_end orelse return error.PeriodEndMissing,
    );
}

test "parseNportP: seriesName == 'N/A' falls through to entry.title" {
    // Single-series trusts (SPY, IVV) write "N/A" in
    // <seriesName>. We must drop it and fall back to the
    // ticker-map title from `entry`.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <genInfo>
        \\      <seriesName>N/A</seriesName>
        \\    </genInfo>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>AAPL</name>
        \\        <pctVal>7.0</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{
        .symbol = "SPY",
        .cik = "0000000000",
        .title = "SPDR S&P 500 ETF Trust",
    };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "SPY", entry, 10);
    defer metrics.deinit(allocator);

    try std.testing.expectEqualStrings(
        "SPDR S&P 500 ETF Trust",
        metrics.series_name orelse return error.SeriesNameMissing,
    );
}

test "parseNportP: empty <seriesName/> falls through to entry.title" {
    // Empty-element form (sn.len == 0) is the same fallback
    // case as N/A - make sure both paths trigger the title
    // fallback.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <genInfo>
        \\      <seriesName></seriesName>
        \\    </genInfo>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>X</name>
        \\        <pctVal>1.0</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{
        .symbol = "X",
        .cik = "0",
        .title = "Fallback Title",
    };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 10);
    defer metrics.deinit(allocator);
    try std.testing.expectEqualStrings(
        "Fallback Title",
        metrics.series_name orelse return error.SeriesNameMissing,
    );
}

test "parseNportP: no <genInfo>, no entry.title -> series_name is null" {
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>X</name>
        \\        <pctVal>1.0</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 10);
    defer metrics.deinit(allocator);
    try std.testing.expect(metrics.series_name == null);
}

test "parseNportP: <netAssets> populated -> net_assets parsed as f64" {
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <fundInfo>
        \\      <netAssets>123456789.50</netAssets>
        \\    </fundInfo>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>X</name>
        \\        <pctVal>1.0</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 10);
    defer metrics.deinit(allocator);
    try std.testing.expect(metrics.net_assets != null);
    try std.testing.expectApproxEqAbs(@as(f64, 123456789.5), metrics.net_assets.?, 0.01);
}

test "parseNportP: <netAssets> with garbage text -> net_assets is null (no error)" {
    // Defensive: we use `catch null` on parseFloat so a malformed
    // value doesn't blow up the whole parse.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <fundInfo>
        \\      <netAssets>not-a-number</netAssets>
        \\    </fundInfo>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>X</name>
        \\        <pctVal>1.0</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 10);
    defer metrics.deinit(allocator);
    try std.testing.expect(metrics.net_assets == null);
}

test "parseNportP: duplicate (assetCat, issuerCat) sums into a single sector" {
    // The aggregation branch: when two holdings share the same
    // (assetCat, issuerCat) bucket, their pcts add into one
    // sector entry rather than producing duplicate rows.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>Holding A</name>
        \\        <pctVal>30.0</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\      </invstOrSec>
        \\      <invstOrSec>
        \\        <name>Holding B</name>
        \\        <pctVal>20.0</pctVal>
        \\        <assetCat>EC</assetCat>
        \\        <issuerCat>CORP</issuerCat>
        \\      </invstOrSec>
        \\      <invstOrSec>
        \\        <name>Holding C</name>
        \\        <pctVal>50.0</pctVal>
        \\        <assetCat>DBT</assetCat>
        \\        <issuerCat>UST</issuerCat>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 10);
    defer metrics.deinit(allocator);

    // Expect 2 distinct sectors despite 3 holdings.
    try std.testing.expectEqual(@as(usize, 2), metrics.sectors.len);

    // Find the EC/CORP bucket and verify it sums to 50.0.
    var ec_corp_pct: ?f64 = null;
    for (metrics.sectors) |s| {
        if (std.mem.eql(u8, s.code, "EC/CORP")) ec_corp_pct = s.pct_of_portfolio;
    }
    try std.testing.expect(ec_corp_pct != null);
    try std.testing.expectApproxEqAbs(@as(f64, 50.0), ec_corp_pct.?, 0.001);
}

test "parseNportP: missing assetCat/issuerCat -> '?' bucket key" {
    // When a holding lacks one or both category tags, we still
    // bucket it (under "?") rather than dropping it. Verifies
    // the `orelse "?"` branches.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <invstOrSecs>
        \\      <invstOrSec>
        \\        <name>Mystery Holding</name>
        \\        <pctVal>10.0</pctVal>
        \\      </invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 10);
    defer metrics.deinit(allocator);

    try std.testing.expectEqual(@as(usize, 1), metrics.sectors.len);
    try std.testing.expectEqualStrings("?/?", metrics.sectors[0].code);
}

test "parseNportP: top_n_holdings caps the holdings array, frees rest" {
    // When more holdings than top_n exist, we keep the top N by
    // pctVal and free the rest. testing.allocator catches any
    // leak in the discard path.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <invstOrSecs>
        \\      <invstOrSec><name>A</name><pctVal>10.0</pctVal><assetCat>EC</assetCat><issuerCat>CORP</issuerCat></invstOrSec>
        \\      <invstOrSec><name>B</name><pctVal>20.0</pctVal><assetCat>EC</assetCat><issuerCat>CORP</issuerCat></invstOrSec>
        \\      <invstOrSec><name>C</name><pctVal>30.0</pctVal><assetCat>EC</assetCat><issuerCat>CORP</issuerCat></invstOrSec>
        \\      <invstOrSec><name>D</name><pctVal>40.0</pctVal><assetCat>EC</assetCat><issuerCat>CORP</issuerCat></invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 2);
    defer metrics.deinit(allocator);

    try std.testing.expectEqual(@as(usize, 2), metrics.holdings.len);
    try std.testing.expectEqualStrings("D", metrics.holdings[0].name); // pct 40.0
    try std.testing.expectEqualStrings("C", metrics.holdings[1].name); // pct 30.0
}

test "parseNportP: holding with non-parseable pctVal is skipped" {
    // The `parseFloat ... catch continue` branch.
    const allocator = std.testing.allocator;
    const xml_fixture =
        \\<?xml version="1.0" encoding="UTF-8"?>
        \\<edgarSubmission>
        \\  <formData>
        \\    <invstOrSecs>
        \\      <invstOrSec><name>Bad</name><pctVal>not-a-number</pctVal><assetCat>EC</assetCat><issuerCat>CORP</issuerCat></invstOrSec>
        \\      <invstOrSec><name>Good</name><pctVal>50.0</pctVal><assetCat>EC</assetCat><issuerCat>CORP</issuerCat></invstOrSec>
        \\    </invstOrSecs>
        \\  </formData>
        \\</edgarSubmission>
    ;
    const entry = GenericTickerEntry{ .symbol = "X", .cik = "0" };
    var metrics = try parseNportP(std.testing.io, allocator, xml_fixture, "X", entry, 10);
    defer metrics.deinit(allocator);

    try std.testing.expectEqual(@as(usize, 1), metrics.holdings.len);
    try std.testing.expectEqualStrings("Good", metrics.holdings[0].name);
}

test "appendEtfMetricRecords decomposes one fund into profile + sectors + holdings" {
    const allocator = std.testing.allocator;

    // Build a minimal EtfMetrics by hand. Strings are owned; deinit
    // matches what `parseNportP` would do.
    const sectors = try allocator.alloc(SectorWeight, 2);
    sectors[0] = .{
        .code = try allocator.dupe(u8, "EC/CORP"),
        .description = try allocator.dupe(u8, "Equity / Corporate"),
        .pct_of_portfolio = 98.5,
    };
    sectors[1] = .{
        .code = try allocator.dupe(u8, "STIV/CORP"),
        .description = try allocator.dupe(u8, "Short-Term Investment Vehicle / Corporate"),
        .pct_of_portfolio = 1.5,
    };
    const holdings = try allocator.alloc(Holding, 3);
    holdings[0] = .{
        .name = try allocator.dupe(u8, "Apple Inc"),
        .ticker = try allocator.dupe(u8, "AAPL"),
        .cusip = try allocator.dupe(u8, "037833100"),
        .lei = null,
        .country = try allocator.dupe(u8, "US"),
        .pct_of_portfolio = 7.0,
    };
    holdings[1] = .{
        .name = try allocator.dupe(u8, "Microsoft Corp"),
        .ticker = try allocator.dupe(u8, "MSFT"),
        .cusip = try allocator.dupe(u8, "594918104"),
        .lei = null,
        .country = try allocator.dupe(u8, "US"),
        .pct_of_portfolio = 6.0,
    };
    holdings[2] = .{
        .name = try allocator.dupe(u8, "NVIDIA Corp"),
        .ticker = try allocator.dupe(u8, "NVDA"),
        .cusip = try allocator.dupe(u8, "67066G104"),
        .lei = null,
        .country = try allocator.dupe(u8, "US"),
        .pct_of_portfolio = 5.0,
    };
    var metrics = EtfMetrics{
        .symbol = try allocator.dupe(u8, "VTI"),
        .series_name = try allocator.dupe(u8, "VANGUARD TOTAL STOCK MARKET INDEX FUND"),
        .cik = try allocator.dupe(u8, "0000036405"),
        .series_id = try allocator.dupe(u8, "S000002848"),
        .net_assets = 2_000_000_000_000.0,
        .period_end = try allocator.dupe(u8, "2025-12-31"),
        .as_of = try allocator.dupe(u8, "2026-05-25"),
        .holdings = holdings,
        .sectors = sectors,
    };
    defer metrics.deinit(allocator);

    var out: std.ArrayList(EtfMetricRecord) = .empty;
    defer {
        for (out.items) |*r| r.deinit(allocator);
        out.deinit(allocator);
    }

    try appendEtfMetricRecords(allocator, &out, metrics);

    // Expect 1 profile + 2 sectors + 3 holdings = 6 records.
    try std.testing.expectEqual(@as(usize, 6), out.items.len);

    // First is profile.
    try std.testing.expect(out.items[0] == .profile);
    try std.testing.expectEqualStrings("VTI", out.items[0].profile.symbol);
    try std.testing.expectEqualStrings("0000036405", out.items[0].profile.cik);
    try std.testing.expectEqualStrings("S000002848", out.items[0].profile.series_id orelse return error.SeriesIdMissing);

    // Next two are sectors.
    try std.testing.expect(out.items[1] == .sector);
    try std.testing.expectEqualStrings("EC/CORP", out.items[1].sector.code);
    try std.testing.expectEqualStrings("Equity / Corporate", out.items[1].sector.description);
    try std.testing.expect(out.items[2] == .sector);

    // Last three are holdings.
    try std.testing.expect(out.items[3] == .holding);
    try std.testing.expect(out.items[4] == .holding);
    try std.testing.expect(out.items[5] == .holding);
    try std.testing.expectEqualStrings("AAPL", out.items[3].holding.ticker orelse return error.TickerMissing);
    try std.testing.expectEqualStrings("VTI", out.items[3].holding.symbol); // fund symbol, not holding's
}

// ── TickerEntry / TickerMap tests ────────────────────────────

test "MutualFundTickerEntry.deinit frees all owned strings" {
    // Round-trip via testing.allocator: any leak fails the test.
    const allocator = std.testing.allocator;
    const entry: MutualFundTickerEntry = .{
        .symbol = try allocator.dupe(u8, "VTI"),
        .cik = try allocator.dupe(u8, "0000036405"),
        .series_id = try allocator.dupe(u8, "S000002848"),
        .class_id = try allocator.dupe(u8, "C000007808"),
        .title = try allocator.dupe(u8, "Vanguard Total Stock Market"),
    };
    entry.deinit(allocator);
}

test "MutualFundTickerEntry.deinit handles all-null optionals" {
    const allocator = std.testing.allocator;
    const entry: MutualFundTickerEntry = .{
        .symbol = try allocator.dupe(u8, "VTI"),
        .cik = try allocator.dupe(u8, "0000036405"),
    };
    entry.deinit(allocator);
}

test "MutualFundTickerEntry.freeSlice frees each entry and the slice" {
    const allocator = std.testing.allocator;
    var list: std.ArrayList(MutualFundTickerEntry) = .empty;
    errdefer list.deinit(allocator);
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "VTI"),
        .cik = try allocator.dupe(u8, "0000036405"),
        .series_id = try allocator.dupe(u8, "S000002848"),
        .class_id = try allocator.dupe(u8, "C000007808"),
    });
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "AGG"),
        .cik = try allocator.dupe(u8, "0001100663"),
    });
    const entries = try list.toOwnedSlice(allocator);
    MutualFundTickerEntry.freeSlice(allocator, entries);
}

test "CompanyTickerEntry.deinit frees all owned strings" {
    const allocator = std.testing.allocator;
    const entry: CompanyTickerEntry = .{
        .symbol = try allocator.dupe(u8, "SPY"),
        .cik = try allocator.dupe(u8, "0000078462"),
        .title = try allocator.dupe(u8, "SPDR S&P 500 ETF Trust"),
    };
    entry.deinit(allocator);
}

test "CompanyTickerEntry.freeSlice frees each entry and the slice" {
    const allocator = std.testing.allocator;
    var list: std.ArrayList(CompanyTickerEntry) = .empty;
    errdefer list.deinit(allocator);
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "SPY"),
        .cik = try allocator.dupe(u8, "0000078462"),
        .title = try allocator.dupe(u8, "SPDR S&P 500 ETF Trust"),
    });
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "GLD"),
        .cik = try allocator.dupe(u8, "0001222333"),
    });
    const entries = try list.toOwnedSlice(allocator);
    CompanyTickerEntry.freeSlice(allocator, entries);
}

test "TickerMap.fromEntries: get returns matching entry by symbol" {
    const allocator = std.testing.allocator;
    var list: std.ArrayList(MutualFundTickerEntry) = .empty;
    errdefer list.deinit(allocator);
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "VTI"),
        .cik = try allocator.dupe(u8, "0000036405"),
        .series_id = try allocator.dupe(u8, "S000002848"),
        .class_id = try allocator.dupe(u8, "C000007808"),
    });
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "AGG"),
        .cik = try allocator.dupe(u8, "0001100663"),
        .series_id = try allocator.dupe(u8, "S000004362"),
        .class_id = try allocator.dupe(u8, "C000012092"),
    });
    const entries = try list.toOwnedSlice(allocator);
    var map = try TickerMap(MutualFundTickerEntry).fromEntries(allocator, entries);
    defer map.deinit();

    const vti = map.get("VTI") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("0000036405", vti.cik);

    const agg = map.get("AGG") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("0001100663", agg.cik);

    try std.testing.expect(map.get("DOES_NOT_EXIST") == null);
}

test "TickerMap.fromEntries: first-wins on duplicate symbols" {
    // A duplicate ticker (same symbol, different CIK) keeps the
    // first occurrence and silently shadows the second. Mirrors
    // the behavior of the old in-parser hashmap.
    const allocator = std.testing.allocator;
    var list: std.ArrayList(MutualFundTickerEntry) = .empty;
    errdefer list.deinit(allocator);
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "DUP"),
        .cik = try allocator.dupe(u8, "0000000001"),
    });
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "DUP"),
        .cik = try allocator.dupe(u8, "0000000002"),
    });
    const entries = try list.toOwnedSlice(allocator);
    var map = try TickerMap(MutualFundTickerEntry).fromEntries(allocator, entries);
    defer map.deinit();

    const dup = map.get("DUP") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("0000000001", dup.cik);
}

test "TickerMap.fromEntries: empty slice produces empty map" {
    const allocator = std.testing.allocator;
    const entries: []MutualFundTickerEntry = &.{};
    var map = try TickerMap(MutualFundTickerEntry).fromEntries(allocator, try allocator.dupe(MutualFundTickerEntry, entries));
    defer map.deinit();

    try std.testing.expect(map.get("VTI") == null);
}

test "TickerMap(CompanyTickerEntry): borrow + lookup round-trip" {
    const allocator = std.testing.allocator;
    var list: std.ArrayList(CompanyTickerEntry) = .empty;
    errdefer list.deinit(allocator);
    try list.append(allocator, .{
        .symbol = try allocator.dupe(u8, "SPY"),
        .cik = try allocator.dupe(u8, "0000078462"),
        .title = try allocator.dupe(u8, "SPDR S&P 500 ETF Trust"),
    });
    const entries = try list.toOwnedSlice(allocator);
    var map = try TickerMap(CompanyTickerEntry).fromEntries(allocator, entries);
    defer map.deinit();

    const spy = map.get("SPY") orelse return error.TestFailed;
    try std.testing.expectEqualStrings("SPY", spy.symbol);
    try std.testing.expectEqualStrings("SPDR S&P 500 ETF Trust", spy.title orelse return error.TitleMissing);
}

test "parseTickerMap: duplicate symbol rows produce one entry (first-wins)" {
    // Real-world: SEC's mutual-fund file occasionally has multiple
    // class IDs sharing a ticker. We keep the first row and skip
    // the rest rather than emitting both and letting TickerMap
    // dedupe - the slice itself is the cache, so we want it
    // canonical.
    const fixture =
        \\{"fields":["cik","seriesId","classId","symbol"],"data":[
        \\  [36405,"S000002848","C000007808","DUP"],
        \\  [99999,"S000099999","C000099999","DUP"]
        \\]}
    ;
    const allocator = std.testing.allocator;
    const entries = try parseTickerMap(allocator, fixture);
    defer MutualFundTickerEntry.freeSlice(allocator, entries);

    try std.testing.expectEqual(@as(usize, 1), entries.len);
    try std.testing.expectEqualStrings("0000036405", entries[0].cik);
}

test "parseStockTickerMap: duplicate symbol rows produce one entry (first-wins)" {
    const fixture =
        \\{
        \\  "0":{"cik_str":78462,"ticker":"DUP","title":"First"},
        \\  "1":{"cik_str":99999,"ticker":"DUP","title":"Second"}
        \\}
    ;
    const allocator = std.testing.allocator;
    const entries = try parseStockTickerMap(allocator, fixture);
    defer CompanyTickerEntry.freeSlice(allocator, entries);

    try std.testing.expectEqual(@as(usize, 1), entries.len);
    try std.testing.expectEqualStrings("First", entries[0].title orelse return error.TitleMissing);
}

// ── parseLatestNportPFromSearch ────────────────────────────────

test "parseLatestNportPFromSearch: picks newest by file_date and builds canonical URL" {
    // SEC EDGAR full-text-search returns multiple NPORT-P
    // filings; we want the latest by file_date (lex-max on
    // YYYY-MM-DD). The URL is built from CIK (zero-stripped)
    // and adsh (dashes stripped).
    const fixture =
        \\{
        \\  "hits": {
        \\    "hits": [
        \\      {"_source":{"file_date":"2024-08-15","adsh":"0000123456-24-000001","ciks":["0001378872"]}},
        \\      {"_source":{"file_date":"2025-02-20","adsh":"0001234567-25-000099","ciks":["0001378872"]}},
        \\      {"_source":{"file_date":"2024-11-10","adsh":"0009999999-24-000050","ciks":["0001378872"]}}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const url = try parseLatestNportPFromSearch(allocator, fixture);
    defer if (url) |u| allocator.free(u);

    try std.testing.expect(url != null);
    try std.testing.expectEqualStrings(
        "https://www.sec.gov/Archives/edgar/data/1378872/000123456725000099/primary_doc.xml",
        url.?,
    );
}

test "parseLatestNportPFromSearch: empty hits returns null" {
    const fixture =
        \\{"hits":{"hits":[]}}
    ;
    const allocator = std.testing.allocator;
    const url = try parseLatestNportPFromSearch(allocator, fixture);
    defer if (url) |u| allocator.free(u);
    try std.testing.expect(url == null);
}

test "parseLatestNportPFromSearch: missing hits.hits returns null" {
    const fixture =
        \\{"hits":{}}
    ;
    const allocator = std.testing.allocator;
    const url = try parseLatestNportPFromSearch(allocator, fixture);
    defer if (url) |u| allocator.free(u);
    try std.testing.expect(url == null);
}

test "parseLatestNportPFromSearch: non-object root returns null" {
    const fixture = "[1,2,3]";
    const allocator = std.testing.allocator;
    const url = try parseLatestNportPFromSearch(allocator, fixture);
    defer if (url) |u| allocator.free(u);
    try std.testing.expect(url == null);
}

test "parseLatestNportPFromSearch: hit with non-object _source skipped, falls through" {
    // Defensive: if a hit has malformed _source, we skip it and
    // continue. As long as at least one valid hit exists, we
    // return its URL.
    const fixture =
        \\{
        \\  "hits": {
        \\    "hits": [
        \\      {"_source":"malformed"},
        \\      {"_source":{"file_date":"2025-01-01","adsh":"0000000001-25-000001","ciks":["0000000001"]}}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const url = try parseLatestNportPFromSearch(allocator, fixture);
    defer if (url) |u| allocator.free(u);
    try std.testing.expect(url != null);
    try std.testing.expect(std.mem.indexOf(u8, url.?, "0000000001") != null);
}

test "parseLatestNportPFromSearch: empty ciks array returns null" {
    const fixture =
        \\{
        \\  "hits": {
        \\    "hits": [
        \\      {"_source":{"file_date":"2025-01-01","adsh":"0000000001-25-000001","ciks":[]}}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const url = try parseLatestNportPFromSearch(allocator, fixture);
    defer if (url) |u| allocator.free(u);
    try std.testing.expect(url == null);
}

test "parseLatestNportPFromSearch: CIK with all leading zeros except one preserved" {
    // Edge case: CIK like "0000000001" should strip to "1",
    // not "" (the loop condition is `len > 1`).
    const fixture =
        \\{
        \\  "hits": {
        \\    "hits": [
        \\      {"_source":{"file_date":"2025-01-01","adsh":"abc-25-001","ciks":["0000000001"]}}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const url = try parseLatestNportPFromSearch(allocator, fixture);
    defer if (url) |u| allocator.free(u);
    try std.testing.expect(url != null);
    try std.testing.expect(std.mem.indexOf(u8, url.?, "/data/1/") != null);
}

// ── parseSharesOutstanding ────────────────────────────────────

test "parseSharesOutstanding: picks latest by `end` date and returns value+form" {
    const fixture =
        \\{
        \\  "units": {
        \\    "shares": [
        \\      {"end":"2024-03-31","val":15000000000,"form":"10-Q"},
        \\      {"end":"2025-06-30","val":15500000000,"form":"10-Q"},
        \\      {"end":"2024-12-31","val":15300000000,"form":"10-K"}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    try std.testing.expect(result != null);
    defer result.?.deinit(allocator);

    try std.testing.expectEqual(@as(u64, 15500000000), result.?.value);
    try std.testing.expectEqualStrings("2025-06-30", result.?.period_end);
    try std.testing.expectEqualStrings("10-Q", result.?.form);
}

test "parseSharesOutstanding: float val coerces to u64" {
    // EDGAR sometimes serializes huge counts as floats. Should
    // round-trip cleanly.
    const fixture =
        \\{
        \\  "units": {
        \\    "shares": [
        \\      {"end":"2025-01-01","val":1234567890.0,"form":"10-K"}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    try std.testing.expect(result != null);
    defer result.?.deinit(allocator);
    try std.testing.expectEqual(@as(u64, 1234567890), result.?.value);
}

test "parseSharesOutstanding: negative val rejected (returns null)" {
    const fixture =
        \\{
        \\  "units": {
        \\    "shares": [
        \\      {"end":"2025-01-01","val":-1,"form":"10-K"}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    if (result) |r| r.deinit(allocator);
    try std.testing.expect(result == null);
}

test "parseSharesOutstanding: missing units -> null" {
    const fixture = "{}";
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    if (result) |r| r.deinit(allocator);
    try std.testing.expect(result == null);
}

test "parseSharesOutstanding: empty units object -> null" {
    const fixture =
        \\{"units":{}}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    if (result) |r| r.deinit(allocator);
    try std.testing.expect(result == null);
}

test "parseSharesOutstanding: empty rows array -> null" {
    const fixture =
        \\{"units":{"shares":[]}}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    if (result) |r| r.deinit(allocator);
    try std.testing.expect(result == null);
}

test "parseSharesOutstanding: missing form field defaults to empty string" {
    const fixture =
        \\{
        \\  "units": {
        \\    "shares": [
        \\      {"end":"2025-01-01","val":1000}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    try std.testing.expect(result != null);
    defer result.?.deinit(allocator);
    try std.testing.expectEqualStrings("", result.?.form);
}

test "parseSharesOutstanding: non-object root -> null" {
    const fixture = "[]";
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    if (result) |r| r.deinit(allocator);
    try std.testing.expect(result == null);
}

test "parseSharesOutstanding: row with non-string end skipped" {
    // Defensive: malformed end falls through; if no valid rows
    // remain, returns null.
    const fixture =
        \\{
        \\  "units": {
        \\    "shares": [
        \\      {"end":12345,"val":1000,"form":"X"}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    if (result) |r| r.deinit(allocator);
    try std.testing.expect(result == null);
}

test "parseSharesOutstanding: takes first units key when multiple present" {
    // Defensive: if EDGAR ever serves multiple unit keys (it
    // usually doesn't for shares), we take the first non-empty
    // one rather than crashing.
    const fixture =
        \\{
        \\  "units": {
        \\    "USD/shares": [],
        \\    "shares": [
        \\      {"end":"2025-01-01","val":42,"form":"10-K"}
        \\    ]
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSharesOutstanding(allocator, fixture);
    try std.testing.expect(result != null);
    defer result.?.deinit(allocator);
    try std.testing.expectEqual(@as(u64, 42), result.?.value);
}

// ── parseSubmissionsFeed (incl. findNportPUrlInSubmissions) ────

test "parseSubmissionsFeed: extracts entity metadata + latest NPORT-P URL" {
    const fixture =
        \\{
        \\  "name":"Acme Funds Trust",
        \\  "entityType":"investment company",
        \\  "sicDescription":"Investment Companies",
        \\  "filings":{
        \\    "recent":{
        \\      "form":["10-K","NPORT-P","NPORT-P","8-K"],
        \\      "accessionNumber":["0000000001-25-000001","0000000002-25-000001","0000000003-25-000050","0000000004-25-000099"],
        \\      "filingDate":["2025-01-01","2024-08-15","2025-02-20","2025-03-01"]
        \\    }
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSubmissionsFeed(allocator, fixture, "0001378872");
    defer result.deinit(allocator);

    try std.testing.expectEqualStrings("Acme Funds Trust", result.entity_name.?);
    try std.testing.expectEqualStrings("investment company", result.entity_type.?);
    try std.testing.expectEqualStrings("Investment Companies", result.sic_description.?);
    // Latest NPORT-P is index 2 (filingDate 2025-02-20), not
    // index 1. URL builds from accession 0000000003-25-000050.
    try std.testing.expect(result.latest_nport_p_url != null);
    try std.testing.expectEqualStrings(
        "https://www.sec.gov/Archives/edgar/data/1378872/000000000325000050/primary_doc.xml",
        result.latest_nport_p_url.?,
    );
}

test "parseSubmissionsFeed: no NPORT-P filings -> latest_nport_p_url is null" {
    const fixture =
        \\{
        \\  "name":"Plain Stock Inc",
        \\  "entityType":"operating",
        \\  "filings":{
        \\    "recent":{
        \\      "form":["10-K","8-K","10-Q"],
        \\      "accessionNumber":["a","b","c"],
        \\      "filingDate":["2025-01-01","2025-02-01","2025-03-01"]
        \\    }
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSubmissionsFeed(allocator, fixture, "0000123456");
    defer result.deinit(allocator);

    try std.testing.expectEqualStrings("Plain Stock Inc", result.entity_name.?);
    try std.testing.expect(result.latest_nport_p_url == null);
}

test "parseSubmissionsFeed: empty sicDescription not duped" {
    // Defensive: SEC sometimes returns an empty string for
    // sicDescription. We should leave the field null rather
    // than dupe an empty string.
    const fixture =
        \\{
        \\  "name":"X",
        \\  "sicDescription":"",
        \\  "filings":{"recent":{"form":[],"accessionNumber":[],"filingDate":[]}}
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSubmissionsFeed(allocator, fixture, "0000000001");
    defer result.deinit(allocator);
    try std.testing.expect(result.sic_description == null);
}

test "parseSubmissionsFeed: missing filings object returns metadata-only summary" {
    const fixture =
        \\{"name":"Bare Entity","entityType":"trust"}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSubmissionsFeed(allocator, fixture, "0000000001");
    defer result.deinit(allocator);
    try std.testing.expectEqualStrings("Bare Entity", result.entity_name.?);
    try std.testing.expect(result.latest_nport_p_url == null);
}

test "parseSubmissionsFeed: non-object root returns empty summary" {
    const fixture = "[]";
    const allocator = std.testing.allocator;
    const result = try parseSubmissionsFeed(allocator, fixture, "0000000001");
    defer result.deinit(allocator);
    try std.testing.expect(result.entity_name == null);
    try std.testing.expect(result.latest_nport_p_url == null);
}

test "parseSubmissionsFeed: NPORT-P with date-array shorter than form-array skips OOB" {
    // Defensive: form/date/accession arrays should be parallel,
    // but if `dates` is short we must not OOB-read. The
    // `if (i >= dates.len) continue` guard exercises this.
    const fixture =
        \\{
        \\  "name":"X",
        \\  "filings":{
        \\    "recent":{
        \\      "form":["NPORT-P","NPORT-P"],
        \\      "accessionNumber":["a","b"],
        \\      "filingDate":["2025-01-01"]
        \\    }
        \\  }
        \\}
    ;
    const allocator = std.testing.allocator;
    const result = try parseSubmissionsFeed(allocator, fixture, "0000000001");
    defer result.deinit(allocator);
    // Index 0 has a matching date, so we still find a URL.
    try std.testing.expect(result.latest_nport_p_url != null);
}