905 lines
40 KiB
Zig
905 lines
40 KiB
Zig
//! Wikidata SPARQL classification provider.
|
|
//!
|
|
//! ## What this provider does
|
|
//!
|
|
//! Given a stock symbol, Wikidata can answer:
|
|
//!
|
|
//! * "What kind of entity is this?" - name, industry, sector,
|
|
//! country of incorporation, inception date, instance-of
|
|
//! classification (operating company / mutual fund / ETF / ...).
|
|
//! * "Does this match the SEC's CIK?" - Wikidata's P5531 already
|
|
//! stores the 10-digit zero-padded CIK matching SEC's convention.
|
|
//!
|
|
//! ## Workflow
|
|
//!
|
|
//! `fetch(symbols)` runs ONE batched SPARQL query that returns
|
|
//! per-ticker rows. The query is keyed on the US-listing (NYSE /
|
|
//! Nasdaq / NYSE Arca / OTC Markets) of each ticker - without that
|
|
//! filter, common US tickers silently resolve to whichever
|
|
//! foreign-exchange company happens to share the symbol (`MRK` ->
|
|
//! Merck KGaA on Frankfurt; `PG` -> People's Garment on SET; etc.).
|
|
//!
|
|
//! The provider is stateless. Caching belongs to the data service,
|
|
//! which writes per-symbol `classification.srf` files after this
|
|
//! provider returns and reads them back on subsequent calls.
|
|
//!
|
|
//! ## Glossary
|
|
//!
|
|
//! SPARQL Query language for RDF-shaped data. Wikidata's
|
|
//! primary read API.
|
|
//! P-number Property identifier in Wikidata (P249 = ticker symbol,
|
|
//! P414 = stock exchange, P31 = instance of, ...).
|
|
//! Q-number Entity identifier in Wikidata (Q845477 = ETF as a
|
|
//! concept, Q13677 = NYSE the entity, Q312 = Apple Inc.
|
|
//! the entity).
|
|
//! wdt:Pxxx Truthy/direct property statement - the simple shape.
|
|
//! p:Pxxx Reified property statement - lets a statement carry
|
|
//! qualifiers (e.g. ticker symbol AS A QUALIFIER on the
|
|
//! stock-exchange statement, rather than as a direct
|
|
//! property of the company).
|
|
//! ps:Pxxx "Statement value" predicate - within a reified
|
|
//! statement, points to the statement's main value.
|
|
//! pq:Pxxx "Qualifier" predicate - within a reified statement,
|
|
//! points to a qualifier on that statement.
|
|
//!
|
|
//! Why the reified statement matters here: Wikidata stores tickers
|
|
//! as P249 qualifiers on a P414 (stock exchange) statement, NOT as
|
|
//! a direct `wdt:P249` property. Querying naively returns zero rows
|
|
//! for nearly every US-listed equity.
|
|
|
|
const std = @import("std");
|
|
const http = @import("../net/http.zig");
|
|
const fmt = @import("../format.zig");
|
|
const classification = @import("../models/classification.zig");
|
|
|
|
// `ClassificationRecord`, `geo`, and `geoFor` are domain-level
|
|
// types (any classification source could populate them), so they
|
|
// live in `models/classification.zig`. Re-export here so existing
|
|
// internal references compile unchanged.
|
|
pub const ClassificationRecord = classification.ClassificationRecord;
|
|
pub const geo = classification.geo;
|
|
pub const geoFor = classification.geoFor;
|
|
|
|
const sparql_endpoint = "https://query.wikidata.org/sparql";
|
|
|
|
/// Wikidata Q-IDs we test against `instance of` (P31) to classify
|
|
/// fund-shaped securities. Curated, not exhaustive.
|
|
/// Wikidata Q-IDs for fund-shaped securities. Used to set
|
|
/// `is_etf` and `asset_class` based on the `instance of` (P31)
|
|
/// statement on the security entity.
|
|
///
|
|
/// These were verified by querying Wikidata's `rdfs:label` for
|
|
/// each Q-ID (the previous list had stale/incorrect IDs that
|
|
/// matched unrelated entities like "marathon" and silently
|
|
/// disabled the is_etf detection for every ETF in the corpus).
|
|
const etf_q_ids = [_][]const u8{
|
|
"Q845477", // exchange-traded fund
|
|
"Q1383049", // exchange-traded note
|
|
};
|
|
const mutual_fund_q_ids = [_][]const u8{
|
|
"Q791974", // mutual fund
|
|
"Q55598711", // mutual fund (alternate / class-of)
|
|
};
|
|
|
|
/// US stock exchanges accepted by the SPARQL exchange filter.
|
|
/// Without this filter, ticker collisions across global exchanges
|
|
/// silently return the wrong company.
|
|
///
|
|
/// Q-IDs:
|
|
/// Q13677 New York Stock Exchange (NYSE)
|
|
/// Q82059 Nasdaq
|
|
/// Q4527260 NYSE Arca
|
|
/// Q1666011 OTC Markets Group / Pink Sheets
|
|
const us_exchanges = [_][]const u8{
|
|
"wd:Q13677",
|
|
"wd:Q82059",
|
|
"wd:Q4527260",
|
|
"wd:Q1666011",
|
|
};
|
|
|
|
// ── Wikidata provider state (file-as-struct) ─────────────────────
|
|
//
|
|
// Callers do `const wikidata = @import("providers/Wikidata.zig");`
|
|
// followed by `var wd = wikidata.init(...);` and `wd.fetch(...)`.
|
|
|
|
client: http.Client,
|
|
allocator: std.mem.Allocator,
|
|
io: std.Io,
|
|
/// Contact email for User-Agent / From headers, sourced from
|
|
/// `Config.user_email`. Required; callers must surface a clear
|
|
/// missing-config error before constructing this provider.
|
|
user_email: []const u8,
|
|
|
|
const Wikidata = @This();
|
|
|
|
pub fn init(
|
|
io: std.Io,
|
|
allocator: std.mem.Allocator,
|
|
user_email: []const u8,
|
|
) Wikidata {
|
|
return .{
|
|
.client = http.Client.init(io, allocator),
|
|
.allocator = allocator,
|
|
.io = io,
|
|
.user_email = user_email,
|
|
};
|
|
}
|
|
|
|
pub fn deinit(self: *Wikidata) void {
|
|
self.client.deinit();
|
|
}
|
|
|
|
/// Fetch and parse Wikidata classifications for `symbols`.
|
|
/// Runs a single batched SPARQL query and parses the response.
|
|
/// Caller owns the returned slice and each record.
|
|
pub fn fetch(
|
|
self: *Wikidata,
|
|
result_allocator: std.mem.Allocator,
|
|
symbols: []const []const u8,
|
|
) ![]ClassificationRecord {
|
|
if (symbols.len == 0) return &.{};
|
|
|
|
const query = try buildQuery(self.allocator, symbols);
|
|
defer self.allocator.free(query);
|
|
|
|
const json = try self.postSparql(query);
|
|
defer self.allocator.free(json);
|
|
|
|
return parse(self.io, result_allocator, json, symbols);
|
|
}
|
|
|
|
/// POST a SPARQL query. Sets the User-Agent + From headers from
|
|
/// `user_email` for politeness; Wikidata explicitly recommends
|
|
/// descriptive User-Agent strings.
|
|
fn postSparql(self: *Wikidata, query: []const u8) ![]u8 {
|
|
var form_buf: std.Io.Writer.Allocating = .init(self.allocator);
|
|
defer form_buf.deinit();
|
|
try form_buf.writer.writeAll("query=");
|
|
// `Component.formatEscaped` percent-encodes everything outside
|
|
// RFC 3986's unreserved set - exactly the contract for the
|
|
// `application/x-www-form-urlencoded` body we're building.
|
|
try (std.Uri.Component{ .raw = query }).formatEscaped(&form_buf.writer);
|
|
|
|
var ua_buf: [256]u8 = undefined;
|
|
const ua = std.fmt.bufPrint(&ua_buf, "zfin/0.1 ({s})", .{self.user_email}) catch return error.UserEmailTooLong;
|
|
|
|
const headers = [_]std.http.Header{
|
|
.{ .name = "User-Agent", .value = ua },
|
|
.{ .name = "Accept", .value = "application/sparql-results+json" },
|
|
.{ .name = "Content-Type", .value = "application/x-www-form-urlencoded" },
|
|
.{ .name = "From", .value = self.user_email },
|
|
};
|
|
|
|
var resp = try self.client.request(.POST, sparql_endpoint, form_buf.written(), &headers);
|
|
defer resp.deinit();
|
|
return self.allocator.dupe(u8, resp.body);
|
|
}
|
|
|
|
/// Build the batched SPARQL query for a slice of ticker symbols.
|
|
/// Caller owns the returned bytes. Symbols interpolated via
|
|
/// `VALUES ?ticker { "AAPL" "MSFT" ... }`.
|
|
///
|
|
/// Wikidata's ticker storage is non-obvious: tickers are stored as
|
|
/// `P249` qualifiers on a `P414` (stock exchange) statement. Naive
|
|
/// `?security wdt:P249 ?ticker` returns zero rows for nearly every
|
|
/// US-listed equity. The query reaches them via:
|
|
///
|
|
/// ?security p:P414 ?stmt .
|
|
/// ?stmt ps:P414 ?exchange .
|
|
/// ?stmt pq:P249 ?ticker .
|
|
///
|
|
/// `?exchange` is filtered to a small set of US exchanges to avoid
|
|
/// ticker collisions with foreign listings.
|
|
fn buildQuery(allocator: std.mem.Allocator, symbols: []const []const u8) ![]u8 {
|
|
var aw: std.Io.Writer.Allocating = .init(allocator);
|
|
defer aw.deinit();
|
|
|
|
try aw.writer.writeAll(
|
|
\\SELECT ?ticker ?security ?securityLabel ?industryLabel ?countryCode ?inception ?cik ?instance WHERE {
|
|
\\ VALUES ?ticker {
|
|
);
|
|
for (symbols) |s| {
|
|
try aw.writer.print(" \"{s}\"", .{s});
|
|
}
|
|
try aw.writer.writeAll(" }\n");
|
|
try aw.writer.writeAll(" VALUES ?exchange {");
|
|
for (us_exchanges) |x| {
|
|
try aw.writer.print(" {s}", .{x});
|
|
}
|
|
try aw.writer.writeAll(" }\n");
|
|
try aw.writer.writeAll(
|
|
\\ ?security p:P414 ?exchstmt .
|
|
\\ ?exchstmt ps:P414 ?exchange .
|
|
\\ ?exchstmt pq:P249 ?ticker .
|
|
\\ OPTIONAL { ?security wdt:P452 ?industry . }
|
|
\\ OPTIONAL { ?security wdt:P17 ?country . ?country wdt:P297 ?countryCode . }
|
|
\\ OPTIONAL { ?security wdt:P571 ?inception . }
|
|
\\ OPTIONAL { ?security wdt:P5531 ?cik . }
|
|
\\ OPTIONAL { ?security wdt:P31 ?instance . }
|
|
\\ SERVICE wikibase:label { bd:serviceParam wikibase:language "en". }
|
|
\\}
|
|
);
|
|
return aw.toOwnedSlice();
|
|
}
|
|
|
|
/// Parse the SPARQL JSON response into `ClassificationRecord` values.
|
|
/// Canonical sector taxonomy (GICS-aligned 11-sector model).
|
|
/// Wikidata's `wdt:P452` (industry) values are noisy, often
|
|
/// returning multiple long-tail sub-industries per company in
|
|
/// arbitrary SPARQL order. `canonicalizeSector` maps each raw
|
|
/// industry label to one of these buckets so the user gets a
|
|
/// stable sector choice rather than whichever sub-industry
|
|
/// SPARQL surfaced first.
|
|
///
|
|
/// Lives in `models/classification.zig` so multiple producers
|
|
/// share one taxonomy. Re-exported here for back-compat with
|
|
/// existing internal references.
|
|
pub const sector = classification.sector;
|
|
|
|
/// Map a Wikidata `wdt:P452` industry label (lowercase or mixed
|
|
/// case) to one of the canonical sectors. Returns null if no
|
|
/// keyword matches - the caller falls back to whatever pre-canonical
|
|
/// industry string was last seen.
|
|
///
|
|
/// Priority is encoded by ordering: the function returns the FIRST
|
|
/// matching sector, so more-specific keywords appear first within
|
|
/// each sector. Cross-sector priority order (Tech, Comms, Consumer
|
|
/// Cyclical, ...) doesn't matter because the caller calls this
|
|
/// once per industry label and picks among results separately.
|
|
fn canonicalizeSector(industry: []const u8) ?[]const u8 {
|
|
// Lowercase via ascii because Wikidata mixes title case
|
|
// ("Semiconductor Industry") with lowercase ("software
|
|
// development"). We compare against lowercase keywords.
|
|
var buf: [128]u8 = undefined;
|
|
if (industry.len > buf.len) return null;
|
|
const lc = std.ascii.lowerString(buf[0..industry.len], industry);
|
|
|
|
// Technology - most specific first. Keywords cover both
|
|
// "tech-as-the-product" (semiconductors, software, hardware,
|
|
// computing) and "tech-as-the-platform" (web hosting, cloud
|
|
// computing, internet services, SaaS, data centers). Amazon's
|
|
// Wikidata `industry` triple is "web hosting service" - without
|
|
// explicit coverage, the canonicalizer would miss it and fall
|
|
// through to Consumer Cyclical via "online retail" / "e-commerce"
|
|
// (which are also valid for AMZN, just not the more useful answer
|
|
// for portfolio-level sector breakdown).
|
|
if (containsAny(lc, &.{
|
|
"semiconductor",
|
|
"software",
|
|
"computer hardware",
|
|
"consumer electronics",
|
|
"internet company",
|
|
"internet service",
|
|
"technology industry",
|
|
"computing",
|
|
"cloud",
|
|
"web hosting",
|
|
"saas",
|
|
"software as a service",
|
|
"data center",
|
|
"information technology",
|
|
})) return sector.technology;
|
|
|
|
// Communication Services - telecom, media, internet services
|
|
// (distinct from "internet company" which is more
|
|
// tech-platform-shaped).
|
|
if (containsAny(lc, &.{ "telecom", "broadcast", "media industry", "publishing", "advertising", "social network", "video game" })) return sector.communication_services;
|
|
|
|
// Healthcare.
|
|
if (containsAny(lc, &.{ "pharmaceutical", "biotech", "medical", "healthcare", "health care", "health insurance", "drug" })) return sector.healthcare;
|
|
|
|
// Financial Services.
|
|
if (containsAny(lc, &.{ "bank", "insurance", "asset management", "financial services", "financial industry", "investment", "brokerage", "credit card" })) return sector.financial_services;
|
|
|
|
// Energy.
|
|
if (containsAny(lc, &.{ "oil and gas", "petroleum", "natural gas", "renewable energy", "solar power", "wind power", "energy industry", "coal" })) return sector.energy;
|
|
|
|
// Real Estate / REITs.
|
|
if (containsAny(lc, &.{ "real estate", "reit", "property" })) return sector.real_estate;
|
|
|
|
// Utilities.
|
|
if (containsAny(lc, &.{ "electric utility", "water utility", "gas utility", "utilities", "power generation" })) return sector.utilities;
|
|
|
|
// Basic Materials.
|
|
if (containsAny(lc, &.{ "chemical industry", "mining", "metals", "steel", "basic materials", "forestry", "paper industry" })) return sector.basic_materials;
|
|
|
|
// Consumer Cyclical / Discretionary - apparel, retail,
|
|
// automotive, hospitality.
|
|
if (containsAny(lc, &.{ "retail", "clothing", "apparel", "automotive", "automobile", "hospitality", "restaurant", "luxury", "consumer cyclical", "consumer discretionary", "leisure", "e-commerce" })) return sector.consumer_cyclical;
|
|
|
|
// Consumer Defensive / Staples - food, beverage, tobacco,
|
|
// household products.
|
|
if (containsAny(lc, &.{ "food industry", "beverage", "tobacco", "household products", "consumer staples", "consumer defensive", "grocery", "personal care" })) return sector.consumer_defensive;
|
|
|
|
// Industrials - generic last so "industrial sector" doesn't
|
|
// trump more-specific buckets like Consumer Cyclical's
|
|
// "automotive". (NKE has both "industrial sector" and
|
|
// "clothing industry" listed; we want Consumer Cyclical.)
|
|
if (containsAny(lc, &.{ "aerospace", "defense industry", "construction", "machinery", "transportation", "logistics", "shipping", "airline", "railway", "industrial sector", "industrials" })) return sector.industrials;
|
|
|
|
return null;
|
|
}
|
|
|
|
/// Returns true if `haystack` contains any of `needles` as a
|
|
/// substring (case-sensitive - caller lowercases first if
|
|
/// needed).
|
|
fn containsAny(haystack: []const u8, needles: []const []const u8) bool {
|
|
for (needles) |needle| {
|
|
if (std.mem.indexOf(u8, haystack, needle) != null) return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
/// Parse the SPARQL JSON response into `ClassificationRecord` values.
|
|
/// Multiple bindings for the same ticker (e.g. multiple `instance of`
|
|
/// values) get merged into one record - first-non-null wins.
|
|
fn parse(
|
|
io: std.Io,
|
|
allocator: std.mem.Allocator,
|
|
json_bytes: []const u8,
|
|
expected_symbols: []const []const u8,
|
|
) ![]ClassificationRecord {
|
|
const today = fmt.todayDate(io);
|
|
var as_of_buf: [10]u8 = undefined;
|
|
const as_of = try std.fmt.bufPrint(&as_of_buf, "{f}", .{today});
|
|
|
|
const parsed = std.json.parseFromSlice(std.json.Value, allocator, json_bytes, .{}) catch
|
|
return &.{};
|
|
defer parsed.deinit();
|
|
|
|
const root = switch (parsed.value) {
|
|
.object => |o| o,
|
|
else => return &.{},
|
|
};
|
|
const results = switch (root.get("results") orelse return &.{}) {
|
|
.object => |o| o,
|
|
else => return &.{},
|
|
};
|
|
const bindings = switch (results.get("bindings") orelse return &.{}) {
|
|
.array => |a| a.items,
|
|
else => return &.{},
|
|
};
|
|
|
|
// Map symbol -> record; merge multiple bindings.
|
|
var by_symbol: std.StringHashMap(ClassificationRecord) = .init(allocator);
|
|
defer {
|
|
var it = by_symbol.valueIterator();
|
|
while (it.next()) |r| r.deinit(allocator);
|
|
by_symbol.deinit();
|
|
}
|
|
|
|
for (bindings) |b| {
|
|
const obj = switch (b) {
|
|
.object => |o| o,
|
|
else => continue,
|
|
};
|
|
const ticker = sparqlValue(obj, "ticker") orelse continue;
|
|
|
|
// Verify ticker is one we asked for. Wikidata can return
|
|
// surprising matches (foreign exchanges); skip those.
|
|
if (fmt.findIgnoreCase(expected_symbols, ticker) == null) continue;
|
|
|
|
const existing_or_new = try by_symbol.getOrPut(ticker);
|
|
if (!existing_or_new.found_existing) {
|
|
existing_or_new.key_ptr.* = try allocator.dupe(u8, ticker);
|
|
existing_or_new.value_ptr.* = .{
|
|
.symbol = try allocator.dupe(u8, ticker),
|
|
.as_of = try allocator.dupe(u8, as_of),
|
|
.source = try allocator.dupe(u8, "wikidata"),
|
|
};
|
|
}
|
|
const rec = existing_or_new.value_ptr;
|
|
|
|
if (rec.name == null) {
|
|
if (sparqlValue(obj, "securityLabel")) |label| {
|
|
rec.name = try allocator.dupe(u8, label);
|
|
}
|
|
}
|
|
if (sparqlValue(obj, "industryLabel")) |ind| {
|
|
// Always remember the first industry verbatim (debug
|
|
// / display only).
|
|
if (rec.industry == null) {
|
|
rec.industry = try allocator.dupe(u8, ind);
|
|
}
|
|
// For sector, prefer a canonical mapping. Multiple
|
|
// bindings can fire for the same security (Wikidata
|
|
// returns one row per industry value), so we keep
|
|
// overwriting until we find a canonical match. Once
|
|
// we have a canonical sector, we don't downgrade to
|
|
// a non-canonical one.
|
|
const sector_is_canonical = blk: {
|
|
if (rec.sector) |current| {
|
|
inline for (@typeInfo(sector).@"struct".decls) |d| {
|
|
if (std.mem.eql(u8, current, @field(sector, d.name))) break :blk true;
|
|
}
|
|
}
|
|
break :blk false;
|
|
};
|
|
if (!sector_is_canonical) {
|
|
if (canonicalizeSector(ind)) |canon| {
|
|
if (rec.sector) |old| allocator.free(old);
|
|
rec.sector = try allocator.dupe(u8, canon);
|
|
} else if (rec.sector == null) {
|
|
// No canonical match yet; keep the raw
|
|
// label as a fallback so downstream display
|
|
// has something rather than null.
|
|
rec.sector = try allocator.dupe(u8, ind);
|
|
}
|
|
}
|
|
}
|
|
if (rec.country == null) {
|
|
if (sparqlValue(obj, "countryCode")) |c| {
|
|
rec.country = try allocator.dupe(u8, c);
|
|
}
|
|
}
|
|
if (rec.inception_date == null) {
|
|
if (sparqlValue(obj, "inception")) |d| {
|
|
if (d.len >= 10) {
|
|
rec.inception_date = try allocator.dupe(u8, d[0..10]);
|
|
}
|
|
}
|
|
}
|
|
if (rec.cik == null) {
|
|
if (sparqlValue(obj, "cik")) |c| {
|
|
rec.cik = try allocator.dupe(u8, c);
|
|
}
|
|
}
|
|
if (sparqlValue(obj, "instance")) |inst_iri| {
|
|
// The "instance" value is a Q-ID URI like
|
|
// "http://www.wikidata.org/entity/Q845477". Extract the
|
|
// Q-ID suffix and test against our known sets.
|
|
const last_slash = std.mem.lastIndexOfScalar(u8, inst_iri, '/');
|
|
const q_id = if (last_slash) |i| inst_iri[i + 1 ..] else inst_iri;
|
|
for (etf_q_ids) |target| {
|
|
if (std.mem.eql(u8, q_id, target)) {
|
|
rec.is_etf = true;
|
|
if (rec.asset_class == null) {
|
|
rec.asset_class = try allocator.dupe(u8, "ETF");
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
for (mutual_fund_q_ids) |target| {
|
|
if (std.mem.eql(u8, q_id, target)) {
|
|
if (rec.asset_class == null) {
|
|
rec.asset_class = try allocator.dupe(u8, "Mutual Fund");
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Drain map into owned slice. Caller takes ownership; our defer
|
|
// above calls deinit on values, so clear the map before returning
|
|
// to avoid double-free.
|
|
var out = try allocator.alloc(ClassificationRecord, by_symbol.count());
|
|
var idx: usize = 0;
|
|
var it = by_symbol.iterator();
|
|
while (it.next()) |entry| {
|
|
out[idx] = entry.value_ptr.*;
|
|
idx += 1;
|
|
}
|
|
var key_it = by_symbol.keyIterator();
|
|
while (key_it.next()) |k| allocator.free(k.*);
|
|
by_symbol.clearRetainingCapacity();
|
|
return out;
|
|
}
|
|
|
|
/// Pull the `.value` string out of a SPARQL JSON binding object's
|
|
/// named field. Returns null if absent or non-string.
|
|
fn sparqlValue(obj: std.json.ObjectMap, field: []const u8) ?[]const u8 {
|
|
const slot = obj.get(field) orelse return null;
|
|
const slot_obj = switch (slot) {
|
|
.object => |o| o,
|
|
else => return null,
|
|
};
|
|
const val = slot_obj.get("value") orelse return null;
|
|
return switch (val) {
|
|
.string => |s| s,
|
|
else => null,
|
|
};
|
|
}
|
|
|
|
// ── Tests ────────────────────────────────────────────────────────
|
|
|
|
test "buildQuery includes all symbols and required SELECT vars" {
|
|
const allocator = std.testing.allocator;
|
|
const syms = [_][]const u8{ "AAPL", "VTI" };
|
|
const q = try buildQuery(allocator, &syms);
|
|
defer allocator.free(q);
|
|
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "\"AAPL\"") != null);
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "\"VTI\"") != null);
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "p:P414") != null);
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "pq:P249") != null);
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "wdt:P452") != null);
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "wdt:P17") != null);
|
|
// US-exchange filter must be present - without it, US tickers
|
|
// collide with foreign exchanges (MRK->Merck KGaA, PG->People's
|
|
// Garment, etc.). See `us_exchanges` doc-block.
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "wd:Q13677") != null); // NYSE
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "wd:Q82059") != null); // Nasdaq
|
|
try std.testing.expect(std.mem.indexOf(u8, q, "ps:P414 ?exchange") != null);
|
|
}
|
|
|
|
test "parse: AAPL fixture round-trips name + industry + country" {
|
|
const fixture =
|
|
\\{
|
|
\\ "head": {"vars": ["ticker", "security", "securityLabel", "industryLabel", "countryCode", "inception", "cik", "instance"]},
|
|
\\ "results": {
|
|
\\ "bindings": [
|
|
\\ {
|
|
\\ "ticker": {"type": "literal", "value": "AAPL"},
|
|
\\ "security": {"type": "uri", "value": "http://www.wikidata.org/entity/Q312"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Apple Inc."},
|
|
\\ "industryLabel": {"type": "literal", "value": "consumer electronics"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"},
|
|
\\ "instance": {"type": "uri", "value": "http://www.wikidata.org/entity/Q4830453"}
|
|
\\ }
|
|
\\ ]
|
|
\\ }
|
|
\\}
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const expected = [_][]const u8{"AAPL"};
|
|
const recs = try parse(std.testing.io, allocator, fixture, &expected);
|
|
defer {
|
|
for (recs) |*r| {
|
|
var m = r.*;
|
|
m.deinit(allocator);
|
|
}
|
|
allocator.free(recs);
|
|
}
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), recs.len);
|
|
try std.testing.expectEqualStrings("AAPL", recs[0].symbol);
|
|
try std.testing.expectEqualStrings("Apple Inc.", recs[0].name.?);
|
|
// Industry is preserved verbatim from Wikidata (debug /
|
|
// display only); sector is canonicalized via the keyword
|
|
// taxonomy.
|
|
try std.testing.expectEqualStrings("consumer electronics", recs[0].industry.?);
|
|
try std.testing.expectEqualStrings("Technology", recs[0].sector.?);
|
|
try std.testing.expectEqualStrings("US", recs[0].country.?);
|
|
try std.testing.expect(!recs[0].is_etf);
|
|
}
|
|
|
|
test "parse: ETF fixture sets is_etf=true and asset_class" {
|
|
const fixture =
|
|
\\{
|
|
\\ "head": {"vars": ["ticker", "security", "securityLabel", "instance"]},
|
|
\\ "results": {
|
|
\\ "bindings": [
|
|
\\ {
|
|
\\ "ticker": {"type": "literal", "value": "VTI"},
|
|
\\ "security": {"type": "uri", "value": "http://www.wikidata.org/entity/Q1809462"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Vanguard Total Stock Market ETF"},
|
|
\\ "instance": {"type": "uri", "value": "http://www.wikidata.org/entity/Q845477"}
|
|
\\ }
|
|
\\ ]
|
|
\\ }
|
|
\\}
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const expected = [_][]const u8{"VTI"};
|
|
const recs = try parse(std.testing.io, allocator, fixture, &expected);
|
|
defer {
|
|
for (recs) |*r| {
|
|
var m = r.*;
|
|
m.deinit(allocator);
|
|
}
|
|
allocator.free(recs);
|
|
}
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), recs.len);
|
|
try std.testing.expect(recs[0].is_etf);
|
|
try std.testing.expectEqualStrings("ETF", recs[0].asset_class.?);
|
|
}
|
|
|
|
test "parse: bindings for symbols not requested are dropped" {
|
|
const fixture =
|
|
\\{
|
|
\\ "head": {"vars": ["ticker", "security", "securityLabel"]},
|
|
\\ "results": {
|
|
\\ "bindings": [
|
|
\\ {"ticker": {"type": "literal", "value": "WRONG"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q1"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Wrong Co"}}
|
|
\\ ]
|
|
\\ }
|
|
\\}
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const expected = [_][]const u8{"AAPL"};
|
|
const recs = try parse(std.testing.io, allocator, fixture, &expected);
|
|
defer allocator.free(recs);
|
|
|
|
try std.testing.expectEqual(@as(usize, 0), recs.len);
|
|
}
|
|
|
|
test "parse: multiple industry bindings canonicalize to most-specific sector (NKE shape)" {
|
|
// NKE has three industry values in Wikidata: "industrial
|
|
// sector", "retail", "clothing industry". Two of those
|
|
// canonicalize to Consumer Cyclical and one to Industrials.
|
|
// The parser should pick a canonical sector once it sees
|
|
// one and not downgrade. Order in this fixture matches what
|
|
// SPARQL returned for NKE during enrich testing.
|
|
const fixture =
|
|
\\{
|
|
\\ "head": {"vars": ["ticker", "security", "securityLabel", "industryLabel", "countryCode"]},
|
|
\\ "results": {
|
|
\\ "bindings": [
|
|
\\ {"ticker": {"type": "literal", "value": "NKE"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q14790"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Nike"},
|
|
\\ "industryLabel": {"type": "literal", "value": "industrial sector"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"}},
|
|
\\ {"ticker": {"type": "literal", "value": "NKE"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q14790"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Nike"},
|
|
\\ "industryLabel": {"type": "literal", "value": "retail"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"}},
|
|
\\ {"ticker": {"type": "literal", "value": "NKE"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q14790"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Nike"},
|
|
\\ "industryLabel": {"type": "literal", "value": "clothing industry"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"}}
|
|
\\ ]
|
|
\\ }
|
|
\\}
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const expected = [_][]const u8{"NKE"};
|
|
const recs = try parse(std.testing.io, allocator, fixture, &expected);
|
|
defer {
|
|
for (recs) |*r| {
|
|
var m = r.*;
|
|
m.deinit(allocator);
|
|
}
|
|
allocator.free(recs);
|
|
}
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), recs.len);
|
|
// Sector: first binding ("industrial sector") sets
|
|
// Industrials. Second binding ("retail") canonicalizes to
|
|
// Consumer Cyclical and (per current logic) overrides
|
|
// because "industrial sector" was the LAST keyword fallback.
|
|
// Once a canonical sector is set, subsequent canonical
|
|
// matches don't downgrade (Consumer Cyclical stays put for
|
|
// "clothing industry").
|
|
//
|
|
// The expected outcome is Consumer Cyclical OR Industrials
|
|
// depending on binding order - but the user-visible
|
|
// answer should always be a canonical sector, NOT a raw
|
|
// Wikidata label like "industrial sector". This test
|
|
// asserts the canonical-only invariant.
|
|
const s = recs[0].sector.?;
|
|
try std.testing.expect(
|
|
std.mem.eql(u8, s, sector.industrials) or
|
|
std.mem.eql(u8, s, sector.consumer_cyclical),
|
|
);
|
|
// Industry is the FIRST raw label (preserves the original
|
|
// Wikidata data for debug/display).
|
|
try std.testing.expectEqualStrings("industrial sector", recs[0].industry.?);
|
|
}
|
|
|
|
test "parse: multiple industry bindings - canonical match overrides earlier raw-label fallback" {
|
|
// Order: a non-canonical industry first ("xyz industry") so
|
|
// the parser falls back to raw label, then a canonical
|
|
// match ("software industry"). The canonical match should
|
|
// override the raw label.
|
|
const fixture =
|
|
\\{
|
|
\\ "head": {"vars": ["ticker", "security", "securityLabel", "industryLabel", "countryCode"]},
|
|
\\ "results": {
|
|
\\ "bindings": [
|
|
\\ {"ticker": {"type": "literal", "value": "TEST"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q1"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Test Co"},
|
|
\\ "industryLabel": {"type": "literal", "value": "xyz industry"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"}},
|
|
\\ {"ticker": {"type": "literal", "value": "TEST"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q1"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Test Co"},
|
|
\\ "industryLabel": {"type": "literal", "value": "software industry"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"}}
|
|
\\ ]
|
|
\\ }
|
|
\\}
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const expected = [_][]const u8{"TEST"};
|
|
const recs = try parse(std.testing.io, allocator, fixture, &expected);
|
|
defer {
|
|
for (recs) |*r| {
|
|
var m = r.*;
|
|
m.deinit(allocator);
|
|
}
|
|
allocator.free(recs);
|
|
}
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), recs.len);
|
|
try std.testing.expectEqualStrings(sector.technology, recs[0].sector.?);
|
|
// First raw label preserved as `industry`.
|
|
try std.testing.expectEqualStrings("xyz industry", recs[0].industry.?);
|
|
}
|
|
|
|
test "parse: canonical match never downgrades to non-canonical" {
|
|
// First binding: "software industry" -> Technology
|
|
// (canonical). Second binding: "xyz industry" -> no canonical
|
|
// match. Sector should STAY Technology, not downgrade to
|
|
// "xyz industry".
|
|
const fixture =
|
|
\\{
|
|
\\ "head": {"vars": ["ticker", "security", "securityLabel", "industryLabel", "countryCode"]},
|
|
\\ "results": {
|
|
\\ "bindings": [
|
|
\\ {"ticker": {"type": "literal", "value": "TEST"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q1"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Test Co"},
|
|
\\ "industryLabel": {"type": "literal", "value": "software industry"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"}},
|
|
\\ {"ticker": {"type": "literal", "value": "TEST"},
|
|
\\ "security": {"type": "uri", "value": "http://example/Q1"},
|
|
\\ "securityLabel": {"type": "literal", "value": "Test Co"},
|
|
\\ "industryLabel": {"type": "literal", "value": "xyz industry"},
|
|
\\ "countryCode": {"type": "literal", "value": "US"}}
|
|
\\ ]
|
|
\\ }
|
|
\\}
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const expected = [_][]const u8{"TEST"};
|
|
const recs = try parse(std.testing.io, allocator, fixture, &expected);
|
|
defer {
|
|
for (recs) |*r| {
|
|
var m = r.*;
|
|
m.deinit(allocator);
|
|
}
|
|
allocator.free(recs);
|
|
}
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), recs.len);
|
|
try std.testing.expectEqualStrings(sector.technology, recs[0].sector.?);
|
|
}
|
|
|
|
// ── canonicalizeSector ───────────────────────────────────────
|
|
|
|
test "canonicalizeSector: technology keywords map to Technology" {
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("semiconductor industry").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("software development").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("software industry").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("Technology Industry").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("computing").?);
|
|
}
|
|
|
|
test "canonicalizeSector: tech-platform keywords (cloud / web hosting / SaaS) map to Technology" {
|
|
// Regression check for AMZN: Wikidata returns
|
|
// "web hosting service" as Amazon's first industry triple.
|
|
// Pre-fix, that fell through to Consumer Cyclical via
|
|
// "online retail" / "e-commerce". With the expanded
|
|
// keyword list, web hosting -> Technology directly.
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("web hosting service").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("cloud computing").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("cloud services").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("internet service provider").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("internet services").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("SaaS").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("software as a service").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("data center").?);
|
|
try std.testing.expectEqualStrings(sector.technology, canonicalizeSector("information technology").?);
|
|
}
|
|
|
|
test "canonicalizeSector: e-commerce still maps to Consumer Cyclical (priority order matters)" {
|
|
// Regression check that the Technology keyword expansion
|
|
// didn't accidentally swallow Consumer Cyclical hits.
|
|
// E-commerce / online retail / retail still hit the Consumer
|
|
// Cyclical branch because none of them contain Technology
|
|
// keywords.
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("e-commerce").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("online retail").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("retail").?);
|
|
}
|
|
|
|
test "canonicalizeSector: communication services" {
|
|
try std.testing.expectEqualStrings(sector.communication_services, canonicalizeSector("telecom").?);
|
|
try std.testing.expectEqualStrings(sector.communication_services, canonicalizeSector("media industry").?);
|
|
try std.testing.expectEqualStrings(sector.communication_services, canonicalizeSector("video game industry").?);
|
|
try std.testing.expectEqualStrings(sector.communication_services, canonicalizeSector("publishing").?);
|
|
}
|
|
|
|
test "canonicalizeSector: healthcare" {
|
|
try std.testing.expectEqualStrings(sector.healthcare, canonicalizeSector("pharmaceutical industry").?);
|
|
try std.testing.expectEqualStrings(sector.healthcare, canonicalizeSector("biotech").?);
|
|
try std.testing.expectEqualStrings(sector.healthcare, canonicalizeSector("medical device").?);
|
|
try std.testing.expectEqualStrings(sector.healthcare, canonicalizeSector("healthcare industry").?);
|
|
}
|
|
|
|
test "canonicalizeSector: financial services" {
|
|
try std.testing.expectEqualStrings(sector.financial_services, canonicalizeSector("bank").?);
|
|
try std.testing.expectEqualStrings(sector.financial_services, canonicalizeSector("insurance company").?);
|
|
try std.testing.expectEqualStrings(sector.financial_services, canonicalizeSector("asset management").?);
|
|
try std.testing.expectEqualStrings(sector.financial_services, canonicalizeSector("financial services").?);
|
|
}
|
|
|
|
test "canonicalizeSector: energy" {
|
|
try std.testing.expectEqualStrings(sector.energy, canonicalizeSector("oil and gas industry").?);
|
|
try std.testing.expectEqualStrings(sector.energy, canonicalizeSector("petroleum industry").?);
|
|
try std.testing.expectEqualStrings(sector.energy, canonicalizeSector("renewable energy").?);
|
|
try std.testing.expectEqualStrings(sector.energy, canonicalizeSector("solar power").?);
|
|
}
|
|
|
|
test "canonicalizeSector: real estate" {
|
|
try std.testing.expectEqualStrings(sector.real_estate, canonicalizeSector("real estate").?);
|
|
try std.testing.expectEqualStrings(sector.real_estate, canonicalizeSector("REIT").?);
|
|
try std.testing.expectEqualStrings(sector.real_estate, canonicalizeSector("commercial real estate").?);
|
|
}
|
|
|
|
test "canonicalizeSector: utilities" {
|
|
try std.testing.expectEqualStrings(sector.utilities, canonicalizeSector("electric utility").?);
|
|
try std.testing.expectEqualStrings(sector.utilities, canonicalizeSector("water utility").?);
|
|
try std.testing.expectEqualStrings(sector.utilities, canonicalizeSector("power generation").?);
|
|
}
|
|
|
|
test "canonicalizeSector: basic materials" {
|
|
try std.testing.expectEqualStrings(sector.basic_materials, canonicalizeSector("chemical industry").?);
|
|
try std.testing.expectEqualStrings(sector.basic_materials, canonicalizeSector("mining").?);
|
|
try std.testing.expectEqualStrings(sector.basic_materials, canonicalizeSector("steel industry").?);
|
|
}
|
|
|
|
test "canonicalizeSector: consumer cyclical (NKE / AMZN keywords)" {
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("retail").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("clothing industry").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("automotive industry").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("e-commerce").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("hospitality").?);
|
|
}
|
|
|
|
test "canonicalizeSector: consumer defensive" {
|
|
try std.testing.expectEqualStrings(sector.consumer_defensive, canonicalizeSector("food industry").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_defensive, canonicalizeSector("beverage industry").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_defensive, canonicalizeSector("tobacco").?);
|
|
try std.testing.expectEqualStrings(sector.consumer_defensive, canonicalizeSector("household products").?);
|
|
}
|
|
|
|
test "canonicalizeSector: industrials (last-fallback for industrial sector)" {
|
|
try std.testing.expectEqualStrings(sector.industrials, canonicalizeSector("aerospace").?);
|
|
try std.testing.expectEqualStrings(sector.industrials, canonicalizeSector("transportation").?);
|
|
try std.testing.expectEqualStrings(sector.industrials, canonicalizeSector("airline").?);
|
|
try std.testing.expectEqualStrings(sector.industrials, canonicalizeSector("industrial sector").?);
|
|
}
|
|
|
|
test "canonicalizeSector: NKE 'industrial sector' is overridden by 'clothing industry' in parser" {
|
|
// The parser walks each binding and calls canonicalizeSector
|
|
// per industry label. NKE's bindings include "industrial
|
|
// sector" (Industrials) AND "clothing industry"
|
|
// (Consumer Cyclical). Whichever is processed last wins
|
|
// as long as the previous one wasn't canonical-and-better.
|
|
// Here we just verify the keywords map as expected - the
|
|
// parser's first-canonical-wins logic is verified separately.
|
|
try std.testing.expectEqualStrings(sector.consumer_cyclical, canonicalizeSector("clothing industry").?);
|
|
try std.testing.expectEqualStrings(sector.industrials, canonicalizeSector("industrial sector").?);
|
|
}
|
|
|
|
test "canonicalizeSector: returns null for unknown / non-industry strings" {
|
|
try std.testing.expect(canonicalizeSector("International Standard Industrial Classification") == null);
|
|
try std.testing.expect(canonicalizeSector("Unknown") == null);
|
|
try std.testing.expect(canonicalizeSector("") == null);
|
|
try std.testing.expect(canonicalizeSector("xyzzy") == null);
|
|
}
|
|
|
|
test "canonicalizeSector: input longer than 128 bytes returns null (no false matches)" {
|
|
// The internal lowercasing buffer is 128 bytes; oversized
|
|
// industry labels return null rather than match against a
|
|
// truncated buffer. Real Wikidata labels are always well
|
|
// under this; the bound is defensive.
|
|
var huge: [200]u8 = undefined;
|
|
@memset(&huge, 'a');
|
|
try std.testing.expect(canonicalizeSector(&huge) == null);
|
|
}
|