//! Reads a C struct definition into a `struct_layout.Definition` (FR-3.1). //! //! The point is pasting a struct from real code, so the input is C rather than a //! language of our own. It is a defined subset, and anything outside it is an error //! that says where, never a guess: //! //! - `struct [tag] { ... }` and `typedef struct [tag] { ... } Name;`, optionally with //! a trailing `;`. //! - Members of any scalar type, in C spellings (`unsigned long long`, `uint32_t`, //! `size_t`, `wchar_t`, `__int128`, `enum color`) or short ones (`u32`, `f64`, //! `ptr`, `usize`), and pointers to anything (`struct node *`, `void (*)(int)`). //! - Several declarators per declaration (`int a, *b, c[4];`), arrays of any rank, and //! a flexible array member (`char data[];`). //! - `alignas(N)`, `_Alignas(N)`, `__attribute__((aligned(N)))` and //! `__attribute__((packed))`, on members and on the struct, and //! `#pragma pack(N | push | push, N | pop | )` before it. //! - `//` and `/* */` comments. //! //! What is refused, each with its own error: nested structs and unions by value, //! bitfields, any other preprocessor directive, names it cannot resolve (a typedef or //! a macro from elsewhere), and the types whose layout this module does not model yet //! (`long double`, `_Complex`, `_Atomic`). //! //! It has its own lexer. The expression tokenizer has no braces or brackets, and its //! number rules contradict C's: `1,000` is a grouped number there, `010` is ten, and a //! hex literal continues across a space. const std = @import("std"); const Allocator = std.mem.Allocator; const struct_layout = @import("struct_layout.zig"); const Definition = struct_layout.Definition; const Member = struct_layout.Member; const Primitive = struct_layout.Primitive; const Span = struct_layout.Span; /// The file is the parser, as `Parser.zig` is for expressions. const CStructParser = @This(); /// What parsing can fail with. `UnexpectedToken`, `UnexpectedEnd` and `InvalidNumber` /// are the expression parser's names, so they share its wording. pub const Error = error{ UnexpectedToken, UnexpectedEnd, InvalidNumber, /// `/*` with no `*/`. UnterminatedComment, /// A name that is not a type this parser knows: usually a typedef or a macro /// defined somewhere the input does not include. UnknownType, /// A C type whose layout is not modelled: `long double`, `_Complex`, `_Atomic`, /// or a top-level `union`. UnsupportedType, /// A `void` member. IncompleteType, /// An attribute that could change the layout in a way this parser does not /// model (`mode`, `vector_size`, a bare `aligned`). UnsupportedAttribute, BitfieldNotSupported, /// A struct, union or enum defined inside the struct, or a struct or union /// member by value. NestedTypeNotSupported, /// A preprocessor directive other than `#pragma pack`. PreprocessorNotSupported, /// More than `max_source_bytes` of input. DefinitionTooLong, /// An array dimension of zero. InvalidArrayLength, /// An alignment or pack value that is not a power of two, or a pack value C /// compilers refuse. InvalidAlignment, TooManyMembers, OutOfMemory, }; /// Ceiling on input, so untrusted input cannot ask for unbounded work (NFR-8). A /// struct of a thousand members fits with room to spare. pub const max_source_bytes: usize = 64 * 1024; /// Where a parse failed: the byte range of the offending text in the source. pub const Diagnostic = struct { span: Span = .{ .start = 0, .len = 0 }, }; /// A position a person can find: both 1-based, columns counted in bytes. pub const Location = struct { line: usize, column: usize, }; /// The line and column of a byte offset in `source`. pub fn locate(source: []const u8, offset: usize) Location { const before = source[0..@min(offset, source.len)]; const line_start = if (std.mem.lastIndexOfScalar(u8, before, '\n')) |nl| nl + 1 else 0; return .{ .line = std.mem.count(u8, before, "\n") + 1, .column = before.len - line_start + 1, }; } /// A parsed definition and the memory it lives in. Self-contained: every name is /// copied into the arena, so the source can go away. pub const Parsed = struct { arena: std.heap.ArenaAllocator, definition: Definition, pub fn deinit(self: *Parsed) void { self.arena.deinit(); self.* = undefined; } }; /// Parse one struct definition. /// /// On failure, `diag` (if given) holds the span of the text the error is about. pub fn parse(allocator: Allocator, source: []const u8, diag: ?*Diagnostic) Error!Parsed { if (source.len > max_source_bytes) { if (diag) |d| d.span = .{ .start = max_source_bytes, .len = source.len - max_source_bytes }; return error.DefinitionTooLong; } var arena: std.heap.ArenaAllocator = .init(allocator); errdefer arena.deinit(); var parser: CStructParser = .{ .source = source, .lexer = .{ .source = source }, // SAFETY: `advance` below sets it before anything reads it. .current = undefined, .arena = arena.allocator(), .diag = diag, .pack_stack = .empty, }; try parser.advance(); const definition = try parser.parseInput(); return .{ .arena = arena, .definition = definition }; } // -- Lexer -- const TokenKind = enum { identifier, number, l_brace, r_brace, l_bracket, r_bracket, l_paren, r_paren, semicolon, comma, star, colon, /// A whole preprocessor line, `#` to the end of the line. directive, eof, invalid, }; const Token = struct { kind: TokenKind, start: usize, len: usize, fn span(self: Token) Span { return .{ .start = self.start, .len = self.len }; } }; /// A position in a string. Copying it is a lookahead. const Lexer = struct { source: []const u8, pos: usize = 0, /// The next token, or the span of a comment that never ends. fn next(self: *Lexer) union(enum) { token: Token, unterminated: Span } { const src = self.source; while (self.pos < src.len) { const c = src[self.pos]; if (c == ' ' or c == '\t' or c == '\r' or c == '\n' or c == '\x0b' or c == '\x0c') { self.pos += 1; } else if (std.mem.startsWith(u8, src[self.pos..], "//")) { self.pos = std.mem.indexOfScalarPos(u8, src, self.pos, '\n') orelse src.len; } else if (std.mem.startsWith(u8, src[self.pos..], "/*")) { const end = std.mem.indexOfPos(u8, src, self.pos + 2, "*/") orelse { const start = self.pos; self.pos = src.len; return .{ .unterminated = .{ .start = start, .len = 2 } }; }; self.pos = end + 2; } else break; } const start = self.pos; if (start >= src.len) return .{ .token = .{ .kind = .eof, .start = src.len, .len = 0 } }; const c = src[start]; const kind: TokenKind = switch (c) { 'a'...'z', 'A'...'Z', '_' => blk: { while (self.pos < src.len and isIdentChar(src[self.pos])) self.pos += 1; break :blk .identifier; }, '0'...'9' => blk: { // The whole pp-number, suffix and all; the parser decides whether it // is a valid literal. while (self.pos < src.len and isIdentChar(src[self.pos])) self.pos += 1; break :blk .number; }, '#' => blk: { self.pos = std.mem.indexOfScalarPos(u8, src, self.pos, '\n') orelse src.len; break :blk .directive; }, else => blk: { self.pos += 1; break :blk switch (c) { '{' => .l_brace, '}' => .r_brace, '[' => .l_bracket, ']' => .r_bracket, '(' => .l_paren, ')' => .r_paren, ';' => .semicolon, ',' => .comma, '*' => .star, ':' => .colon, else => .invalid, }; }, }; return .{ .token = .{ .kind = kind, .start = start, .len = self.pos - start } }; } }; fn isIdentChar(c: u8) bool { return std.ascii.isAlphanumeric(c) or c == '_'; } // -- Parser state -- source: []const u8, lexer: Lexer, current: Token, arena: Allocator, diag: ?*Diagnostic, /// `#pragma pack` in effect, and what `push` saved. pack: ?u32 = null, pack_stack: std.ArrayList(?u32), fn text(self: *const CStructParser, tok: Token) []const u8 { return self.source[tok.start..][0..tok.len]; } /// Record where an error is and return it. fn fail(self: *CStructParser, err: Error, span: Span) Error { if (self.diag) |d| d.span = span; return err; } /// The error for the current token not being what the grammar needs here. fn unexpected(self: *CStructParser) Error { const err: Error = if (self.current.kind == .eof) error.UnexpectedEnd else error.UnexpectedToken; return self.fail(err, self.current.span()); } fn advance(self: *CStructParser) Error!void { switch (self.lexer.next()) { .token => |tok| self.current = tok, .unterminated => |span| return self.fail(error.UnterminatedComment, span), } } fn expect(self: *CStructParser, kind: TokenKind) Error!Token { if (self.current.kind != kind) return self.unexpected(); const tok = self.current; try self.advance(); return tok; } fn isWord(self: *const CStructParser, word: []const u8) bool { return self.current.kind == .identifier and std.mem.eql(u8, self.text(self.current), word); } fn eatWord(self: *CStructParser, word: []const u8) Error!bool { if (!self.isWord(word)) return false; try self.advance(); return true; } // -- Grammar -- /// Attributes that apply to whatever they are attached to. const Attributes = struct { aligned: ?u32 = null, @"packed": bool = false, fn merge(self: *Attributes, other: Attributes) void { if (other.aligned) |n| self.aligned = @max(self.aligned orelse 0, n); if (other.@"packed") self.@"packed" = true; } }; /// The whole input: directives, one struct, directives. fn parseInput(self: *CStructParser) Error!Definition { try self.parseDirectives(); const definition = try self.parseTop(); try self.parseDirectives(); if (self.current.kind != .eof) return self.unexpected(); return definition; } fn parseDirectives(self: *CStructParser) Error!void { while (self.current.kind == .directive) { try self.parseDirective(self.current); try self.advance(); } } /// `#pragma pack(...)`, the one directive that matters to a layout. Everything else a /// header might hold (`#include`, `#define`, `#if`) is refused rather than skipped, /// because skipping `#if 0` would count the members inside it. fn parseDirective(self: *CStructParser, line: Token) Error!void { var sub: CStructParser = .{ .source = self.source, .lexer = .{ .source = self.source[0 .. line.start + line.len], .pos = line.start + 1 }, // SAFETY: the first `advance` below sets it before anything reads it. .current = undefined, .arena = self.arena, .diag = self.diag, .pack_stack = self.pack_stack, }; defer self.pack_stack = sub.pack_stack; sub.pack = self.pack; defer self.pack = sub.pack; try sub.advance(); if (!try sub.eatWord("pragma") or !try sub.eatWord("pack")) { return self.fail(error.PreprocessorNotSupported, line.span()); } _ = try sub.expect(.l_paren); if (try sub.eatWord("push")) { try sub.pack_stack.append(sub.arena, sub.pack); if (sub.current.kind == .comma) { try sub.advance(); sub.pack = try sub.parsePackValue(); } } else if (sub.isWord("pop")) { try sub.advance(); sub.pack = sub.pack_stack.pop() orelse null; } else if (sub.current.kind == .number) { sub.pack = try sub.parsePackValue(); } else if (sub.current.kind != .r_paren) { return sub.unexpected(); } else { // `#pragma pack()` restores the default. sub.pack = null; } _ = try sub.expect(.r_paren); if (sub.current.kind != .eof) return sub.unexpected(); } /// The values clang and GCC accept for `#pragma pack`. fn parsePackValue(self: *CStructParser) Error!u32 { const tok = self.current; const n = try self.parseInteger(); return switch (n) { 1, 2, 4, 8, 16 => @intCast(n), else => return self.fail(error.InvalidAlignment, tok.span()), }; } /// `struct ...;` or `typedef struct ... Name;`. fn parseTop(self: *CStructParser) Error!Definition { const is_typedef = try self.eatWord("typedef"); if (self.isWord("union")) return self.fail(error.UnsupportedType, self.current.span()); if (!self.isWord("struct")) return self.unexpected(); try self.advance(); var attrs: Attributes = .{}; attrs.merge(try self.parseAttributes()); var tag: ?[]const u8 = null; if (self.current.kind == .identifier and !isAttributeWord(self.text(self.current))) { tag = try self.arena.dupe(u8, self.text(self.current)); try self.advance(); } attrs.merge(try self.parseAttributes()); // The pack in effect where the struct is defined is the one it uses. const pack = self.pack; const members = try self.parseBody(); attrs.merge(try self.parseAttributes()); var name = tag; var name_kind: Definition.NameKind = .tag; if (is_typedef) { const name_tok = try self.expect(.identifier); name = try self.arena.dupe(u8, self.text(name_tok)); name_kind = .typedef; attrs.merge(try self.parseAttributes()); _ = try self.expect(.semicolon); } else if (self.current.kind == .semicolon) { try self.advance(); } return .{ .name = name, .name_kind = name_kind, .members = members, .@"packed" = attrs.@"packed", .pack = pack, .aligned = attrs.aligned, }; } /// `{ declaration* }`. fn parseBody(self: *CStructParser) Error![]const Member { _ = try self.expect(.l_brace); var members: std.ArrayList(Member) = .empty; while (self.current.kind != .r_brace) { switch (self.current.kind) { .eof => return self.unexpected(), .directive => return self.fail(error.PreprocessorNotSupported, self.current.span()), // A stray `;` is an empty declaration, which C compilers accept with a // warning at most. .semicolon => try self.advance(), else => try self.parseDeclaration(&members), } } try self.advance(); return members.items; } /// The type part of a declaration, shared by every declarator in it. const Specifiers = struct { /// What the specifiers name, before any pointer in a declarator. base: Base, /// The words as written, joined by single spaces, for display. words: []const u8, attrs: Attributes, span: Span, const Base = union(enum) { primitive: Primitive, /// `void`, which only a pointer may be. void, /// `struct X` or `union X`, which only a pointer may be here. aggregate, }; }; /// `specifiers declarator (, declarator)* ;` fn parseDeclaration(self: *CStructParser, members: *std.ArrayList(Member)) Error!void { const specs = try self.parseSpecifiers(); while (true) { if (members.items.len >= struct_layout.max_members) { return self.fail(error.TooManyMembers, self.current.span()); } try members.append(self.arena, try self.parseDeclarator(specs)); if (self.current.kind == .colon) return self.fail(error.BitfieldNotSupported, self.current.span()); if (self.current.kind != .comma) break; try self.advance(); } _ = try self.expect(.semicolon); } /// Which word a type keyword is, for counting. const Keyword = enum { signed, unsigned, char, short, int, long, float, double, void, bool, int128 }; const keywords = std.StaticStringMap(Keyword).initComptime(.{ .{ "signed", .signed }, .{ "__signed__", .signed }, .{ "unsigned", .unsigned }, .{ "char", .char }, .{ "short", .short }, .{ "int", .int }, .{ "long", .long }, .{ "float", .float }, .{ "double", .double }, .{ "void", .void }, .{ "_Bool", .bool }, .{ "bool", .bool }, .{ "__int128", .int128 }, }); /// Type names that stand alone. The C ones are the standard library's typedefs whose /// layout every profile agrees on by kind; the short ones are this calculator's. const named_types = std.StaticStringMap(Primitive).initComptime(.{ .{ "int8_t", .char }, .{ "uint8_t", .char }, .{ "int16_t", .short }, .{ "uint16_t", .short }, .{ "int32_t", .int }, .{ "uint32_t", .int }, .{ "int64_t", .long_long }, .{ "uint64_t", .long_long }, .{ "__int128_t", .int128 }, .{ "__uint128_t", .int128 }, .{ "intptr_t", .pointer }, .{ "uintptr_t", .pointer }, .{ "size_t", .pointer }, .{ "ssize_t", .pointer }, .{ "ptrdiff_t", .pointer }, .{ "wchar_t", .wchar }, .{ "char16_t", .short }, .{ "char32_t", .int }, .{ "u8", .char }, .{ "i8", .char }, .{ "u16", .short }, .{ "i16", .short }, .{ "u32", .int }, .{ "i32", .int }, .{ "u64", .long_long }, .{ "i64", .long_long }, .{ "u128", .int128 }, .{ "i128", .int128 }, .{ "f32", .float }, .{ "f64", .double }, .{ "usize", .pointer }, .{ "isize", .pointer }, .{ "ptr", .pointer }, }); /// Words that change nothing about a layout. const qualifiers = std.StaticStringMap(void).initComptime(.{ .{"const"}, .{"volatile"}, .{"restrict"}, .{"__restrict"}, .{"__restrict__"}, .{"__const"}, .{"__volatile__"}, .{"__extension__"}, }); /// C types that exist and are not modelled. Recognised so the error can say so, /// rather than calling `long double`'s second word an unknown type. const unsupported_types = std.StaticStringMap(void).initComptime(.{ .{"_Complex"}, .{"_Imaginary"}, .{"_Atomic"}, .{"_Float16"}, .{"__fp16"}, .{"__bf16"}, .{"_Float128"}, .{"__float128"}, .{"_Decimal32"}, .{"_Decimal64"}, .{"_Decimal128"}, .{"_BitInt"}, }); fn isAttributeWord(word: []const u8) bool { return std.mem.eql(u8, word, "__attribute__") or std.mem.eql(u8, word, "__attribute") or std.mem.eql(u8, word, "alignas") or std.mem.eql(u8, word, "_Alignas"); } fn parseSpecifiers(self: *CStructParser) Error!Specifiers { const first = self.current; var words: std.ArrayList(u8) = .empty; var attrs: Attributes = .{}; var counts: std.EnumArray(Keyword, u8) = .initFill(0); var named: ?Primitive = null; var base: ?Specifiers.Base = null; var last_end = first.start; while (self.current.kind == .identifier) { const tok = self.current; const word = self.text(tok); if (isAttributeWord(word)) { attrs.merge(try self.parseAttributes()); continue; } if (std.mem.eql(u8, word, "struct") or std.mem.eql(u8, word, "union") or std.mem.eql(u8, word, "enum")) { if (base != null or named != null or hasKeywords(counts)) return self.unexpected(); try self.advance(); const tag = self.current; if (tag.kind == .l_brace) return self.fail(error.NestedTypeNotSupported, tok.span()); _ = try self.expect(.identifier); if (self.current.kind == .l_brace) return self.fail(error.NestedTypeNotSupported, tok.span()); base = if (word[0] == 'e') .{ .primitive = .enumeration } else .aggregate; try appendWord(self.arena, &words, word); try appendWord(self.arena, &words, self.text(tag)); last_end = tag.start + tag.len; continue; } if (qualifiers.has(word)) { try appendWord(self.arena, &words, word); } else if (unsupported_types.has(word)) { return self.fail(error.UnsupportedType, tok.span()); } else if (keywords.get(word)) |kw| { if (base != null or named != null) return self.unexpected(); counts.set(kw, counts.get(kw) + 1); try appendWord(self.arena, &words, word); } else if (base != null or named != null or hasKeywords(counts)) { // A type has been named, so this is the declarator's name. break; } else if (named_types.get(word)) |p| { named = p; try appendWord(self.arena, &words, word); } else { return self.fail(error.UnknownType, tok.span()); } last_end = tok.start + tok.len; try self.advance(); } const span: Span = .{ .start = first.start, .len = last_end - first.start }; const resolved: Specifiers.Base = if (base) |b| b else if (named) |p| .{ .primitive = p } else if (hasKeywords(counts)) try self.resolveKeywords(counts, span) else return self.unexpected(); return .{ .base = resolved, .words = words.items, .attrs = attrs, .span = span }; } fn hasKeywords(counts: std.EnumArray(Keyword, u8)) bool { for (counts.values) |n| if (n > 0) return true; return false; } fn appendWord(arena: Allocator, words: *std.ArrayList(u8), word: []const u8) Error!void { if (words.items.len > 0) try words.append(arena, ' '); try words.appendSlice(arena, word); } /// What a combination of type keywords means, C's way: `unsigned` alone is /// `unsigned int`, `long long int` is `long long`, and `long double` exists but is /// not modelled. fn resolveKeywords(self: *CStructParser, counts: std.EnumArray(Keyword, u8), span: Span) Error!Specifiers.Base { // `long double` is a real type, so it gets its own error rather than being // called an invalid combination. if (counts.get(.double) == 1 and counts.get(.long) == 1 and counts.get(.signed) + counts.get(.unsigned) + counts.get(.short) + counts.get(.int) == 0) { return self.fail(error.UnsupportedType, span); } return combine(counts) orelse self.fail(error.UnexpectedToken, span); } /// The type a set of keywords names, or null for a combination C rejects. fn combine(counts: std.EnumArray(Keyword, u8)) ?Specifiers.Base { // No keyword may repeat except `long`, once. for (std.enums.values(Keyword)) |kw| { const limit: u8 = if (kw == .long) 2 else 1; if (counts.get(kw) > limit) return null; } const signs = counts.get(.signed) + counts.get(.unsigned); if (signs > 1) return null; const short = counts.get(.short); const long = counts.get(.long); const int = counts.get(.int); const others = counts.get(.char) + counts.get(.float) + counts.get(.double) + counts.get(.void) + counts.get(.bool) + counts.get(.int128); if (others > 1) return null; // These take no sign and no size. const bare = signs + short + long + int == 0; if (counts.get(.void) == 1) return if (bare) .void else null; if (counts.get(.bool) == 1) return if (bare) .{ .primitive = .bool } else null; if (counts.get(.float) == 1) return if (bare) .{ .primitive = .float } else null; if (counts.get(.double) == 1) return if (bare) .{ .primitive = .double } else null; // These take a sign and no size. if (counts.get(.char) == 1) return if (short + long + int == 0) .{ .primitive = .char } else null; if (counts.get(.int128) == 1) return if (short + long + int == 0) .{ .primitive = .int128 } else null; if (short == 1) return if (long == 0) .{ .primitive = .short } else null; return switch (long) { 0 => .{ .primitive = .int }, 1 => .{ .primitive = .long }, else => .{ .primitive = .long_long }, }; } /// `__attribute__((...))`, `alignas(N)` and `_Alignas(N)`, as many as are written. fn parseAttributes(self: *CStructParser) Error!Attributes { var attrs: Attributes = .{}; while (self.current.kind == .identifier) { const word = self.text(self.current); if (std.mem.eql(u8, word, "alignas") or std.mem.eql(u8, word, "_Alignas")) { try self.advance(); _ = try self.expect(.l_paren); const tok = self.current; if (tok.kind != .number) return self.unexpected(); const n = try self.parseInteger(); // C11: an alignment of zero has no effect. if (n != 0) attrs.merge(.{ .aligned = try self.alignmentValue(n, tok) }); _ = try self.expect(.r_paren); } else if (std.mem.eql(u8, word, "__attribute__") or std.mem.eql(u8, word, "__attribute")) { try self.advance(); _ = try self.expect(.l_paren); _ = try self.expect(.l_paren); while (self.current.kind != .r_paren) { attrs.merge(try self.parseAttribute()); if (self.current.kind != .comma) break; try self.advance(); } _ = try self.expect(.r_paren); _ = try self.expect(.r_paren); } else break; } return attrs; } /// One entry of an `__attribute__` list. fn parseAttribute(self: *CStructParser) Error!Attributes { const tok = try self.expect(.identifier); const name = trimUnderscores(self.text(tok)); if (std.mem.eql(u8, name, "packed")) return .{ .@"packed" = true }; if (std.mem.eql(u8, name, "aligned")) { // A bare `aligned` means "the largest alignment the target uses", which // differs by target in a way the profiles do not record. if (self.current.kind != .l_paren) return self.fail(error.UnsupportedAttribute, tok.span()); try self.advance(); const value = self.current; if (value.kind != .number) return self.unexpected(); const n = try self.parseInteger(); _ = try self.expect(.r_paren); return .{ .aligned = try self.alignmentValue(n, value) }; } // Attributes that mean nothing to a layout are accepted, with any arguments. for ([_][]const u8{ "unused", "used", "deprecated", "may_alias", "nonstring" }) |harmless| { if (std.mem.eql(u8, name, harmless)) { if (self.current.kind == .l_paren) try self.skipBalanced(); return .{}; } } // Anything else (`mode`, `vector_size`, `designated_init`, ...) might change the // type or the layout, and a guess would be a wrong answer. return self.fail(error.UnsupportedAttribute, tok.span()); } /// `__packed__` is `packed`: GCC accepts every attribute with or without the /// surrounding double underscores. fn trimUnderscores(name: []const u8) []const u8 { if (name.len > 4 and std.mem.startsWith(u8, name, "__") and std.mem.endsWith(u8, name, "__")) { return name[2 .. name.len - 2]; } return name; } fn alignmentValue(self: *CStructParser, n: u64, tok: Token) Error!u32 { if (n == 0 or n > 1 << 28 or !std.math.isPowerOfTwo(n)) return self.fail(error.InvalidAlignment, tok.span()); return @intCast(n); } /// Skip a parenthesised group, nested groups included. fn skipBalanced(self: *CStructParser) Error!void { var depth: usize = 0; while (true) { switch (self.current.kind) { .l_paren => depth += 1, .r_paren => depth -= 1, .eof => return self.unexpected(), else => {}, } try self.advance(); if (depth == 0) return; } } /// One declarator: `name`, `*name`, `name[2][3]`, `name[]`, `(*name)(...)`, plus any /// attributes after it. fn parseDeclarator(self: *CStructParser, specs: Specifiers) Error!Member { var attrs = specs.attrs; attrs.merge(try self.parseAttributes()); var stars: usize = 0; while (self.current.kind == .star) { stars += 1; try self.advance(); // `char *const p`: a qualified pointer is still a pointer. while (self.current.kind == .identifier and qualifiers.has(self.text(self.current))) try self.advance(); } // SAFETY: both branches below assign it before it is read. var name_tok: Token = undefined; var dims: []const u64 = &.{}; var flexible = false; var function_pointer = false; var array_pointer = false; if (self.current.kind == .l_paren) { // `(*name)(params)` or `(*name)[N]`: a pointer whatever the outside says, // and possibly an array of them, `(*handlers[4])(int)`. try self.advance(); if (self.current.kind != .star) return self.unexpected(); while (self.current.kind == .star) { try self.advance(); while (self.current.kind == .identifier and qualifiers.has(self.text(self.current))) try self.advance(); } name_tok = try self.expect(.identifier); const inner = try self.parseDimensions(); dims = inner.dims; flexible = inner.flexible; _ = try self.expect(.r_paren); switch (self.current.kind) { .l_paren => { try self.skipBalanced(); function_pointer = true; }, .l_bracket => { // The pointee's dimensions are not part of this member's size. _ = try self.parseDimensions(); array_pointer = true; }, else => {}, } stars += 1; } else { name_tok = try self.expect(.identifier); if (self.current.kind == .l_paren) { // `int f(void);` declares a function, which a struct cannot hold. return self.unexpected(); } const outer = try self.parseDimensions(); dims = outer.dims; flexible = outer.flexible; } attrs.merge(try self.parseAttributes()); const primitive: Primitive = if (stars > 0) .pointer else switch (specs.base) { .primitive => |p| p, .void => return self.fail(error.IncompleteType, specs.span), .aggregate => return self.fail(error.NestedTypeNotSupported, specs.span), }; var type_name: std.ArrayList(u8) = .empty; try type_name.appendSlice(self.arena, specs.words); if (function_pointer or array_pointer) { try type_name.append(self.arena, ' '); try type_name.appendNTimes(self.arena, '*', stars - 1); try type_name.appendSlice(self.arena, if (function_pointer) "(*)()" else "(*)[]"); } else if (stars > 0) { try type_name.append(self.arena, ' '); try type_name.appendNTimes(self.arena, '*', stars); } return .{ .name = try self.arena.dupe(u8, self.text(name_tok)), .type_name = type_name.items, .primitive = primitive, .dims = dims, .flexible = flexible, .aligned = attrs.aligned, .@"packed" = attrs.@"packed", .span = name_tok.span(), }; } /// `[N][M]...`, where only the first may be empty (a flexible array member). fn parseDimensions(self: *CStructParser) Error!struct { dims: []const u64, flexible: bool } { var dims: std.ArrayList(u64) = .empty; var flexible = false; var first = true; while (self.current.kind == .l_bracket) { try self.advance(); if (self.current.kind == .r_bracket and first) { flexible = true; } else { const tok = self.current; if (tok.kind != .number) return self.unexpected(); const n = try self.parseInteger(); if (n == 0) return self.fail(error.InvalidArrayLength, tok.span()); try dims.append(self.arena, n); } _ = try self.expect(.r_bracket); first = false; } return .{ .dims = dims.items, .flexible = flexible }; } /// A C integer literal: decimal, `0x` hex, `0b` binary, or octal with a leading /// zero, with any `u`/`l`/`ll` suffix. `010` is eight, because this is C. fn parseInteger(self: *CStructParser) Error!u64 { const tok = try self.expect(.number); return integerValue(self.text(tok)) orelse self.fail(error.InvalidNumber, tok.span()); } /// The suffixes C allows on an integer literal. `u` takes either case on its own; /// the two letters of `ll` must match each other. const integer_suffixes = std.StaticStringMap(void).initComptime(.{ .{""}, .{"u"}, .{"l"}, .{"L"}, .{"ll"}, .{"LL"}, .{"ul"}, .{"uL"}, .{"lu"}, .{"Lu"}, .{"ull"}, .{"uLL"}, .{"llu"}, .{"LLu"}, }); fn integerValue(literal: []const u8) ?u64 { var suffix_start = literal.len; while (suffix_start > 0 and std.mem.indexOfScalar(u8, "uUlL", literal[suffix_start - 1]) != null) { suffix_start -= 1; } var suffix_buf: [8]u8 = undefined; const suffix = literal[suffix_start..]; if (suffix.len > suffix_buf.len) return null; for (suffix, suffix_buf[0..suffix.len]) |c, *out| out.* = if (c == 'U') 'u' else c; if (!integer_suffixes.has(suffix_buf[0..suffix.len])) return null; var digits = literal[0..suffix_start]; var radix: u8 = 10; if (digits.len > 2 and digits[0] == '0' and (digits[1] == 'x' or digits[1] == 'X')) { radix = 16; digits = digits[2..]; } else if (digits.len > 2 and digits[0] == '0' and (digits[1] == 'b' or digits[1] == 'B')) { radix = 2; digits = digits[2..]; } else if (digits.len > 1 and digits[0] == '0') { radix = 8; digits = digits[1..]; } if (digits.len == 0) return null; return std.fmt.parseInt(u64, digits, radix) catch null; } // -- Tests -- const testing = std.testing; fn expectParseError(expected: Error, source: []const u8, needle: []const u8) !void { var diag: Diagnostic = .{}; try testing.expectError(expected, parse(testing.allocator, source, &diag)); const got = source[diag.span.start..][0..diag.span.len]; // The span is the point of the diagnostic, so it is asserted exactly. try testing.expectEqualStrings(needle, got); } fn expectMember(member: Member, name: []const u8, type_name: []const u8, primitive: Primitive) !void { try testing.expectEqualStrings(name, member.name); try testing.expectEqualStrings(type_name, member.type_name); try testing.expectEqual(primitive, member.primitive); } test "the design document's example" { var parsed = try parse(testing.allocator, "struct { u32 x; u8 flags; u16 id; u64 ts; }", null); defer parsed.deinit(); const def = parsed.definition; try testing.expect(def.name == null); try testing.expectEqual(@as(usize, 4), def.members.len); try expectMember(def.members[0], "x", "u32", .int); try expectMember(def.members[1], "flags", "u8", .char); try expectMember(def.members[2], "id", "u16", .short); try expectMember(def.members[3], "ts", "u64", .long_long); var layout = try struct_layout.compute(testing.allocator, def, struct_layout.default_abi, null); defer layout.deinit(testing.allocator); try testing.expectEqual(@as(u64, 16), layout.size); } test "a struct pasted from C, comments and all" { const source = \\/* A file header. */ \\struct file_header { \\ uint32_t magic; // 'TALY' \\ unsigned short version; \\ const char *name, **aliases; \\ long long int offsets[2][3]; \\ size_t len; \\ enum kind k; \\ struct file_header *next; \\ void (*on_close)(struct file_header *, int); \\ unsigned char data[]; \\}; ; var parsed = try parse(testing.allocator, source, null); defer parsed.deinit(); const def = parsed.definition; try testing.expectEqualStrings("file_header", def.name.?); try testing.expectEqual(Definition.NameKind.tag, def.name_kind); const ms = def.members; try testing.expectEqual(@as(usize, 10), ms.len); try expectMember(ms[0], "magic", "uint32_t", .int); try expectMember(ms[1], "version", "unsigned short", .short); try expectMember(ms[2], "name", "const char *", .pointer); try expectMember(ms[3], "aliases", "const char **", .pointer); try expectMember(ms[4], "offsets", "long long int", .long_long); try testing.expectEqualSlices(u64, &.{ 2, 3 }, ms[4].dims); try expectMember(ms[5], "len", "size_t", .pointer); try expectMember(ms[6], "k", "enum kind", .enumeration); try expectMember(ms[7], "next", "struct file_header *", .pointer); try expectMember(ms[8], "on_close", "void (*)()", .pointer); try expectMember(ms[9], "data", "unsigned char", .char); try testing.expect(ms[9].flexible); try testing.expect(!ms[8].flexible); // Spans point at the member's name, for layout errors to report. const span = ms[7].span.?; try testing.expectEqualStrings("next", source[span.start..][0..span.len]); } test "a typedef names the struct by its typedef" { var parsed = try parse(testing.allocator, "typedef struct node { int v; } node_t;", null); defer parsed.deinit(); try testing.expectEqualStrings("node_t", parsed.definition.name.?); try testing.expectEqual(Definition.NameKind.typedef, parsed.definition.name_kind); var anon = try parse(testing.allocator, "typedef struct { int v; } point;", null); defer anon.deinit(); try testing.expectEqualStrings("point", anon.definition.name.?); } test "C's keyword combinations mean what C says" { const Case = struct { source: []const u8, primitive: Primitive }; for ([_]Case{ .{ .source = "char", .primitive = .char }, .{ .source = "signed char", .primitive = .char }, .{ .source = "unsigned char", .primitive = .char }, .{ .source = "short", .primitive = .short }, .{ .source = "unsigned short int", .primitive = .short }, .{ .source = "int", .primitive = .int }, .{ .source = "unsigned", .primitive = .int }, .{ .source = "signed", .primitive = .int }, .{ .source = "long", .primitive = .long }, .{ .source = "unsigned long", .primitive = .long }, .{ .source = "long int", .primitive = .long }, .{ .source = "long long", .primitive = .long_long }, .{ .source = "unsigned long long int", .primitive = .long_long }, .{ .source = "int long long unsigned", .primitive = .long_long }, .{ .source = "float", .primitive = .float }, .{ .source = "double", .primitive = .double }, .{ .source = "_Bool", .primitive = .bool }, .{ .source = "bool", .primitive = .bool }, .{ .source = "__int128", .primitive = .int128 }, .{ .source = "unsigned __int128", .primitive = .int128 }, .{ .source = "volatile const unsigned", .primitive = .int }, .{ .source = "wchar_t", .primitive = .wchar }, .{ .source = "char16_t", .primitive = .short }, .{ .source = "ptrdiff_t", .primitive = .pointer }, .{ .source = "__uint128_t", .primitive = .int128 }, .{ .source = "i128", .primitive = .int128 }, .{ .source = "f32", .primitive = .float }, .{ .source = "usize", .primitive = .pointer }, .{ .source = "ptr", .primitive = .pointer }, }) |case| { var buf: [96]u8 = undefined; const source = try std.fmt.bufPrint(&buf, "struct {{ {s} x; }}", .{case.source}); var parsed = parse(testing.allocator, source, null) catch |err| { std.debug.print("\"{s}\": {t}\n", .{ case.source, err }); return err; }; defer parsed.deinit(); try testing.expectEqual(case.primitive, parsed.definition.members[0].primitive); try testing.expectEqualStrings(case.source, parsed.definition.members[0].type_name); } } test "keyword combinations C rejects are rejected" { for ([_][]const u8{ "short char", "long short", "long long long", "signed unsigned", "unsigned float", "short double", "long long double", "int int", "char char", "unsigned bool", "long __int128", "void int", "unsigned uint32_t x", "uint32_t int", }) |spec| { var buf: [96]u8 = undefined; const source = try std.fmt.bufPrint(&buf, "struct {{ {s} x; }}", .{spec}); if (parse(testing.allocator, source, null)) |parsed| { var p = parsed; p.deinit(); std.debug.print("\"{s}\" parsed\n", .{spec}); return error.TestUnexpectedResult; } else |err| { try testing.expect(err == error.UnexpectedToken or err == error.UnknownType); } } } test "several declarators share one set of specifiers" { var parsed = try parse(testing.allocator, "struct { int a, *b, c[4], **d; }", null); defer parsed.deinit(); const ms = parsed.definition.members; try expectMember(ms[0], "a", "int", .int); try expectMember(ms[1], "b", "int *", .pointer); try expectMember(ms[2], "c", "int", .int); try testing.expectEqualSlices(u64, &.{4}, ms[2].dims); try expectMember(ms[3], "d", "int **", .pointer); } test "pointers in parentheses are pointers, and arrays of them count" { var parsed = try parse(testing.allocator, \\struct { \\ int (*cb)(void); \\ void (*handlers[4])(int, char *); \\ char (*row)[16]; \\ int (*plain); \\ char *const *const pp; \\} , null); defer parsed.deinit(); const ms = parsed.definition.members; try expectMember(ms[0], "cb", "int (*)()", .pointer); try expectMember(ms[1], "handlers", "void (*)()", .pointer); try testing.expectEqualSlices(u64, &.{4}, ms[1].dims); try expectMember(ms[2], "row", "char (*)[]", .pointer); try testing.expectEqual(@as(usize, 0), ms[2].dims.len); try expectMember(ms[3], "plain", "int *", .pointer); try expectMember(ms[4], "pp", "char **", .pointer); } test "integer literals are read as C reads them" { const Case = struct { literal: []const u8, value: u64 }; for ([_]Case{ .{ .literal = "16", .value = 16 }, .{ .literal = "0x10", .value = 16 }, .{ .literal = "0X1f", .value = 31 }, .{ .literal = "010", .value = 8 }, .{ .literal = "0b101", .value = 5 }, .{ .literal = "4u", .value = 4 }, .{ .literal = "4UL", .value = 4 }, .{ .literal = "4lu", .value = 4 }, .{ .literal = "4ull", .value = 4 }, .{ .literal = "4LLU", .value = 4 }, .{ .literal = "7", .value = 7 }, }) |case| { var buf: [64]u8 = undefined; const source = try std.fmt.bufPrint(&buf, "struct {{ char x[{s}]; }}", .{case.literal}); var parsed = try parse(testing.allocator, source, null); defer parsed.deinit(); try testing.expectEqualSlices(u64, &.{case.value}, parsed.definition.members[0].dims); } for ([_][]const u8{ "08", "0x", "0b2", "4lL", "4uu", "4lul", "1e3", "99999999999999999999999", "0xg" }) |literal| { var buf: [64]u8 = undefined; const source = try std.fmt.bufPrint(&buf, "struct {{ char x[{s}]; }}", .{literal}); try expectParseError(error.InvalidNumber, source, literal); } } test "attributes and alignas reach the members and the struct" { var parsed = try parse(testing.allocator, \\struct __attribute__((packed)) wire { \\ char tag; \\ alignas(8) int a, b; \\ int c __attribute__((aligned(16))); \\ _Alignas(0) short d; \\ int e __attribute__((__packed__, unused)); \\ __attribute__((aligned(4), deprecated("old"))) char f; \\} __attribute__((aligned(32))); , null); defer parsed.deinit(); const def = parsed.definition; try testing.expectEqualStrings("wire", def.name.?); try testing.expect(def.@"packed"); try testing.expectEqual(@as(?u32, 32), def.aligned); const ms = def.members; try testing.expectEqual(@as(?u32, null), ms[0].aligned); try testing.expectEqual(@as(?u32, 8), ms[1].aligned); try testing.expectEqual(@as(?u32, 8), ms[2].aligned); try testing.expectEqual(@as(?u32, 16), ms[3].aligned); try testing.expectEqual(@as(?u32, null), ms[4].aligned); try testing.expect(ms[5].@"packed"); try testing.expect(!ms[4].@"packed"); try testing.expectEqual(@as(?u32, 4), ms[6].aligned); } test "pragma pack is tracked through push and pop" { const Case = struct { source: []const u8, pack: ?u32 }; for ([_]Case{ .{ .source = "#pragma pack(1)\nstruct { int a; };", .pack = 1 }, .{ .source = "#pragma pack(push, 2)\nstruct { int a; };\n#pragma pack(pop)", .pack = 2 }, .{ .source = "#pragma pack(4)\n#pragma pack()\nstruct { int a; };", .pack = null }, .{ .source = "#pragma pack(8)\n#pragma pack(push)\n#pragma pack(1)\n#pragma pack(pop)\nstruct { int a; };", .pack = 8 }, .{ .source = "#pragma pack(pop)\nstruct { int a; };", .pack = null }, .{ .source = " #pragma pack(16) // wide\nstruct { int a; };", .pack = 16 }, .{ .source = "struct { int a; };", .pack = null }, }) |case| { var parsed = try parse(testing.allocator, case.source, null); defer parsed.deinit(); try testing.expectEqual(case.pack, parsed.definition.pack); } } test "the layout of a parsed struct" { // Every piece together: a packed header with an aligned trailer. var parsed = try parse(testing.allocator, \\#pragma pack(push, 2) \\typedef struct { char c; double d; uint16_t s; } pair; \\#pragma pack(pop) , null); defer parsed.deinit(); var layout = try struct_layout.compute(testing.allocator, parsed.definition, struct_layout.default_abi, null); defer layout.deinit(testing.allocator); try testing.expectEqual(@as(u64, 12), layout.size); try testing.expectEqual(@as(u64, 2), layout.placements[1].offset); } test "errors say what and where" { try expectParseError(error.UnknownType, "struct { my_u32 x; }", "my_u32"); try expectParseError(error.UnknownType, "struct {\n int a;\n uint33_t b;\n}", "uint33_t"); try expectParseError(error.UnsupportedType, "struct { long double x; }", "long double"); try expectParseError(error.UnsupportedType, "struct { _Atomic int x; }", "_Atomic"); try expectParseError(error.UnsupportedType, "struct { double _Complex z; }", "_Complex"); try expectParseError(error.UnsupportedType, "union u { int a; float b; };", "union"); try expectParseError(error.IncompleteType, "struct { void v; }", "void"); try expectParseError(error.BitfieldNotSupported, "struct { unsigned a : 3; }", ":"); try expectParseError(error.NestedTypeNotSupported, "struct { struct point p; }", "struct point"); try expectParseError(error.NestedTypeNotSupported, "struct { union { int a; } u; }", "union"); try expectParseError(error.NestedTypeNotSupported, "struct { struct inner { int a; } i; }", "struct"); try expectParseError(error.NestedTypeNotSupported, "struct { enum { A, B } e; }", "enum"); try expectParseError(error.PreprocessorNotSupported, "#include \nstruct { int a; }", "#include "); try expectParseError(error.PreprocessorNotSupported, "struct {\n#if 0\n int a;\n#endif\n}", "#if 0"); try expectParseError(error.PreprocessorNotSupported, "#pragma once\nstruct { int a; }", "#pragma once"); try expectParseError(error.InvalidAlignment, "#pragma pack(3)\nstruct { int a; }", "3"); try expectParseError(error.InvalidAlignment, "#pragma pack(32)\nstruct { int a; }", "32"); try expectParseError(error.InvalidAlignment, "struct { alignas(6) int a; }", "6"); try expectParseError(error.InvalidAlignment, "struct { int a __attribute__((aligned(0))); }", "0"); try expectParseError(error.UnsupportedAttribute, "struct { int a __attribute__((aligned)); }", "aligned"); try expectParseError(error.UnsupportedAttribute, "struct { int a __attribute__((mode(DI))); }", "mode"); try expectParseError(error.InvalidArrayLength, "struct { char a[0]; }", "0"); try expectParseError(error.UnterminatedComment, "struct { int a; /* oops }", "/*"); try expectParseError(error.UnexpectedEnd, "struct { int a;", ""); try expectParseError(error.UnexpectedEnd, "struct { int", ""); try expectParseError(error.UnexpectedEnd, "struct { void (*cb)(int, char *", ""); try expectParseError(error.UnexpectedToken, "struct { int a }", "}"); try expectParseError(error.UnexpectedToken, "struct { int a; } extra", "extra"); try expectParseError(error.UnexpectedToken, "struct { int f(void); }", "("); try expectParseError(error.UnexpectedToken, "struct { int a; } struct { int b; }", "struct"); try expectParseError(error.UnexpectedToken, "struct { int a[]x; }", "x"); try expectParseError(error.UnexpectedToken, "struct { int a[2][]; }", "]"); try expectParseError(error.UnexpectedToken, "struct { int a @ b; }", "@"); try expectParseError(error.UnexpectedToken, "int a;", "int"); try expectParseError(error.UnexpectedToken, "typedef struct { int a; };", ";"); try expectParseError(error.UnexpectedToken, "#pragma pack(push, 2, 3)\nstruct { int a; }", ","); try expectParseError(error.UnexpectedToken, "#pragma pack(show)\nstruct { int a; }", "show"); try expectParseError(error.UnexpectedEnd, "", ""); } test "an empty body parses, and laying it out is what refuses it" { // The parser's job is syntax; `compute` owns the rule that a struct has members, // so every front end gets it. var parsed = try parse(testing.allocator, "struct empty { ; };", null); defer parsed.deinit(); try testing.expectError(error.EmptyStruct, struct_layout.compute(testing.allocator, parsed.definition, struct_layout.default_abi, null)); } test "input past the size limit is refused before it is read" { const big = try testing.allocator.alloc(u8, max_source_bytes + 10); defer testing.allocator.free(big); @memset(big, ' '); var diag: Diagnostic = .{}; try testing.expectError(error.DefinitionTooLong, parse(testing.allocator, big, &diag)); try testing.expectEqual(max_source_bytes, diag.span.start); } test "too many members is refused at the one past the limit" { var source: std.ArrayList(u8) = .empty; defer source.deinit(testing.allocator); try source.appendSlice(testing.allocator, "struct { char "); for (0..struct_layout.max_members + 1) |i| { if (i > 0) try source.appendSlice(testing.allocator, ","); try source.print(testing.allocator, "m{d}", .{i}); } try source.appendSlice(testing.allocator, "; }"); var diag: Diagnostic = .{}; try testing.expectError(error.TooManyMembers, parse(testing.allocator, source.items, &diag)); try testing.expect(std.mem.startsWith(u8, source.items[diag.span.start..], "m1024")); } test "locate turns an offset into a line and column" { const source = "struct {\n int a;\n bad b;\n}"; const at = std.mem.indexOf(u8, source, "bad").?; try testing.expectEqual(Location{ .line = 3, .column = 3 }, locate(source, at)); try testing.expectEqual(Location{ .line = 1, .column = 1 }, locate(source, 0)); try testing.expectEqual(Location{ .line = 4, .column = 2 }, locate(source, source.len)); } test "parse releases everything on any allocation failure" { try testing.checkAllAllocationFailures(testing.allocator, struct { fn run(a: Allocator) !void { var parsed = try parse(a, \\#pragma pack(push, 4) \\typedef struct tag { const char *name, **more; int grid[2][3]; double d; } t; \\#pragma pack(pop) , null); parsed.deinit(); } }.run, .{}); }