centralize module configuration in build.zig

build.zig

@@ -40,23 +40,17 @@ pub fn build(b: *Builder) !void {
         "Skip tests that do not match any of the specified filters",
     ) orelse &.{};
 
-    // TODO: Embed the current git version in the code. We can do this
-    // by looking for .git/HEAD (if it exists, follow the ref to /ref/heads/whatevs,
-    // grab that commit, and use b.addOptions/exe.addOptions to generate the
-    // Options file. See https://github.com/ziglang/zig/issues/14979 for usage
-    // example.
-    //
-    // From there, I'm not sure what the generated file looks like or quite how
-    // to use, but that should be easy. It may also give some ideas on the
-    // code gen piece itself, though it might be nice to leave as a seperate
-    // executable
-    // TODO: This executable should not be built when importing as a package.
-    // It relies on code gen and is all fouled up when getting imported
+    const dep_mods = try getDependencyModules(b, .{
+        .target = target,
+        .optimize = optimize,
+    });
+
     const mod_exe = b.createModule(.{
         .root_source_file = b.path("src/main.zig"),
         .target = target,
         .optimize = optimize,
     });
+    configure(mod_exe, dep_mods, true);
 
     const exe = b.addExecutable(.{
         .name = "demo",
@@ -64,38 +58,6 @@ pub fn build(b: *Builder) !void {
         .use_llvm = !no_llvm,
     });
 
-    // External dependencies
-    const dep_smithy = b.dependency("smithy", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    const mod_smithy = dep_smithy.module("smithy");
-    mod_exe.addImport("smithy", mod_smithy); // not sure this should be here...
-
-    const dep_zeit = b.dependency("zeit", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    const mod_zeit = dep_zeit.module("zeit");
-    mod_exe.addImport("zeit", mod_zeit);
-    // End External dependencies
-
-    // Private modules/dependencies
-    const dep_json = b.dependency("json", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    const mod_json = dep_json.module("json");
-    mod_exe.addImport("json", mod_json);
-
-    const dep_date = b.dependency("date", .{
-        .target = target,
-        .optimize = optimize,
-    });
-    const mod_date = dep_date.module("date");
-    mod_exe.addImport("date", mod_date);
-    // End private modules/dependencies
-
     const run_cmd = b.addRunArtifact(exe);
     run_cmd.step.dependOn(b.getInstallStep());
     if (b.args) |args| {
@@ -113,9 +75,7 @@ pub fn build(b: *Builder) !void {
         .target = b.graph.host,
         .optimize = if (b.verbose) .Debug else .ReleaseSafe,
     });
-    cg_mod.addImport("smithy", mod_smithy);
-    cg_mod.addImport("date", mod_date);
-    cg_mod.addImport("json", mod_json);
+    configure(cg_mod, dep_mods, false);
 
     const cg_exe = b.addExecutable(.{
         .name = "codegen",
@@ -160,10 +120,7 @@ pub fn build(b: *Builder) !void {
         .target = target,
         .optimize = optimize,
     });
-    service_manifest_module.addImport("smithy", mod_smithy);
-    service_manifest_module.addImport("date", mod_date);
-    service_manifest_module.addImport("json", mod_json);
-    service_manifest_module.addImport("zeit", mod_zeit);
+    configure(service_manifest_module, dep_mods, true);
 
     mod_exe.addImport("service_manifest", service_manifest_module);
 
@@ -173,19 +130,14 @@ pub fn build(b: *Builder) !void {
         .target = target,
         .optimize = optimize,
     });
-    mod_aws.addImport("smithy", mod_smithy);
     mod_aws.addImport("service_manifest", service_manifest_module);
-    mod_aws.addImport("date", mod_date);
-    mod_aws.addImport("json", mod_json);
-    mod_aws.addImport("zeit", mod_zeit);
+    configure(mod_aws, dep_mods, true);
 
     // Expose module to others
     const mod_aws_signing = b.addModule("aws-signing", .{
         .root_source_file = b.path("src/aws_signing.zig"),
     });
-    mod_aws_signing.addImport("date", mod_date);
-    mod_aws_signing.addImport("smithy", mod_smithy);
-    mod_aws_signing.addImport("json", mod_json);
+    configure(mod_aws_signing, dep_mods, false);
 
     // Similar to creating the run step earlier, this exposes a `test` step to
     // the `zig build --help` menu, providing a way for the user to request
@@ -214,11 +166,8 @@ pub fn build(b: *Builder) !void {
             .target = b.resolveTargetQuery(t),
             .optimize = optimize,
         });
-        mod_unit_tests.addImport("smithy", mod_smithy);
         mod_unit_tests.addImport("service_manifest", service_manifest_module);
-        mod_unit_tests.addImport("date", mod_date);
-        mod_unit_tests.addImport("zeit", mod_zeit);
-        mod_unit_tests.addImport("json", mod_json);
+        configure(mod_unit_tests, dep_mods, true);
 
         // Creates a step for unit testing. This only builds the test executable
         // but does not run it.
@@ -261,3 +210,36 @@ pub fn build(b: *Builder) !void {
         b.installArtifact(exe);
     }
 }
+
+fn configure(compile: *std.Build.Module, modules: std.StringHashMap(*std.Build.Module), include_time: bool) void {
+    compile.addImport("smithy", modules.get("smithy").?);
+    compile.addImport("date", modules.get("date").?);
+    compile.addImport("json", modules.get("json").?);
+    if (include_time) compile.addImport("zeit", modules.get("zeit").?);
+}
+
+fn getDependencyModules(b: *std.Build, args: anytype) !std.StringHashMap(*std.Build.Module) {
+    var result = std.StringHashMap(*std.Build.Module).init(b.allocator);
+
+    // External dependencies
+    const dep_smithy = b.dependency("smithy", args);
+    const mod_smithy = dep_smithy.module("smithy");
+    try result.putNoClobber("smithy", mod_smithy);
+
+    const dep_zeit = b.dependency("zeit", args);
+    const mod_zeit = dep_zeit.module("zeit");
+    try result.putNoClobber("zeit", mod_zeit);
+    // End External dependencies
+
+    // Private modules/dependencies
+    const dep_json = b.dependency("json", args);
+    const mod_json = dep_json.module("json");
+    try result.putNoClobber("json", mod_json);
+
+    const dep_date = b.dependency("date", args);
+    const mod_date = dep_date.module("date");
+    try result.putNoClobber("date", mod_date);
+    // End private modules/dependencies
+
+    return result;
+}

src/xml.zig

@@ -25,6 +25,7 @@ pub const Element = struct {
     tag: []const u8,
     attributes: AttributeList,
     children: ContentList,
+    next_sibling: ?*Element = null,
 
     fn init(tag: []const u8, alloc: Allocator) Element {
         return .{
@@ -347,7 +348,7 @@ fn parseDocument(ctx: *ParseContext, backing_allocator: Allocator) !Document {
     _ = ctx.eatWs();
     try trySkipComments(ctx, allocator);
 
-    doc.root = (try tryParseElement(ctx, allocator)) orelse return error.InvalidDocument;
+    doc.root = (try tryParseElement(ctx, allocator, null)) orelse return error.InvalidDocument;
     _ = ctx.eatWs();
     try trySkipComments(ctx, allocator);
 
@@ -415,12 +416,12 @@ fn tryParseCharData(ctx: *ParseContext, alloc: Allocator) !?[]const u8 {
     return try dupeAndUnescape(alloc, ctx.source[begin..end]);
 }
 
-fn parseContent(ctx: *ParseContext, alloc: Allocator) ParseError!Content {
+fn parseContent(ctx: *ParseContext, alloc: Allocator, parent: ?*Element) ParseError!Content {
     if (try tryParseCharData(ctx, alloc)) |cd| {
         return Content{ .CharData = cd };
     } else if (try tryParseComment(ctx, alloc)) |comment| {
         return Content{ .Comment = comment };
-    } else if (try tryParseElement(ctx, alloc)) |elem| {
+    } else if (try tryParseElement(ctx, alloc, parent)) |elem| {
         return Content{ .Element = elem };
     } else {
         return error.UnexpectedCharacter;
@@ -440,7 +441,7 @@ fn tryParseAttr(ctx: *ParseContext, alloc: Allocator) !?*Attribute {
     return attr;
 }
 
-fn tryParseElement(ctx: *ParseContext, alloc: Allocator) !?*Element {
+fn tryParseElement(ctx: *ParseContext, alloc: Allocator, parent: ?*Element) !?*Element {
     const start = ctx.offset;
     if (!ctx.eat('<')) return null;
     const tag = parseNameNoDupe(ctx) catch {
@@ -469,7 +470,7 @@ fn tryParseElement(ctx: *ParseContext, alloc: Allocator) !?*Element {
             break;
         }
 
-        const content = try parseContent(ctx, alloc);
+        const content = try parseContent(ctx, alloc, element);
         try element.children.append(content);
     }
 
@@ -480,6 +481,23 @@ fn tryParseElement(ctx: *ParseContext, alloc: Allocator) !?*Element {
 
     _ = ctx.eatWs();
     try ctx.expect('>');
+
+    if (parent) |p| {
+        var last_element: ?*Element = null;
+
+        for (0..p.children.items.len) |i| {
+            const child = p.children.items[p.children.items.len - i - 1];
+            if (child == .Element) {
+                last_element = child.Element;
+                break;
+            }
+        }
+
+        if (last_element) |lc| {
+            lc.next_sibling = element;
+        }
+    }
+
     return element;
 }
 
@@ -490,13 +508,13 @@ test "tryParseElement" {
 
     {
         var ctx = ParseContext.init("<= a='b'/>");
-        try testing.expectEqual(@as(?*Element, null), try tryParseElement(&ctx, alloc));
+        try testing.expectEqual(@as(?*Element, null), try tryParseElement(&ctx, alloc, null));
         try testing.expectEqual(@as(?u8, '<'), ctx.peek());
     }
 
     {
         var ctx = ParseContext.init("<python size='15' color = \"green\"/>");
-        const elem = try tryParseElement(&ctx, alloc);
+        const elem = try tryParseElement(&ctx, alloc, null);
         try testing.expectEqualSlices(u8, elem.?.tag, "python");
 
         const size_attr = elem.?.attributes.items[0];
@@ -510,14 +528,14 @@ test "tryParseElement" {
 
     {
         var ctx = ParseContext.init("<python>test</python>");
-        const elem = try tryParseElement(&ctx, alloc);
+        const elem = try tryParseElement(&ctx, alloc, null);
         try testing.expectEqualSlices(u8, elem.?.tag, "python");
         try testing.expectEqualSlices(u8, elem.?.children.items[0].CharData, "test");
     }
 
     {
         var ctx = ParseContext.init("<a>b<c/>d<e/>f<!--g--></a>");
-        const elem = try tryParseElement(&ctx, alloc);
+        const elem = try tryParseElement(&ctx, alloc, null);
         try testing.expectEqualSlices(u8, elem.?.tag, "a");
         try testing.expectEqualSlices(u8, elem.?.children.items[0].CharData, "b");
         try testing.expectEqualSlices(u8, elem.?.children.items[1].Element.tag, "c");

src/xml_shaper.zig

@@ -1,6 +1,7 @@
 const std = @import("std");
 const xml = @import("xml.zig");
 const date = @import("date");
+const sm = @import("service_manifest");
 
 const log = std.log.scoped(.xml_shaper);
 
@@ -94,6 +95,52 @@ pub fn parse(comptime T: type, source: []const u8, options: ParseOptions) !Parse
     return Parsed(T).init(arena_allocator, try parseInternal(T, root, opts), parsed);
 }
 
+pub const XmlArrayStyle = enum {
+    collection, // Has a container element and list of child elements
+    repeated_root, // Repeats the same element without a container, e.g. S3 ListBucketResult
+};
+
+fn detectArrayStyle(comptime T: type, element: *xml.Element, options: ParseOptions) !XmlArrayStyle {
+    _ = options;
+
+    if (@typeInfo(T) != .@"struct") {
+        return .collection;
+    }
+
+    // does the element have child elements that match our expected struct?
+    const field_names = comptime blk: {
+        var result: [std.meta.fieldNames(T).len]struct {
+            []const u8,
+        } = undefined;
+
+        for (std.meta.fieldNames(T), 0..) |field_name, i| {
+            const key = if (@hasDecl(T, "fieldNameFor"))
+                T.fieldNameFor(undefined, field_name)
+            else
+                field_name;
+
+            result[i] = .{key};
+        }
+
+        break :blk std.StaticStringMap(void).initComptime(result);
+    };
+
+    var matching_fields: usize = 0;
+    var element_iterator = element.elements();
+
+    while (element_iterator.next()) |el| {
+        if (field_names.has(el.tag)) {
+            matching_fields += 1;
+        }
+    }
+
+    if (matching_fields > 0) {
+        return .repeated_root;
+    }
+
+    return .collection;
+}
+
 fn parseInternal(comptime T: type, element: *xml.Element, options: ParseOptions) !T {
     switch (@typeInfo(T)) {
         .bool => {
@@ -330,23 +377,31 @@ fn parseInternal(comptime T: type, element: *xml.Element, options: ParseOptions)
                     //   <Item>bar</Item>
                     // <Items>
                     if (ptr_info.child != u8) {
-                        log.debug("type = {s}, ptr_info.child == {s}, element = {s}", .{ @typeName(T), @typeName(ptr_info.child), element.tag });
-                        var iterator = element.elements();
+                        const array_style = try detectArrayStyle(ptr_info.child, element, options);
+
+                        log.debug("type = {s}, style = {s}, ptr_info.child == {s}, element = {s}", .{ @typeName(T), @tagName(array_style), @typeName(ptr_info.child), element.tag });
+
                         var children = std.ArrayList(ptr_info.child).init(allocator);
                         defer children.deinit();
+
+                        switch (array_style) {
+                            .collection => {
+                                var iterator = element.elements();
                                 while (iterator.next()) |child_element| {
                                     try children.append(try parseInternal(ptr_info.child, child_element, options));
                                 }
+                            },
+                            .repeated_root => {
+                                var current: ?*Element = element;
+                                while (current) |el| : (current = el.next_sibling) {
+                                    if (!std.mem.eql(u8, el.tag, element.tag)) continue;
+
+                                    try children.append(try parseInternal(ptr_info.child, el, options));
+                                }
+                            },
+                        }
+
                         return children.toOwnedSlice();
-                        // var inx: usize = 0;
-                        // while (inx < children.len) {
-                        //     switch (element.children.items[inx]) {
-                        //         .Element => children[inx] = try parseInternal(ptr_info.child, element.children.items[inx].Element, options),
-                        //         .CharData => children[inx] = try allocator.dupe(u8, element.children.items[inx].CharData),
-                        //         .Comment => children[inx] = try allocator.dupe(u8, element.children.items[inx].Comment), // This might be an error...
-                        //     }
-                        //     inx += 1;
-                        // }
                     }
                     return try allocator.dupe(u8, element.children.items[0].CharData);
                 },
@@ -738,3 +793,33 @@ test "compiler assertion failure 2" {
     defer parsed_data.deinit();
     try testing.expect(parsed_data.parsed_value.key_group_list.?.quantity == 42);
 }
+
+test "can parse list objects" {
+    const data =
+        \\<?xml version="1.0" encoding="UTF-8"?>
+        \\<ListBucketResult>
+        \\    <Contents>
+        \\        <Key>file1.txt</Key>
+        \\        <Size>1024</Size>
+        \\    </Contents>
+        \\    <Contents>
+        \\        <Key>file2.jpg</Key>
+        \\        <Size>2048</Size>
+        \\    </Contents>
+        \\</ListBucketResult>
+    ;
+
+    const Response = sm.s3.list_objects_v2.Response;
+
+    const parsed_data = try parse(Response, data, .{ .allocator = testing.allocator });
+    defer parsed_data.deinit();
+
+    const response: Response = parsed_data.parsed_value;
+    const s3_objects: []sm.s3.Object = response.contents.?;
+
+    try testing.expectEqual(2, s3_objects.len);
+    try testing.expectEqualStrings(s3_objects[0].key.?, "file1.txt");
+    try testing.expectEqualStrings(s3_objects[1].key.?, "file2.jpg");
+    try testing.expectEqual(s3_objects[0].size.?, 1024);
+    try testing.expectEqual(s3_objects[1].size.?, 2048);
+}