EC2 support

README.md

@@ -2,15 +2,13 @@
 
 [![Build Status](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/status.svg?ref=refs/heads/master)](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/)
 
-This SDK currently supports all AWS services except EC2 and S3. These two
+This SDK currently supports all AWS services except services using the restXml
-services only support XML, and more work is needed to parse and integrate
+protocol (4 services including S3). See TODO list below.
-type hydration from the base parsing. S3 also requires some plumbing tweaks
-in the signature calculation. Examples of usage are in src/main.zig.
 
-Current executable size for the demo is 953k (90k of which is the AWS PEM file)
+Current executable size for the demo is 1.6M (90k of which is the AWS PEM file,
-after compiling with -Drelease-safe and
+and approximately 600K for XML services) after compiling with -Drelease-safe and
 [stripping the executable after compilation](https://github.com/ziglang/zig/issues/351).
-This is for x86_linux. Tested targets:
+This is for x86_linux, and will vary based on services used. Tested targets:
 
 * x86_64-linux
 * riscv64-linux
@@ -41,8 +39,7 @@ require passing in a client option to specify an different TLS root certificate
 (pass null to disable certificate verification).
 
 The [old branch](https://github.com/elerch/aws-sdk-for-zig/tree/aws-crt) exists
-for posterity, and supports x86_64 linux. This branch is recommended moving
+for posterity, and supports x86_64 linux. The old branch is deprecated.
-forward.
 
 ## Limitations
 
@@ -52,13 +49,8 @@ implemented.
 
 TODO List:
 
-* Complete integration of Xml responses with remaining code base
 * Implement [AWS restXml protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-restxml-protocol.html).
-  Includes S3. Total service count 4. This may be blocked due to the same issue as EC2.
+  Includes S3. Total service count 4.
-* Implement [AWS EC2 query protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-ec2-query-protocol.html).
-  Includes EC2. Total service count 1. This may be blocked on a compiler bug,
-  though has not been tested with zig 0.9.0. More details and llvm ir log can be found in the
-  [XML branch](https://git.lerch.org/lobo/aws-sdk-for-zig/src/branch/xml).
 * Implement sigv4a signing
 * Implement jitter/exponential backoff
 * Implement timeouts and other TODO's in the code

src/aws.zig

@@ -5,7 +5,7 @@ const json = @import("json.zig");
 const url = @import("url.zig");
 const case = @import("case.zig");
 const servicemodel = @import("servicemodel.zig");
-// const xml_shaper = @import("xml_shaper.zig");
+const xml_shaper = @import("xml_shaper.zig");
 
 const log = std.log.scoped(.aws);
 
@@ -175,8 +175,6 @@ pub fn Request(comptime action: anytype) type {
         // handle lists and maps properly anyway yet, so we'll go for it and see
         // where it breaks. PRs and/or failing test cases appreciated.
         fn callQuery(request: ActionRequest, options: Options) !FullResponseType {
-            if (Self.service_meta.aws_protocol == .ec2_query)
-                @compileError("XML responses from EC2 blocked due to zig compiler bug scheduled to be fixed no earlier than 0.10.0");
             var buffer = std.ArrayList(u8).init(options.client.allocator);
             defer buffer.deinit();
             const writer = buffer.writer();
@@ -250,10 +248,9 @@ pub fn Request(comptime action: anytype) type {
                 }
             }
 
-            // TODO: Handle XML
+            if (!isJson) return try xmlReturn(options, response);
-            if (!isJson) return error.XmlUnimplemented;
 
-            const SResponse = if (Self.service_meta.aws_protocol != .query and Self.service_meta.aws_protocol != .ec2_query)
+            const SResponse = if (Self.service_meta.aws_protocol != .query)
                 action.Response
             else
                 ServerResponse(action);
@@ -272,7 +269,7 @@ pub fn Request(comptime action: anytype) type {
                     .response_metadata = .{
                         .request_id = try requestIdFromHeaders(aws_request, response, options),
                     },
-                    .parser_options = parser_options,
+                    .parser_options = .{ .json = parser_options },
                     .raw_parsed = .{ .raw = .{} },
                 };
 
@@ -294,13 +291,25 @@ pub fn Request(comptime action: anytype) type {
                 return e;
             };
 
-            if (Self.service_meta.aws_protocol != .query and Self.service_meta.aws_protocol != .ec2_query) {
+            // TODO: Figure out this hack
+            // the code setting the response about 10 lines down will trigger
+            // an error because the first field may not be a struct when
+            // XML processing is happening above, which we only know at runtime.
+            //
+            // We could simply force .ec2_query and .rest_xml above rather than
+            // isJson, but it would be nice to automatically support json if
+            // these services start returning that like we'd like them to.
+            //
+            // Otherwise, the compiler gets down here thinking this will be
+            // processed. If it is, then we have a problem when the field name
+            // may not be a struct.
+            if (Self.service_meta.aws_protocol != .query or Self.service_meta.aws_protocol == .ec2_query) {
                 return FullResponseType{
                     .response = parsed_response,
                     .response_metadata = .{
                         .request_id = try requestIdFromHeaders(aws_request, response, options),
                     },
-                    .parser_options = parser_options,
+                    .parser_options = .{ .json = parser_options },
                     .raw_parsed = .{ .raw = parsed_response },
                 };
             }
@@ -320,10 +329,53 @@ pub fn Request(comptime action: anytype) type {
                 .response_metadata = .{
                     .request_id = try options.client.allocator.dupe(u8, real_response.ResponseMetadata.RequestId),
                 },
-                .parser_options = parser_options,
+                .parser_options = .{ .json = parser_options },
                 .raw_parsed = .{ .server = parsed_response },
             };
         }
+
+        fn xmlReturn(options: Options, result: awshttp.HttpResult) !FullResponseType {
+            // Server shape be all like:
+            //
+            // <?xml version="1.0" encoding="UTF-8"?>
+            // <DescribeRegionsResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
+            //     <requestId>0efe31c6-cad5-4882-b275-dfea478cf039</requestId>
+            //     <regionInfo>
+            //         <item>
+            //             <regionName>eu-north-1</regionName>
+            //             <regionEndpoint>ec2.eu-north-1.amazonaws.com</regionEndpoint>
+            //             <optInStatus>opt-in-not-required</optInStatus>
+            //         </item>
+            //     </regionInfo>
+            // </DescribeRegionsResponse>
+            //
+            // While our stuff be like:
+            //
+            // struct {
+            //   regions: []struct {
+            //     region_name: []const u8,
+            //   }
+            // }
+            //
+            // Big thing is that requestid, which we'll need to fetch "manually"
+            const xml_options = xml_shaper.ParseOptions{ .allocator = options.client.allocator };
+            const parsed = try xml_shaper.parse(action.Response, result.body, xml_options);
+            // This needs to get into FullResponseType somehow: defer parsed.deinit();
+            const request_id = blk: {
+                if (parsed.document.root.getCharData("requestId")) |elem|
+                    break :blk elem;
+                return error.RequestIdNotFound;
+            };
+
+            return FullResponseType{
+                .response = parsed.parsed_value,
+                .response_metadata = .{
+                    .request_id = try options.client.allocator.dupe(u8, request_id),
+                },
+                .parser_options = .{ .xml = xml_options },
+                .raw_parsed = .{ .xml = parsed },
+            };
+        }
     };
 }
 
@@ -397,21 +449,32 @@ fn FullResponse(comptime action: anytype) type {
         response_metadata: struct {
             request_id: []u8,
         },
-        parser_options: json.ParseOptions,
+        parser_options: union(enum) {
+            json: json.ParseOptions,
+            xml: xml_shaper.ParseOptions,
+        },
         raw_parsed: union(enum) {
             server: ServerResponse(action),
             raw: action.Response,
+            xml: xml_shaper.Parsed(action.Response),
         },
-        // raw_parsed: ServerResponse(request),
 
         const Self = @This();
         pub fn deinit(self: Self) void {
             switch (self.raw_parsed) {
-                .server => json.parseFree(ServerResponse(action), self.raw_parsed.server, self.parser_options),
+                // Server is json only (so far)
-                .raw => json.parseFree(action.Response, self.raw_parsed.raw, self.parser_options),
+                .server => json.parseFree(ServerResponse(action), self.raw_parsed.server, self.parser_options.json),
+                // Raw is json only (so far)
+                .raw => json.parseFree(action.Response, self.raw_parsed.raw, self.parser_options.json),
+                .xml => |xml| xml.deinit(),
             }
 
-            self.parser_options.allocator.?.free(self.response_metadata.request_id);
+            var allocator: std.mem.Allocator = undefined;
+            switch (self.parser_options) {
+                .json => |j| allocator = j.allocator.?,
+                .xml => |x| allocator = x.allocator.?,
+            }
+            allocator.free(self.response_metadata.request_id);
         }
     };
 }

src/main.zig

@@ -14,6 +14,15 @@ pub fn log(
     // Ignore aws_signing messages
     if (verbose < 2 and scope == .aws_signing and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
         return;
+    // Ignore aws_credentials messages
+    if (verbose < 2 and scope == .aws_credentials and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
+        return;
+    // Ignore xml_shaper messages
+    if (verbose < 2 and scope == .xml_shaper and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
+        return;
+    // Ignore date messages
+    if (verbose < 2 and scope == .date and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
+        return;
     // Ignore awshttp messages
     if (verbose < 1 and scope == .awshttp and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
         return;
@@ -169,18 +178,17 @@ pub fn main() anyerror!void {
                     std.log.err("no functions to work with", .{});
                 }
             },
-            // TODO: This test fails with broken LLVM module
             .ec2_query_no_input => {
-                std.log.err("EC2 Test disabled due to compiler bug", .{});
                 // Describe regions is a simpler request and easier to debug
-                // const instances = try client.call(services.ec2.describe_regions.Request{}, options);
+                const result = try client.call(services.ec2.describe_regions.Request{}, options);
-                // defer instances.deinit();
+                defer result.deinit();
-                // std.log.info("region count: {d}", .{instances.response.regions.?.len});
+                std.log.info("request id: {s}", .{result.response_metadata.request_id});
+                std.log.info("region count: {d}", .{result.response.regions.?.len});
 
                 // Describe instances is more interesting
-                // const instances = try client.call(services.ec2.describe_instances.Request{}, options);
+                const instances = try client.call(services.ec2.describe_instances.Request{}, options);
-                // defer instances.deinit();
+                defer instances.deinit();
-                // std.log.info("reservation count: {d}", .{instances.response.reservations.len});
+                std.log.info("reservation count: {d}", .{instances.response.reservations.?.len});
             },
         }
         std.log.info("===== End Test: {s} =====\n", .{@tagName(t)});

src/xml.zig

@@ -45,7 +45,7 @@ pub const Element = struct {
     }
 
     pub fn getCharData(self: *Element, child_tag: []const u8) ?[]const u8 {
-        const child = self.findChildByTag(child_tag) orelse return null;
+        const child = (self.findChildByTag(child_tag) catch return null) orelse return null;
         if (child.children.items.len != 1) {
             return null;
         }

src/xml_shaper.zig

@@ -242,6 +242,7 @@ fn parseInternal(comptime T: type, element: *xml.Element, options: ParseOptions)
                 if (@typeInfo(field.field_type) == .Optional and !found_value) {
                     // @compileLog("Optional: Field name ", field.name, ", type ", field.field_type);
                     @field(r, field.name) = null;
+                    fields_set = fields_set + 1;
                     found_value = true;
                 }
                 // Using this else clause breaks zig, so we'll use a boolean instead