update readme

README.md

@@ -100,7 +100,7 @@ TODO List:
 * Move to compiler on tagged release (hopefully 0.8.1)
 (new 2021-05-29. I will proceed in this order unless I get other requests)
 * ✓ Implement [AWS query protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-query-protocol.html). This is the protocol in use by sts.getcalleridentity. Total service count 18
-* Implement [AWS Json 1.0 protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-json-1_0-protocol.html). Includes dynamodb. Total service count 18
+* ✓ Implement [AWS Json 1.0 protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-json-1_0-protocol.html). Includes dynamodb. Total service count 18
 * Implement [AWS Json 1.1 protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-json-1_1-protocol.html). Includes ecs. Total service count 105
 * Implement [AWS restXml protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-restxml-protocol.html). Includes S3. Total service count 4. This may be blocked due to the same issue as EC2.
 * Implement [AWS EC2 query protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-ec2-query-protocol.html). Includes EC2. Total service count 1. This is currently blocked, probably on self-hosted compiler coming in zig 0.9.0 (January 2022) due to compiler bug discovered. More details and llvm ir log can be found in the [XML branch](https://git.lerch.org/lobo/aws-sdk-for-zig/src/branch/xml).

codegen/src/main.zig

@@ -176,7 +176,7 @@ fn generateOperation(allocator: *std.mem.Allocator, operation: smithy.ShapeInfo,
     _ = try writer.write(",\n");
     _ = try writer.write("    Response: type = ");
     if (operation.shape.operation.output) |member| {
-        try generateTypeFor(allocator, member, shapes, writer, "    ", true, &type_stack, true);
+        try generateTypeFor(allocator, member, shapes, writer, "    ", false, &type_stack, true);
     } else _ = try writer.write("struct {}"); // we want to maintain consistency with other ops
     _ = try writer.write(",\n");
 

src/aws.zig

@@ -56,25 +56,72 @@ pub const Aws = struct {
 
         // It seems as though there are 3 major branches of the 6 protocols.
         // 1. query/ec2_query, which are identical until you get to complex
-        //    structures. TBD if the shortcut we're taking for query to make
-        //    it return json will work on EC2, but my guess is yes.
+        //    structures. EC2 query does not allow us to request json though,
+        //    so we need to handle xml returns from this.
         // 2. *json*: These three appear identical for input (possible difference
         //    for empty body serialization), but differ in error handling.
         //    We're not doing a lot of error handling here, though.
         // 3. rest_xml: This is a one-off for S3, never used since
         switch (service_meta.aws_protocol) {
-            .query, .ec2_query => return self.callQuery(request, service_meta, action, options),
-            .rest_json_1, .json_1_0, .json_1_1 => @compileError("REST Json, Json 1.0/1.1 protocol not yet supported"),
-            .rest_xml => @compileError("REST XML protocol not yet supported"),
+            .query => return self.callQuery(request, service_meta, action, options),
+            // .query, .ec2_query => return self.callQuery(request, service_meta, action, options),
+            .rest_json_1, .json_1_0, .json_1_1 => return self.callJson(request, service_meta, action, options),
+            .ec2_query, .rest_xml => @compileError("XML responses may be blocked on a zig compiler bug scheduled to be fixed in 0.9.0"),
         }
     }
 
+    /// Calls using one of the json protocols (rest_json_1, json_1_0, json_1_1
+    fn callJson(self: Self, comptime request: anytype, comptime service_meta: anytype, action: anytype, options: Options) !FullResponse(request) {
+        // Target might be a problem. The smithy docs differ fairly significantly
+        // from the REST API examples. Here I'm following the REST API examples
+        // as they have not yet led me astray. Whether they're consistent
+        // across other services is another matter...
+        var version = try self.allocator.alloc(u8, service_meta.version.len);
+        defer self.allocator.free(version);
+        const replacements = std.mem.replace(u8, service_meta.version, "-", "", version);
+        // Resize the version, otherwise the junk at the end will mess with allocPrint
+        version = try self.allocator.resize(version, version.len - replacements);
+        const target =
+            try std.fmt.allocPrint(self.allocator, "{s}_{s}.{s}", .{
+            service_meta.sdk_id,
+            version,
+            action.action_name,
+        });
+        defer self.allocator.free(target);
+
+        var buffer = std.ArrayList(u8).init(self.allocator);
+        defer buffer.deinit();
+
+        // The transformer needs to allocate stuff out of band, but we
+        // can guarantee we don't need the memory after this call completes,
+        // so we'll use an arena allocator to whack everything.
+        // TODO: Determine if sending in null values is ok, or if we need another
+        //       tweak to the stringify function to exclude
+        var nameAllocator = std.heap.ArenaAllocator.init(self.allocator);
+        defer nameAllocator.deinit();
+        try json.stringify(request, .{ .whitespace = .{}, .allocator = &nameAllocator.allocator, .nameTransform = pascalTransformer }, buffer.writer());
+
+        var content_type: []const u8 = undefined;
+        switch (service_meta.aws_protocol) {
+            .rest_json_1 => content_type = "application/json",
+            .json_1_0 => content_type = "application/x-amz-json-1.0",
+            .json_1_1 => content_type = "application/x-amz-json-1.1",
+            else => unreachable,
+        }
+        return try self.callAws(request, service_meta, .{
+            .query = "",
+            .body = buffer.items,
+            .content_type = content_type,
+            .headers = &[_]awshttp.Header{.{ .name = "X-Amz-Target", .value = target }},
+        }, options);
+    }
+
     // Call using query protocol. This is documented as an XML protocol, but
     // throwing a JSON accept header seems to work. EC2Query is very simliar to
     // Query, so we'll handle both here. Realistically we probably don't effectively
     // handle lists and maps properly anyway yet, so we'll go for it and see
     // where it breaks. PRs and/or failing test cases appreciated.
-    fn callQuery(self: Self, comptime request: anytype, service_meta: anytype, action: anytype, options: Options) !FullResponse(request) {
+    fn callQuery(self: Self, comptime request: anytype, comptime service_meta: anytype, action: anytype, options: Options) !FullResponse(request) {
         var buffer = std.ArrayList(u8).init(self.allocator);
         defer buffer.deinit();
         const writer = buffer.writer();
@@ -103,31 +150,28 @@ pub const Aws = struct {
         else // EC2
             try std.fmt.allocPrint(self.allocator, "{s}", .{buffer.items});
         defer self.allocator.free(body);
+        return try self.callAws(request, service_meta, .{
+            .query = query,
+            .body = body,
+            .content_type = "application/x-www-form-urlencoded",
+        }, options);
+    }
 
+    fn callAws(self: Self, comptime request: anytype, comptime service_meta: anytype, aws_request: awshttp.HttpRequest, options: Options) !FullResponse(request) {
         const FullR = FullResponse(request);
         const response = try self.aws_http.callApi(
             service_meta.endpoint_prefix,
-            .{
-                .body = body,
-                .query = query,
-            },
+            aws_request,
             .{
                 .region = options.region,
                 .dualstack = options.dualstack,
                 .sigv4_service_name = service_meta.sigv4_name,
             },
         );
-        // TODO: Can response handling be reused?
         defer response.deinit();
+        // try self.reportTraffic("", aws_request, response, log.debug);
         if (response.response_code != 200) {
-            log.err("call failed! return status: {d}", .{response.response_code});
-            log.err("Request Query:\n  |{s}\n", .{query});
-            log.err("Request Body:\n  |{s}\n", .{body});
-
-            log.err("Response Headers:\n", .{});
-            for (response.headers) |h|
-                log.err("\t{s}:{s}\n", .{ h.name, h.value });
-            log.err("Response Body:\n  |{s}", .{response.body});
+            try self.reportTraffic("Call Failed", aws_request, response, log.err);
             return error.HttpFailure;
         }
         // EC2 ignores our accept type, but technically query protocol only
@@ -138,6 +182,10 @@ pub const Aws = struct {
             if (std.mem.eql(u8, "Content-Type", h.name)) {
                 if (std.mem.startsWith(u8, h.value, "application/json")) {
                     isJson = true;
+                } else if (std.mem.startsWith(u8, h.value, "application/x-amz-json-1.0")) {
+                    isJson = true;
+                } else if (std.mem.startsWith(u8, h.value, "application/x-amz-json-1.1")) {
+                    isJson = true;
                 } else if (std.mem.startsWith(u8, h.value, "text/xml")) {
                     isJson = false;
                 } else {
@@ -160,19 +208,53 @@ pub const Aws = struct {
             .allow_unknown_fields = true, // new option. Cannot yet handle non-struct fields though
             .allow_missing_fields = false, // new option. Cannot yet handle non-struct fields though
         };
-        const SResponse = ServerResponse(request);
+
+        // const SResponse = ServerResponse(request);
+        const SResponse = if (service_meta.aws_protocol != .query and service_meta.aws_protocol != .ec2_query)
+            Response(request)
+        else
+            ServerResponse(request);
+
         const parsed_response = json.parse(SResponse, &stream, parser_options) catch |e| {
             log.err(
                 \\Call successful, but unexpected response from service.
                 \\This could be the result of a bug or a stale set of code generated
-                \\service models. Response from server:
+                \\service models.
+                \\
+                \\Model Type: {s}
+                \\
+                \\Response from server:
                 \\
                 \\{s}
                 \\
-            , .{response.body});
+            , .{ SResponse, response.body });
             return e;
         };
 
+        if (service_meta.aws_protocol != .query and service_meta.aws_protocol != .ec2_query) {
+            var request_id: []u8 = undefined;
+            var found = false;
+            for (response.headers) |h| {
+                if (std.ascii.eqlIgnoreCase(h.name, "X-Amzn-RequestId")) {
+                    found = true;
+                    request_id = try std.fmt.allocPrint(self.allocator, "{s}", .{h.value}); // will be freed in FullR.deinit()
+                }
+            }
+            if (!found) {
+                try self.reportTraffic("Request ID not found", aws_request, response, log.err);
+                return error.RequestIdNotFound;
+            }
+
+            return FullR{
+                .response = parsed_response,
+                .response_metadata = .{
+                    .request_id = request_id,
+                },
+                .parser_options = parser_options,
+                .raw_parsed = .{ .raw = parsed_response },
+            };
+        }
+
         // Grab the first (and only) object from the server. Server shape expected to be:
         // { ActionResponse: {ActionResult: {...}, ResponseMetadata: {...} } }
         //                   ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -189,9 +271,36 @@ pub const Aws = struct {
                 .request_id = real_response.ResponseMetadata.RequestId,
             },
             .parser_options = parser_options,
-            .raw_parsed = parsed_response,
+            .raw_parsed = .{ .server = parsed_response },
         };
     }
+
+    fn reportTraffic(self: Self, info: []const u8, request: awshttp.HttpRequest, response: awshttp.HttpResult, comptime reporter: fn (comptime []const u8, anytype) void) !void {
+        var msg = std.ArrayList(u8).init(self.allocator);
+        defer msg.deinit();
+        const writer = msg.writer();
+        try writer.print("{s}\n\n", .{info});
+        try writer.print("Return status: {d}\n\n", .{response.response_code});
+        if (request.query.len > 0) try writer.print("Request Query:\n  \t{s}\n", .{request.query});
+        _ = try writer.write("Unique Request Headers:\n");
+        if (request.headers.len > 0) {
+            for (request.headers) |h|
+                try writer.print("\t{s}: {s}\n", .{ h.name, h.value });
+        }
+        try writer.print("\tContent-Type: {s}\n\n", .{request.content_type});
+
+        _ = try writer.write("Request Body:\n");
+        try writer.print("-------------\n{s}\n", .{request.body});
+        _ = try writer.write("-------------\n");
+        _ = try writer.write("Response Headers:\n");
+        for (response.headers) |h|
+            try writer.print("\t{s}: {s}\n", .{ h.name, h.value });
+
+        _ = try writer.write("Response Body:\n");
+        try writer.print("--------------\n{s}\n", .{response.body});
+        _ = try writer.write("--------------\n");
+        reporter("{s}\n", .{msg.items});
+    }
 };
 
 fn ServerResponse(comptime request: anytype) type {
@@ -250,11 +359,20 @@ fn FullResponse(comptime request: anytype) type {
             request_id: []u8,
         },
         parser_options: json.ParseOptions,
-        raw_parsed: ServerResponse(request),
+        raw_parsed: union(enum) {
+            server: ServerResponse(request),
+            raw: Response(request),
+        },
+        // raw_parsed: ServerResponse(request),
 
         const Self = @This();
         pub fn deinit(self: Self) void {
-            json.parseFree(ServerResponse(request), self.raw_parsed, self.parser_options);
+            switch (self.raw_parsed) {
+                .server => json.parseFree(ServerResponse(request), self.raw_parsed.server, self.parser_options),
+                .raw => json.parseFree(Response(request), self.raw_parsed.raw, self.parser_options),
+            }
+
+            self.parser_options.allocator.?.free(self.response_metadata.request_id);
         }
     };
 }
@@ -265,6 +383,9 @@ fn queryFieldTransformer(field_name: []const u8, encoding_options: url.EncodingO
     return try case.snakeToPascal(encoding_options.allocator.?, field_name);
 }
 
+fn pascalTransformer(field_name: []const u8, options: json.StringifyOptions) anyerror![]const u8 {
+    return try case.snakeToPascal(options.allocator.?, field_name);
+}
 // Use for debugging json responses of specific requests
 // test "dummy request" {
 //     const allocator = std.testing.allocator;

src/awshttp.zig

@@ -74,14 +74,15 @@ const SigningOptions = struct {
     service: []const u8,
 };
 
-const HttpRequest = struct {
+pub const HttpRequest = struct {
     path: []const u8 = "/",
     query: []const u8 = "",
     body: []const u8 = "",
     method: []const u8 = "POST",
-    // headers: []Header = .{},
+    content_type: []const u8 = "application/json", // Can we get away with this?
+    headers: []Header = &[_]Header{},
 };
-const HttpResult = struct {
+pub const HttpResult = struct {
     response_code: u16, // actually 3 digits can fit in u10
     body: []const u8,
     headers: []Header,
@@ -99,7 +100,7 @@ const HttpResult = struct {
     }
 };
 
-const Header = struct {
+pub const Header = struct {
     name: []const u8,
     value: []const u8,
 };
@@ -320,7 +321,7 @@ pub const AwsHttp = struct {
         var tls_connection_options: ?*c.aws_tls_connection_options = null;
         const host = try self.allocator.dupeZ(u8, endpoint.host);
         defer self.allocator.free(host);
-        try self.addHeaders(http_request.?, host, request.body);
+        try self.addHeaders(http_request.?, host, request.body, request.content_type, request.headers);
         if (std.mem.eql(u8, endpoint.scheme, "https")) {
             // TODO: Figure out why this needs to be inline vs function call
             // tls_connection_options = try self.setupTls(host);
@@ -631,7 +632,7 @@ pub const AwsHttp = struct {
         async_result.sync.store(false, .SeqCst);
     }
 
-    fn addHeaders(self: Self, request: *c.aws_http_message, host: []const u8, body: []const u8) !void {
+    fn addHeaders(self: Self, request: *c.aws_http_message, host: []const u8, body: []const u8, content_type: []const u8, additional_headers: []Header) !void {
         const accept_header = c.aws_http_header{
             .name = c.aws_byte_cursor_from_c_str("Accept"),
             .value = c.aws_byte_cursor_from_c_str("application/json"),
@@ -662,22 +663,37 @@ pub const AwsHttp = struct {
         // const accept_encoding_header = c.aws_http_header{
         //     .name = c.aws_byte_cursor_from_c_str("Accept-Encoding"),
         //     .value = c.aws_byte_cursor_from_c_str("identity"),
-        //     .compression = .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE,
+        //     .compression = 0, //.AWS_HTTP_HEADER_COMPRESSION_USE_CACHE,
         // };
         // if (c.aws_http_message_add_header(request, accept_encoding_header) != c.AWS_OP_SUCCESS)
         //     return AwsError.AddHeaderError;
 
         // AWS *does* seem to care about Content-Type. I don't think this header
         // will hold for all APIs
-        // TODO: Work out Content-type
+        const c_type = try std.fmt.allocPrintZ(self.allocator, "{s}", .{content_type});
+        defer self.allocator.free(c_type);
         const content_type_header = c.aws_http_header{
             .name = c.aws_byte_cursor_from_c_str("Content-Type"),
-            .value = c.aws_byte_cursor_from_c_str("application/x-www-form-urlencoded"),
+            .value = c.aws_byte_cursor_from_c_str(c_type),
             .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE,
         };
         if (c.aws_http_message_add_header(request, content_type_header) != c.AWS_OP_SUCCESS)
             return AwsError.AddHeaderError;
 
+        for (additional_headers) |h| {
+            const name = try std.fmt.allocPrintZ(self.allocator, "{s}", .{h.name});
+            defer self.allocator.free(name);
+            const value = try std.fmt.allocPrintZ(self.allocator, "{s}", .{h.value});
+            defer self.allocator.free(value);
+            const c_header = c.aws_http_header{
+                .name = c.aws_byte_cursor_from_c_str(name),
+                .value = c.aws_byte_cursor_from_c_str(value),
+                .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE,
+            };
+            if (c.aws_http_message_add_header(request, c_header) != c.AWS_OP_SUCCESS)
+                return AwsError.AddHeaderError;
+        }
+
         if (body.len > 0) {
             const len = try std.fmt.allocPrintZ(self.allocator, "{d}", .{body.len});
             // This defer seems to work ok, but I'm a bit concerned about why

src/json.zig

@@ -2656,6 +2656,15 @@ pub const StringifyOptions = struct {
 
     string: StringOptions = StringOptions{ .String = .{} },
 
+    nameTransform: fn ([]const u8, StringifyOptions) anyerror![]const u8 = nullTransform,
+
+    /// Not used by stringify - might be needed for your name transformer
+    allocator: ?*std.mem.Allocator = null,
+
+    fn nullTransform(name: []const u8, _: StringifyOptions) ![]const u8 {
+        return name;
+    }
+
     /// Should []u8 be serialised as a string? or an array?
     pub const StringOptions = union(enum) {
         Array,
@@ -2699,7 +2708,7 @@ pub fn stringify(
     value: anytype,
     options: StringifyOptions,
     out_stream: anytype,
-) @TypeOf(out_stream).Error!void {
+) !void {
     const T = @TypeOf(value);
     switch (@typeInfo(T)) {
         .Float, .ComptimeFloat => {
@@ -2768,7 +2777,10 @@ pub fn stringify(
                     try out_stream.writeByte('\n');
                     try child_whitespace.outputIndent(out_stream);
                 }
-                try stringify(Field.name, options, out_stream);
+                const name = child_options.nameTransform(Field.name, options) catch {
+                    return error.NameTransformationError;
+                };
+                try stringify(name, options, out_stream);
                 try out_stream.writeByte(':');
                 if (child_options.whitespace) |child_whitespace| {
                     if (child_whitespace.separator) {

src/main.zig

@@ -27,6 +27,8 @@ const Tests = enum {
     query_no_input,
     query_with_input,
     ec2_query_no_input,
+    json_1_0_query_with_input,
+    json_1_0_query_no_input,
 };
 
 pub fn main() anyerror!void {
@@ -65,7 +67,7 @@ pub fn main() anyerror!void {
     var client = aws.Aws.init(allocator);
     defer client.deinit();
 
-    const services = aws.Services(.{ .sts, .ec2 }){};
+    const services = aws.Services(.{ .sts, .ec2, .dynamo_db }){};
 
     for (tests.items) |t| {
         std.log.info("===== Start Test: {s} =====", .{@tagName(t)});
@@ -84,12 +86,27 @@ pub fn main() anyerror!void {
                     .duration_seconds = 900,
                 }, options);
                 defer access.deinit();
-                std.log.info("access key: {s}", .{access.response.credentials.access_key_id});
+                std.log.info("access key: {s}", .{access.response.credentials.?.access_key_id});
+            },
+            .json_1_0_query_with_input => {
+                // TODO: Find test without sensitive info
+                const tables = try client.call(services.dynamo_db.list_tables.Request{
+                    .limit = 1,
+                }, options);
+                defer tables.deinit();
+                std.log.info("request id: {s}", .{tables.response_metadata.request_id});
+                std.log.info("account has tables: {b}", .{tables.response.table_names.?.len > 0});
+            },
+            .json_1_0_query_no_input => {
+                const limits = try client.call(services.dynamo_db.describe_limits.Request{}, options);
+                defer limits.deinit();
+                std.log.info("account read capacity limit: {d}", .{limits.response.account_max_read_capacity_units});
             },
             .ec2_query_no_input => {
-                const instances = try client.call(services.ec2.describe_instances.Request{}, options);
-                defer instances.deinit();
-                std.log.info("reservation count: {d}", .{instances.response.reservations.len});
+                std.log.err("EC2 Test disabled due to compiler bug", .{});
+                // const instances = try client.call(services.ec2.describe_instances.Request{}, options);
+                // defer instances.deinit();
+                // std.log.info("reservation count: {d}", .{instances.response.reservations.len});
             },
         }
         std.log.info("===== End Test: {s} =====\n", .{@tagName(t)});