support for EC2Query protocol requests (response is XML and not yet implemented)

build.zig

@@ -17,6 +17,12 @@ pub fn build(b: *Builder) !void {
     // https://github.com/ziglang/zig/issues/855
     exe.addPackagePath("smithy", "smithy/src/smithy.zig");
 
+    // This bitfield workaround will end up requiring a bunch of headers that
+    // currently mean building in the docker container is the best way to build
+    // TODO: Determine if it's a good idea to copy these files out of our
+    // docker container to the local fs so we can just build even outside
+    // the container. And maybe, just maybe these even get committed to
+    // source control?
     exe.addCSourceFile("src/bitfield-workaround.c", &[_][]const u8{"-std=c99"});
     const c_include_dirs = .{
         "./src/",
@@ -54,7 +60,10 @@ pub fn build(b: *Builder) !void {
     // https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.321+15a030ef3.tar.xz
     exe.linkage = .static;
 
-    const is_strip = b.option(bool, "strip", "strip exe") orelse true;
+    // TODO: Strip doesn't actually fully strip the executable. If we're on
+    //       linux we can run strip on the result, probably at the expense
+    //       of busting cache logic
+    const is_strip = b.option(bool, "strip", "strip exe [true]") orelse true;
     exe.strip = is_strip;
 
     const run_cmd = exe.run();
@@ -84,10 +93,19 @@ pub fn build(b: *Builder) !void {
     }
 
     // TODO: Support > linux
-    // TODO: Get a better cache in place
     if (std.builtin.os.tag == .linux) {
         const codegen = b.step("gen", "Generate zig service code from smithy models");
         codegen.dependOn(&b.addSystemCommand(&.{ "/bin/sh", "-c", "cd codegen && zig build" }).step);
+        // Since codegen binary is built every time, if it's newer than our
+        // service manifest we know it needs to be regenerated. So this step
+        // will remove the service manifest if codegen has been touched, thereby
+        // triggering the re-gen
+        codegen.dependOn(&b.addSystemCommand(&.{
+            "/bin/sh", "-c",
+            \\ [ ! -f src/models/service_manifest.zig ] || \
+            \\ [ src/models/service_manifest.zig -nt codegen/codegen ] || \
+            \\ rm src/models/service_manifest.zig
+        }).step);
         codegen.dependOn(&b.addSystemCommand(&.{
             "/bin/sh", "-c",
             \\ mkdir -p src/models/ && \

codegen/src/main.zig

@@ -106,7 +106,7 @@ fn generateServices(allocator: *std.mem.Allocator, comptime _: []const u8, file:
         // Service struct
         // name of the field will be snake_case of whatever comes in from
         // sdk_id. Not sure this will simple...
-        const constant_name = try snake.fromPascalCase(allocator, sdk_id);
+        const constant_name = try constantName(allocator, sdk_id);
         try constant_names.append(constant_name);
         try writer.print("const Self = @This();\n", .{});
         try writer.print("pub const version: []const u8 = \"{s}\";\n", .{version});
@@ -132,6 +132,26 @@ fn generateServices(allocator: *std.mem.Allocator, comptime _: []const u8, file:
     }
     return constant_names.toOwnedSlice();
 }
+fn constantName(allocator: *std.mem.Allocator, id: []const u8) ![]const u8 {
+    // There are some ids that don't follow consistent rules, so we'll
+    // look for the exceptions and, if not found, revert to the snake case
+    // algorithm
+
+    // This one might be a bug in snake, but it's the only example so HPDL
+    if (std.mem.eql(u8, id, "SESv2")) return try std.fmt.allocPrint(allocator, "ses_v2", .{});
+    // IoT is an acryonym, but snake wouldn't know that. Interestingly not all
+    // iot services are capitalizing that way.
+    if (std.mem.eql(u8, id, "IoTSiteWise")) return try std.fmt.allocPrint(allocator, "iot_site_wise", .{}); //sitewise?
+    if (std.mem.eql(u8, id, "IoTFleetHub")) return try std.fmt.allocPrint(allocator, "iot_fleet_hub", .{});
+    if (std.mem.eql(u8, id, "IoTSecureTunneling")) return try std.fmt.allocPrint(allocator, "iot_secure_tunneling", .{});
+    if (std.mem.eql(u8, id, "IoTThingsGraph")) return try std.fmt.allocPrint(allocator, "iot_things_graph", .{});
+    // snake turns this into dev_ops, which is a little weird
+    if (std.mem.eql(u8, id, "DevOps Guru")) return try std.fmt.allocPrint(allocator, "devops_guru", .{});
+    if (std.mem.eql(u8, id, "FSx")) return try std.fmt.allocPrint(allocator, "fsx", .{});
+
+    // Not a special case - just snake it
+    return try snake.fromPascalCase(allocator, id);
+}
 fn generateOperation(allocator: *std.mem.Allocator, operation: smithy.ShapeInfo, shapes: anytype, writer: anytype, service: []const u8) !void {
     const snake_case_name = try snake.fromPascalCase(allocator, operation.name);
     defer allocator.free(snake_case_name);

codegen/src/snake.zig

@@ -2,59 +2,97 @@ const std = @import("std");
 const expectEqualStrings = std.testing.expectEqualStrings;
 
 pub fn fromPascalCase(allocator: *std.mem.Allocator, name: []const u8) ![]u8 {
+    const rc = try allocator.alloc(u8, name.len * 2); // This is overkill, but is > the maximum length possibly needed
+    errdefer allocator.free(rc);
     var utf8_name = (std.unicode.Utf8View.init(name) catch unreachable).iterator();
     var target_inx: u64 = 0;
-    var previous_codepoint: ?u21 = null;
-    var cp = utf8_name.nextCodepoint();
-    if (cp == null) {
-        return try allocator.dupeZ(u8, name);
-    } // TODO: fix bug if single letter uppercase
-    var codepoint = cp.?;
-    const rc = try allocator.alloc(u8, name.len * 2); // This is overkill, but is > the maximum length possibly needed
-    while (utf8_name.nextCodepoint()) |next_codepoint| {
-        if (codepoint > 0xff) return error{UnicodeNotSupported}.UnicodeNotSupported;
-        if (next_codepoint > 0xff) return error{UnicodeNotSupported}.UnicodeNotSupported;
-        const ascii_char = @truncate(u8, codepoint);
-        if (next_codepoint == ' ') continue; // ignore all spaces in name
-        if (ascii_char >= 'A' and ascii_char < 'Z') {
-            const lowercase_char = ascii_char + ('a' - 'A');
-            if (previous_codepoint == null) {
-                rc[target_inx] = lowercase_char;
-                target_inx = target_inx + 1;
+    var curr_char = (try isAscii(utf8_name.nextCodepoint())).?;
+    target_inx = setNext(lowercase(curr_char), rc, target_inx);
+    var prev_char = curr_char;
+    if (try isAscii(utf8_name.nextCodepoint())) |ch| {
+        curr_char = ch;
     } else {
-                if (next_codepoint >= 'A' and next_codepoint <= 'Z' and previous_codepoint.? >= 'A' and previous_codepoint.? <= 'Z') {
+        // Single character only - we're done here
+        _ = setNext(0, rc, target_inx);
+        return rc[0..target_inx];
+    }
+    while (try isAscii(utf8_name.nextCodepoint())) |next_char| {
+        if (next_char == ' ') {
+            // a space shouldn't be happening. But if it does, it clues us
+            // in pretty well:
+            //
+            // MyStuff Is Awesome
+            //       |^
+            //       |next_char
+            //       ^
+            //       prev_codepoint/ascii_prev_char (and target_inx)
+            target_inx = setNext(lowercase(curr_char), rc, target_inx);
+            target_inx = setNext('_', rc, target_inx);
+            curr_char = (try isAscii(utf8_name.nextCodepoint())).?;
+            target_inx = setNext(lowercase(curr_char), rc, target_inx);
+            prev_char = curr_char;
+            curr_char = (try isAscii(utf8_name.nextCodepoint())).?;
+            continue;
+        }
+        if (between(curr_char, 'A', 'Z')) {
+            if (isAcronym(curr_char, next_char)) {
+                // We could be in an acronym at the start of a word. This
+                // is the only case where we actually need to look back at the
+                // previous character, and if that's the case, throw in an
+                // underscore
+                // "SAMLMySAMLAcronymThing");
+                if (between(prev_char, 'a', 'z'))
+                    target_inx = setNext('_', rc, target_inx);
+
                 //we are in an acronym - don't snake, just lower
-                    rc[target_inx] = lowercase_char;
-                    target_inx = target_inx + 1;
+                target_inx = setNext(lowercase(curr_char), rc, target_inx);
             } else {
-                    rc[target_inx] = '_';
-                    rc[target_inx + 1] = lowercase_char;
-                    target_inx = target_inx + 2;
-                }
+                target_inx = setNext('_', rc, target_inx);
+                target_inx = setNext(lowercase(curr_char), rc, target_inx);
             }
         } else {
-            // if (ascii_char == ' ') {
-            //     rc[target_inx] = '_';
-            // } else {
-            rc[target_inx] = ascii_char;
-            // }
-            target_inx = target_inx + 1;
+            target_inx = setNext(curr_char, rc, target_inx);
         }
-        previous_codepoint = codepoint;
-        codepoint = next_codepoint;
+        prev_char = curr_char;
+        curr_char = next_char;
     }
     // work in the last codepoint - force lowercase
-    rc[target_inx] = @truncate(u8, codepoint);
-    if (rc[target_inx] >= 'A' and rc[target_inx] <= 'Z') {
-        const lowercase_char = rc[target_inx] + ('a' - 'A');
-        rc[target_inx] = lowercase_char;
-    }
-    target_inx = target_inx + 1;
+    target_inx = setNext(lowercase(curr_char), rc, target_inx);
 
     rc[target_inx] = 0;
     return rc[0..target_inx];
 }
 
+fn isAcronym(char1: u8, char2: u8) bool {
+    return isAcronymChar(char1) and isAcronymChar(char2);
+}
+fn isAcronymChar(char: u8) bool {
+    return between(char, 'A', 'Z') or between(char, '0', '9');
+}
+fn isAscii(codepoint: ?u21) !?u8 {
+    if (codepoint) |cp| {
+        if (cp > 0xff) return error.UnicodeNotSupported;
+        return @truncate(u8, cp);
+    }
+    return null;
+}
+
+fn setNext(ascii: u8, slice: []u8, inx: u64) u64 {
+    slice[inx] = ascii;
+    return inx + 1;
+}
+
+fn lowercase(ascii: u8) u8 {
+    var lowercase_char = ascii;
+    if (between(ascii, 'A', 'Z'))
+        lowercase_char = ascii + ('a' - 'A');
+    return lowercase_char;
+}
+
+fn between(char: u8, from: u8, to: u8) bool {
+    return char >= from and char <= to;
+}
+
 test "converts from PascalCase to snake_case" {
     const allocator = std.testing.allocator;
     const snake_case = try fromPascalCase(allocator, "MyPascalCaseThing");
@@ -73,3 +111,26 @@ test "spaces in the name" {
     defer allocator.free(snake_case);
     try expectEqualStrings("api_gateway", snake_case);
 }
+
+test "S3" {
+    const allocator = std.testing.allocator;
+    const snake_case = try fromPascalCase(allocator, "S3");
+    defer allocator.free(snake_case);
+    try expectEqualStrings("s3", snake_case);
+}
+
+test "ec2" {
+    const allocator = std.testing.allocator;
+    const snake_case = try fromPascalCase(allocator, "EC2");
+    defer allocator.free(snake_case);
+    try expectEqualStrings("ec2", snake_case);
+}
+
+test "IoT 1Click Devices Service" {
+    const allocator = std.testing.allocator;
+    const snake_case = try fromPascalCase(allocator, "IoT 1Click Devices Service");
+    defer allocator.free(snake_case);
+    // NOTE: There is some debate amoung humans about what this should
+    // turn into. Should it be iot_1click_... or iot_1_click...?
+    try expectEqualStrings("iot_1_click_devices_service", snake_case);
+}

src/aws.zig

@@ -84,12 +84,33 @@ pub const Aws = struct {
         });
         const continuation = if (buffer.items.len > 0) "&" else "";
 
-        const body = try std.fmt.allocPrint(self.allocator, "Action={s}&Version={s}{s}{s}\n", .{ action.action_name, service_meta.version, continuation, buffer.items });
+        const query = if (service_meta.aws_protocol == .query)
+            try std.fmt.allocPrint(self.allocator, "", .{})
+        else // EC2
+            try std.fmt.allocPrint(self.allocator, "?Action={s}&Version={s}", .{
+                action.action_name,
+                service_meta.version,
+            });
+        defer self.allocator.free(query);
+
+        const body = if (service_meta.aws_protocol == .query)
+            try std.fmt.allocPrint(self.allocator, "Action={s}&Version={s}{s}{s}", .{
+                action.action_name,
+                service_meta.version,
+                continuation,
+                buffer.items,
+            })
+        else // EC2
+            try std.fmt.allocPrint(self.allocator, "{s}", .{buffer.items});
         defer self.allocator.free(body);
+
         const FullR = FullResponse(request);
         const response = try self.aws_http.callApi(
             service_meta.endpoint_prefix,
-            body,
+            .{
+                .body = body,
+                .query = query,
+            },
             .{
                 .region = options.region,
                 .dualstack = options.dualstack,
@@ -100,11 +121,36 @@ pub const Aws = struct {
         defer response.deinit();
         if (response.response_code != 200) {
             log.err("call failed! return status: {d}", .{response.response_code});
-            log.err("Request:\n  |{s}\nResponse:\n  |{s}", .{ body, response.body });
+            log.err("Request Query:\n  |{s}\n", .{query});
+            log.err("Request Body:\n  |{s}\n", .{body});
+
+            log.err("Response Headers:\n", .{});
+            for (response.headers) |h|
+                log.err("\t{s}:{s}\n", .{ h.name, h.value });
+            log.err("Response Body:\n  |{s}", .{response.body});
             return error.HttpFailure;
         }
-        // log.debug("Successful return from server:\n  |{s}", .{response.body});
-        // TODO: Check status code for badness
+        // EC2 ignores our accept type, but technically query protocol only
+        // returns XML as well. So, we'll ignore the protocol here and just
+        // look at the return type
+        var isJson: bool = undefined;
+        for (response.headers) |h| {
+            if (std.mem.eql(u8, "Content-Type", h.name)) {
+                if (std.mem.startsWith(u8, h.value, "application/json")) {
+                    isJson = true;
+                } else if (std.mem.startsWith(u8, h.value, "text/xml")) {
+                    isJson = false;
+                } else {
+                    log.err("Unexpected content type: {s}", .{h.value});
+                    return error.UnexpectedContentType;
+                }
+                break;
+            }
+        }
+
+        // TODO: Handle XML
+        if (!isJson) return error.XmlUnimplemented;
+
         var stream = json.TokenStream.init(response.body);
 
         const parser_options = json.ParseOptions{

src/awshttp.zig

@@ -74,13 +74,26 @@ const SigningOptions = struct {
     service: []const u8,
 };
 
+const HttpRequest = struct {
+    path: []const u8 = "/",
+    query: []const u8 = "",
+    body: []const u8 = "",
+    method: []const u8 = "POST",
+    // headers: []Header = .{},
+};
 const HttpResult = struct {
     response_code: u16, // actually 3 digits can fit in u10
     body: []const u8,
+    headers: []Header,
     allocator: *std.mem.Allocator,
 
     pub fn deinit(self: HttpResult) void {
         self.allocator.free(self.body);
+        for (self.headers) |h| {
+            self.allocator.free(h.name);
+            self.allocator.free(h.value);
+        }
+        self.allocator.free(self.headers);
         httplog.debug("http result deinit complete", .{});
         return;
     }
@@ -235,16 +248,15 @@ pub const AwsHttp = struct {
     /// It will calculate the appropriate endpoint and action parameters for the
     /// service called, and will set up the signing options. The return
     /// value is simply a raw HttpResult
-    pub fn callApi(self: Self, service: []const u8, body: []const u8, options: Options) !HttpResult {
+    pub fn callApi(self: Self, service: []const u8, request: HttpRequest, options: Options) !HttpResult {
         const endpoint = try regionSubDomain(self.allocator, service, options.region, options.dualstack);
         defer endpoint.deinit();
         httplog.debug("Calling endpoint {s}", .{endpoint.uri});
-        httplog.debug("Body\n====\n{s}\n====", .{body});
         const signing_options: SigningOptions = .{
             .region = options.region,
             .service = if (options.sigv4_service_name) |name| name else service,
         };
-        return try self.makeRequest(endpoint, "POST", "/", body, signing_options);
+        return try self.makeRequest(endpoint, request, signing_options);
     }
 
     /// makeRequest is a low level http/https function that can be used inside
@@ -265,15 +277,20 @@ pub const AwsHttp = struct {
     /// Return value is an HttpResult, which will need the caller to deinit().
     /// HttpResult currently contains the body only. The addition of Headers
     /// and return code would be a relatively minor change
-    pub fn makeRequest(self: Self, endpoint: EndPoint, method: []const u8, path: []const u8, body: []const u8, signing_options: ?SigningOptions) !HttpResult {
+    pub fn makeRequest(self: Self, endpoint: EndPoint, request: HttpRequest, signing_options: ?SigningOptions) !HttpResult {
         // Since we're going to pass these into C-land, we need to make sure
         // our inputs have sentinals
-        const method_z = try self.allocator.dupeZ(u8, method);
+        const method_z = try self.allocator.dupeZ(u8, request.method);
         defer self.allocator.free(method_z);
-        const path_z = try self.allocator.dupeZ(u8, path);
+        // Path contains both path and query
+        const path_z = try std.fmt.allocPrintZ(self.allocator, "{s}{s}", .{ request.path, request.query });
         defer self.allocator.free(path_z);
-        const body_z = try self.allocator.dupeZ(u8, body);
+        const body_z = try self.allocator.dupeZ(u8, request.body);
         defer self.allocator.free(body_z);
+        httplog.debug("Path: {s}", .{path_z});
+        httplog.debug("Method: {s}", .{request.method});
+        httplog.debug("body length: {d}", .{request.body.len});
+        httplog.debug("Body\n====\n{s}\n====", .{request.body});
         // TODO: Try to re-encapsulate this
         // var http_request = try createRequest(method, path, body);
 
@@ -287,13 +304,11 @@ pub const AwsHttp = struct {
         if (c.aws_http_message_set_request_path(http_request, c.aws_byte_cursor_from_c_str(@ptrCast([*c]const u8, path_z))) != c.AWS_OP_SUCCESS)
             return AwsError.SetRequestPathError;
 
-        httplog.debug("body length: {d}", .{body.len});
         const body_cursor = c.aws_byte_cursor_from_c_str(@ptrCast([*c]const u8, body_z));
         const request_body = c.aws_input_stream_new_from_cursor(c_allocator, &body_cursor);
         defer c.aws_input_stream_destroy(request_body);
-        if (body.len > 0) {
+        if (request.body.len > 0)
             c.aws_http_message_set_body_stream(http_request, request_body);
-        }
 
         // End CreateRequest. This should return a struct with a deinit function that can do
         // destroys, etc
@@ -305,7 +320,7 @@ pub const AwsHttp = struct {
         var tls_connection_options: ?*c.aws_tls_connection_options = null;
         const host = try self.allocator.dupeZ(u8, endpoint.host);
         defer self.allocator.free(host);
-        try self.addHeaders(http_request.?, host, body);
+        try self.addHeaders(http_request.?, host, request.body);
         if (std.mem.eql(u8, endpoint.scheme, "https")) {
             // TODO: Figure out why this needs to be inline vs function call
             // tls_connection_options = try self.setupTls(host);
@@ -449,6 +464,7 @@ pub const AwsHttp = struct {
         const rc = HttpResult{
             .response_code = context.response_code.?,
             .body = final_body,
+            .headers = context.headers.?.toOwnedSlice(),
             .allocator = self.allocator,
         };
         return rc;
@@ -956,12 +972,12 @@ const RequestContext = struct {
     pub fn appendToBody(self: *Self, fragment: []const u8) !void {
         var orig_body: []const u8 = "";
         if (self.body) |b| {
-            orig_body = try self.allocator.dupeZ(u8, b);
+            orig_body = try self.allocator.dupe(u8, b);
             self.allocator.free(b);
             self.body = null;
         }
         defer self.allocator.free(orig_body);
-        self.body = try std.fmt.allocPrintZ(self.allocator, "{s}{s}", .{ orig_body, fragment });
+        self.body = try std.fmt.allocPrint(self.allocator, "{s}{s}", .{ orig_body, fragment });
     }
 
     pub fn addHeader(self: *Self, name: []const u8, value: []const u8) !void {

src/case.zig

@@ -5,7 +5,8 @@ pub fn snakeToCamel(allocator: *std.mem.Allocator, name: []const u8) ![]u8 {
     var utf8_name = (std.unicode.Utf8View.init(name) catch unreachable).iterator();
     var target_inx: u64 = 0;
     var previous_ascii: u8 = 0;
-    const rc = try allocator.alloc(u8, name.len); // This is slightly overkill, will need <= number of input chars
+    // A single word will take the entire length plus our sentinel
+    const rc = try allocator.alloc(u8, name.len + 1);
     while (utf8_name.nextCodepoint()) |cp| {
         if (cp > 0xff) return error.UnicodeNotSupported;
         const ascii_char = @truncate(u8, cp);
@@ -38,3 +39,9 @@ test "converts from snake to camelCase" {
     defer allocator.free(camel);
     try expectEqualStrings("accessKeyId", camel);
 }
+test "single word" {
+    const allocator = std.testing.allocator;
+    const camel = try snakeToCamel(allocator, "word");
+    defer allocator.free(camel);
+    try expectEqualStrings("word", camel);
+}

src/json.zig

@@ -1625,6 +1625,8 @@ fn parseInternal(comptime T: type, token: Token, tokens: *TokenStream, options:
             var r: T = undefined;
             var fields_seen = [_]bool{false} ** structInfo.fields.len;
             errdefer {
+                // TODO: why so high here? This was needed for ec2 describe instances
+                @setEvalBranchQuota(100000);
                 inline for (structInfo.fields) |field, i| {
                     if (fields_seen[i] and !field.is_comptime) {
                         parseFree(field.field_type, @field(r, field.name), options);

src/main.zig

@@ -65,7 +65,7 @@ pub fn main() anyerror!void {
     var client = aws.Aws.init(allocator);
     defer client.deinit();
 
-    const services = aws.Services(.{.sts}){};
+    const services = aws.Services(.{ .sts, .ec2 }){};
 
     for (tests.items) |t| {
         std.log.info("===== Start Test: {s} =====", .{@tagName(t)});
@@ -87,7 +87,9 @@ pub fn main() anyerror!void {
                 std.log.info("access key: {s}", .{access.response.credentials.access_key_id});
             },
             .ec2_query_no_input => {
-                // TODO: Find test
+                const instances = try client.call(services.ec2.describe_instances.Request{}, options);
+                defer instances.deinit();
+                std.log.info("reservation count: {d}", .{instances.response.reservations.len});
             },
         }
         std.log.info("===== End Test: {s} =====\n", .{@tagName(t)});