src/aws.zig

@@ -79,10 +79,21 @@ pub const Aws = struct {
         var buffer = std.ArrayList(u8).init(self.allocator);
         defer buffer.deinit();
         const writer = buffer.writer();
-        try url.encode(request, writer, .{
+        // TODO: transformation function should be refactored for operation
-            .field_name_transformer = &queryFieldTransformer,
+        // with a Writer passed in so we don't have to allocate
-            .allocator = self.allocator,
+        const transformer = struct {
-        });
+            allocator: *std.mem.Allocator,
+
+            const This = @This();
+
+            pub fn transform(this: This, name: []const u8) ![]const u8 {
+                return try case.snakeToPascal(this.allocator, name);
+            }
+            pub fn transform_deinit(this: This, name: []const u8) void {
+                this.allocator.free(name);
+            }
+        }{ .allocator = self.allocator };
+        try url.encode(request, writer, .{ .field_name_transformer = transformer });
         const continuation = if (buffer.items.len > 0) "&" else "";
 
         const body = try std.fmt.allocPrint(self.allocator, "Action={s}&Version={s}{s}{s}\n", .{ action.action_name, service.version, continuation, buffer.items });
@@ -104,7 +115,6 @@ pub const Aws = struct {
             log.err("Request:\n  |{s}\nResponse:\n  |{s}", .{ body, response.body });
             return error.HttpFailure;
         }
-        // log.debug("Successful return from server:\n  |{s}", .{response.body});
         // TODO: Check status code for badness
         var stream = json.TokenStream.init(response.body);
 
@@ -113,20 +123,9 @@ pub const Aws = struct {
             .allow_camel_case_conversion = true, // new option
             .allow_snake_case_conversion = true, // new option
             .allow_unknown_fields = true, // new option. Cannot yet handle non-struct fields though
-            .allow_missing_fields = false, // new option. Cannot yet handle non-struct fields though
         };
         const SResponse = ServerResponse(request);
-        const parsed_response = json.parse(SResponse, &stream, parser_options) catch |e| {
+        const parsed_response = try json.parse(SResponse, &stream, parser_options);
-            log.err(
-                \\Call successful, but unexpected response from service.
-                \\This could be the result of a bug or a stale set of code generated
-                \\service models. Response from server:
-                \\
-                \\{s}
-                \\
-            , .{response.body});
-            return e;
-        };
 
         // Grab the first (and only) object from the server. Server shape expected to be:
         // { ActionResponse: {ActionResult: {...}, ResponseMetadata: {...} } }
@@ -216,29 +215,3 @@ fn FullResponse(comptime request: anytype) type {
 fn Response(comptime request: anytype) type {
     return request.metaInfo().action.Response;
 }
-fn queryFieldTransformer(field_name: []const u8, encoding_options: url.EncodingOptions) anyerror![]const u8 {
-    return try case.snakeToPascal(encoding_options.allocator.?, field_name);
-}
-
-// Use for debugging json responses of specific requests
-// test "dummy request" {
-//     const allocator = std.testing.allocator;
-//     const svs = Services(.{.sts}){};
-//     const request = svs.sts.get_session_token.Request{
-//         .duration_seconds = 900,
-//     };
-//     const FullR = FullResponse(request);
-//     const response =
-//     var stream = json.TokenStream.init(response);
-//
-//     const parser_options = json.ParseOptions{
-//         .allocator = allocator,
-//         .allow_camel_case_conversion = true, // new option
-//         .allow_snake_case_conversion = true, // new option
-//         .allow_unknown_fields = true, // new option. Cannot yet handle non-struct fields though
-//         .allow_missing_fields = false, // new option. Cannot yet handle non-struct fields though
-//     };
-//     const SResponse = ServerResponse(request);
-//     const r = try json.parse(SResponse, &stream, parser_options);
-//     json.parseFree(SResponse, r, parser_options);
-// }

src/json.zig

@@ -1454,7 +1454,6 @@ pub const ParseOptions = struct {
     allow_camel_case_conversion: bool = false,
     allow_snake_case_conversion: bool = false,
     allow_unknown_fields: bool = false,
-    allow_missing_fields: bool = false,
 };
 
 fn camelCaseComp(field: []const u8, key: []const u8, options: ParseOptions) !bool {
@@ -1472,17 +1471,6 @@ fn camelCaseComp(field: []const u8, key: []const u8, options: ParseOptions) !boo
     }
     return std.mem.eql(u8, field, key);
 }
-
-test "snake" {
-    const allocator = testing.allocator;
-    const options = ParseOptions{
-        .allocator = allocator,
-        .allow_camel_case_conversion = true,
-        .allow_snake_case_conversion = true,
-        .allow_unknown_fields = true,
-    };
-    try std.testing.expect(try snakeCaseComp("access_key_id", "AccessKeyId", options));
-}
 fn snakeCaseComp(field: []const u8, key: []const u8, options: ParseOptions) !bool {
     // snake case is much more intricate. Input:
     // Field: user_id
@@ -1492,8 +1480,10 @@ fn snakeCaseComp(field: []const u8, key: []const u8, options: ParseOptions) !boo
     // Then compare
     var found: u32 = 0;
     for (field) |ch| {
-        if (ch == '_')
+        if (ch == '_') {
             found = found + 1;
+            break;
+        }
     }
     if (found == 0)
         return std.mem.eql(u8, field, key);
@@ -1525,11 +1515,6 @@ fn snakeCaseComp(field: []const u8, key: []const u8, options: ParseOptions) !boo
         }
         inx = inx + 1;
     }
-    // std.debug.print("comp_field, len {d}: {s}\n", .{ comp_field.len, comp_field });
-    // std.debug.print("normalized_key, len {d}: {s}\n", .{ normalized_key.len, normalized_key });
-    // std.debug.print("comp_field, last: {d}\n", .{comp_field[comp_field.len - 1]});
-    // std.debug.print("normalized_key, last: {d}\n", .{normalized_key[normalized_key.len - 1]});
-
     return std.mem.eql(u8, comp_field, normalized_key);
 }
 
@@ -1554,13 +1539,8 @@ fn parseInternal(comptime T: type, token: Token, tokens: *TokenStream, options:
                 .Number => |n| n,
                 else => return error.UnexpectedToken,
             };
-            // This is a bug. you can still potentially have an integer that has exponents
+            if (!numberToken.is_integer) return error.UnexpectedToken;
-            // if (!numberToken.is_integer) return error.UnexpectedToken;
+            return try std.fmt.parseInt(T, numberToken.slice(tokens.slice, tokens.i - 1), 10);
-            if (numberToken.is_integer)
-                return try std.fmt.parseInt(T, numberToken.slice(tokens.slice, tokens.i - 1), 10);
-            const float = try std.fmt.parseFloat(f128, numberToken.slice(tokens.slice, tokens.i - 1));
-            if (std.math.round(float) != float) return error.InvalidNumber;
-            return @floatToInt(T, float);
         },
         .Optional => |optionalInfo| {
             if (token == .Null) {
@@ -1699,8 +1679,7 @@ fn parseInternal(comptime T: type, token: Token, tokens: *TokenStream, options:
                             @field(r, field.name) = default;
                         }
                     } else {
-                        if (!options.allow_missing_fields)
+                        return error.MissingField;
-                            return error.MissingField;
                     }
                 }
             }
@@ -2018,11 +1997,6 @@ test "parse into struct with no fields" {
     const T = struct {};
     try testing.expectEqual(T{}, try parse(T, &TokenStream.init("{}"), ParseOptions{}));
 }
-test "parse exponential into int" {
-    const T = struct { int: i64 };
-    const r = try parse(T, &TokenStream.init("{ \"int\": 4.2e2 }"), ParseOptions{});
-    try testing.expectEqual(@as(i64, 420), r.int);
-}
 
 test "parse into struct with misc fields" {
     @setEvalBranchQuota(10000);

src/main.zig

@@ -2,8 +2,6 @@ const std = @import("std");
 const aws = @import("aws.zig");
 const json = @import("json.zig");
 
-var verbose = false;
-
 pub fn log(
     comptime level: std.log.Level,
     comptime scope: @TypeOf(.EnumLiteral),
@@ -11,7 +9,7 @@ pub fn log(
     args: anytype,
 ) void {
     // Ignore awshttp messages
-    if (!verbose and scope == .awshttp and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
+    if (scope == .awshttp and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
         return;
     const scope_prefix = "(" ++ @tagName(scope) ++ "): ";
     const prefix = "[" ++ @tagName(level) ++ "] " ++ scope_prefix;
@@ -23,90 +21,60 @@ pub fn log(
     nosuspend stderr.print(prefix ++ format ++ "\n", args) catch return;
 }
 
-const Tests = enum {
-    query_no_input,
-    query_with_input,
-    ec2_query_no_input,
-};
-
 pub fn main() anyerror!void {
+    // Uncomment if you want to log allocations
+    // const file = try std.fs.cwd().createFile("/tmp/allocations.log", .{ .truncate = true });
+    // defer file.close();
+    // var child_allocator = std.heap.c_allocator;
+    // const allocator = &std.heap.loggingAllocator(child_allocator, file.writer()).allocator;
+
+    // Flip to true to run a second time. This will help debug
+    // allocation/deallocation issues
+    const test_twice = false;
+
+    // Flip to true to run through the json parsing changes made to stdlib
+    const test_json = false;
+    if (test_json) try jsonFun();
+
     const c_allocator = std.heap.c_allocator;
     var gpa = std.heap.GeneralPurposeAllocator(.{}){
         .backing_allocator = c_allocator,
     };
-    defer _ = gpa.deinit();
+    defer if (!gpa.deinit()) @panic("memory leak detected");
     const allocator = &gpa.allocator;
-    var tests = std.ArrayList(Tests).init(allocator);
+    // const allocator = std.heap.c_allocator;
-    defer tests.deinit();
-    var args = std.process.args();
-    while (args.next(allocator)) |arg_or_error| {
-        const arg = try arg_or_error;
-        defer allocator.free(arg);
-        if (std.mem.eql(u8, "-v", arg)) {
-            verbose = true;
-            continue;
-        }
-        inline for (@typeInfo(Tests).Enum.fields) |f| {
-            if (std.mem.eql(u8, f.name, arg)) {
-                try tests.append(@field(Tests, f.name));
-                break;
-            }
-        }
-    }
-    if (tests.items.len == 0) {
-        inline for (@typeInfo(Tests).Enum.fields) |f|
-            try tests.append(@field(Tests, f.name));
-    }
 
     const options = aws.Options{
         .region = "us-west-2",
     };
-    std.log.info("Start\n", .{});
+    std.log.info("Start", .{});
+
     var client = aws.Aws.init(allocator);
     defer client.deinit();
-
     const services = aws.Services(.{.sts}){};
+    const resp = try client.call(services.sts.get_caller_identity.Request{}, options);
+    // TODO: This is a bit wonky. Root cause is lack of declarations in
+    //       comptime-generated types
+    defer resp.deinit();
 
-    for (tests.items) |t| {
+    if (test_twice) {
-        std.log.info("===== Start Test: {s} =====", .{@tagName(t)});
+        std.time.sleep(1000 * std.time.ns_per_ms);
-        switch (t) {
+        std.log.info("second request", .{});
-            .query_no_input => {
+
-                const resp = try client.call(services.sts.get_caller_identity.Request{}, options);
+        var client2 = aws.Aws.init(allocator);
-                defer resp.deinit();
+        defer client2.deinit();
-                std.log.info("arn: {s}", .{resp.response.arn});
+        const resp2 = try client2.call(services.sts.get_caller_identity.Request{}, options); // catch here and try alloc?
-                std.log.info("id: {s}", .{resp.response.user_id});
+        defer resp2.deinit();
-                std.log.info("account: {s}", .{resp.response.account});
-                std.log.info("requestId: {s}", .{resp.response_metadata.request_id});
-            },
-            .query_with_input => {
-                // TODO: Find test without sensitive info
-                const access = try client.call(services.sts.get_session_token.Request{
-                    .duration_seconds = 900,
-                }, options);
-                defer access.deinit();
-                std.log.info("access key: {s}", .{access.response.credentials.access_key_id});
-            },
-            .ec2_query_no_input => {
-                // TODO: Find test
-            },
-        }
-        std.log.info("===== End Test: {s} =====\n", .{@tagName(t)});
     }
 
-    // if (test_twice) {
+    std.log.info("arn: {s}", .{resp.response.arn});
-    //     std.time.sleep(1000 * std.time.ns_per_ms);
+    std.log.info("id: {s}", .{resp.response.user_id});
-    //     std.log.info("second request", .{});
+    std.log.info("account: {s}", .{resp.response.account});
-    //
+    std.log.info("requestId: {s}", .{resp.response_metadata.request_id});
-    //     var client2 = aws.Aws.init(allocator);
-    //     defer client2.deinit();
-    //     const resp2 = try client2.call(services.sts.get_caller_identity.Request{}, options); // catch here and try alloc?
-    //     defer resp2.deinit();
-    // }
 
-    std.log.info("===== Tests complete =====", .{});
+    std.log.info("Departing main", .{});
 }
 
-// TODO: Move into json.zig
 pub fn jsonFun() !void {
     // Standard behavior
     const payload =

src/url.zig

@@ -1,81 +1,29 @@
 const std = @import("std");
 
-fn defaultTransformer(field_name: []const u8, options: EncodingOptions) anyerror![]const u8 {
+pub fn encode(obj: anytype, writer: anytype, options: anytype) !void {
-    return field_name;
+    try encodeStruct("", obj, writer, options);
 }
 
-pub const FieldNameTransformer = fn ([]const u8, EncodingOptions) anyerror![]const u8;
+fn encodeStruct(parent: []const u8, obj: anytype, writer: anytype, options: anytype) !void {
-
+    var first = true;
-pub const EncodingOptions = struct {
-    allocator: ?*std.mem.Allocator = null,
-    field_name_transformer: *const FieldNameTransformer = &defaultTransformer,
-};
-
-pub fn encode(obj: anytype, writer: anytype, options: EncodingOptions) !void {
-    _ = try encodeInternal("", "", true, obj, writer, options);
-}
-
-fn encodeStruct(parent: []const u8, first: bool, obj: anytype, writer: anytype, options: EncodingOptions) !bool {
-    var rc = first;
     inline for (@typeInfo(@TypeOf(obj)).Struct.fields) |field| {
-        const field_name = try options.field_name_transformer.*(field.name, options);
+        const field_name = if (@hasField(@TypeOf(options), "field_name_transformer")) try options.field_name_transformer.transform(field.name) else field.name;
-        defer if (options.field_name_transformer.* != defaultTransformer)
+        defer {
-            if (options.allocator) |a| a.free(field_name);
+            if (@hasField(@TypeOf(options), "field_name_transformer"))
-        // @compileLog(@typeInfo(field.field_type).Pointer);
+                options.field_name_transformer.transform_deinit(field_name);
-        rc = try encodeInternal(parent, field_name, rc, @field(obj, field.name), writer, options);
+        }
+        if (!first) _ = try writer.write("&");
+        switch (@typeInfo(field.field_type)) {
+            .Struct => {
+                try encodeStruct(field_name ++ ".", @field(obj, field.name), writer);
+            },
+            else => try writer.print("{s}{s}={s}", .{ parent, field_name, @field(obj, field.name) }),
+        }
+        first = false;
     }
-    return rc;
 }
 
-pub fn encodeInternal(parent: []const u8, field_name: []const u8, first: bool, obj: anytype, writer: anytype, options: EncodingOptions) !bool {
+fn testencode(expected: []const u8, value: anytype, options: anytype) !void {
-    // @compileLog(@typeInfo(@TypeOf(obj)));
-    var rc = first;
-    switch (@typeInfo(@TypeOf(obj))) {
-        .Optional => if (obj) |o| {
-            rc = try encodeInternal(parent, field_name, first, o, writer, options);
-        },
-        .Pointer => |ti| if (ti.size == .One) {
-            rc = try encodeInternal(parent, field_name, first, obj.*, writer, options);
-        } else {
-            if (!first) _ = try writer.write("&");
-            try writer.print("{s}{s}={s}", .{ parent, field_name, obj });
-            rc = false;
-        },
-        .Struct => if (std.mem.eql(u8, "", field_name)) {
-            rc = try encodeStruct(parent, first, obj, writer, options);
-        } else {
-            // TODO: It would be lovely if we could concat at compile time or allocPrint at runtime
-            // XOR have compile time allocator support. Alas, neither are possible:
-            // https://github.com/ziglang/zig/issues/868: Comptime detection (feels like foot gun)
-            // https://github.com/ziglang/zig/issues/1291: Comptime allocator
-            const allocator = options.allocator orelse return error.AllocatorRequired;
-            const new_parent = try std.fmt.allocPrint(allocator, "{s}{s}.", .{ parent, field_name });
-            defer allocator.free(new_parent);
-            rc = try encodeStruct(new_parent, first, obj, writer, options);
-            // try encodeStruct(parent ++ field_name ++ ".", first, obj,  writer, options);
-        },
-        .Array => {
-            if (!first) _ = try writer.write("&");
-            try writer.print("{s}{s}={s}", .{ parent, field_name, obj });
-            rc = false;
-        },
-        .Int, .ComptimeInt, .Float, .ComptimeFloat => {
-            if (!first) _ = try writer.write("&");
-            try writer.print("{s}{s}={d}", .{ parent, field_name, obj });
-            rc = false;
-        },
-        // BUGS! any doesn't work - a lot. Check this out:
-        // https://github.com/ziglang/zig/blob/master/lib/std/fmt.zig#L424
-        else => {
-            if (!first) _ = try writer.write("&");
-            try writer.print("{s}{s}={any}", .{ parent, field_name, obj });
-            rc = false;
-        },
-    }
-    return rc;
-}
-
-fn testencode(expected: []const u8, value: anytype, options: EncodingOptions) !void {
     const ValidationWriter = struct {
         const Self = @This();
         pub const Writer = std.io.Writer(*Self, Error, write);
@@ -95,7 +43,7 @@ fn testencode(expected: []const u8, value: anytype, options: EncodingOptions) !v
         }
 
         fn write(self: *Self, bytes: []const u8) Error!usize {
-            // std.debug.print("{s}\n", .{bytes});
+            // std.debug.print("{s}", .{bytes});
             if (self.expected_remaining.len < bytes.len) {
                 std.debug.warn(
                     \\====== expected this output: =========
@@ -132,46 +80,17 @@ fn testencode(expected: []const u8, value: anytype, options: EncodingOptions) !v
     if (vos.expected_remaining.len > 0) return error.NotEnoughData;
 }
 
-test "can urlencode an object" {
+test "can url encode an object" {
     try testencode(
         "Action=GetCallerIdentity&Version=2021-01-01",
         .{ .Action = "GetCallerIdentity", .Version = "2021-01-01" },
         .{},
     );
 }
-test "can urlencode an object with integer" {
+test "can url encode a complex object" {
-    try testencode(
-        "Action=GetCallerIdentity&Duration=32",
-        .{ .Action = "GetCallerIdentity", .Duration = 32 },
-        .{},
-    );
-}
-const UnsetValues = struct {
-    action: ?[]const u8 = null,
-    duration: ?i64 = null,
-    val1: ?i64 = null,
-    val2: ?[]const u8 = null,
-};
-test "can urlencode an object with unset values" {
-    // var buffer = std.ArrayList(u8).init(std.testing.allocator);
-    // defer buffer.deinit();
-    // const writer = buffer.writer();
-    // try encode(
-    //     UnsetValues{ .action = "GetCallerIdentity", .duration = 32 },
-    //     writer,
-    //     .{ .allocator = std.testing.allocator },
-    // );
-    // std.debug.print("{s}", .{buffer.items});
-    try testencode(
-        "action=GetCallerIdentity&duration=32",
-        UnsetValues{ .action = "GetCallerIdentity", .duration = 32 },
-        .{},
-    );
-}
-test "can urlencode a complex object" {
     try testencode(
         "Action=GetCallerIdentity&Version=2021-01-01&complex.innermember=foo",
         .{ .Action = "GetCallerIdentity", .Version = "2021-01-01", .complex = .{ .innermember = "foo" } },
-        .{ .allocator = std.testing.allocator },
+        .{},
     );
 }