update aws models to 2024-02-28 release

README.md

@@ -3,8 +3,9 @@ AWS SDK for Zig
 
 [![Build Status](https://actions-status.lerch.org/lobo/aws-sdk-for-zig/build)](https://git.lerch.org/lobo/aws-sdk-for-zig/actions?workflow=build.yaml&state=closed)
 
-**NOTE: THIS SDK IS CURRENTLY UNUSABLE FOR S3 AND 8 OTHER SERVICES
-            WITHOUT A PROXY. SEE LIMITATIONS SECTION BELOW**
+**NOTE: TLS 1.3 support is still deploying across AWS. Some services, especially S3,
+        may or may not be available without a proxy, depending on the region.
+        See limitations section below**
 
 Current executable size for the demo is 980k after compiling with -Doptimize=ReleaseSmall
 in x86_linux, and will vary based on services used. Tested targets:
@@ -44,32 +45,15 @@ for working with services. For local testing or alternative endpoints, there's
 no real standard, so there is code to look for `AWS_ENDPOINT_URL` environment
 variable that will supersede all other configuration.
 
-Other branches
---------------
-
-The default branch is fully functional but requires TLS 1.3. Until AWS Services
-support TLS 1.3 at the end of 2023, the [0.9.0 branch](https://git.lerch.org/lobo/aws-sdk-for-zig/src/branch/0.9.0)
-may be of use. More details below in limitations. This branch overall is
-superior, as is the 0.11 compiler, but if you need a service that doesn't support
-TLS 1.3 and you need it right away, feel free to use that branch. Note I do not
-intend to update code in the 0.9.0 branch, but will accept PRs.
-
-An [old branch based on aws-crt](https://github.com/elerch/aws-sdk-for-zig/tree/aws-crt) exists
-for posterity, and supports x86_64 linux. The old branch is deprecated, so if
-there are issues you see that work correctly in the aws-crt branch, please
-file an issue. I can't think of a reason to use this branch any more. I do not
-intend to entertain PRs on this branch, but reach out if you think it is important.
-
 Limitations
 -----------
 
-The zig 0.11 HTTP client supports TLS 1.3 only. This, IMHO, is a reasonable
-restriction given its introduction 5 years ago, but is inflicting some short
-term pain on this project as AWS has not yet fully implemented the protocol. AWS has
-committed to [TLS 1.3 support across all services by the end of 2023](https://aws.amazon.com/blogs/security/faster-aws-cloud-connections-with-tls-1-3/), but many (most) services as of August 28th have not yet
-been upgraded. Proxy support has been added, so to get to the services that
+The zig 0.11 HTTP client supports TLS 1.3 only. AWS has committed to
+[TLS 1.3 support across all services by the end of 2023](https://aws.amazon.com/blogs/security/faster-aws-cloud-connections-with-tls-1-3/),
+but a few services as of February 28, 2024 have not been upgraded, and S3 is
+a bit intermittent. Proxy support has been added, so to get to the services that
 do not yet support TLS 1.3, you can use something like [mitmproxy](https://mitmproxy.org/)
-to proxy those requests. Of course, this is not a good production solution...
+to proxy those requests until roll out is complete.
 
 WebIdentityToken is not yet implemented.
 
@@ -87,30 +71,22 @@ TODO List:
 * Implement timeouts and other TODO's in the code
 * Add option to cache signature keys
 
-Compiler wishlist/watchlist:
-
-* [comptime allocations](https://github.com/ziglang/zig/issues/1291) so we can read files, etc (or is there another way)
-
-Services without TLS 1.3 support (9 services out of 255 total)
+Services without TLS 1.3 support (4 services out of 255 total)
 ---------------------------------------------------------------
 
 The following service list is based on limited testing against us-west-2
 region. Your mileage may vary, as there are thousands of endpoints against
-many regions, though it appears the TLS 1.3 rollout is fairly far along at
-this point, with the real remaining issue for most is in S3.
+many regions. It appears the TLS 1.3 rollout is fairly far along at
+this point.
 
 NOTE ON S3: For me, S3 is currently intermittently available using TLS 1.3, so
-it appears deployments are in progress.
+it appears deployments are in progress. The last couple days it has been
+available consistently, so I have removed it from the list.
 
 ```
-cloudsearch
 data.iot
 models.lex
 opsworks
-personalize-runtime
-runtime.lex
-runtime-v2-lex
-s3
 support
 ```
 

build.zig

@@ -3,8 +3,8 @@ const builtin = @import("builtin");
 const Builder = @import("std").build.Builder;
 const Package = @import("Package.zig");
 
-const models_url = "https://github.com/aws/aws-sdk-go-v2/archive/7502ff360b1c3b79cbe117437327f6ff5fb89f65.tar.gz";
-const models_hash: ?[]const u8 = "1220a414719bff14c9362fb1c695e3346fa12ec2e728bae5757a57aae7738916ffd2";
+const models_url = "https://github.com/aws/aws-sdk-go-v2/archive/58cf6509525a12d64fd826da883bfdbacbd2f00e.tar.gz";
+const models_hash: ?[]const u8 = "122017a2f3081ce83c23e0c832feb1b8b4176d507b6077f522855dc774bcf83ee315";
 const models_subdir = "codegen/sdk-codegen/aws-models/"; // note will probably not work on windows
 const models_dir = "p" ++ std.fs.path.sep_str ++ (models_hash orelse "") ++ std.fs.path.sep_str ++ models_subdir;
 

build.zig.zon

@@ -4,8 +4,8 @@
 
     .dependencies = .{
         .smithy = .{
-            .url = "https://git.lerch.org/lobo/smithy/archive/41b61745d25a65817209dd5dddbb5f9b66896a99.tar.gz",
-            .hash = "122087deb0ae309b2258d59b40d82fe5921fdfc35b420bb59033244851f7f276fa34",
+            .url = "https://git.lerch.org/lobo/smithy/archive/d6b6331defdfd33f36258caf26b0b82ce794cd28.tar.gz",
+            .hash = "1220695f5be11b7bd714f6181c60b0e590da5da7411de111ca51cacf1ea4a8169669",
         },
     },
 }

codegen/src/main.zig

@@ -249,12 +249,22 @@ fn addReference(id: []const u8, map: *std.StringHashMap(u64)) !void {
 }
 fn countAllReferences(shape_ids: [][]const u8, shapes: std.StringHashMap(smithy.ShapeInfo), shape_references: *std.StringHashMap(u64), stack: *std.ArrayList([]const u8)) anyerror!void {
     for (shape_ids) |id| {
-        try countReferences(shapes.get(id).?, shapes, shape_references, stack);
+        const shape = shapes.get(id);
+        if (shape == null) {
+            std.log.err("Error - could not find shape with id {s}", .{id});
+            return error.ShapeNotFound;
+        }
+        try countReferences(shape.?, shapes, shape_references, stack);
     }
 }
 fn countTypeMembersReferences(type_members: []smithy.TypeMember, shapes: std.StringHashMap(smithy.ShapeInfo), shape_references: *std.StringHashMap(u64), stack: *std.ArrayList([]const u8)) anyerror!void {
     for (type_members) |m| {
-        try countReferences(shapes.get(m.target).?, shapes, shape_references, stack);
+        const target = shapes.get(m.target);
+        if (target == null) {
+            std.log.err("Error - could not find target {s}", .{m.target});
+            return error.TargetNotFound;
+        }
+        try countReferences(target.?, shapes, shape_references, stack);
     }
 }
 
@@ -285,6 +295,7 @@ fn countReferences(shape: smithy.ShapeInfo, shapes: std.StringHashMap(smithy.Sha
         .bigInteger,
         .bigDecimal,
         .timestamp,
+        .unit,
         => {},
         .document, .member, .resource => {}, // less sure about these?
         .list => |i| try countReferences(shapes.get(i.member_target).?, shapes, shape_references, stack),
@@ -297,10 +308,25 @@ fn countReferences(shape: smithy.ShapeInfo, shapes: std.StringHashMap(smithy.Sha
         .uniontype => |m| try countTypeMembersReferences(m.members, shapes, shape_references, stack),
         .service => |i| try countAllReferences(i.operations, shapes, shape_references, stack),
         .operation => |op| {
-            if (op.input) |i| try countReferences(shapes.get(i).?, shapes, shape_references, stack);
-            if (op.output) |i| try countReferences(shapes.get(i).?, shapes, shape_references, stack);
+            if (op.input) |i| {
+                const val = shapes.get(i);
+                if (val == null) {
+                    std.log.err("Error processing shape with id \"{s}\". Input shape \"{s}\" was not found", .{ shape.id, i });
+                    return error.ShapeNotFound;
+                }
+                try countReferences(val.?, shapes, shape_references, stack);
+            }
+            if (op.output) |i| {
+                const val = shapes.get(i);
+                if (val == null) {
+                    std.log.err("Error processing shape with id \"{s}\". Output shape \"{s}\" was not found", .{ shape.id, i });
+                    return error.ShapeNotFound;
+                }
+                try countReferences(val.?, shapes, shape_references, stack);
+            }
             if (op.errors) |i| try countAllReferences(i, shapes, shape_references, stack);
         },
+        .@"enum" => |m| try countTypeMembersReferences(m.members, shapes, shape_references, stack),
     }
 }
 
@@ -347,8 +373,8 @@ fn generateServices(allocator: std.mem.Allocator, comptime _: []const u8, file:
         var sdk_id: []const u8 = undefined;
         const version: []const u8 = service.shape.service.version;
         const name: []const u8 = service.name;
-        var arn_namespace: []const u8 = undefined;
-        var sigv4_name: []const u8 = undefined;
+        var arn_namespace: ?[]const u8 = undefined;
+        var sigv4_name: ?[]const u8 = null;
         var endpoint_prefix: []const u8 = undefined;
         var aws_protocol: smithy.AwsProtocol = undefined;
         for (service.shape.service.traits) |trait| {
@@ -364,6 +390,11 @@ fn generateServices(allocator: std.mem.Allocator, comptime _: []const u8, file:
                 else => {},
             }
         }
+        if (sigv4_name == null) {
+            // This is true for CodeCatalyst, that operates a bit differently
+            std.log.debug("No sigv4 name found. Service '{s}' cannot be accessed via standard methods. Skipping", .{name});
+            continue;
+        }
 
         // Service struct
         // name of the field will be snake_case of whatever comes in from
@@ -373,18 +404,22 @@ fn generateServices(allocator: std.mem.Allocator, comptime _: []const u8, file:
         try writer.print("const Self = @This();\n", .{});
         try writer.print("pub const version: []const u8 = \"{s}\";\n", .{version});
         try writer.print("pub const sdk_id: []const u8 = \"{s}\";\n", .{sdk_id});
-        try writer.print("pub const arn_namespace: []const u8 = \"{s}\";\n", .{arn_namespace});
+        if (arn_namespace) |a| {
+            try writer.print("pub const arn_namespace: ?[]const u8 = \"{s}\";\n", .{a});
+        } else try writer.print("pub const arn_namespace: ?[]const u8 = null;\n", .{});
         try writer.print("pub const endpoint_prefix: []const u8 = \"{s}\";\n", .{endpoint_prefix});
-        try writer.print("pub const sigv4_name: []const u8 = \"{s}\";\n", .{sigv4_name});
+        try writer.print("pub const sigv4_name: []const u8 = \"{s}\";\n", .{sigv4_name.?});
         try writer.print("pub const name: []const u8 = \"{s}\";\n", .{name});
         // TODO: This really should just be ".whatevs". We're fully qualifying here, which isn't typical
         try writer.print("pub const aws_protocol: smithy.AwsProtocol = {};\n\n", .{aws_protocol});
         _ = try writer.write("pub const service_metadata: struct {\n");
         try writer.print("    version: []const u8 = \"{s}\",\n", .{version});
         try writer.print("    sdk_id: []const u8 = \"{s}\",\n", .{sdk_id});
-        try writer.print("    arn_namespace: []const u8 = \"{s}\",\n", .{arn_namespace});
+        if (arn_namespace) |a| {
+            try writer.print("    arn_namespace: ?[]const u8 = \"{s}\",\n", .{a});
+        } else try writer.print("    arn_namespace: ?[]const u8 = null,\n", .{});
         try writer.print("    endpoint_prefix: []const u8 = \"{s}\",\n", .{endpoint_prefix});
-        try writer.print("    sigv4_name: []const u8 = \"{s}\",\n", .{sigv4_name});
+        try writer.print("    sigv4_name: []const u8 = \"{s}\",\n", .{sigv4_name.?});
         try writer.print("    name: []const u8 = \"{s}\",\n", .{name});
         // TODO: This really should just be ".whatevs". We're fully qualifying here, which isn't typical
         try writer.print("    aws_protocol: smithy.AwsProtocol = {},\n", .{aws_protocol});
@@ -496,20 +531,26 @@ fn generateOperation(allocator: std.mem.Allocator, operation: smithy.ShapeInfo,
     try writer.print("action_name: []const u8 = \"{s}\",\n", .{operation.name});
     try outputIndent(state, writer);
     _ = try writer.write("Request: type = ");
-    if (operation.shape.operation.input) |member| {
+    if (operation.shape.operation.input == null or
+        (try shapeInfoForId(operation.shape.operation.input.?, state)).shape == .unit)
+    {
+        _ = try writer.write("struct {\n");
+        try generateMetadataFunction(operation_name, state, writer);
+    } else if (operation.shape.operation.input) |member| {
         if (try generateTypeFor(member, writer, state, false)) unreachable; // we expect only structs here
         _ = try writer.write("\n");
         try generateMetadataFunction(operation_name, state, writer);
-    } else {
-        _ = try writer.write("struct {\n");
-        try generateMetadataFunction(operation_name, state, writer);
     }
     _ = try writer.write(",\n");
     try outputIndent(state, writer);
     _ = try writer.write("Response: type = ");
-    if (operation.shape.operation.output) |member| {
+    if (operation.shape.operation.output == null or
+        (try shapeInfoForId(operation.shape.operation.output.?, state)).shape == .unit)
+    {
+        _ = try writer.write("struct {}"); // we want to maintain consistency with other ops
+    } else if (operation.shape.operation.output) |member| {
         if (try generateTypeFor(member, writer, state, true)) unreachable; // we expect only structs here
-    } else _ = try writer.write("struct {}"); // we want to maintain consistency with other ops
+    }
     _ = try writer.write(",\n");
 
     if (operation.shape.operation.errors) |errors| {
@@ -589,16 +630,21 @@ fn reuseCommonType(shape: smithy.ShapeInfo, writer: anytype, state: GenerationSt
     }
     return rc;
 }
+fn shapeInfoForId(id: []const u8, state: GenerationState) !smithy.ShapeInfo {
+    return state.file_state.shapes.get(id) orelse {
+        std.debug.print("Shape ID not found. This is most likely a bug. Shape ID: {s}\n", .{id});
+        return error.InvalidType;
+    };
+}
+
 /// return type is anyerror!void as this is a recursive function, so the compiler cannot properly infer error types
 fn generateTypeFor(shape_id: []const u8, writer: anytype, state: GenerationState, end_structure: bool) anyerror!bool {
     var rc = false;
 
     // We assume it must exist
-    const shape_info = state.file_state.shapes.get(shape_id) orelse {
-        std.debug.print("Shape ID not found. This is most likely a bug. Shape ID: {s}\n", .{shape_id});
-        return error.InvalidType;
-    };
+    const shape_info = try shapeInfoForId(shape_id, state);
     const shape = shape_info.shape;
+
     // Check for ourselves up the stack
     var self_occurences: u8 = 0;
     for (state.type_stack.items) |i| {
@@ -653,7 +699,12 @@ fn generateTypeFor(shape_id: []const u8, writer: anytype, state: GenerationState
                 _ = try writer.write("}");
             }
         },
+        // Document is unstructured data, so bag of bytes it is
+        // https://smithy.io/2.0/spec/simple-types.html#document
+        .document => |s| try generateSimpleTypeFor(s, "[]const u8", writer),
         .string => |s| try generateSimpleTypeFor(s, "[]const u8", writer),
+        .unit => |s| try generateSimpleTypeFor(s, "struct {}", writer), // Would be better as void, but doing so creates inconsistency we don't want clients to have to deal with
+        .@"enum" => |s| try generateSimpleTypeFor(s, "[]const u8", writer), // This should be closer to uniontype, but the generated code will look ugly, and Smithy 2.0 requires that enums are open (clients accept unspecified values). So string is the best analog
         .integer => |s| try generateSimpleTypeFor(s, "i64", writer),
         .list => {
             _ = try writer.write("[]");
@@ -908,5 +959,8 @@ fn avoidReserved(snake_name: []const u8) []const u8 {
     if (std.mem.eql(u8, snake_name, "or")) return "@\"or\"";
     if (std.mem.eql(u8, snake_name, "test")) return "@\"test\"";
     if (std.mem.eql(u8, snake_name, "null")) return "@\"null\"";
+    if (std.mem.eql(u8, snake_name, "export")) return "@\"export\"";
+    if (std.mem.eql(u8, snake_name, "union")) return "@\"union\"";
+    if (std.mem.eql(u8, snake_name, "enum")) return "@\"enum\"";
     return snake_name;
 }

codegen/src/snake.zig

@@ -28,10 +28,22 @@ pub fn fromPascalCase(allocator: std.mem.Allocator, name: []const u8) ![]u8 {
             //       prev_codepoint/ascii_prev_char (and target_inx)
             target_inx = setNext(lowercase(curr_char), rc, target_inx);
             target_inx = setNext('_', rc, target_inx);
-            curr_char = (try isAscii(utf8_name.nextCodepoint())).?;
+            var maybe_curr_char = (try isAscii(utf8_name.nextCodepoint()));
+            if (maybe_curr_char == null) {
+                std.log.err("Error on fromPascalCase processing name '{s}'", .{name});
+            }
+            curr_char = maybe_curr_char.?;
+            maybe_curr_char = (try isAscii(utf8_name.nextCodepoint()));
+            if (maybe_curr_char == null) {
+                // We have reached the end of the string (e.g. "Resource Explorer 2")
+                // We need to do this check before we setNext, so that we don't
+                // end up duplicating the last character
+                break;
+                // std.log.err("Error on fromPascalCase processing name '{s}', curr_char = '{}'", .{ name, curr_char });
+            }
             target_inx = setNext(lowercase(curr_char), rc, target_inx);
             prev_char = curr_char;
-            curr_char = (try isAscii(utf8_name.nextCodepoint())).?;
+            curr_char = maybe_curr_char.?;
             continue;
         }
         if (between(curr_char, 'A', 'Z')) {
@@ -60,6 +72,7 @@ pub fn fromPascalCase(allocator: std.mem.Allocator, name: []const u8) ![]u8 {
     target_inx = setNext(lowercase(curr_char), rc, target_inx);
 
     rc[target_inx] = 0;
+    _ = allocator.resize(rc, target_inx);
     return rc[0..target_inx];
 }
 
@@ -134,3 +147,11 @@ test "IoT 1Click Devices Service" {
     // turn into. Should it be iot_1click_... or iot_1_click...?
     try expectEqualStrings("iot_1_click_devices_service", snake_case);
 }
+test "Resource Explorer 2" {
+    const allocator = std.testing.allocator;
+    const snake_case = try fromPascalCase(allocator, "Resource Explorer 2");
+    defer allocator.free(snake_case);
+    // NOTE: There is some debate amoung humans about what this should
+    // turn into. Should it be iot_1click_... or iot_1_click...?
+    try expectEqualStrings("resource_explorer_2", snake_case);
+}