From 8727a4e03810f0ccbbd0b6b3020efe5eac337a0d Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Wed, 16 Feb 2022 14:14:54 -0800 Subject: [PATCH] EC2 support --- README.md | 22 ++++------- src/aws.zig | 93 ++++++++++++++++++++++++++++++++++++++-------- src/main.zig | 24 ++++++++---- src/xml.zig | 2 +- src/xml_shaper.zig | 1 + 5 files changed, 103 insertions(+), 39 deletions(-) diff --git a/README.md b/README.md index 88b4309..650a399 100644 --- a/README.md +++ b/README.md @@ -2,15 +2,13 @@ [![Build Status](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/status.svg?ref=refs/heads/master)](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/) -This SDK currently supports all AWS services except EC2 and S3. These two -services only support XML, and more work is needed to parse and integrate -type hydration from the base parsing. S3 also requires some plumbing tweaks -in the signature calculation. Examples of usage are in src/main.zig. +This SDK currently supports all AWS services except services using the restXml +protocol (4 services including S3). See TODO list below. -Current executable size for the demo is 953k (90k of which is the AWS PEM file) -after compiling with -Drelease-safe and +Current executable size for the demo is 1.6M (90k of which is the AWS PEM file, +and approximately 600K for XML services) after compiling with -Drelease-safe and [stripping the executable after compilation](https://github.com/ziglang/zig/issues/351). -This is for x86_linux. Tested targets: +This is for x86_linux, and will vary based on services used. Tested targets: * x86_64-linux * riscv64-linux @@ -41,8 +39,7 @@ require passing in a client option to specify an different TLS root certificate (pass null to disable certificate verification). The [old branch](https://github.com/elerch/aws-sdk-for-zig/tree/aws-crt) exists -for posterity, and supports x86_64 linux. This branch is recommended moving -forward. +for posterity, and supports x86_64 linux. The old branch is deprecated. ## Limitations @@ -52,13 +49,8 @@ implemented. TODO List: -* Complete integration of Xml responses with remaining code base * Implement [AWS restXml protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-restxml-protocol.html). - Includes S3. Total service count 4. This may be blocked due to the same issue as EC2. -* Implement [AWS EC2 query protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-ec2-query-protocol.html). - Includes EC2. Total service count 1. This may be blocked on a compiler bug, - though has not been tested with zig 0.9.0. More details and llvm ir log can be found in the - [XML branch](https://git.lerch.org/lobo/aws-sdk-for-zig/src/branch/xml). + Includes S3. Total service count 4. * Implement sigv4a signing * Implement jitter/exponential backoff * Implement timeouts and other TODO's in the code diff --git a/src/aws.zig b/src/aws.zig index 4418156..e519433 100644 --- a/src/aws.zig +++ b/src/aws.zig @@ -5,7 +5,7 @@ const json = @import("json.zig"); const url = @import("url.zig"); const case = @import("case.zig"); const servicemodel = @import("servicemodel.zig"); -// const xml_shaper = @import("xml_shaper.zig"); +const xml_shaper = @import("xml_shaper.zig"); const log = std.log.scoped(.aws); @@ -175,8 +175,6 @@ pub fn Request(comptime action: anytype) type { // handle lists and maps properly anyway yet, so we'll go for it and see // where it breaks. PRs and/or failing test cases appreciated. fn callQuery(request: ActionRequest, options: Options) !FullResponseType { - if (Self.service_meta.aws_protocol == .ec2_query) - @compileError("XML responses from EC2 blocked due to zig compiler bug scheduled to be fixed no earlier than 0.10.0"); var buffer = std.ArrayList(u8).init(options.client.allocator); defer buffer.deinit(); const writer = buffer.writer(); @@ -250,10 +248,9 @@ pub fn Request(comptime action: anytype) type { } } - // TODO: Handle XML - if (!isJson) return error.XmlUnimplemented; + if (!isJson) return try xmlReturn(options, response); - const SResponse = if (Self.service_meta.aws_protocol != .query and Self.service_meta.aws_protocol != .ec2_query) + const SResponse = if (Self.service_meta.aws_protocol != .query) action.Response else ServerResponse(action); @@ -272,7 +269,7 @@ pub fn Request(comptime action: anytype) type { .response_metadata = .{ .request_id = try requestIdFromHeaders(aws_request, response, options), }, - .parser_options = parser_options, + .parser_options = .{ .json = parser_options }, .raw_parsed = .{ .raw = .{} }, }; @@ -294,13 +291,25 @@ pub fn Request(comptime action: anytype) type { return e; }; - if (Self.service_meta.aws_protocol != .query and Self.service_meta.aws_protocol != .ec2_query) { + // TODO: Figure out this hack + // the code setting the response about 10 lines down will trigger + // an error because the first field may not be a struct when + // XML processing is happening above, which we only know at runtime. + // + // We could simply force .ec2_query and .rest_xml above rather than + // isJson, but it would be nice to automatically support json if + // these services start returning that like we'd like them to. + // + // Otherwise, the compiler gets down here thinking this will be + // processed. If it is, then we have a problem when the field name + // may not be a struct. + if (Self.service_meta.aws_protocol != .query or Self.service_meta.aws_protocol == .ec2_query) { return FullResponseType{ .response = parsed_response, .response_metadata = .{ .request_id = try requestIdFromHeaders(aws_request, response, options), }, - .parser_options = parser_options, + .parser_options = .{ .json = parser_options }, .raw_parsed = .{ .raw = parsed_response }, }; } @@ -320,10 +329,53 @@ pub fn Request(comptime action: anytype) type { .response_metadata = .{ .request_id = try options.client.allocator.dupe(u8, real_response.ResponseMetadata.RequestId), }, - .parser_options = parser_options, + .parser_options = .{ .json = parser_options }, .raw_parsed = .{ .server = parsed_response }, }; } + + fn xmlReturn(options: Options, result: awshttp.HttpResult) !FullResponseType { + // Server shape be all like: + // + // + // + // 0efe31c6-cad5-4882-b275-dfea478cf039 + // + // + // eu-north-1 + // ec2.eu-north-1.amazonaws.com + // opt-in-not-required + // + // + // + // + // While our stuff be like: + // + // struct { + // regions: []struct { + // region_name: []const u8, + // } + // } + // + // Big thing is that requestid, which we'll need to fetch "manually" + const xml_options = xml_shaper.ParseOptions{ .allocator = options.client.allocator }; + const parsed = try xml_shaper.parse(action.Response, result.body, xml_options); + // This needs to get into FullResponseType somehow: defer parsed.deinit(); + const request_id = blk: { + if (parsed.document.root.getCharData("requestId")) |elem| + break :blk elem; + return error.RequestIdNotFound; + }; + + return FullResponseType{ + .response = parsed.parsed_value, + .response_metadata = .{ + .request_id = try options.client.allocator.dupe(u8, request_id), + }, + .parser_options = .{ .xml = xml_options }, + .raw_parsed = .{ .xml = parsed }, + }; + } }; } @@ -397,21 +449,32 @@ fn FullResponse(comptime action: anytype) type { response_metadata: struct { request_id: []u8, }, - parser_options: json.ParseOptions, + parser_options: union(enum) { + json: json.ParseOptions, + xml: xml_shaper.ParseOptions, + }, raw_parsed: union(enum) { server: ServerResponse(action), raw: action.Response, + xml: xml_shaper.Parsed(action.Response), }, - // raw_parsed: ServerResponse(request), const Self = @This(); pub fn deinit(self: Self) void { switch (self.raw_parsed) { - .server => json.parseFree(ServerResponse(action), self.raw_parsed.server, self.parser_options), - .raw => json.parseFree(action.Response, self.raw_parsed.raw, self.parser_options), + // Server is json only (so far) + .server => json.parseFree(ServerResponse(action), self.raw_parsed.server, self.parser_options.json), + // Raw is json only (so far) + .raw => json.parseFree(action.Response, self.raw_parsed.raw, self.parser_options.json), + .xml => |xml| xml.deinit(), } - self.parser_options.allocator.?.free(self.response_metadata.request_id); + var allocator: std.mem.Allocator = undefined; + switch (self.parser_options) { + .json => |j| allocator = j.allocator.?, + .xml => |x| allocator = x.allocator.?, + } + allocator.free(self.response_metadata.request_id); } }; } diff --git a/src/main.zig b/src/main.zig index 43f0c97..4946920 100644 --- a/src/main.zig +++ b/src/main.zig @@ -14,6 +14,15 @@ pub fn log( // Ignore aws_signing messages if (verbose < 2 and scope == .aws_signing and @enumToInt(level) >= @enumToInt(std.log.Level.debug)) return; + // Ignore aws_credentials messages + if (verbose < 2 and scope == .aws_credentials and @enumToInt(level) >= @enumToInt(std.log.Level.debug)) + return; + // Ignore xml_shaper messages + if (verbose < 2 and scope == .xml_shaper and @enumToInt(level) >= @enumToInt(std.log.Level.debug)) + return; + // Ignore date messages + if (verbose < 2 and scope == .date and @enumToInt(level) >= @enumToInt(std.log.Level.debug)) + return; // Ignore awshttp messages if (verbose < 1 and scope == .awshttp and @enumToInt(level) >= @enumToInt(std.log.Level.debug)) return; @@ -169,18 +178,17 @@ pub fn main() anyerror!void { std.log.err("no functions to work with", .{}); } }, - // TODO: This test fails with broken LLVM module .ec2_query_no_input => { - std.log.err("EC2 Test disabled due to compiler bug", .{}); // Describe regions is a simpler request and easier to debug - // const instances = try client.call(services.ec2.describe_regions.Request{}, options); - // defer instances.deinit(); - // std.log.info("region count: {d}", .{instances.response.regions.?.len}); + const result = try client.call(services.ec2.describe_regions.Request{}, options); + defer result.deinit(); + std.log.info("request id: {s}", .{result.response_metadata.request_id}); + std.log.info("region count: {d}", .{result.response.regions.?.len}); // Describe instances is more interesting - // const instances = try client.call(services.ec2.describe_instances.Request{}, options); - // defer instances.deinit(); - // std.log.info("reservation count: {d}", .{instances.response.reservations.len}); + const instances = try client.call(services.ec2.describe_instances.Request{}, options); + defer instances.deinit(); + std.log.info("reservation count: {d}", .{instances.response.reservations.?.len}); }, } std.log.info("===== End Test: {s} =====\n", .{@tagName(t)}); diff --git a/src/xml.zig b/src/xml.zig index 9f8d063..a8d6b49 100644 --- a/src/xml.zig +++ b/src/xml.zig @@ -45,7 +45,7 @@ pub const Element = struct { } pub fn getCharData(self: *Element, child_tag: []const u8) ?[]const u8 { - const child = self.findChildByTag(child_tag) orelse return null; + const child = (self.findChildByTag(child_tag) catch return null) orelse return null; if (child.children.items.len != 1) { return null; } diff --git a/src/xml_shaper.zig b/src/xml_shaper.zig index 30cb356..c8228e7 100644 --- a/src/xml_shaper.zig +++ b/src/xml_shaper.zig @@ -242,6 +242,7 @@ fn parseInternal(comptime T: type, element: *xml.Element, options: ParseOptions) if (@typeInfo(field.field_type) == .Optional and !found_value) { // @compileLog("Optional: Field name ", field.name, ", type ", field.field_type); @field(r, field.name) = null; + fields_set = fields_set + 1; found_value = true; } // Using this else clause breaks zig, so we'll use a boolean instead