EC2 support
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Emil Lerch 2022-02-16 14:14:54 -08:00
parent 714e7278fd
commit 8727a4e038
Signed by: lobo
GPG Key ID: A7B62D657EF764F8
5 changed files with 103 additions and 39 deletions

View File

@ -2,15 +2,13 @@
[![Build Status](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/status.svg?ref=refs/heads/master)](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/)
This SDK currently supports all AWS services except EC2 and S3. These two
services only support XML, and more work is needed to parse and integrate
type hydration from the base parsing. S3 also requires some plumbing tweaks
in the signature calculation. Examples of usage are in src/main.zig.
This SDK currently supports all AWS services except services using the restXml
protocol (4 services including S3). See TODO list below.
Current executable size for the demo is 953k (90k of which is the AWS PEM file)
after compiling with -Drelease-safe and
Current executable size for the demo is 1.6M (90k of which is the AWS PEM file,
and approximately 600K for XML services) after compiling with -Drelease-safe and
[stripping the executable after compilation](https://github.com/ziglang/zig/issues/351).
This is for x86_linux. Tested targets:
This is for x86_linux, and will vary based on services used. Tested targets:
* x86_64-linux
* riscv64-linux
@ -41,8 +39,7 @@ require passing in a client option to specify an different TLS root certificate
(pass null to disable certificate verification).
The [old branch](https://github.com/elerch/aws-sdk-for-zig/tree/aws-crt) exists
for posterity, and supports x86_64 linux. This branch is recommended moving
forward.
for posterity, and supports x86_64 linux. The old branch is deprecated.
## Limitations
@ -52,13 +49,8 @@ implemented.
TODO List:
* Complete integration of Xml responses with remaining code base
* Implement [AWS restXml protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-restxml-protocol.html).
Includes S3. Total service count 4. This may be blocked due to the same issue as EC2.
* Implement [AWS EC2 query protocol](https://awslabs.github.io/smithy/1.0/spec/aws/aws-ec2-query-protocol.html).
Includes EC2. Total service count 1. This may be blocked on a compiler bug,
though has not been tested with zig 0.9.0. More details and llvm ir log can be found in the
[XML branch](https://git.lerch.org/lobo/aws-sdk-for-zig/src/branch/xml).
Includes S3. Total service count 4.
* Implement sigv4a signing
* Implement jitter/exponential backoff
* Implement timeouts and other TODO's in the code

View File

@ -5,7 +5,7 @@ const json = @import("json.zig");
const url = @import("url.zig");
const case = @import("case.zig");
const servicemodel = @import("servicemodel.zig");
// const xml_shaper = @import("xml_shaper.zig");
const xml_shaper = @import("xml_shaper.zig");
const log = std.log.scoped(.aws);
@ -175,8 +175,6 @@ pub fn Request(comptime action: anytype) type {
// handle lists and maps properly anyway yet, so we'll go for it and see
// where it breaks. PRs and/or failing test cases appreciated.
fn callQuery(request: ActionRequest, options: Options) !FullResponseType {
if (Self.service_meta.aws_protocol == .ec2_query)
@compileError("XML responses from EC2 blocked due to zig compiler bug scheduled to be fixed no earlier than 0.10.0");
var buffer = std.ArrayList(u8).init(options.client.allocator);
defer buffer.deinit();
const writer = buffer.writer();
@ -250,10 +248,9 @@ pub fn Request(comptime action: anytype) type {
}
}
// TODO: Handle XML
if (!isJson) return error.XmlUnimplemented;
if (!isJson) return try xmlReturn(options, response);
const SResponse = if (Self.service_meta.aws_protocol != .query and Self.service_meta.aws_protocol != .ec2_query)
const SResponse = if (Self.service_meta.aws_protocol != .query)
action.Response
else
ServerResponse(action);
@ -272,7 +269,7 @@ pub fn Request(comptime action: anytype) type {
.response_metadata = .{
.request_id = try requestIdFromHeaders(aws_request, response, options),
},
.parser_options = parser_options,
.parser_options = .{ .json = parser_options },
.raw_parsed = .{ .raw = .{} },
};
@ -294,13 +291,25 @@ pub fn Request(comptime action: anytype) type {
return e;
};
if (Self.service_meta.aws_protocol != .query and Self.service_meta.aws_protocol != .ec2_query) {
// TODO: Figure out this hack
// the code setting the response about 10 lines down will trigger
// an error because the first field may not be a struct when
// XML processing is happening above, which we only know at runtime.
//
// We could simply force .ec2_query and .rest_xml above rather than
// isJson, but it would be nice to automatically support json if
// these services start returning that like we'd like them to.
//
// Otherwise, the compiler gets down here thinking this will be
// processed. If it is, then we have a problem when the field name
// may not be a struct.
if (Self.service_meta.aws_protocol != .query or Self.service_meta.aws_protocol == .ec2_query) {
return FullResponseType{
.response = parsed_response,
.response_metadata = .{
.request_id = try requestIdFromHeaders(aws_request, response, options),
},
.parser_options = parser_options,
.parser_options = .{ .json = parser_options },
.raw_parsed = .{ .raw = parsed_response },
};
}
@ -320,10 +329,53 @@ pub fn Request(comptime action: anytype) type {
.response_metadata = .{
.request_id = try options.client.allocator.dupe(u8, real_response.ResponseMetadata.RequestId),
},
.parser_options = parser_options,
.parser_options = .{ .json = parser_options },
.raw_parsed = .{ .server = parsed_response },
};
}
fn xmlReturn(options: Options, result: awshttp.HttpResult) !FullResponseType {
// Server shape be all like:
//
// <?xml version="1.0" encoding="UTF-8"?>
// <DescribeRegionsResponse xmlns="http://ec2.amazonaws.com/doc/2016-11-15/">
// <requestId>0efe31c6-cad5-4882-b275-dfea478cf039</requestId>
// <regionInfo>
// <item>
// <regionName>eu-north-1</regionName>
// <regionEndpoint>ec2.eu-north-1.amazonaws.com</regionEndpoint>
// <optInStatus>opt-in-not-required</optInStatus>
// </item>
// </regionInfo>
// </DescribeRegionsResponse>
//
// While our stuff be like:
//
// struct {
// regions: []struct {
// region_name: []const u8,
// }
// }
//
// Big thing is that requestid, which we'll need to fetch "manually"
const xml_options = xml_shaper.ParseOptions{ .allocator = options.client.allocator };
const parsed = try xml_shaper.parse(action.Response, result.body, xml_options);
// This needs to get into FullResponseType somehow: defer parsed.deinit();
const request_id = blk: {
if (parsed.document.root.getCharData("requestId")) |elem|
break :blk elem;
return error.RequestIdNotFound;
};
return FullResponseType{
.response = parsed.parsed_value,
.response_metadata = .{
.request_id = try options.client.allocator.dupe(u8, request_id),
},
.parser_options = .{ .xml = xml_options },
.raw_parsed = .{ .xml = parsed },
};
}
};
}
@ -397,21 +449,32 @@ fn FullResponse(comptime action: anytype) type {
response_metadata: struct {
request_id: []u8,
},
parser_options: json.ParseOptions,
parser_options: union(enum) {
json: json.ParseOptions,
xml: xml_shaper.ParseOptions,
},
raw_parsed: union(enum) {
server: ServerResponse(action),
raw: action.Response,
xml: xml_shaper.Parsed(action.Response),
},
// raw_parsed: ServerResponse(request),
const Self = @This();
pub fn deinit(self: Self) void {
switch (self.raw_parsed) {
.server => json.parseFree(ServerResponse(action), self.raw_parsed.server, self.parser_options),
.raw => json.parseFree(action.Response, self.raw_parsed.raw, self.parser_options),
// Server is json only (so far)
.server => json.parseFree(ServerResponse(action), self.raw_parsed.server, self.parser_options.json),
// Raw is json only (so far)
.raw => json.parseFree(action.Response, self.raw_parsed.raw, self.parser_options.json),
.xml => |xml| xml.deinit(),
}
self.parser_options.allocator.?.free(self.response_metadata.request_id);
var allocator: std.mem.Allocator = undefined;
switch (self.parser_options) {
.json => |j| allocator = j.allocator.?,
.xml => |x| allocator = x.allocator.?,
}
allocator.free(self.response_metadata.request_id);
}
};
}

View File

@ -14,6 +14,15 @@ pub fn log(
// Ignore aws_signing messages
if (verbose < 2 and scope == .aws_signing and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
return;
// Ignore aws_credentials messages
if (verbose < 2 and scope == .aws_credentials and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
return;
// Ignore xml_shaper messages
if (verbose < 2 and scope == .xml_shaper and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
return;
// Ignore date messages
if (verbose < 2 and scope == .date and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
return;
// Ignore awshttp messages
if (verbose < 1 and scope == .awshttp and @enumToInt(level) >= @enumToInt(std.log.Level.debug))
return;
@ -169,18 +178,17 @@ pub fn main() anyerror!void {
std.log.err("no functions to work with", .{});
}
},
// TODO: This test fails with broken LLVM module
.ec2_query_no_input => {
std.log.err("EC2 Test disabled due to compiler bug", .{});
// Describe regions is a simpler request and easier to debug
// const instances = try client.call(services.ec2.describe_regions.Request{}, options);
// defer instances.deinit();
// std.log.info("region count: {d}", .{instances.response.regions.?.len});
const result = try client.call(services.ec2.describe_regions.Request{}, options);
defer result.deinit();
std.log.info("request id: {s}", .{result.response_metadata.request_id});
std.log.info("region count: {d}", .{result.response.regions.?.len});
// Describe instances is more interesting
// const instances = try client.call(services.ec2.describe_instances.Request{}, options);
// defer instances.deinit();
// std.log.info("reservation count: {d}", .{instances.response.reservations.len});
const instances = try client.call(services.ec2.describe_instances.Request{}, options);
defer instances.deinit();
std.log.info("reservation count: {d}", .{instances.response.reservations.?.len});
},
}
std.log.info("===== End Test: {s} =====\n", .{@tagName(t)});

View File

@ -45,7 +45,7 @@ pub const Element = struct {
}
pub fn getCharData(self: *Element, child_tag: []const u8) ?[]const u8 {
const child = self.findChildByTag(child_tag) orelse return null;
const child = (self.findChildByTag(child_tag) catch return null) orelse return null;
if (child.children.items.len != 1) {
return null;
}

View File

@ -242,6 +242,7 @@ fn parseInternal(comptime T: type, element: *xml.Element, options: ParseOptions)
if (@typeInfo(field.field_type) == .Optional and !found_value) {
// @compileLog("Optional: Field name ", field.name, ", type ", field.field_type);
@field(r, field.name) = null;
fields_set = fields_set + 1;
found_value = true;
}
// Using this else clause breaks zig, so we'll use a boolean instead