provide ability to handle requests built at runtime
Some checks failed
continuous-integration/drone/push Build is failing

This commit adds a new interface that is capable of runtime
use. By calling Request(action).call(request, options), the request
object can now be built at runtime. This change also moves the client
object into the options structure. It also moves the metaInfo generated
function to a type-based function rather than requiring an instance for
binding.
This commit is contained in:
Emil Lerch 2021-08-24 17:02:28 -07:00
parent 866a68777e
commit 7f178bcc91
Signed by: lobo
GPG Key ID: A7B62D657EF764F8
3 changed files with 376 additions and 313 deletions

View File

@ -211,7 +211,7 @@ fn generateMetadataFunction(_: []const u8, operation_name: []const u8, comptime
// }
// We want to add a short "get my parents" function into the response
try writer.print("{s} ", .{prefix});
_ = try writer.write("pub fn metaInfo(_: @This()) struct { ");
_ = try writer.write("pub fn metaInfo() struct { ");
try writer.print("service_metadata: @TypeOf(service_metadata), action: @TypeOf({s})", .{operation_name});
_ = try writer.write(" } {\n" ++ prefix ++ " return .{ .service_metadata = service_metadata, ");
try writer.print(".action = {s}", .{operation_name});

View File

@ -12,6 +12,7 @@ pub const Options = struct {
region: []const u8 = "aws-global",
dualstack: bool = false,
success_http_code: i64 = 200,
client: Client,
};
/// Using this constant may blow up build times. Recommed using Services()
@ -24,7 +25,7 @@ pub const services = servicemodel.services;
/// This will give you a constant with service data for sts, ec2, s3 and ddb only
pub const Services = servicemodel.Services;
pub const Aws = struct {
pub const Client = struct {
allocator: *std.mem.Allocator,
aws_http: awshttp.AwsHttp,
@ -36,16 +37,34 @@ pub const Aws = struct {
.aws_http = awshttp.AwsHttp.init(allocator),
};
}
pub fn deinit(self: *Aws) void {
pub fn deinit(self: *Client) void {
self.aws_http.deinit();
}
pub fn call(self: Self, comptime request: anytype, options: Options) !FullResponse(request) {
/// Calls AWS. Use a comptime request and options. For a runtime interface,
/// see Request
pub fn call(_: Self, comptime request: anytype, options: Options) !FullResponse(@TypeOf(request).metaInfo().action) {
const action = @TypeOf(request).metaInfo().action;
return Request(action).call(request, options);
}
};
/// Establish an AWS request that can be later called with runtime-known
/// parameters. If all parameters are known at comptime, the call function
/// may be simpler to use. request parameter here refers to the action
/// constant from the model, e.g. Request(services.lambda.list_functions)
pub fn Request(comptime action: anytype) type {
return struct {
const ActionRequest = action.Request;
const FullResponseType = FullResponse(action);
const Self = @This();
const action = action;
pub fn call(request: ActionRequest, options: Options) !FullResponseType {
// every codegenned request object includes a metaInfo function to get
// pointers to service and action
const meta_info = request.metaInfo();
const meta_info = ActionRequest.metaInfo();
const service_meta = meta_info.service_metadata;
const action = meta_info.action;
log.debug("call: prefix {s}, sigv4 {s}, version {s}, action {s}", .{
service_meta.endpoint_prefix,
@ -64,16 +83,16 @@ pub const Aws = struct {
// We're not doing a lot of error handling here, though.
// 3. rest_xml: This is a one-off for S3, never used since
switch (service_meta.aws_protocol) {
.query => return self.callQuery(request, service_meta, action, options),
.query => return Self.callQuery(request, options),
// .query, .ec2_query => return self.callQuery(request, service_meta, action, options),
.json_1_0, .json_1_1 => return self.callJson(request, service_meta, action, options),
.rest_json_1 => return self.callRestJson(request, service_meta, action, options),
.json_1_0, .json_1_1 => return Self.callJson(request, options),
.rest_json_1 => return Self.callRestJson(request, options),
.ec2_query, .rest_xml => @compileError("XML responses may be blocked on a zig compiler bug scheduled to be fixed in 0.9.0"),
}
}
/// Rest Json is the most complex and so we handle this seperately
fn callRestJson(self: Self, comptime request: anytype, comptime service_meta: anytype, action: anytype, options: Options) !FullResponse(request) {
fn callRestJson(request: ActionRequest, options: Options) !FullResponseType {
const Action = @TypeOf(action);
var aws_request: awshttp.HttpRequest = .{
.method = Action.http_config.method,
@ -85,35 +104,37 @@ pub const Aws = struct {
log.debug("Rest JSON v1 success code: {d}", .{Action.http_config.success_code});
log.debug("Rest JSON v1 raw uri: {s}", .{Action.http_config.uri});
aws_request.query = try buildQuery(self.allocator, request);
aws_request.query = try buildQuery(options.client.allocator, request);
log.debug("Rest JSON v1 query: {s}", .{aws_request.query});
defer self.allocator.free(aws_request.query);
defer options.client.allocator.free(aws_request.query);
// We don't know if we need a body...guessing here, this should cover most
var buffer = std.ArrayList(u8).init(self.allocator);
var buffer = std.ArrayList(u8).init(options.client.allocator);
defer buffer.deinit();
var nameAllocator = std.heap.ArenaAllocator.init(self.allocator);
var nameAllocator = std.heap.ArenaAllocator.init(options.client.allocator);
defer nameAllocator.deinit();
if (std.mem.eql(u8, "PUT", aws_request.method) or std.mem.eql(u8, "POST", aws_request.method)) {
try json.stringify(request, .{ .whitespace = .{} }, buffer.writer());
}
return try self.callAws(request, service_meta, aws_request, .{
return try Self.callAws(aws_request, .{
.success_http_code = Action.http_config.success_code,
.region = options.region,
.dualstack = options.dualstack,
.client = options.client,
});
}
/// Calls using one of the json protocols (json_1_0, json_1_1)
fn callJson(self: Self, comptime request: anytype, comptime service_meta: anytype, action: anytype, options: Options) !FullResponse(request) {
fn callJson(request: ActionRequest, options: Options) !FullResponseType {
const service_meta = ActionRequest.metaInfo().service_metadata;
const target =
try std.fmt.allocPrint(self.allocator, "{s}.{s}", .{
try std.fmt.allocPrint(options.client.allocator, "{s}.{s}", .{
service_meta.name,
action.action_name,
});
defer self.allocator.free(target);
defer options.client.allocator.free(target);
var buffer = std.ArrayList(u8).init(self.allocator);
var buffer = std.ArrayList(u8).init(options.client.allocator);
defer buffer.deinit();
// The transformer needs to allocate stuff out of band, but we
@ -125,7 +146,7 @@ pub const Aws = struct {
// for a boxed member with no observable difference." But we're
// seeing a lot of differences here between spec and reality
//
var nameAllocator = std.heap.ArenaAllocator.init(self.allocator);
var nameAllocator = std.heap.ArenaAllocator.init(options.client.allocator);
defer nameAllocator.deinit();
try json.stringify(request, .{ .whitespace = .{} }, buffer.writer());
@ -135,7 +156,7 @@ pub const Aws = struct {
.json_1_1 => content_type = "application/x-amz-json-1.1",
else => unreachable,
}
return try self.callAws(request, service_meta, .{
return try Self.callAws(.{
.query = "",
.body = buffer.items,
.content_type = content_type,
@ -148,45 +169,46 @@ pub const Aws = struct {
// Query, so we'll handle both here. Realistically we probably don't effectively
// handle lists and maps properly anyway yet, so we'll go for it and see
// where it breaks. PRs and/or failing test cases appreciated.
fn callQuery(self: Self, comptime request: anytype, comptime service_meta: anytype, action: anytype, options: Options) !FullResponse(request) {
var buffer = std.ArrayList(u8).init(self.allocator);
fn callQuery(request: ActionRequest, options: Options) !FullResponseType {
var buffer = std.ArrayList(u8).init(options.client.allocator);
defer buffer.deinit();
const writer = buffer.writer();
try url.encode(request, writer, .{
.field_name_transformer = &queryFieldTransformer,
.allocator = self.allocator,
.allocator = options.client.allocator,
});
const continuation = if (buffer.items.len > 0) "&" else "";
const service_meta = ActionRequest.metaInfo().service_metadata;
const query = if (service_meta.aws_protocol == .query)
try std.fmt.allocPrint(self.allocator, "", .{})
try std.fmt.allocPrint(options.client.allocator, "", .{})
else // EC2
try std.fmt.allocPrint(self.allocator, "?Action={s}&Version={s}", .{
try std.fmt.allocPrint(options.client.allocator, "?Action={s}&Version={s}", .{
action.action_name,
service_meta.version,
});
defer self.allocator.free(query);
defer options.client.allocator.free(query);
const body = if (service_meta.aws_protocol == .query)
try std.fmt.allocPrint(self.allocator, "Action={s}&Version={s}{s}{s}", .{
try std.fmt.allocPrint(options.client.allocator, "Action={s}&Version={s}{s}{s}", .{
action.action_name,
service_meta.version,
continuation,
buffer.items,
})
else // EC2
try std.fmt.allocPrint(self.allocator, "{s}", .{buffer.items});
defer self.allocator.free(body);
return try self.callAws(request, service_meta, .{
try std.fmt.allocPrint(options.client.allocator, "{s}", .{buffer.items});
defer options.client.allocator.free(body);
return try Self.callAws(.{
.query = query,
.body = body,
.content_type = "application/x-www-form-urlencoded",
}, options);
}
fn callAws(self: Self, comptime request: anytype, comptime service_meta: anytype, aws_request: awshttp.HttpRequest, options: Options) !FullResponse(request) {
const FullR = FullResponse(request);
const response = try self.aws_http.callApi(
fn callAws(aws_request: awshttp.HttpRequest, options: Options) !FullResponseType {
const service_meta = ActionRequest.metaInfo().service_metadata;
const response = try options.client.aws_http.callApi(
service_meta.endpoint_prefix,
aws_request,
.{
@ -196,9 +218,8 @@ pub const Aws = struct {
},
);
defer response.deinit();
// try self.reportTraffic("", aws_request, response, log.debug);
if (response.response_code != 200) {
try self.reportTraffic("Call Failed", aws_request, response, log.err);
if (response.response_code != options.success_http_code) {
try reportTraffic(options.client.allocator, "Call Failed", aws_request, response, log.err);
return error.HttpFailure;
}
// EC2 ignores our accept type, but technically query protocol only
@ -229,7 +250,7 @@ pub const Aws = struct {
var stream = json.TokenStream.init(response.body);
const parser_options = json.ParseOptions{
.allocator = self.allocator,
.allocator = options.client.allocator,
.allow_camel_case_conversion = true, // new option
.allow_snake_case_conversion = true, // new option
.allow_unknown_fields = true, // new option. Cannot yet handle non-struct fields though
@ -238,9 +259,9 @@ pub const Aws = struct {
// const SResponse = ServerResponse(request);
const SResponse = if (service_meta.aws_protocol != .query and service_meta.aws_protocol != .ec2_query)
Response(request)
action.Response
else
ServerResponse(request);
ServerResponse(action);
const parsed_response = json.parse(SResponse, &stream, parser_options) catch |e| {
log.err(
@ -264,15 +285,15 @@ pub const Aws = struct {
for (response.headers) |h| {
if (std.ascii.eqlIgnoreCase(h.name, "X-Amzn-RequestId")) {
found = true;
request_id = try std.fmt.allocPrint(self.allocator, "{s}", .{h.value}); // will be freed in FullR.deinit()
request_id = try std.fmt.allocPrint(options.client.allocator, "{s}", .{h.value}); // will be freed in FullR.deinit()
}
}
if (!found) {
try self.reportTraffic("Request ID not found", aws_request, response, log.err);
try reportTraffic(options.client.allocator, "Request ID not found", aws_request, response, log.err);
return error.RequestIdNotFound;
}
return FullR{
return FullResponseType{
.response = parsed_response,
.response_metadata = .{
.request_id = request_id,
@ -292,47 +313,20 @@ pub const Aws = struct {
// We can grab index [0] as structs are guaranteed by zig to be returned in the order
// declared, and we're declaring in that order in ServerResponse().
const real_response = @field(parsed_response, @typeInfo(SResponse).Struct.fields[0].name);
return FullR{
return FullResponseType{
.response = @field(real_response, @typeInfo(@TypeOf(real_response)).Struct.fields[0].name),
.response_metadata = .{
.request_id = try self.allocator.dupe(u8, real_response.ResponseMetadata.RequestId),
.request_id = try options.client.allocator.dupe(u8, real_response.ResponseMetadata.RequestId),
},
.parser_options = parser_options,
.raw_parsed = .{ .server = parsed_response },
};
}
};
}
fn reportTraffic(self: Self, info: []const u8, request: awshttp.HttpRequest, response: awshttp.HttpResult, comptime reporter: fn (comptime []const u8, anytype) void) !void {
var msg = std.ArrayList(u8).init(self.allocator);
defer msg.deinit();
const writer = msg.writer();
try writer.print("{s}\n\n", .{info});
try writer.print("Return status: {d}\n\n", .{response.response_code});
if (request.query.len > 0) try writer.print("Request Query:\n \t{s}\n", .{request.query});
_ = try writer.write("Unique Request Headers:\n");
if (request.headers.len > 0) {
for (request.headers) |h|
try writer.print("\t{s}: {s}\n", .{ h.name, h.value });
}
try writer.print("\tContent-Type: {s}\n\n", .{request.content_type});
_ = try writer.write("Request Body:\n");
try writer.print("-------------\n{s}\n", .{request.body});
_ = try writer.write("-------------\n");
_ = try writer.write("Response Headers:\n");
for (response.headers) |h|
try writer.print("\t{s}: {s}\n", .{ h.name, h.value });
_ = try writer.write("Response Body:\n");
try writer.print("--------------\n{s}\n", .{response.body});
_ = try writer.write("--------------\n");
reporter("{s}\n", .{msg.items});
}
};
fn ServerResponse(comptime request: anytype) type {
const T = Response(request);
const action = request.metaInfo().action;
fn ServerResponse(comptime action: anytype) type {
const T = action.Response;
// NOTE: The non-standard capitalization here is used as a performance
// enhancement and to reduce allocations in json.zig. These fields are
// not (nor are they ever intended to be) exposed outside this codebase
@ -379,49 +373,49 @@ fn ServerResponse(comptime request: anytype) type {
},
});
}
fn FullResponse(comptime request: anytype) type {
fn FullResponse(comptime action: anytype) type {
return struct {
response: Response(request),
response: action.Response,
response_metadata: struct {
request_id: []u8,
},
parser_options: json.ParseOptions,
raw_parsed: union(enum) {
server: ServerResponse(request),
raw: Response(request),
server: ServerResponse(action),
raw: action.Response,
},
// raw_parsed: ServerResponse(request),
const Self = @This();
pub fn deinit(self: Self) void {
switch (self.raw_parsed) {
.server => json.parseFree(ServerResponse(request), self.raw_parsed.server, self.parser_options),
.raw => json.parseFree(Response(request), self.raw_parsed.raw, self.parser_options),
.server => json.parseFree(ServerResponse(action), self.raw_parsed.server, self.parser_options),
.raw => json.parseFree(action.Response, self.raw_parsed.raw, self.parser_options),
}
self.parser_options.allocator.?.free(self.response_metadata.request_id);
}
};
}
fn Response(comptime request: anytype) type {
return request.metaInfo().action.Response;
}
fn queryFieldTransformer(field_name: []const u8, encoding_options: url.EncodingOptions) anyerror![]const u8 {
return try case.snakeToPascal(encoding_options.allocator.?, field_name);
}
fn buildQuery(allocator: *std.mem.Allocator, comptime request: anytype) ![]const u8 {
fn buildQuery(allocator: *std.mem.Allocator, request: anytype) ![]const u8 {
// query should look something like this:
// pub const http_query = .{
// .master_region = "MasterRegion",
// .function_version = "FunctionVersion",
// .marker = "Marker",
// };
const query_arguments = @TypeOf(request).http_query;
var buffer = std.ArrayList(u8).init(allocator);
const writer = buffer.writer();
defer buffer.deinit();
var has_begun = false;
const Req = @TypeOf(request);
if (std.meta.fieldIndex(Req, "http_query") == null)
return buffer.toOwnedSlice();
const query_arguments = Req.http_query;
inline for (@typeInfo(@TypeOf(query_arguments)).Struct.fields) |arg| {
const val = @field(request, arg.name);
if (@typeInfo(@TypeOf(val)) == .Optional) {
@ -446,6 +440,32 @@ fn addQueryArg(key: []const u8, value: anytype, writer: anytype, start: bool) !v
try writer.print("{s}=", .{key});
try json.stringify(value, .{}, writer);
}
fn reportTraffic(allocator: *std.mem.Allocator, info: []const u8, request: awshttp.HttpRequest, response: awshttp.HttpResult, comptime reporter: fn (comptime []const u8, anytype) void) !void {
var msg = std.ArrayList(u8).init(allocator);
defer msg.deinit();
const writer = msg.writer();
try writer.print("{s}\n\n", .{info});
try writer.print("Return status: {d}\n\n", .{response.response_code});
if (request.query.len > 0) try writer.print("Request Query:\n \t{s}\n", .{request.query});
_ = try writer.write("Unique Request Headers:\n");
if (request.headers.len > 0) {
for (request.headers) |h|
try writer.print("\t{s}: {s}\n", .{ h.name, h.value });
}
try writer.print("\tContent-Type: {s}\n\n", .{request.content_type});
_ = try writer.write("Request Body:\n");
try writer.print("-------------\n{s}\n", .{request.body});
_ = try writer.write("-------------\n");
_ = try writer.write("Response Headers:\n");
for (response.headers) |h|
try writer.print("\t{s}: {s}\n", .{ h.name, h.value });
_ = try writer.write("Response Body:\n");
try writer.print("--------------\n{s}\n", .{response.body});
_ = try writer.write("--------------\n");
reporter("{s}\n", .{msg.items});
}
test "REST Json v1 builds proper queries" {
const allocator = std.testing.allocator;

View File

@ -33,6 +33,7 @@ const Tests = enum {
json_1_1_query_no_input,
rest_json_1_query_no_input,
rest_json_1_query_with_input,
rest_json_1_work_with_lambda,
};
pub fn main() anyerror!void {
@ -64,11 +65,12 @@ pub fn main() anyerror!void {
try tests.append(@field(Tests, f.name));
}
std.log.info("Start\n", .{});
var client = aws.Client.init(allocator);
const options = aws.Options{
.region = "us-west-2",
.client = client,
};
std.log.info("Start\n", .{});
var client = aws.Aws.init(allocator);
defer client.deinit();
const services = aws.Services(.{ .sts, .ec2, .dynamo_db, .ecs, .lambda }){};
@ -77,7 +79,8 @@ pub fn main() anyerror!void {
std.log.info("===== Start Test: {s} =====", .{@tagName(t)});
switch (t) {
.query_no_input => {
const call = try client.call(services.sts.get_caller_identity.Request{}, options);
const call = try aws.Request(services.sts.get_caller_identity).call(.{}, options);
// const call = try client.call(services.sts.get_caller_identity.Request{}, options);
defer call.deinit();
std.log.info("arn: {s}", .{call.response.arn});
std.log.info("id: {s}", .{call.response.user_id});
@ -90,7 +93,7 @@ pub fn main() anyerror!void {
.duration_seconds = 900,
}, options);
defer call.deinit();
std.log.info("call key: {s}", .{call.response.credentials.?.access_key_id});
std.log.info("access key: {s}", .{call.response.credentials.?.access_key_id});
},
.json_1_0_query_with_input => {
const call = try client.call(services.dynamo_db.list_tables.Request{
@ -98,7 +101,7 @@ pub fn main() anyerror!void {
}, options);
defer call.deinit();
std.log.info("request id: {s}", .{call.response_metadata.request_id});
std.log.info("account has call: {b}", .{call.response.table_names.?.len > 0});
std.log.info("account has tables: {b}", .{call.response.table_names.?.len > 0});
},
.json_1_0_query_no_input => {
const call = try client.call(services.dynamo_db.describe_limits.Request{}, options);
@ -111,13 +114,13 @@ pub fn main() anyerror!void {
}, options);
defer call.deinit();
std.log.info("request id: {s}", .{call.response_metadata.request_id});
std.log.info("account has call: {b}", .{call.response.cluster_arns.?.len > 0});
std.log.info("account has clusters: {b}", .{call.response.cluster_arns.?.len > 0});
},
.json_1_1_query_no_input => {
const call = try client.call(services.ecs.list_clusters.Request{}, options);
defer call.deinit();
std.log.info("request id: {s}", .{call.response_metadata.request_id});
std.log.info("account has call: {b}", .{call.response.cluster_arns.?.len > 0});
std.log.info("account has clusters: {b}", .{call.response.cluster_arns.?.len > 0});
},
.rest_json_1_query_with_input => {
const call = try client.call(services.lambda.list_functions.Request{
@ -125,13 +128,40 @@ pub fn main() anyerror!void {
}, options);
defer call.deinit();
std.log.info("request id: {s}", .{call.response_metadata.request_id});
std.log.info("account has call: {b}", .{call.response.functions.?.len > 0});
std.log.info("account has functions: {b}", .{call.response.functions.?.len > 0});
},
.rest_json_1_query_no_input => {
const call = try client.call(services.lambda.list_functions.Request{}, options);
defer call.deinit();
std.log.info("request id: {s}", .{call.response_metadata.request_id});
std.log.info("account has call: {b}", .{call.response.functions.?.len > 0});
std.log.info("account has functions: {b}", .{call.response.functions.?.len > 0});
},
.rest_json_1_work_with_lambda => {
const call = try client.call(services.lambda.list_functions.Request{}, options);
defer call.deinit();
std.log.info("list request id: {s}", .{call.response_metadata.request_id});
if (call.response.functions) |fns| {
if (fns.len > 0) {
const func = fns[0];
const arn = func.function_arn.?;
var tags = try std.ArrayList(@typeInfo(try typeForField(services.lambda.tag_resource.Request, "tags")).Pointer.child).initCapacity(allocator, 1);
defer tags.deinit();
tags.appendAssumeCapacity(.{ .key = "Foo", .value = "Bar" });
const req = services.lambda.tag_resource.Request{ .resource = arn, .tags = tags.items };
const addtag = try aws.Request(services.lambda.tag_resource).call(req, options);
// const addtag = try client.call(services.lambda.tag_resource.Request{ .resource = arn, .tags = &.{.{ .key = "Foo", .value = "Bar" }} }, options);
std.log.info("add tag request id: {s}", .{addtag.response_metadata.request_id});
var tag_keys = try std.ArrayList([]const u8).initCapacity(allocator, 1);
defer tag_keys.deinit();
tag_keys.appendAssumeCapacity("Foo");
const deletetag = try aws.Request(services.lambda.untag_resource).call(.{ .tag_keys = tag_keys.items, .resource = arn }, options);
std.log.info("delete tag request id: {s}", .{deletetag.response_metadata.request_id});
} else {
std.log.err("no functions to work with", .{});
}
} else {
std.log.err("no functions to work with", .{});
}
},
.ec2_query_no_input => {
std.log.err("EC2 Test disabled due to compiler bug", .{});
@ -155,6 +185,19 @@ pub fn main() anyerror!void {
std.log.info("===== Tests complete =====", .{});
}
fn typeForField(comptime T: type, field_name: []const u8) !type {
const ti = @typeInfo(T);
switch (ti) {
.Struct => {
inline for (ti.Struct.fields) |field| {
if (std.mem.eql(u8, field.name, field_name))
return field.field_type;
}
},
else => return error.TypeIsNotAStruct, // should not hit this
}
return error.FieldNotFound;
}
// TODO: Move into json.zig
pub fn jsonFun() !void {