From a5b78384f5395ee2e0e46137f52485aa60804296 Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Wed, 12 Jan 2022 09:18:16 -0800 Subject: [PATCH] build works, c_allocator no longer required --- src/aws.zig | 2 +- src/aws_authentication.zig | 6 + src/aws_credentials.zig | 23 +++ src/aws_http.zig | 295 +++++++++++++------------------------ src/aws_http_base.zig | 33 +++++ src/aws_signing.zig | 80 ++++++++++ src/main.zig | 5 +- 7 files changed, 245 insertions(+), 199 deletions(-) create mode 100644 src/aws_authentication.zig create mode 100644 src/aws_credentials.zig create mode 100644 src/aws_http_base.zig create mode 100644 src/aws_signing.zig diff --git a/src/aws.zig b/src/aws.zig index 117797f..6c5bea1 100644 --- a/src/aws.zig +++ b/src/aws.zig @@ -1,6 +1,6 @@ const std = @import("std"); -const awshttp = @import("awshttp.zig"); +const awshttp = @import("aws_http.zig"); const json = @import("json.zig"); const url = @import("url.zig"); const case = @import("case.zig"); diff --git a/src/aws_authentication.zig b/src/aws_authentication.zig new file mode 100644 index 0000000..580c93d --- /dev/null +++ b/src/aws_authentication.zig @@ -0,0 +1,6 @@ +pub const Credentials = struct { + access_key: []const u8, + secret_key: []const u8, + session_token: ?[]const u8, + // uint64_t expiration_timepoint_seconds); +}; diff --git a/src/aws_credentials.zig b/src/aws_credentials.zig new file mode 100644 index 0000000..f7c1497 --- /dev/null +++ b/src/aws_credentials.zig @@ -0,0 +1,23 @@ +//! Implements the standard credential chain: +//! 1. Environment variables +//! 2. Web identity token from STS +//! 3. Credentials/config files +//! 4. ECS Container credentials, using AWS_CONTAINER_CREDENTIALS_RELATIVE_URI +//! 5. EC2 instance profile credentials +const std = @import("std"); +const auth = @import("aws_authentication.zig"); + +pub fn getCredentials(allocator: std.mem.Allocator) !auth.Credentials { + _ = allocator; + if (getEnvironmentCredentials()) |cred| return cred; + // TODO: 2-5 + return error.NotImplemented; +} + +fn getEnvironmentCredentials() ?auth.Credentials { + return auth.Credentials{ + .access_key = std.os.getenv("AWS_ACCESS_KEY_ID") orelse return null, + .secret_key = std.os.getenv("AWS_SECRET_ACCESS_KEY") orelse return null, + .session_token = std.os.getenv("AWS_SESSION_TOKEN"), + }; +} diff --git a/src/aws_http.zig b/src/aws_http.zig index 1d5e3ca..ef38146 100644 --- a/src/aws_http.zig +++ b/src/aws_http.zig @@ -8,13 +8,17 @@ //! const result = client.callApi (or client.makeRequest) //! defer result.deinit(); const std = @import("std"); +const base = @import("aws_http_base.zig"); +const signing = @import("aws_signing.zig"); +const credentials = @import("aws_credentials.zig"); +const zfetch = @import("zfetch"); const CN_NORTH_1_HASH = std.hash_map.hashString("cn-north-1"); const CN_NORTHWEST_1_HASH = std.hash_map.hashString("cn-northwest-1"); const US_ISO_EAST_1_HASH = std.hash_map.hashString("us-iso-east-1"); const US_ISOB_EAST_1_HASH = std.hash_map.hashString("us-isob-east-1"); -const httplog = std.log.scoped(.awshttp); +const log = std.log.scoped(.awshttp); pub const AwsError = error{ AddHeaderError, @@ -38,41 +42,9 @@ pub const Options = struct { sigv4_service_name: ?[]const u8 = null, }; -const SigningOptions = struct { - region: []const u8 = "aws-global", - service: []const u8, -}; - -pub const HttpRequest = struct { - path: []const u8 = "/", - query: []const u8 = "", - body: []const u8 = "", - method: []const u8 = "POST", - content_type: []const u8 = "application/json", // Can we get away with this? - headers: []Header = &[_]Header{}, -}; -pub const HttpResult = struct { - response_code: u16, // actually 3 digits can fit in u10 - body: []const u8, - headers: []Header, - allocator: std.mem.Allocator, - - pub fn deinit(self: HttpResult) void { - self.allocator.free(self.body); - for (self.headers) |h| { - self.allocator.free(h.name); - self.allocator.free(h.value); - } - self.allocator.free(self.headers); - httplog.debug("http result deinit complete", .{}); - return; - } -}; - -pub const Header = struct { - name: []const u8, - value: []const u8, -}; +pub const Header = base.Header; +pub const HttpRequest = base.Request; +pub const HttpResult = base.Result; const EndPoint = struct { uri: []const u8, @@ -99,7 +71,8 @@ pub const AwsHttp = struct { } pub fn deinit(self: *AwsHttp) void { - httplog.debug("Deinit complete", .{}); + _ = self; + log.debug("Deinit complete", .{}); } /// callApi allows the calling of AWS APIs through a higher-level interface. @@ -109,12 +82,15 @@ pub const AwsHttp = struct { pub fn callApi(self: Self, service: []const u8, request: HttpRequest, options: Options) !HttpResult { const endpoint = try regionSubDomain(self.allocator, service, options.region, options.dualstack); defer endpoint.deinit(); - httplog.debug("Calling endpoint {s}", .{endpoint.uri}); - const signing_options: SigningOptions = .{ + log.debug("Calling endpoint {s}", .{endpoint.uri}); + const creds = try credentials.getCredentials(self.allocator); + // defer allocator.free(), except sometimes we don't need freeing... + const signing_config: signing.Config = .{ .region = options.region, - .service = if (options.sigv4_service_name) |name| name else service, + .service = options.sigv4_service_name orelse service, + .credentials = creds, }; - return try self.makeRequest(endpoint, request, signing_options); + return try self.makeRequest(endpoint, request, signing_config); } /// makeRequest is a low level http/https function that can be used inside @@ -135,76 +111,70 @@ pub const AwsHttp = struct { /// Return value is an HttpResult, which will need the caller to deinit(). /// HttpResult currently contains the body only. The addition of Headers /// and return code would be a relatively minor change - pub fn makeRequest(self: Self, endpoint: EndPoint, request: HttpRequest, signing_options: ?SigningOptions) !HttpResult { - httplog.debug("Path: {s}", .{request.path}); - httplog.debug("Query: {s}", .{request.query}); - httplog.debug("Method: {s}", .{request.method}); - httplog.debug("body length: {d}", .{request.body.len}); - httplog.debug("Body\n====\n{s}\n====", .{request.body}); + pub fn makeRequest(self: Self, endpoint: EndPoint, request: HttpRequest, signing_config: ?signing.Config) !HttpResult { + log.debug("Path: {s}", .{request.path}); + log.debug("Query: {s}", .{request.query}); + log.debug("Method: {s}", .{request.method}); + log.debug("body length: {d}", .{request.body.len}); + log.debug("Body\n====\n{s}\n====", .{request.body}); // End CreateRequest. This should return a struct with a deinit function that can do // destroys, etc - var context = RequestContext{ - .allocator = self.allocator, - }; - try self.addHeaders(http_request.?, host, request.body, request.content_type, request.headers); - if (signing_options) |opts| try self.signRequest(http_request.?, opts); + // TODO: Add headers + _ = endpoint; + //try self.addHeaders(endpoint.host, request.body, request.content_type, request.headers); + if (signing_config) |opts| try signing.signRequest(self.allocator, request, opts); // TODO: make req - // TODO: Timeout - httplog.debug("request_complete. Response code {d}", .{context.response_code.?}); - httplog.debug("headers:", .{}); - for (context.headers.?.items) |h| { - httplog.debug(" {s}: {s}", .{ h.name, h.value }); + try zfetch.init(); // This only does anything on Windows. Not sure how performant it is to do this on every request + defer zfetch.deinit(); + var headers = zfetch.Headers.init(self.allocator); + defer headers.deinit(); + for (request.headers) |header| + try headers.appendValue(header.name, header.value); + + // TODO: Construct URL with endpoint and request info + var req = try zfetch.Request.init(self.allocator, "https://www.lerch.org", null); + + // TODO: http method as requested + // TODO: payload + try req.do(.GET, headers, null); + + // TODO: Timeout - is this now above us? + log.debug("request_complete. Response code {d}: {s}", .{ req.status.code, req.status.reason }); + log.debug("headers:", .{}); + var resp_headers = try std.ArrayList(Header).initCapacity(self.allocator, req.headers.list.items.len); + for (req.headers.list.items) |h| { + log.debug(" {s}: {s}", .{ h.name, h.value }); + resp_headers.appendAssumeCapacity(.{ .name = h.name, .value = h.value }); } - httplog.debug("raw response body:\n{s}", .{context.body}); + const reader = req.reader(); + // TODO: Get content length and use that to allocate the buffer + var buf: [65535]u8 = undefined; + while (true) { + const read = try reader.read(&buf); + if (read == 0) break; + } + log.debug("raw response body:\n{s}", .{buf}); // Headers would need to be allocated/copied into HttpResult similar // to RequestContext, so we'll leave this as a later excercise // if it becomes necessary const rc = HttpResult{ - .response_code = context.response_code.?, - .body = final_body, - .headers = context.headers.?.toOwnedSlice(), + .response_code = req.status.code, + .body = "change me", // TODO: work this all out + .headers = resp_headers.toOwnedSlice(), .allocator = self.allocator, }; return rc; } - fn signRequest(self: Self, http_request: *c.aws_http_message, options: SigningOptions) !void { - const creds = try self.getCredentials(); - httplog.debug("Signing with access key: {s}", .{c.aws_string_c_str(access_key)}); - - // const signing_region = try std.fmt.allocPrintZ(self.allocator, "{s}", .{options.region}); - // defer self.allocator.free(signing_region); - // const signing_service = try std.fmt.allocPrintZ(self.allocator, "{s}", .{options.service}); - // defer self.allocator.free(signing_service); - // const temp_signing_config = c.bitfield_workaround_aws_signing_config_aws{ - // .algorithm = 0, // .AWS_SIGNING_ALGORITHM_V4, // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L38 - // .config_type = 1, // .AWS_SIGNING_CONFIG_AWS, // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L24 - // .signature_type = 0, // .AWS_ST_HTTP_REQUEST_HEADERS, // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L49 - // .region = c.aws_byte_cursor_from_c_str(@ptrCast([*c]const u8, signing_region)), - // .service = c.aws_byte_cursor_from_c_str(@ptrCast([*c]const u8, signing_service)), - // .should_sign_header = null, - // .should_sign_header_ud = null, - // // TODO: S3 does not double uri encode. Also not sure why normalizing - // // the path here is a flag - seems like it should always do this? - // .flags = c.bitfield_workaround_aws_signing_config_aws_flags{ - // .use_double_uri_encode = 1, - // .should_normalize_uri_path = 1, - // .omit_session_token = 1, - // }, - // .signed_body_value = c.aws_byte_cursor_from_c_str(""), - // .signed_body_header = 1, // .AWS_SBHT_X_AMZ_CONTENT_SHA256, //or 0 = AWS_SBHT_NONE // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L131 - // .credentials = creds, - // .credentials_provider = self.credentialsProvider, - // .expiration_in_seconds = 0, - // }; - // return AwsError.SignableError; - } - - - fn addHeaders(self: Self, request: *c.aws_http_message, host: []const u8, body: []const u8, content_type: []const u8, additional_headers: []Header) !void { + fn addHeaders(self: Self, host: []const u8, body: []const u8, content_type: []const u8, additional_headers: []Header) !void { + _ = self; + _ = host; + _ = body; + _ = content_type; + _ = additional_headers; // const accept_header = c.aws_http_header{ // .name = c.aws_byte_cursor_from_c_str("Accept"), // .value = c.aws_byte_cursor_from_c_str("application/json"), @@ -225,52 +195,40 @@ pub const AwsHttp = struct { // AWS *does* seem to care about Content-Type. I don't think this header // will hold for all APIs - const c_type = try std.fmt.allocPrintZ(self.allocator, "{s}", .{content_type}); - defer self.allocator.free(c_type); - const content_type_header = c.aws_http_header{ - .name = c.aws_byte_cursor_from_c_str("Content-Type"), - .value = c.aws_byte_cursor_from_c_str(c_type), - .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE, - }; - - for (additional_headers) |h| { - const name = try std.fmt.allocPrintZ(self.allocator, "{s}", .{h.name}); - defer self.allocator.free(name); - const value = try std.fmt.allocPrintZ(self.allocator, "{s}", .{h.value}); - defer self.allocator.free(value); - const c_header = c.aws_http_header{ - .name = c.aws_byte_cursor_from_c_str(name), - .value = c.aws_byte_cursor_from_c_str(value), - .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE, - }; - if (c.aws_http_message_add_header(request, c_header) != c.AWS_OP_SUCCESS) - return AwsError.AddHeaderError; - } - - if (body.len > 0) { - const len = try std.fmt.allocPrintZ(self.allocator, "{d}", .{body.len}); - // This defer seems to work ok, but I'm a bit concerned about why - defer self.allocator.free(len); - const content_length_header = c.aws_http_header{ - .name = c.aws_byte_cursor_from_c_str("Content-Length"), - .value = c.aws_byte_cursor_from_c_str(@ptrCast([*c]const u8, len)), - .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE, - }; - if (c.aws_http_message_add_header(request, content_length_header) != c.AWS_OP_SUCCESS) - return AwsError.AddHeaderError; - } - } - - - fn getCredentials(self: Self) !*c.aws_credentials { - // const get_async_result = - _ = c.aws_credentials_provider_get_credentials(self.credentialsProvider, callback, &callback_results); - - if (credential_result.error_code != c.AWS_ERROR_SUCCESS) { - httplog.err("Could not acquire credentials: {s}:{s}", .{ c.aws_error_name(credential_result.error_code), c.aws_error_str(credential_result.error_code) }); - return AwsError.CredentialsError; - } - return credential_result.result orelse unreachable; + // const c_type = try std.fmt.allocPrintZ(self.allocator, "{s}", .{content_type}); + // defer self.allocator.free(c_type); + // const content_type_header = c.aws_http_header{ + // .name = c.aws_byte_cursor_from_c_str("Content-Type"), + // .value = c.aws_byte_cursor_from_c_str(c_type), + // .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE, + // }; + // + // for (additional_headers) |h| { + // const name = try std.fmt.allocPrintZ(self.allocator, "{s}", .{h.name}); + // defer self.allocator.free(name); + // const value = try std.fmt.allocPrintZ(self.allocator, "{s}", .{h.value}); + // defer self.allocator.free(value); + // const c_header = c.aws_http_header{ + // .name = c.aws_byte_cursor_from_c_str(name), + // .value = c.aws_byte_cursor_from_c_str(value), + // .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE, + // }; + // if (c.aws_http_message_add_header(request, c_header) != c.AWS_OP_SUCCESS) + // return AwsError.AddHeaderError; + // } + // + // if (body.len > 0) { + // const len = try std.fmt.allocPrintZ(self.allocator, "{d}", .{body.len}); + // // This defer seems to work ok, but I'm a bit concerned about why + // defer self.allocator.free(len); + // const content_length_header = c.aws_http_header{ + // .name = c.aws_byte_cursor_from_c_str("Content-Length"), + // .value = c.aws_byte_cursor_from_c_str(@ptrCast([*c]const u8, len)), + // .compression = 0, // .AWS_HTTP_HEADER_COMPRESSION_USE_CACHE, + // }; + // if (c.aws_http_message_add_header(request, content_length_header) != c.AWS_OP_SUCCESS) + // return AwsError.AddHeaderError; + // } } }; @@ -297,7 +255,7 @@ fn regionSubDomain(allocator: std.mem.Allocator, service: []const u8, region: [] const uri = try std.fmt.allocPrintZ(allocator, "https://{s}{s}.{s}.{s}", .{ service, dualstack, realregion, domain }); const host = uri["https://".len..]; - httplog.debug("host: {s}, scheme: {s}, port: {}", .{ host, "https", 443 }); + log.debug("host: {s}, scheme: {s}, port: {}", .{ host, "https", 443 }); return EndPoint{ .uri = uri, .host = host, @@ -346,7 +304,7 @@ fn endPointFromUri(allocator: std.mem.Allocator, uri: []const u8) !EndPoint { } host = uri[host_start..host_end]; - httplog.debug("host: {s}, scheme: {s}, port: {}", .{ host, scheme, port }); + log.debug("host: {s}, scheme: {s}, port: {}", .{ host, scheme, port }); return EndPoint{ .uri = uri, .host = host, @@ -355,54 +313,3 @@ fn endPointFromUri(allocator: std.mem.Allocator, uri: []const u8) !EndPoint { .port = port, }; } - -const RequestContext = struct { - connection: ?*c.aws_http_connection = null, - connection_complete: std.atomic.Atomic(bool) = std.atomic.Atomic(bool).init(false), - request_complete: std.atomic.Atomic(bool) = std.atomic.Atomic(bool).init(false), - return_error: ?AwsError = null, - allocator: std.mem.Allocator, - body: ?[]const u8 = null, - response_code: ?u16 = null, - headers: ?std.ArrayList(Header) = null, - - const Self = @This(); - - pub fn deinit(self: Self) void { - // We're going to leave it to the caller to free the body - // if (self.body) |b| self.allocator.free(b); - if (self.headers) |hs| { - for (hs.items) |h| { - // deallocate the copied values - self.allocator.free(h.name); - self.allocator.free(h.value); - } - // deallocate the structure itself - hs.deinit(); - } - } - - pub fn appendToBody(self: *Self, fragment: []const u8) !void { - var orig_body: []const u8 = ""; - if (self.body) |b| { - orig_body = try self.allocator.dupe(u8, b); - self.allocator.free(b); - self.body = null; - } - defer self.allocator.free(orig_body); - self.body = try std.fmt.allocPrint(self.allocator, "{s}{s}", .{ orig_body, fragment }); - } - - pub fn addHeader(self: *Self, name: []const u8, value: []const u8) !void { - if (self.headers == null) - self.headers = std.ArrayList(Header).init(self.allocator); - - const name_copy = try self.allocator.dupeZ(u8, name); - const value_copy = try self.allocator.dupeZ(u8, value); - - try self.headers.?.append(.{ - .name = name_copy, - .value = value_copy, - }); - } -}; diff --git a/src/aws_http_base.zig b/src/aws_http_base.zig new file mode 100644 index 0000000..5b05cf5 --- /dev/null +++ b/src/aws_http_base.zig @@ -0,0 +1,33 @@ +//! This module provides base data structures for aws http requests +const std = @import("std"); +const log = std.log.scoped(.aws_base); +pub const Request = struct { + path: []const u8 = "/", + query: []const u8 = "", + body: []const u8 = "", + method: []const u8 = "POST", + content_type: []const u8 = "application/json", // Can we get away with this? + headers: []Header = &[_]Header{}, +}; +pub const Result = struct { + response_code: u16, // actually 3 digits can fit in u10 + body: []const u8, + headers: []Header, + allocator: std.mem.Allocator, + + pub fn deinit(self: Result) void { + self.allocator.free(self.body); + for (self.headers) |h| { + self.allocator.free(h.name); + self.allocator.free(h.value); + } + self.allocator.free(self.headers); + log.debug("http result deinit complete", .{}); + return; + } +}; + +pub const Header = struct { + name: []const u8, + value: []const u8, +}; diff --git a/src/aws_signing.zig b/src/aws_signing.zig new file mode 100644 index 0000000..9ec907f --- /dev/null +++ b/src/aws_signing.zig @@ -0,0 +1,80 @@ +const std = @import("std"); +const base = @import("aws_http_base.zig"); +const auth = @import("aws_authentication.zig"); + +const log = std.log.scoped(.aws_signing); + +// see https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L186-L207 +const ConfigFlags = packed struct { + // We assume the uri will be encoded once in preparation for transmission. Certain services + // do not decode before checking signature, requiring us to actually double-encode the uri in the canonical + // request in order to pass a signature check. + + use_double_uri_encode: bool = true, + + // Controls whether or not the uri paths should be normalized when building the canonical request + should_normalize_uri_path: bool = true, + + // Controls whether "X-Amz-Security-Token" is omitted from the canonical request. + // "X-Amz-Security-Token" is added during signing, as a header or + // query param, when credentials have a session token. + // If false (the default), this parameter is included in the canonical request. + // If true, this parameter is still added, but omitted from the canonical request. + omit_session_token: bool = true, +}; + +pub const Config = struct { + // These two should be all you need to set most of the time + service: []const u8, + credentials: auth.Credentials, + + region: []const u8 = "aws-global", + // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L38 + algorithm: enum { v4, v4a } = .v4, + // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L24 + // config_type: ?? // CRT only has one value. We'll ignore for now + // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L49 + signature_type: enum { + headers, // we only support this + query_params, + request_chunk, + request_event, // not implemented by CRT + canonical_request_headers, + canonical_request_query_params, + } = .headers, + signing_time: ?i64 = null, // Used for testing. If null, will use current time + + // In the CRT, should_sign_header is a function to allow header filtering. + // The _ud would be a anyopaque user defined data for the function to use + // .should_sign_header = null, + // .should_sign_header_ud = null, + + // In the CRT, this is only used if the body has been precalculated. We don't have + // this use case, and we'll ignore + // .signed_body_value = c.aws_byte_cursor_from_c_str(""), + signed_body_header: enum { sha256, none } = .sha256, // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L131 + + // This is more complex in the CRT. We'll just take the creds. Someone + // else can use a provider and get them in advance + // https://github.com/awslabs/aws-c-auth/blob/ace1311f8ef6ea890b26dd376031bed2721648eb/include/aws/auth/signing_config.h#L225-L251 + // If non-zero and the signing transform is query param, then signing will add X-Amz-Expires to the query + // string, equal to the value specified here. If this value is zero or if header signing is being used then + // this parameter has no effect. + expiration_in_seconds: u64 = 0, +}; + +pub const SigningError = error{ + NotImplemented, +}; + +pub fn signRequest(allocator: std.mem.Allocator, http_request: base.Request, config: Config) SigningError!void { + _ = allocator; + _ = http_request; + try validateConfig(config); + log.debug("Signing with access key: {s}", .{config.credentials.access_key}); +} + +fn validateConfig(config: Config) SigningError!void { + _ = config; + return SigningError.NotImplemented; +} diff --git a/src/main.zig b/src/main.zig index 7b36b2d..343f8f4 100644 --- a/src/main.zig +++ b/src/main.zig @@ -37,10 +37,7 @@ const Tests = enum { }; pub fn main() anyerror!void { - const c_allocator = std.heap.c_allocator; - var gpa = std.heap.GeneralPurposeAllocator(.{}){ - .backing_allocator = c_allocator, - }; + var gpa = std.heap.GeneralPurposeAllocator(.{}){}; defer _ = gpa.deinit(); const allocator = gpa.allocator(); var tests = std.ArrayList(Tests).init(allocator);