handle s3 key paths
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Emil Lerch 2022-06-29 09:24:16 -07:00
parent a662f6f674
commit 8d852e8084
Signed by: lobo
GPG Key ID: A7B62D657EF764F8
3 changed files with 26 additions and 20 deletions

View File

@ -2,8 +2,6 @@
[![Build Status](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/status.svg?ref=refs/heads/master)](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/) [![Build Status](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/status.svg?ref=refs/heads/master)](https://drone.lerch.org/api/badges/lobo/aws-sdk-for-zig/)
This SDK currently supports all AWS services. S3 has basic support
Current executable size for the demo is 1.7M (90k of which is the AWS PEM file, Current executable size for the demo is 1.7M (90k of which is the AWS PEM file,
and approximately 600K for XML services) after compiling with -Drelease-safe and and approximately 600K for XML services) after compiling with -Drelease-safe and
[stripping the executable after compilation](https://github.com/ziglang/zig/issues/351). [stripping the executable after compilation](https://github.com/ziglang/zig/issues/351).
@ -42,13 +40,10 @@ for posterity, and supports x86_64 linux. The old branch is deprecated.
## Limitations ## Limitations
There are many nuances of AWS V4 signature calculation, and not all edge cases WebIdentityToken is not yet implemented.
of S3 are handled. WebIdentityToken is not yet implemented.
TODO List: TODO List:
* Implement more robust S3 support. Keys with slashes in the name are currently
causing a SignatureDoesNotMatch error
* Bump to zig 0.9.1. iguanaTLS, used in zFetch is still [working out 0.9.1 issues](https://github.com/alexnask/iguanaTLS/pull/29) * Bump to zig 0.9.1. iguanaTLS, used in zFetch is still [working out 0.9.1 issues](https://github.com/alexnask/iguanaTLS/pull/29)
* Implement sigv4a signing * Implement sigv4a signing
* Implement jitter/exponential backoff * Implement jitter/exponential backoff

View File

@ -110,7 +110,13 @@ pub fn Request(comptime action: anytype) type {
log.debug("Rest method: '{s}'", .{aws_request.method}); log.debug("Rest method: '{s}'", .{aws_request.method});
log.debug("Rest success code: '{d}'", .{Action.http_config.success_code}); log.debug("Rest success code: '{d}'", .{Action.http_config.success_code});
log.debug("Rest raw uri: '{s}'", .{Action.http_config.uri}); log.debug("Rest raw uri: '{s}'", .{Action.http_config.uri});
aws_request.path = try buildPath(options.client.allocator, Action.http_config.uri, ActionRequest, request); aws_request.path = try buildPath(
options.client.allocator,
Action.http_config.uri,
ActionRequest,
request,
!std.mem.eql(u8, Self.service_meta.sdk_id, "S3"),
);
defer options.client.allocator.free(aws_request.path); defer options.client.allocator.free(aws_request.path);
log.debug("Rest processed uri: '{s}'", .{aws_request.path}); log.debug("Rest processed uri: '{s}'", .{aws_request.path});
// TODO: Make sure this doesn't get escaped here for S3 // TODO: Make sure this doesn't get escaped here for S3
@ -883,7 +889,13 @@ fn queryFieldTransformer(field_name: []const u8, encoding_options: url.EncodingO
return try case.snakeToPascal(encoding_options.allocator.?, field_name); return try case.snakeToPascal(encoding_options.allocator.?, field_name);
} }
fn buildPath(allocator: std.mem.Allocator, raw_uri: []const u8, comptime ActionRequest: type, request: anytype) ![]const u8 { fn buildPath(
allocator: std.mem.Allocator,
raw_uri: []const u8,
comptime ActionRequest: type,
request: anytype,
encode_slash: bool,
) ![]const u8 {
var buffer = try std.ArrayList(u8).initCapacity(allocator, raw_uri.len); var buffer = try std.ArrayList(u8).initCapacity(allocator, raw_uri.len);
// const writer = buffer.writer(); // const writer = buffer.writer();
defer buffer.deinit(); defer buffer.deinit();
@ -916,7 +928,7 @@ fn buildPath(allocator: std.mem.Allocator, raw_uri: []const u8, comptime ActionR
replacement_writer, replacement_writer,
); );
const trimmed_replacement_val = std.mem.trim(u8, replacement_buffer.items, "\""); const trimmed_replacement_val = std.mem.trim(u8, replacement_buffer.items, "\"");
try uriEncode(trimmed_replacement_val, encoded_buffer.writer()); try uriEncode(trimmed_replacement_val, encoded_buffer.writer(), encode_slash);
try buffer.appendSlice(encoded_buffer.items); try buffer.appendSlice(encoded_buffer.items);
} }
} }
@ -929,12 +941,12 @@ fn buildPath(allocator: std.mem.Allocator, raw_uri: []const u8, comptime ActionR
return buffer.toOwnedSlice(); return buffer.toOwnedSlice();
} }
fn uriEncode(input: []const u8, writer: anytype) !void { fn uriEncode(input: []const u8, writer: anytype, encode_slash: bool) !void {
for (input) |c| for (input) |c|
try uriEncodeByte(c, writer); try uriEncodeByte(c, writer, encode_slash);
} }
fn uriEncodeByte(char: u8, writer: anytype) !void { fn uriEncodeByte(char: u8, writer: anytype, encode_slash: bool) !void {
switch (char) { switch (char) {
'!' => _ = try writer.write("%21"), '!' => _ = try writer.write("%21"),
'#' => _ = try writer.write("%23"), '#' => _ = try writer.write("%23"),
@ -946,7 +958,7 @@ fn uriEncodeByte(char: u8, writer: anytype) !void {
'*' => _ = try writer.write("%2A"), '*' => _ = try writer.write("%2A"),
'+' => _ = try writer.write("%2B"), '+' => _ = try writer.write("%2B"),
',' => _ = try writer.write("%2C"), ',' => _ = try writer.write("%2C"),
'/' => _ = try writer.write("%2F"), '/' => _ = if (encode_slash) try writer.write("%2F") else try writer.write("/"),
':' => _ = try writer.write("%3A"), ':' => _ = try writer.write("%3A"),
';' => _ = try writer.write("%3B"), ';' => _ = try writer.write("%3B"),
'=' => _ = try writer.write("%3D"), '=' => _ = try writer.write("%3D"),
@ -1030,7 +1042,7 @@ fn addQueryArg(comptime ValueType: type, prefix: []const u8, key: []const u8, va
fn addBasicQueryArg(prefix: []const u8, key: []const u8, value: anytype, writer: anytype) !bool { fn addBasicQueryArg(prefix: []const u8, key: []const u8, value: anytype, writer: anytype) !bool {
_ = try writer.write(prefix); _ = try writer.write(prefix);
// TODO: url escaping // TODO: url escaping
try uriEncode(key, writer); try uriEncode(key, writer, true);
_ = try writer.write("="); _ = try writer.write("=");
try json.stringify(value, .{}, ignoringWriter(uriEncodingWriter(writer).writer(), '"').writer()); try json.stringify(value, .{}, ignoringWriter(uriEncodingWriter(writer).writer(), '"').writer());
return true; return true;
@ -1050,7 +1062,7 @@ pub fn UriEncodingWriter(comptime WriterType: type) type {
const Self = @This(); const Self = @This();
pub fn write(self: *Self, bytes: []const u8) Error!usize { pub fn write(self: *Self, bytes: []const u8) Error!usize {
try uriEncode(bytes, self.child_stream); try uriEncode(bytes, self.child_stream, true);
return bytes.len; // We say that all bytes are "written", even if they're not, as caller may be retrying return bytes.len; // We say that all bytes are "written", even if they're not, as caller may be retrying
} }
@ -1193,7 +1205,7 @@ test "REST Json v1 buildpath substitutes" {
.max_items = 1, .max_items = 1,
}; };
const input_path = "https://myhost/{MaxItems}/"; const input_path = "https://myhost/{MaxItems}/";
const output_path = try buildPath(allocator, input_path, @TypeOf(request), request); const output_path = try buildPath(allocator, input_path, @TypeOf(request), request, true);
defer allocator.free(output_path); defer allocator.free(output_path);
try std.testing.expectEqualStrings("https://myhost/1/", output_path); try std.testing.expectEqualStrings("https://myhost/1/", output_path);
} }
@ -1204,7 +1216,7 @@ test "REST Json v1 buildpath handles restricted characters" {
.marker = ":", .marker = ":",
}; };
const input_path = "https://myhost/{Marker}/"; const input_path = "https://myhost/{Marker}/";
const output_path = try buildPath(allocator, input_path, @TypeOf(request), request); const output_path = try buildPath(allocator, input_path, @TypeOf(request), request, true);
defer allocator.free(output_path); defer allocator.free(output_path);
try std.testing.expectEqualStrings("https://myhost/%3A/", output_path); try std.testing.expectEqualStrings("https://myhost/%3A/", output_path);
} }

View File

@ -235,9 +235,8 @@ pub fn main() anyerror!void {
std.log.info("key group quantity: {d}", .{list.quantity}); std.log.info("key group quantity: {d}", .{list.quantity});
}, },
.rest_xml_work_with_s3 => { .rest_xml_work_with_s3 => {
// TODO: Fix signature calculation mismatch with slashes const key = "i/am/a/teapot/foo";
// const key = "i/am/a/teapot/foo"; // const key = "foo";
const key = "foo";
const bucket = blk: { const bucket = blk: {
const result = try client.call(services.s3.list_buckets.Request{}, options); const result = try client.call(services.s3.list_buckets.Request{}, options);