Compare commits

..

No commits in common. "74704506d85fd468687996d54160afe9ccd24582" and "8d399cb8a6679609e33033d3a34438e8a775f56c" have entirely different histories.

6 changed files with 1533 additions and 1508 deletions

File diff suppressed because it is too large Load diff

View file

@ -90,37 +90,8 @@ pub const Options = struct {
dualstack: bool = false,
sigv4_service_name: ?[]const u8 = null,
mock: ?Mock = null,
};
/// mocking methods for isolated testing
pub const Mock = struct {
/// Used to provide consistent signing
signing_time: ?i64,
/// context is desiged to be type-erased pointer (@intFromPtr)
context: usize = 0,
request_fn: *const fn (
usize,
std.http.Method,
std.Uri,
std.http.Client.RequestOptions,
) std.http.Client.RequestError!std.http.Client.Request,
send_body_complete: *const fn (usize, []u8) std.Io.Writer.Error!void,
receive_head: *const fn (usize) std.http.Client.Request.ReceiveHeadError!std.http.Client.Response,
reader_decompressing: *const fn (usize) *std.Io.Reader,
fn request(m: Mock, method: std.http.Method, uri: std.Uri, options: std.http.Client.RequestOptions) std.http.Client.RequestError!std.http.Client.Request {
return m.request_fn(m.context, method, uri, options);
}
fn sendBodyComplete(m: Mock, body: []u8) std.Io.Writer.Error!void {
return m.send_body_complete(m.context, body);
}
fn receiveHead(m: Mock) std.http.Client.Request.ReceiveHeadError!std.http.Client.Response {
return m.receive_head(m.context);
}
fn readerDecompressing(m: Mock) *std.Io.Reader {
return m.reader_decompressing(m.context);
}
/// Used for testing to provide consistent signing. If null, will use current time
signing_time: ?i64 = null,
};
pub const Header = std.http.Header;
@ -192,9 +163,9 @@ pub const AwsHttp = struct {
.region = getRegion(service, options.region),
.service = options.sigv4_service_name orelse service,
.credentials = creds,
.signing_time = if (options.mock) |m| m.signing_time else null,
.signing_time = options.signing_time,
};
return try self.makeRequest(endpoint, request, signing_config, options);
return try self.makeRequest(endpoint, request, signing_config);
}
/// makeRequest is a low level http/https function that can be used inside
@ -213,13 +184,7 @@ pub const AwsHttp = struct {
/// Content-Length: (length of body)
///
/// Return value is an HttpResult, which will need the caller to deinit().
pub fn makeRequest(
self: Self,
endpoint: EndPoint,
request: HttpRequest,
signing_config: ?signing.Config,
options: Options,
) !HttpResult {
pub fn makeRequest(self: Self, endpoint: EndPoint, request: HttpRequest, signing_config: ?signing.Config) !HttpResult {
var request_cp = request;
log.debug("Request Path: {s}", .{request_cp.path});
@ -262,16 +227,13 @@ pub const AwsHttp = struct {
log.debug("Request url: {s}", .{url});
// TODO: Fix this proxy stuff. This is all a kludge just to compile, but std.http.Client has it all built in now
var cl = std.http.Client{ .allocator = self.allocator, .https_proxy = if (self.proxy) |*p| @constCast(p) else null };
// Not sure this if statement is correct here. deinit seems to assume
// that client.request was called at least once, but we don't do that
// if we're in a test harness
defer cl.deinit(); // TODO: Connection pooling
const method = std.meta.stringToEnum(std.http.Method, request_cp.method).?;
// Fetch API in 0.15.1 is insufficient as it does not provide
// server headers. We'll construct and send the request ourselves
const uri = try std.Uri.parse(url);
const req_options: std.http.Client.RequestOptions = .{
var req = try cl.request(method, try std.Uri.parse(url), .{
// we need full control over most headers. I wish libraries would do a
// better job of having default headers as an opt-in...
.headers = .{
@ -283,13 +245,7 @@ pub const AwsHttp = struct {
.content_type = .omit,
},
.extra_headers = headers.items,
};
var req = if (options.mock) |m|
try m.request(method, uri, req_options) // This will call the test harness
else
try cl.request(method, uri, req_options);
});
// TODO: Need to test for payloads > 2^14. I believe one of our tests does this, but not sure
// if (request_cp.body.len > 0) {
// // Workaround for https://github.com/ziglang/zig/issues/15626
@ -311,14 +267,10 @@ pub const AwsHttp = struct {
// in the chain then does actually modify the body of the request
// so we'll need to duplicate it here
const req_body = try self.allocator.dupe(u8, request_cp.body);
defer self.allocator.free(req_body); // docs for sendBodyComplete say it flushes, so no need to outlive this
if (options.mock) |m|
try m.sendBodyComplete(req_body)
else
try req.sendBodyComplete(req_body);
} else if (options.mock == null) try req.sendBodiless();
try req.sendBodyComplete(req_body);
} else try req.sendBodiless();
var response = if (options.mock) |m| try m.receiveHead() else try req.receiveHead(&.{});
var response = try req.receiveHead(&.{});
// TODO: Timeout - is this now above us?
log.debug(

View file

@ -348,10 +348,10 @@ pub fn freeSignedRequest(allocator: std.mem.Allocator, request: *base.Request, c
pub const credentialsFn = *const fn ([]const u8) ?Credentials;
pub fn verifyServerRequest(allocator: std.mem.Allocator, request: *std.http.Server.Request, credentials_fn: credentialsFn) !bool {
pub fn verifyServerRequest(allocator: std.mem.Allocator, request: *std.http.Server.Request, request_body_reader: anytype, credentials_fn: credentialsFn) !bool {
var unverified_request = try UnverifiedRequest.init(allocator, request);
defer unverified_request.deinit();
return verify(allocator, unverified_request, credentials_fn);
return verify(allocator, unverified_request, request_body_reader, credentials_fn);
}
pub const UnverifiedRequest = struct {
@ -359,19 +359,17 @@ pub const UnverifiedRequest = struct {
target: []const u8,
method: std.http.Method,
allocator: std.mem.Allocator,
raw: *std.http.Server.Request,
pub fn init(allocator: std.mem.Allocator, request: *std.http.Server.Request) !UnverifiedRequest {
var al = std.ArrayList(std.http.Header){};
defer al.deinit(allocator);
var al = std.ArrayList(std.http.Header).init(allocator);
defer al.deinit();
var it = request.iterateHeaders();
while (it.next()) |h| try al.append(allocator, h);
while (it.next()) |h| try al.append(h);
return .{
.target = request.head.target,
.method = request.head.method,
.headers = try al.toOwnedSlice(allocator),
.headers = try al.toOwnedSlice(),
.allocator = allocator,
.raw = request,
};
}
@ -389,7 +387,7 @@ pub const UnverifiedRequest = struct {
}
};
pub fn verify(allocator: std.mem.Allocator, request: UnverifiedRequest, credentials_fn: credentialsFn) !bool {
pub fn verify(allocator: std.mem.Allocator, request: UnverifiedRequest, request_body_reader: anytype, credentials_fn: credentialsFn) !bool {
var arena = std.heap.ArenaAllocator.init(allocator);
defer arena.deinit();
const aa = arena.allocator();
@ -422,6 +420,7 @@ pub fn verify(allocator: std.mem.Allocator, request: UnverifiedRequest, credenti
return verifyParsedAuthorization(
aa,
request,
request_body_reader,
credential.?,
signed_headers.?,
signature.?,
@ -432,6 +431,7 @@ pub fn verify(allocator: std.mem.Allocator, request: UnverifiedRequest, credenti
fn verifyParsedAuthorization(
allocator: std.mem.Allocator,
request: UnverifiedRequest,
request_body_reader: anytype,
credential: []const u8,
signed_headers: []const u8,
signature: []const u8,
@ -494,8 +494,7 @@ fn verifyParsedAuthorization(
.content_type = request.getFirstHeaderValue("content-type").?,
};
signed_request.query = request.target[signed_request.path.len..]; // TODO: should this be +1? query here would include '?'
// TODO: This is almost certainly not what we want here long term, but will get tests working
signed_request.body = try request.raw.server.reader.in.allocRemaining(allocator, .unlimited);
signed_request.body = try request_body_reader.readAllAlloc(allocator, std.math.maxInt(usize));
defer allocator.free(signed_request.body);
signed_request = try signRequest(allocator, signed_request, config);
defer freeSignedRequest(allocator, &signed_request, config);
@ -1011,13 +1010,13 @@ test "canonical query" {
test "canonical headers" {
const allocator = std.testing.allocator;
var headers = try std.ArrayList(std.http.Header).initCapacity(allocator, 5);
defer headers.deinit(allocator);
try headers.append(allocator, .{ .name = "Host", .value = "iam.amazonaws.com" });
try headers.append(allocator, .{ .name = "Content-Type", .value = "application/x-www-form-urlencoded; charset=utf-8" });
try headers.append(allocator, .{ .name = "User-Agent", .value = "This header should be skipped" });
try headers.append(allocator, .{ .name = "My-header1", .value = " a b c " });
try headers.append(allocator, .{ .name = "X-Amz-Date", .value = "20150830T123600Z" });
try headers.append(allocator, .{ .name = "My-header2", .value = " \"a b c\" " });
defer headers.deinit();
try headers.append(.{ .name = "Host", .value = "iam.amazonaws.com" });
try headers.append(.{ .name = "Content-Type", .value = "application/x-www-form-urlencoded; charset=utf-8" });
try headers.append(.{ .name = "User-Agent", .value = "This header should be skipped" });
try headers.append(.{ .name = "My-header1", .value = " a b c " });
try headers.append(.{ .name = "X-Amz-Date", .value = "20150830T123600Z" });
try headers.append(.{ .name = "My-header2", .value = " \"a b c\" " });
const expected =
\\content-type:application/x-www-form-urlencoded; charset=utf-8
\\host:iam.amazonaws.com
@ -1036,12 +1035,12 @@ test "canonical headers" {
test "canonical request" {
const allocator = std.testing.allocator;
var headers = try std.ArrayList(std.http.Header).initCapacity(allocator, 5);
defer headers.deinit(allocator);
try headers.append(allocator, .{ .name = "User-agent", .value = "c sdk v1.0" });
defer headers.deinit();
try headers.append(.{ .name = "User-agent", .value = "c sdk v1.0" });
// In contrast to AWS CRT (aws-c-auth), we add the date as part of the
// signing operation. They add it as part of the canonicalization
try headers.append(allocator, .{ .name = "X-Amz-Date", .value = "20150830T123600Z" });
try headers.append(allocator, .{ .name = "Host", .value = "example.amazonaws.com" });
try headers.append(.{ .name = "X-Amz-Date", .value = "20150830T123600Z" });
try headers.append(.{ .name = "Host", .value = "example.amazonaws.com" });
const req = base.Request{
.path = "/",
.method = "GET",
@ -1096,10 +1095,10 @@ test "can sign" {
const allocator = std.testing.allocator;
var headers = try std.ArrayList(std.http.Header).initCapacity(allocator, 5);
defer headers.deinit(allocator);
try headers.append(allocator, .{ .name = "Content-Type", .value = "application/x-www-form-urlencoded; charset=utf-8" });
try headers.append(allocator, .{ .name = "Content-Length", .value = "13" });
try headers.append(allocator, .{ .name = "Host", .value = "example.amazonaws.com" });
defer headers.deinit();
try headers.append(.{ .name = "Content-Type", .value = "application/x-www-form-urlencoded; charset=utf-8" });
try headers.append(.{ .name = "Content-Length", .value = "13" });
try headers.append(.{ .name = "Host", .value = "example.amazonaws.com" });
const req = base.Request{
.path = "/",
.query = "",
@ -1166,25 +1165,25 @@ test "can verify server request" {
"X-Amz-Date: 20230908T170252Z\r\n" ++
"x-amz-content-sha256: fcde2b2edba56bf408601fb721fe9b5c338d10ee429ea04fae5511b68fbf8fb9\r\n" ++
"Authorization: AWS4-HMAC-SHA256 Credential=ACCESS/20230908/us-west-2/s3/aws4_request, SignedHeaders=accept;content-length;content-type;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class, Signature=fcc43ce73a34c9bd1ddf17e8a435f46a859812822f944f9eeb2aabcd64b03523\r\n\r\nbar";
var reader = std.Io.Reader.fixed(req);
var read_buffer: [1024]u8 = undefined;
@memcpy(read_buffer[0..req.len], req);
var server: std.http.Server = .{
.out = undefined, // We're not sending a response here
.reader = .{
.in = &reader,
.interface = undefined,
.state = .received_head,
.max_head_len = req.len,
},
.connection = undefined,
.state = .ready,
.read_buffer = &read_buffer,
.read_buffer_len = req.len,
.next_request_start = 0,
};
var request: std.http.Server.Request = .{
.server = &server,
.head = try std.http.Server.Request.Head.parse(req),
.head_buffer = req,
.head_end = req.len - 3,
.head = try std.http.Server.Request.Head.parse(read_buffer[0 .. req.len - 3]),
.reader_state = undefined,
};
// std.testing.log_level = .debug;
if (true) return error.SkipZigTest;
try std.testing.expect(try verifyServerRequest(allocator, &request, struct {
var fbs = std.io.fixedBufferStream("bar");
try std.testing.expect(try verifyServerRequest(allocator, &request, fbs.reader(), struct {
cred: Credentials,
const Self = @This();
@ -1222,24 +1221,22 @@ test "can verify server request without x-amz-content-sha256" {
const req_data = head ++ body;
var read_buffer: [2048]u8 = undefined;
@memcpy(read_buffer[0..req_data.len], req_data);
var reader = std.Io.Reader.fixed(&read_buffer);
var server: std.http.Server = .{
.out = undefined, // We're not sending a response here
.reader = .{
.interface = undefined,
.in = &reader,
.state = .received_head,
.max_head_len = 1024,
},
.connection = undefined,
.state = .ready,
.read_buffer = &read_buffer,
.read_buffer_len = req_data.len,
.next_request_start = 0,
};
var request: std.http.Server.Request = .{
.server = &server,
.head = try std.http.Server.Request.Head.parse(head),
.head_buffer = head,
.head_end = head.len,
.head = try std.http.Server.Request.Head.parse(read_buffer[0..head.len]),
.reader_state = undefined,
};
{
var h = try std.ArrayList(std.http.Header).initCapacity(allocator, 4);
defer h.deinit(allocator);
var h = std.ArrayList(std.http.Header).init(allocator);
defer h.deinit();
const signed_headers = &[_][]const u8{ "content-type", "host", "x-amz-date", "x-amz-target" };
var it = request.iterateHeaders();
while (it.next()) |source| {
@ -1248,7 +1245,7 @@ test "can verify server request without x-amz-content-sha256" {
match = std.ascii.eqlIgnoreCase(s, source.name);
if (match) break;
}
if (match) try h.append(allocator, .{ .name = source.name, .value = source.value });
if (match) try h.append(.{ .name = source.name, .value = source.value });
}
const req = base.Request{
.path = "/",
@ -1285,8 +1282,9 @@ test "can verify server request without x-amz-content-sha256" {
}
{ // verification
if (true) return error.SkipZigTest;
try std.testing.expect(try verifyServerRequest(allocator, &request, struct {
var fis = std.io.fixedBufferStream(body[0..]);
try std.testing.expect(try verifyServerRequest(allocator, &request, fis.reader(), struct {
cred: Credentials,
const Self = @This();

File diff suppressed because it is too large Load diff

View file

@ -59,7 +59,6 @@ pub fn encodeInternal(
switch (ti.child) {
// TODO: not sure this first one is valid. How should [][]const u8 be serialized here?
[]const u8 => {
// if (true) @panic("panic at the disco!");
std.log.warn(
"encoding object of type [][]const u8...pretty sure this is wrong {s}{s}={any}",
.{ parent, field_name, obj },
@ -104,29 +103,78 @@ pub fn encodeInternal(
return rc;
}
fn testencode(allocator: std.mem.Allocator, expected: []const u8, value: anytype, comptime options: EncodingOptions) !void {
const ValidationWriter = struct {
const Self = @This();
pub const Writer = std.io.Writer(*Self, Error, write);
pub const Error = error{
TooMuchData,
DifferentData,
};
expected_remaining: []const u8,
fn init(exp: []const u8) Self {
return .{ .expected_remaining = exp };
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
fn write(self: *Self, bytes: []const u8) Error!usize {
// std.debug.print("{s}\n", .{bytes});
if (self.expected_remaining.len < bytes.len) {
std.log.warn(
\\====== expected this output: =========
\\{s}
\\======== instead found this: =========
\\{s}
\\======================================
, .{
self.expected_remaining,
bytes,
});
return error.TooMuchData;
}
if (!std.mem.eql(u8, self.expected_remaining[0..bytes.len], bytes)) {
std.log.warn(
\\====== expected this output: =========
\\{s}
\\======== instead found this: =========
\\{s}
\\======================================
, .{
self.expected_remaining[0..bytes.len],
bytes,
});
return error.DifferentData;
}
self.expected_remaining = self.expected_remaining[bytes.len..];
return bytes.len;
}
};
var vos = ValidationWriter.init(expected);
try encode(allocator, value, vos.writer(), options);
if (vos.expected_remaining.len > 0) return error.NotEnoughData;
}
test "can urlencode an object" {
const expected = "Action=GetCallerIdentity&Version=2021-01-01";
var aw = std.Io.Writer.Allocating.init(std.testing.allocator);
defer aw.deinit();
try encode(
try testencode(
std.testing.allocator,
"Action=GetCallerIdentity&Version=2021-01-01",
.{ .Action = "GetCallerIdentity", .Version = "2021-01-01" },
&aw.writer,
.{},
);
try std.testing.expectEqualStrings(expected, aw.written());
}
test "can urlencode an object with integer" {
const expected = "Action=GetCallerIdentity&Duration=32";
var aw = std.Io.Writer.Allocating.init(std.testing.allocator);
defer aw.deinit();
try encode(
try testencode(
std.testing.allocator,
"Action=GetCallerIdentity&Duration=32",
.{ .Action = "GetCallerIdentity", .Duration = 32 },
&aw.writer,
.{},
);
try std.testing.expectEqualStrings(expected, aw.written());
}
const UnsetValues = struct {
action: ?[]const u8 = null,
@ -135,28 +183,30 @@ const UnsetValues = struct {
val2: ?[]const u8 = null,
};
test "can urlencode an object with unset values" {
const expected = "action=GetCallerIdentity&duration=32";
var aw = std.Io.Writer.Allocating.init(std.testing.allocator);
defer aw.deinit();
try encode(
// var buffer = std.ArrayList(u8).init(std.testing.allocator);
// defer buffer.deinit();
// const writer = buffer.writer();
// try encode(
// std.testing.allocator,
// UnsetValues{ .action = "GetCallerIdentity", .duration = 32 },
// writer,
// .{},
// );
// std.debug.print("\n\nEncoded as '{s}'\n", .{buffer.items});
try testencode(
std.testing.allocator,
"action=GetCallerIdentity&duration=32",
UnsetValues{ .action = "GetCallerIdentity", .duration = 32 },
&aw.writer,
.{},
);
try std.testing.expectEqualStrings(expected, aw.written());
}
test "can urlencode a complex object" {
const expected = "Action=GetCallerIdentity&Version=2021-01-01&complex.innermember=foo";
var aw = std.Io.Writer.Allocating.init(std.testing.allocator);
defer aw.deinit();
try encode(
try testencode(
std.testing.allocator,
"Action=GetCallerIdentity&Version=2021-01-01&complex.innermember=foo",
.{ .Action = "GetCallerIdentity", .Version = "2021-01-01", .complex = .{ .innermember = "foo" } },
&aw.writer,
.{},
);
try std.testing.expectEqualStrings(expected, aw.written());
}
const Filter = struct {
@ -179,28 +229,26 @@ const Request: type = struct {
all_regions: ?bool = null,
};
test "can urlencode an EC2 Filter" {
// TODO: This is a strange test, mainly to document current behavior
// EC2 filters are supposed to be something like
// Filter.Name=foo&Filter.Values=bar or, when there is more, something like
// Filter.1.Name=instance-type&Filter.1.Value.1=m1.small&Filter.1.Value.2=m1.large&Filter.2.Name=block-device-mapping.status&Filter.2.Value.1=attached
//
// This looks like a real PITA, so until it is actually needed, this is
// a placeholder test to track what actual encoding is happening. This
// changed between zig 0.14.x and 0.15.1, and I'm not entirely sure why
// yet, but because the remaining functionality is fine, we're going with
// this
const zig_14x_expected = "filters={ url.Filter{ .name = { 102, 111, 111 }, .values = { { ... } } } }";
_ = zig_14x_expected;
const expected = "filters={ .{ .name = { 102, 111, 111 }, .values = { { ... } } } }";
var aw = std.Io.Writer.Allocating.init(std.testing.allocator);
defer aw.deinit();
try encode(
// TODO: Fix this encoding...
testencode(
std.testing.allocator,
"filters={ url.Filter{ .name = { 102, 111, 111 }, .values = { { ... } } } }",
Request{
.filters = @constCast(&[_]Filter{.{ .name = "foo", .values = @constCast(&[_][]const u8{"bar"}) }}),
},
&aw.writer,
.{},
);
try std.testing.expectEqualStrings(expected, aw.written());
) catch |err| {
var al = std.ArrayList(u8).init(std.testing.allocator);
defer al.deinit();
try encode(
std.testing.allocator,
Request{
.filters = @constCast(&[_]Filter{.{ .name = "foo", .values = @constCast(&[_][]const u8{"bar"}) }}),
},
al.writer(),
.{},
);
std.log.warn("Error found. Full encoding is '{s}'", .{al.items});
return err;
};
}

View file

@ -413,8 +413,7 @@ test "stringify basic types" {
{
const result = try stringifyAlloc(allocator, 3.14, .{});
defer allocator.free(result);
// zig 0.14.x outputs 3.14e0, but zig 0.15.1 outputs 3.14. Either *should* be acceptable
try testing.expectEqualStrings("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<root>3.14</root>", result);
try testing.expectEqualStrings("<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<root>3.14e0</root>", result);
}
// Test string