Compare commits

...

10 Commits

9 changed files with 378 additions and 26 deletions

View File

@ -0,0 +1,33 @@
name: AWS-Zig Build
run-name: ${{ github.actor }} building Universal Lambda
on: [push]
env:
ACTIONS_RUNTIME_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ACTIONS_RUNTIME_URL: ${{ env.GITHUB_SERVER_URL }}/api/actions_pipeline/
jobs:
build-zig-0.11.0-amd64-host:
runs-on: ubuntu-latest
env:
ZIG_VERSION: 0.11.0
ARCH: x86_64
steps:
- name: Check out repository code
uses: actions/checkout@v3
# ARCH is fine, but we can't substitute directly because zig
# uses x86_64 instead of amd64. They also use aarch64 instead of arm64.
#
# However, arm64/linux isn't quite fully tier 1 yet, so this is more of a
# TODO: https://github.com/ziglang/zig/issues/2443
- run: wget -q https://ziglang.org/download/${ZIG_VERSION}/zig-linux-${ARCH}-${ZIG_VERSION}.tar.xz
- run: tar x -C /usr/local -f zig-linux-${ARCH}-${ZIG_VERSION}.tar.xz
- run: ln -s /usr/local/zig-linux-${ARCH}-${ZIG_VERSION}/zig /usr/local/bin/zig
- run: zig build test
- run: zig build test -Dtarget=arm-linux # we want to know we can build for 32 bit
- name: Notify
uses: https://git.lerch.org/lobo/action-notify-ntfy@v2
if: always()
with:
host: ${{ secrets.NTFY_HOST }}
topic: ${{ secrets.NTFY_TOPIC }}
user: ${{ secrets.NTFY_USER }}
password: ${{ secrets.NTFY_PASSWORD }}

View File

@ -3,7 +3,7 @@ const std = @import("std");
// Although this function looks imperative, note that its job is to
// declaratively construct a build graph that will be executed by an external
// runner.
pub fn build(b: *std.Build) void {
pub fn build(b: *std.Build) !void {
// Standard target options allows the person running `zig build` to choose
// what target to build for. Here we do not override the defaults, which
// means any target is allowed, and the default is native. Other options
@ -23,6 +23,11 @@ pub fn build(b: *std.Build) void {
.target = target,
.optimize = optimize,
});
// Because we are...well, ourselves, we'll manually override the module
// root (we are not a module here).
const ulb = @import("src/universal_lambda_build.zig");
ulb.module_root = "";
_ = try ulb.createOptionsModule(b, lib);
// This declares intent for the library to be installed into the standard
// location when the user invokes the "install" step (the default step when
@ -32,12 +37,14 @@ pub fn build(b: *std.Build) void {
// Creates a step for unit testing. This only builds the test executable
// but does not run it.
const main_tests = b.addTest(.{
.root_source_file = .{ .path = "src/main.zig" },
.root_source_file = .{ .path = "src/universal_lambda.zig" },
.target = target,
.optimize = optimize,
});
_ = try ulb.createOptionsModule(b, main_tests);
const run_main_tests = b.addRunArtifact(main_tests);
var run_main_tests = b.addRunArtifact(main_tests);
run_main_tests.skip_foreign_checks = true;
// This creates a build step. It will be visible in the `zig build --help` menu,
// and can be selected like this: `zig build test`

140
src/flexilib-interface.zig Normal file
View File

@ -0,0 +1,140 @@
const std = @import("std");
// C interfaces between main and libraries
pub const Header = extern struct {
name_ptr: [*]u8,
name_len: usize,
value_ptr: [*]u8,
value_len: usize,
};
pub const Response = extern struct {
ptr: [*]u8,
len: usize,
headers: [*]Header,
headers_len: usize,
};
pub const Request = extern struct {
method: [*:0]u8,
method_len: usize,
content: [*]u8,
content_len: usize,
headers: [*]Header,
headers_len: usize,
};
// If the library is Zig, we can use these helpers
threadlocal var allocator: ?*std.mem.Allocator = null;
const log = std.log.scoped(.interface);
pub const ZigRequest = struct {
method: [:0]u8,
content: []u8,
headers: []Header,
};
pub const ZigHeader = struct {
name: []u8,
value: []u8,
};
pub const ZigResponse = struct {
body: *std.ArrayList(u8),
headers: *std.StringHashMap([]const u8),
};
pub const ZigRequestHandler = *const fn (std.mem.Allocator, ZigRequest, ZigResponse) anyerror!void;
/// This function is optional and can be exported by zig libraries for
/// initialization. If exported, it will be called once in the beginning of
/// a request and will be provided a pointer to std.mem.Allocator, which is
/// useful for reusing the parent allocator. If you're planning on using
/// the handleRequest helper below, you must use zigInit or otherwise
/// set the interface allocator in your own version of zigInit
pub fn zigInit(parent_allocator: *anyopaque) callconv(.C) void {
allocator = @ptrCast(@alignCast(parent_allocator));
}
pub fn toZigHeader(header: Header) ZigHeader {
return .{
.name = header.name_ptr[0..header.name_len],
.value = header.value_ptr[0..header.value_len],
};
}
/// Converts a StringHashMap to the structure necessary for passing through the
/// C boundary. This will be called automatically for you via the handleRequest function
/// and is also used by the main processing loop to coerce request headers
fn toHeaders(alloc: std.mem.Allocator, headers: std.StringHashMap([]const u8)) ![*]Header {
var header_array = try std.ArrayList(Header).initCapacity(alloc, headers.count());
var iterator = headers.iterator();
while (iterator.next()) |kv| {
header_array.appendAssumeCapacity(.{
.name_ptr = @constCast(kv.key_ptr.*).ptr,
.name_len = kv.key_ptr.*.len,
.value_ptr = @constCast(kv.value_ptr.*).ptr,
.value_len = kv.value_ptr.*.len,
});
}
return header_array.items.ptr;
}
/// handles a request, implementing the C interface to communicate between the
/// main program and a zig library. Most importantly, it will catch/report
/// errors appropriately and allow zig code to use standard Zig error semantics
pub fn handleRequest(request: *Request, zigRequestHandler: ZigRequestHandler) ?*Response {
// TODO: implement another library in C or Rust or something to show
// that anything using a C ABI can be successful
var alloc = if (allocator) |a| a.* else {
log.err("zigInit not called prior to handle_request. This is a coding error", .{});
return null;
};
// setup response body
var response = std.ArrayList(u8).init(alloc);
// setup headers
var headers = std.StringHashMap([]const u8).init(alloc);
zigRequestHandler(
alloc,
.{
.method = request.method[0..request.method_len :0],
.content = request.content[0..request.content_len],
.headers = request.headers[0..request.headers_len],
},
.{
.body = &response,
.headers = &headers,
},
) catch |e| {
log.err("Unexpected error processing request: {any}", .{e});
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
return null;
};
// Marshall data back for handling by server
var rc = alloc.create(Response) catch {
log.err("Could not allocate memory for response object. This may be fatal", .{});
return null;
};
rc.ptr = response.items.ptr;
rc.len = response.items.len;
rc.headers = toHeaders(alloc, headers) catch |e| {
log.err("Unexpected error processing request: {any}", .{e});
if (@errorReturnTrace()) |trace| {
std.debug.dumpStackTrace(trace.*);
}
return null;
};
rc.headers_len = headers.count();
return rc;
}

107
src/flexilib.zig Normal file
View File

@ -0,0 +1,107 @@
const std = @import("std");
const interface = @import("flexilib-interface.zig"); // TODO: pull in flexilib directly
const testing = std.testing;
const log = std.log.scoped(.@"main-lib");
const client_handler = @import("flexilib_handler");
// The main program will look for exports during the request lifecycle:
// zigInit (optional): called at the beginning of a request, includes pointer to an allocator
// handle_request: called with request data, expects response data
// request_deinit (optional): called at the end of a request to allow resource cleanup
//
// Setup for these is aided by the interface library as shown below
// zigInit is an optional export called at the beginning of a request. It will
// be passed an allocator (which...shh...is an arena allocator). Since the
// interface library provides a request handler that requires a built-in allocator,
// if you are using the interface's handleRequest function as shown above,
// you will need to also include this export. To customize, just do something
// like this:
//
// export fn zigInit(parent_allocator: *anyopaque) callconv(.C) void {
// // your code here, just include the next line
// interface.zigInit(parent_allocator);
// }
//
comptime {
@export(interface.zigInit, .{ .name = "zigInit", .linkage = .Strong });
}
/// handle_request will be called on a single request, but due to the preservation
/// of restrictions imposed by the calling interface, it should generally be more
/// useful to call into the interface library to let it do the conversion work
/// on your behalf
export fn handle_request(request: *interface.Request) callconv(.C) ?*interface.Response {
// The interface library provides a handleRequest function that will handle
// marshalling data back and forth from the C format used for the interface
// to a more Zig friendly format. It also allows usage of zig errors. To
// use, pass in the request and the zig function used to handle the request
// (here called "handleRequest"). The function signature must be:
//
// fn (std.mem.Allocator, interface.ZigRequest, interface.ZigResponse) !void
//
return interface.handleRequest(request, handleRequest);
}
// request_deinit is an optional export and will be called a the end of the
// request. Useful for deallocating memory. Since this is zig code and the
// allocator used is an arena allocator, all allocated memory will be automatically
// cleaned up by the main program at the end of a request
//
// export fn request_deinit() void {
// }
// ************************************************************************
// Boilerplate ^^, Custom code vv
// ************************************************************************
//
// handleRequest function here is the last line of boilerplate and the
// entry to a request
fn handleRequest(allocator: std.mem.Allocator, request: interface.ZigRequest, response: interface.ZigResponse) !void {
// setup
var response_writer = response.body.writer();
try response_writer.writeAll(try client_handler.handler(allocator, request.content, .{}));
// real work
for (request.headers) |h| {
const header = interface.toZigHeader(h);
// std.debug.print("\n{s}: {s}\n", .{ header.name, header.value });
if (std.ascii.eqlIgnoreCase(header.name, "host") and std.mem.startsWith(u8, header.value, "iam")) {
try response_writer.print("iam response", .{});
return;
}
if (std.ascii.eqlIgnoreCase(header.name, "x-slow")) {
std.time.sleep(std.time.ns_per_ms * (std.fmt.parseInt(usize, header.value, 10) catch 1000));
try response_writer.print("i am slow\n\n", .{});
return;
}
}
try response.headers.put("X-custom-foo", "bar");
log.info("handlerequest header count {d}", .{response.headers.count()});
}
// Need to figure out how tests would work
test "handle_request" {
var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena.deinit();
var aa = arena.allocator();
interface.zigInit(&aa);
var headers: []interface.Header = @constCast(&[_]interface.Header{.{
.name_ptr = @ptrCast(@constCast("GET".ptr)),
.name_len = 3,
.value_ptr = @ptrCast(@constCast("GET".ptr)),
.value_len = 3,
}});
var req = interface.Request{
.method = @ptrCast(@constCast("GET".ptr)),
.method_len = 3,
.content = @ptrCast(@constCast("GET".ptr)),
.content_len = 3,
.headers = headers.ptr,
.headers_len = 1,
};
const response = handle_request(&req).?;
try testing.expectEqualStrings(" 1", response.ptr[0..response.len]);
try testing.expectEqualStrings("X-custom-foo", response.headers[0].name_ptr[0..response.headers[0].name_len]);
try testing.expectEqualStrings("bar", response.headers[0].value_ptr[0..response.headers[0].value_len]);
}

47
src/flexilib_build.zig Normal file
View File

@ -0,0 +1,47 @@
const std = @import("std");
const builtin = @import("builtin");
/// flexilib will create a dynamic library for use with flexilib.
/// Flexilib will need to get the exe compiled as a library
/// For flexilib, we will need the main file to have a pub fn named
/// "handler". If it is not called that, a pub const handler = ... can be
/// used instead
pub fn configureBuild(b: *std.build.Builder, exe: *std.Build.Step.Compile, build_root_src: []const u8) !void {
const package_step = b.step("flexilib", "Create a flexilib dynamic library");
// const exe = b.addExecutable(.{
// .name = "universal-lambda-example",
// // In this case the main source file is merely a path, however, in more
// // complicated build scripts, this could be a generated file.
// .root_source_file = .{ .path = "src/main.zig" },
// .target = target,
// .optimize = optimize,
// });
//
const lib = b.addSharedLibrary(.{
.name = exe.name,
.root_source_file = .{ .path = b.pathJoin(&[_][]const u8{ build_root_src, "flexilib.zig" }) },
.target = exe.target,
.optimize = exe.optimize,
});
// We will not free this, as the rest of the build system will use it.
// This should be ok because our allocator is, I believe, an arena
var module_dependencies = try b.allocator.alloc(std.Build.ModuleDependency, exe.modules.count());
var iterator = exe.modules.iterator();
var i: usize = 0;
while (iterator.next()) |entry| : (i += 1) {
module_dependencies[i] = .{
.name = entry.key_ptr.*,
.module = entry.value_ptr.*,
};
lib.addModule(entry.key_ptr.*, entry.value_ptr.*);
}
lib.addAnonymousModule("flexilib_handler", .{
// Source file can be anywhere on disk, does not need to be a subdirectory
.source_file = exe.root_src.?,
.dependencies = module_dependencies,
});
package_step.dependOn(&b.addInstallArtifact(lib, .{}).step);
}

View File

@ -2,6 +2,7 @@ const std = @import("std");
const builtin = @import("builtin");
const HandlerFn = @import("universal_lambda.zig").HandlerFn;
const Context = @import("universal_lambda.zig").Context;
const log = std.log.scoped(.lambda);
@ -183,9 +184,14 @@ const Event = struct {
var req = try cl.request(.POST, response_uri, empty_headers, .{});
// var req = try client.?.request(.POST, response_uri, empty_headers, .{});
defer req.deinit();
// Lambda does different things, depending on the runtime. Go 1.x takes
// any return value but escapes double quotes. Custom runtimes can
// do whatever they want. node I believe wraps as a json object. We're
// going to leave the return value up to the handler, and they can
// use a seperate API for normalization so we're explicit.
const response_content = try std.fmt.allocPrint(
self.allocator,
"{{ \"content\": \"{s}\" }}",
"{s}",
.{event_response},
);
defer self.allocator.free(response_content);
@ -261,12 +267,16 @@ fn getEvent(allocator: std.mem.Allocator, event_data_uri: std.Uri) !?Event {
const req_id = request_id.?;
log.debug("got lambda request with id {s}", .{req_id});
var resp_payload = try std.ArrayList(u8).initCapacity(allocator, content_length.?);
defer resp_payload.deinit();
try resp_payload.resize(content_length.?);
var response_data = try resp_payload.toOwnedSlice();
errdefer allocator.free(response_data);
_ = try req.readAll(response_data);
var response_data: []u8 =
if (req.response.transfer_encoding) |_| // the only value here is "chunked"
try req.reader().readAllAlloc(allocator, std.math.maxInt(usize))
else blk: {
// content length
var tmp_data = try allocator.alloc(u8, content_length.?);
errdefer allocator.free(tmp_data);
_ = try req.readAll(tmp_data);
break :blk tmp_data;
};
return Event.init(
allocator,
@ -357,7 +367,7 @@ fn processRequest(allocator: std.mem.Allocator, server: *std.http.Server) !void
var response_bytes: []const u8 = errbuf[0..];
if (res.request.content_length) |l|
server_request_aka_lambda_response = try res.reader().readAllAlloc(allocator, @as(usize, l));
server_request_aka_lambda_response = try res.reader().readAllAlloc(allocator, @as(usize, @intCast(l)));
log.debug(
"tid {d} (server): {d} bytes read from request",
@ -391,8 +401,9 @@ fn serve(allocator: std.mem.Allocator, res: *std.http.Server.Response) ![]const
return server_response;
}
fn handler(allocator: std.mem.Allocator, event_data: []const u8) ![]const u8 {
fn handler(allocator: std.mem.Allocator, event_data: []const u8, context: Context) ![]const u8 {
_ = allocator;
_ = context;
return event_data;
}
fn test_run(allocator: std.mem.Allocator, event_handler: HandlerFn) !std.Thread {
@ -458,9 +469,8 @@ test "basic request" {
\\{"foo": "bar", "baz": "qux"}
;
// This is what's actually coming back. Is this right?
const expected_response =
\\{ "content": "{"foo": "bar", "baz": "qux"}" }
\\{"foo": "bar", "baz": "qux"}
;
const lambda_response = try lambda_request(allocator, request, 1);
defer deinit();
@ -475,9 +485,8 @@ test "several requests do not fail" {
\\{"foo": "bar", "baz": "qux"}
;
// This is what's actually coming back. Is this right?
const expected_response =
\\{ "content": "{"foo": "bar", "baz": "qux"}" }
\\{"foo": "bar", "baz": "qux"}
;
const lambda_response = try lambda_request(allocator, request, 5);
defer deinit();

View File

@ -120,3 +120,10 @@ fn processRequest(aa: std.mem.Allocator, server: *std.http.Server, event_handler
_ = try res.writer().writeAll(response_bytes);
try res.finish();
}
test {
std.testing.refAllDecls(@This()); // standalone, standalone web server
std.testing.refAllDecls(@import("lambda.zig")); // lambda
// TODO: re-enable
// std.testing.refAllDecls(@import("flexilib.zig")); // flexilib
}

View File

@ -12,22 +12,24 @@ pub const BuildType = enum {
flexilib,
};
pub fn configureBuild(b: *std.Build, exe: *std.Build.Step.Compile) !void {
pub var module_root: ?[]const u8 = null;
pub fn configureBuild(b: *std.Build, cs: *std.Build.Step.Compile) !void {
const file_location = try findFileLocation(b);
// Add module
exe.addAnonymousModule("universal_lambda_handler", .{
cs.addAnonymousModule("universal_lambda_handler", .{
// Source file can be anywhere on disk, does not need to be a subdirectory
.source_file = .{ .path = b.pathJoin(&[_][]const u8{ file_location, "universal_lambda.zig" }) },
.dependencies = &[_]std.Build.ModuleDependency{.{
.name = "build_options",
.module = try createOptionsModule(b, exe),
.module = try createOptionsModule(b, cs),
}},
});
// Add steps
try @import("lambdabuild.zig").configureBuild(b, exe);
try @import("standalone_server_build.zig").configureBuild(b, exe);
try @import("flexilib_build.zig").configureBuild(b, exe, file_location);
try @import("lambda_build.zig").configureBuild(b, cs);
try @import("standalone_server_build.zig").configureBuild(b, cs);
try @import("flexilib_build.zig").configureBuild(b, cs, file_location);
// Add options module so we can let our universal_lambda know what
// type of interface is necessary
@ -58,6 +60,7 @@ pub fn configureBuild(b: *std.Build, exe: *std.Build.Step.Compile) !void {
/// for the example build.zig to simply import the file directly than it is
/// to pull from a download location and update hashes every time we change
fn findFileLocation(b: *std.Build) ![]const u8 {
if (module_root) |r| return b.pathJoin(&[_][]const u8{ r, "src" });
const build_root = b.option([]const u8, "universal_lambda_build_root", "Build root for universal lambda (development of universal lambda only)");
if (build_root) |br| {
return b.pathJoin(&[_][]const u8{ br, "src" });
@ -72,7 +75,7 @@ fn findFileLocation(b: *std.Build) ![]const u8 {
/// Make our target platform visible to runtime through an import
/// called "build_options". This will also be available to the consuming
/// executable if needed
fn createOptionsModule(b: *std.Build, exe: *std.Build.Step.Compile) !*std.Build.Module {
pub fn createOptionsModule(b: *std.Build, cs: *std.Build.Step.Compile) !*std.Build.Module {
// We need to go through the command line args, look for argument(s)
// between "build" and anything prefixed with "-". First take, blow up
// if there is more than one. That's the step we're rolling with
@ -83,14 +86,13 @@ fn createOptionsModule(b: *std.Build, exe: *std.Build.Step.Compile) !*std.Build.
defer b.allocator.free(args);
const options = b.addOptions();
options.addOption(BuildType, "build_type", findBuildType(args) orelse .exe_run);
exe.addOptions("build_options", options);
return exe.modules.get("build_options").?;
cs.addOptions("build_options", options);
return cs.modules.get("build_options").?;
}
fn findBuildType(build_args: [][:0]u8) ?BuildType {
var rc: ?BuildType = null;
for (build_args[1..]) |arg| {
if (std.mem.startsWith(u8, arg, "-")) break; // we're done as soon as we get to options
inline for (std.meta.fields(BuildType)) |field| {
if (std.mem.startsWith(u8, arg, field.name)) {
if (rc != null)