initial CI workflow

.gitea/workflows/build.yaml

@@ -0,0 +1,33 @@
+name: AWS-Zig Build
+run-name: ${{ github.actor }} building Universal Lambda
+on: [push]
+env:
+  ACTIONS_RUNTIME_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  ACTIONS_RUNTIME_URL: ${{ env.GITHUB_SERVER_URL }}/api/actions_pipeline/
+jobs:
+  build-zig-0.11.0-amd64-host:
+    runs-on: ubuntu-latest
+    env:
+      ZIG_VERSION: 0.11.0
+      ARCH: x86_64
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v3
+      # ARCH is fine, but we can't substitute directly because zig
+      # uses x86_64 instead of amd64. They also use aarch64 instead of arm64.
+      #
+      # However, arm64/linux isn't quite fully tier 1 yet, so this is more of a
+      # TODO: https://github.com/ziglang/zig/issues/2443
+      - run: wget -q https://ziglang.org/download/${ZIG_VERSION}/zig-linux-${ARCH}-${ZIG_VERSION}.tar.xz
+      - run: tar x -C /usr/local -f zig-linux-${ARCH}-${ZIG_VERSION}.tar.xz
+      - run: ln -s /usr/local/zig-linux-${ARCH}-${ZIG_VERSION}/zig /usr/local/bin/zig
+      - run: zig build test
+      - run: zig build test -Dtarget=arm-linux # we want to know we can build for 32 bit 
+      - name: Notify
+        uses: https://git.lerch.org/lobo/action-notify-ntfy@v2
+        if: always()
+        with:
+          host: ${{ secrets.NTFY_HOST }}
+          topic: ${{ secrets.NTFY_TOPIC }}
+          user: ${{ secrets.NTFY_USER }}
+          password: ${{ secrets.NTFY_PASSWORD }}

build.zig

@@ -3,7 +3,7 @@ const std = @import("std");
 // Although this function looks imperative, note that its job is to
 // declaratively construct a build graph that will be executed by an external
 // runner.
-pub fn build(b: *std.Build) void {
+pub fn build(b: *std.Build) !void {
     // Standard target options allows the person running `zig build` to choose
     // what target to build for. Here we do not override the defaults, which
     // means any target is allowed, and the default is native. Other options
@@ -23,6 +23,11 @@ pub fn build(b: *std.Build) void {
         .target = target,
         .optimize = optimize,
     });
+    // Because we are...well, ourselves, we'll manually override the module
+    // root (we are not a module here).
+    const ulb = @import("src/universal_lambda_build.zig");
+    ulb.module_root = "";
+    _ = try ulb.createOptionsModule(b, lib);
 
     // This declares intent for the library to be installed into the standard
     // location when the user invokes the "install" step (the default step when
@@ -32,12 +37,14 @@ pub fn build(b: *std.Build) void {
     // Creates a step for unit testing. This only builds the test executable
     // but does not run it.
     const main_tests = b.addTest(.{
-        .root_source_file = .{ .path = "src/main.zig" },
+        .root_source_file = .{ .path = "src/universal_lambda.zig" },
         .target = target,
         .optimize = optimize,
     });
+    _ = try ulb.createOptionsModule(b, main_tests);
 
-    const run_main_tests = b.addRunArtifact(main_tests);
+    var run_main_tests = b.addRunArtifact(main_tests);
+    run_main_tests.skip_foreign_checks = true;
 
     // This creates a build step. It will be visible in the `zig build --help` menu,
     // and can be selected like this: `zig build test`

src/flexilib-interface.zig

@@ -0,0 +1,140 @@
+const std = @import("std");
+
+// C interfaces between main and libraries
+pub const Header = extern struct {
+    name_ptr: [*]u8,
+    name_len: usize,
+
+    value_ptr: [*]u8,
+    value_len: usize,
+};
+pub const Response = extern struct {
+    ptr: [*]u8,
+    len: usize,
+
+    headers: [*]Header,
+    headers_len: usize,
+};
+
+pub const Request = extern struct {
+    method: [*:0]u8,
+    method_len: usize,
+
+    content: [*]u8,
+    content_len: usize,
+
+    headers: [*]Header,
+    headers_len: usize,
+};
+
+// If the library is Zig, we can use these helpers
+threadlocal var allocator: ?*std.mem.Allocator = null;
+
+const log = std.log.scoped(.interface);
+
+pub const ZigRequest = struct {
+    method: [:0]u8,
+    content: []u8,
+    headers: []Header,
+};
+
+pub const ZigHeader = struct {
+    name: []u8,
+    value: []u8,
+};
+
+pub const ZigResponse = struct {
+    body: *std.ArrayList(u8),
+    headers: *std.StringHashMap([]const u8),
+};
+
+pub const ZigRequestHandler = *const fn (std.mem.Allocator, ZigRequest, ZigResponse) anyerror!void;
+
+/// This function is optional and can be exported by zig libraries for
+/// initialization. If exported, it will be called once in the beginning of
+/// a request and will be provided a pointer to std.mem.Allocator, which is
+/// useful for reusing the parent allocator. If you're planning on using
+/// the handleRequest helper below, you must use zigInit or otherwise
+/// set the interface allocator in your own version of zigInit
+pub fn zigInit(parent_allocator: *anyopaque) callconv(.C) void {
+    allocator = @ptrCast(@alignCast(parent_allocator));
+}
+
+pub fn toZigHeader(header: Header) ZigHeader {
+    return .{
+        .name = header.name_ptr[0..header.name_len],
+        .value = header.value_ptr[0..header.value_len],
+    };
+}
+
+/// Converts a StringHashMap to the structure necessary for passing through the
+/// C boundary. This will be called automatically for you via the handleRequest function
+/// and is also used by the main processing loop to coerce request headers
+fn toHeaders(alloc: std.mem.Allocator, headers: std.StringHashMap([]const u8)) ![*]Header {
+    var header_array = try std.ArrayList(Header).initCapacity(alloc, headers.count());
+    var iterator = headers.iterator();
+    while (iterator.next()) |kv| {
+        header_array.appendAssumeCapacity(.{
+            .name_ptr = @constCast(kv.key_ptr.*).ptr,
+            .name_len = kv.key_ptr.*.len,
+
+            .value_ptr = @constCast(kv.value_ptr.*).ptr,
+            .value_len = kv.value_ptr.*.len,
+        });
+    }
+    return header_array.items.ptr;
+}
+
+/// handles a request, implementing the C interface to communicate between the
+/// main program and a zig library. Most importantly, it will catch/report
+/// errors appropriately and allow zig code to use standard Zig error semantics
+pub fn handleRequest(request: *Request, zigRequestHandler: ZigRequestHandler) ?*Response {
+    // TODO: implement another library in C or Rust or something to show
+    // that anything using a C ABI can be successful
+    var alloc = if (allocator) |a| a.* else {
+        log.err("zigInit not called prior to handle_request. This is a coding error", .{});
+        return null;
+    };
+
+    // setup response body
+    var response = std.ArrayList(u8).init(alloc);
+
+    // setup headers
+    var headers = std.StringHashMap([]const u8).init(alloc);
+    zigRequestHandler(
+        alloc,
+        .{
+            .method = request.method[0..request.method_len :0],
+            .content = request.content[0..request.content_len],
+            .headers = request.headers[0..request.headers_len],
+        },
+        .{
+            .body = &response,
+            .headers = &headers,
+        },
+    ) catch |e| {
+        log.err("Unexpected error processing request: {any}", .{e});
+        if (@errorReturnTrace()) |trace| {
+            std.debug.dumpStackTrace(trace.*);
+        }
+        return null;
+    };
+
+    // Marshall data back for handling by server
+
+    var rc = alloc.create(Response) catch {
+        log.err("Could not allocate memory for response object. This may be fatal", .{});
+        return null;
+    };
+    rc.ptr = response.items.ptr;
+    rc.len = response.items.len;
+    rc.headers = toHeaders(alloc, headers) catch |e| {
+        log.err("Unexpected error processing request: {any}", .{e});
+        if (@errorReturnTrace()) |trace| {
+            std.debug.dumpStackTrace(trace.*);
+        }
+        return null;
+    };
+    rc.headers_len = headers.count();
+    return rc;
+}

src/flexilib.zig

@@ -0,0 +1,107 @@
+const std = @import("std");
+const interface = @import("flexilib-interface.zig"); // TODO: pull in flexilib directly
+const testing = std.testing;
+
+const log = std.log.scoped(.@"main-lib");
+
+const client_handler = @import("flexilib_handler");
+
+// The main program will look for exports during the request lifecycle:
+// zigInit (optional): called at the beginning of a request, includes pointer to an allocator
+// handle_request: called with request data, expects response data
+// request_deinit (optional): called at the end of a request to allow resource cleanup
+//
+// Setup for these is aided by the interface library as shown below
+
+// zigInit is an optional export called at the beginning of a request. It will
+// be passed an allocator (which...shh...is an arena allocator). Since the
+// interface library provides a request handler that requires a built-in allocator,
+// if you are using the interface's handleRequest function as shown above,
+// you will need to also include this export. To customize, just do something
+// like this:
+//
+// export fn zigInit(parent_allocator: *anyopaque) callconv(.C) void {
+//   // your code here, just include the next line
+//   interface.zigInit(parent_allocator);
+// }
+//
+comptime {
+    @export(interface.zigInit, .{ .name = "zigInit", .linkage = .Strong });
+}
+
+/// handle_request will be called on a single request, but due to the preservation
+/// of restrictions imposed by the calling interface, it should generally be more
+/// useful to call into the interface library to let it do the conversion work
+/// on your behalf
+export fn handle_request(request: *interface.Request) callconv(.C) ?*interface.Response {
+    // The interface library provides a handleRequest function that will handle
+    // marshalling data back and forth from the C format used for the interface
+    // to a more Zig friendly format. It also allows usage of zig errors. To
+    // use, pass in the request and the zig function used to handle the request
+    // (here called "handleRequest"). The function signature must be:
+    //
+    // fn (std.mem.Allocator, interface.ZigRequest, interface.ZigResponse) !void
+    //
+    return interface.handleRequest(request, handleRequest);
+}
+
+// request_deinit is an optional export and will be called a the end of the
+// request. Useful for deallocating memory. Since this is zig code and the
+// allocator used is an arena allocator, all allocated memory will be automatically
+// cleaned up by the main program at the end of a request
+//
+// export fn request_deinit() void {
+// }
+
+// ************************************************************************
+// Boilerplate ^^, Custom code vv
+// ************************************************************************
+//
+// handleRequest function here is the last line of boilerplate and the
+// entry to a request
+fn handleRequest(allocator: std.mem.Allocator, request: interface.ZigRequest, response: interface.ZigResponse) !void {
+    // setup
+    var response_writer = response.body.writer();
+    try response_writer.writeAll(try client_handler.handler(allocator, request.content, .{}));
+    // real work
+    for (request.headers) |h| {
+        const header = interface.toZigHeader(h);
+        // std.debug.print("\n{s}: {s}\n", .{ header.name, header.value });
+        if (std.ascii.eqlIgnoreCase(header.name, "host") and std.mem.startsWith(u8, header.value, "iam")) {
+            try response_writer.print("iam response", .{});
+            return;
+        }
+        if (std.ascii.eqlIgnoreCase(header.name, "x-slow")) {
+            std.time.sleep(std.time.ns_per_ms * (std.fmt.parseInt(usize, header.value, 10) catch 1000));
+            try response_writer.print("i am slow\n\n", .{});
+            return;
+        }
+    }
+    try response.headers.put("X-custom-foo", "bar");
+    log.info("handlerequest header count {d}", .{response.headers.count()});
+}
+// Need to figure out how tests would work
+test "handle_request" {
+    var arena = std.heap.ArenaAllocator.init(std.testing.allocator);
+    defer arena.deinit();
+    var aa = arena.allocator();
+    interface.zigInit(&aa);
+    var headers: []interface.Header = @constCast(&[_]interface.Header{.{
+        .name_ptr = @ptrCast(@constCast("GET".ptr)),
+        .name_len = 3,
+        .value_ptr = @ptrCast(@constCast("GET".ptr)),
+        .value_len = 3,
+    }});
+    var req = interface.Request{
+        .method = @ptrCast(@constCast("GET".ptr)),
+        .method_len = 3,
+        .content = @ptrCast(@constCast("GET".ptr)),
+        .content_len = 3,
+        .headers = headers.ptr,
+        .headers_len = 1,
+    };
+    const response = handle_request(&req).?;
+    try testing.expectEqualStrings(" 1", response.ptr[0..response.len]);
+    try testing.expectEqualStrings("X-custom-foo", response.headers[0].name_ptr[0..response.headers[0].name_len]);
+    try testing.expectEqualStrings("bar", response.headers[0].value_ptr[0..response.headers[0].value_len]);
+}

src/flexilib_build.zig

@@ -0,0 +1,47 @@
+const std = @import("std");
+const builtin = @import("builtin");
+
+/// flexilib will create a dynamic library for use with flexilib.
+/// Flexilib will need to get the exe compiled as a library
+/// For flexilib, we will need the main file to have a pub fn named
+/// "handler". If it is not called that, a pub const handler = ... can be
+/// used instead
+pub fn configureBuild(b: *std.build.Builder, exe: *std.Build.Step.Compile, build_root_src: []const u8) !void {
+    const package_step = b.step("flexilib", "Create a flexilib dynamic library");
+
+    // const exe = b.addExecutable(.{
+    //     .name = "universal-lambda-example",
+    //     // In this case the main source file is merely a path, however, in more
+    //     // complicated build scripts, this could be a generated file.
+    //     .root_source_file = .{ .path = "src/main.zig" },
+    //     .target = target,
+    //     .optimize = optimize,
+    // });
+    //
+    const lib = b.addSharedLibrary(.{
+        .name = exe.name,
+        .root_source_file = .{ .path = b.pathJoin(&[_][]const u8{ build_root_src, "flexilib.zig" }) },
+        .target = exe.target,
+        .optimize = exe.optimize,
+    });
+
+    // We will not free this, as the rest of the build system will use it.
+    // This should be ok because our allocator is, I believe, an arena
+    var module_dependencies = try b.allocator.alloc(std.Build.ModuleDependency, exe.modules.count());
+    var iterator = exe.modules.iterator();
+
+    var i: usize = 0;
+    while (iterator.next()) |entry| : (i += 1) {
+        module_dependencies[i] = .{
+            .name = entry.key_ptr.*,
+            .module = entry.value_ptr.*,
+        };
+        lib.addModule(entry.key_ptr.*, entry.value_ptr.*);
+    }
+    lib.addAnonymousModule("flexilib_handler", .{
+        // Source file can be anywhere on disk, does not need to be a subdirectory
+        .source_file = exe.root_src.?,
+        .dependencies = module_dependencies,
+    });
+    package_step.dependOn(&b.addInstallArtifact(lib, .{}).step);
+}

src/lambda.zig

@@ -2,6 +2,7 @@ const std = @import("std");
 const builtin = @import("builtin");
 
 const HandlerFn = @import("universal_lambda.zig").HandlerFn;
+const Context = @import("universal_lambda.zig").Context;
 
 const log = std.log.scoped(.lambda);
 
@@ -183,9 +184,14 @@ const Event = struct {
         var req = try cl.request(.POST, response_uri, empty_headers, .{});
         // var req = try client.?.request(.POST, response_uri, empty_headers, .{});
         defer req.deinit();
+        // Lambda does different things, depending on the runtime. Go 1.x takes
+        // any return value but escapes double quotes. Custom runtimes can
+        // do whatever they want. node I believe wraps as a json object. We're
+        // going to leave the return value up to the handler, and they can
+        // use a seperate API for normalization so we're explicit.
         const response_content = try std.fmt.allocPrint(
             self.allocator,
-            "{{ \"content\": \"{s}\" }}",
+            "{s}",
             .{event_response},
         );
         defer self.allocator.free(response_content);
@@ -261,12 +267,16 @@ fn getEvent(allocator: std.mem.Allocator, event_data_uri: std.Uri) !?Event {
     const req_id = request_id.?;
     log.debug("got lambda request with id {s}", .{req_id});
 
-    var resp_payload = try std.ArrayList(u8).initCapacity(allocator, content_length.?);
+    var response_data: []u8 =
-    defer resp_payload.deinit();
+        if (req.response.transfer_encoding) |_| // the only value here is "chunked"
-    try resp_payload.resize(content_length.?);
+        try req.reader().readAllAlloc(allocator, std.math.maxInt(usize))
-    var response_data = try resp_payload.toOwnedSlice();
+    else blk: {
-    errdefer allocator.free(response_data);
+        // content length
-    _ = try req.readAll(response_data);
+        var tmp_data = try allocator.alloc(u8, content_length.?);
+        errdefer allocator.free(tmp_data);
+        _ = try req.readAll(tmp_data);
+        break :blk tmp_data;
+    };
 
     return Event.init(
         allocator,
@@ -357,7 +367,7 @@ fn processRequest(allocator: std.mem.Allocator, server: *std.http.Server) !void
     var response_bytes: []const u8 = errbuf[0..];
 
     if (res.request.content_length) |l|
-        server_request_aka_lambda_response = try res.reader().readAllAlloc(allocator, @as(usize, l));
+        server_request_aka_lambda_response = try res.reader().readAllAlloc(allocator, @as(usize, @intCast(l)));
 
     log.debug(
         "tid {d} (server): {d} bytes read from request",
@@ -391,8 +401,9 @@ fn serve(allocator: std.mem.Allocator, res: *std.http.Server.Response) ![]const
     return server_response;
 }
 
-fn handler(allocator: std.mem.Allocator, event_data: []const u8) ![]const u8 {
+fn handler(allocator: std.mem.Allocator, event_data: []const u8, context: Context) ![]const u8 {
     _ = allocator;
+    _ = context;
     return event_data;
 }
 fn test_run(allocator: std.mem.Allocator, event_handler: HandlerFn) !std.Thread {
@@ -458,9 +469,8 @@ test "basic request" {
         \\{"foo": "bar", "baz": "qux"}
     ;
 
-    // This is what's actually coming back. Is this right?
     const expected_response =
-        \\{ "content": "{"foo": "bar", "baz": "qux"}" }
+        \\{"foo": "bar", "baz": "qux"}
     ;
     const lambda_response = try lambda_request(allocator, request, 1);
     defer deinit();
@@ -475,9 +485,8 @@ test "several requests do not fail" {
         \\{"foo": "bar", "baz": "qux"}
     ;
 
-    // This is what's actually coming back. Is this right?
     const expected_response =
-        \\{ "content": "{"foo": "bar", "baz": "qux"}" }
+        \\{"foo": "bar", "baz": "qux"}
     ;
     const lambda_response = try lambda_request(allocator, request, 5);
     defer deinit();

src/universal_lambda.zig

@@ -120,3 +120,10 @@ fn processRequest(aa: std.mem.Allocator, server: *std.http.Server, event_handler
     _ = try res.writer().writeAll(response_bytes);
     try res.finish();
 }
+
+test {
+    std.testing.refAllDecls(@This()); // standalone, standalone web server
+    std.testing.refAllDecls(@import("lambda.zig")); // lambda
+    // TODO: re-enable
+    // std.testing.refAllDecls(@import("flexilib.zig")); // flexilib
+}

src/universal_lambda_build.zig

@@ -12,22 +12,24 @@ pub const BuildType = enum {
     flexilib,
 };
 
-pub fn configureBuild(b: *std.Build, exe: *std.Build.Step.Compile) !void {
+pub var module_root: ?[]const u8 = null;
+
+pub fn configureBuild(b: *std.Build, cs: *std.Build.Step.Compile) !void {
     const file_location = try findFileLocation(b);
     // Add module
-    exe.addAnonymousModule("universal_lambda_handler", .{
+    cs.addAnonymousModule("universal_lambda_handler", .{
         // Source file can be anywhere on disk, does not need to be a subdirectory
         .source_file = .{ .path = b.pathJoin(&[_][]const u8{ file_location, "universal_lambda.zig" }) },
         .dependencies = &[_]std.Build.ModuleDependency{.{
             .name = "build_options",
-            .module = try createOptionsModule(b, exe),
+            .module = try createOptionsModule(b, cs),
         }},
     });
 
     // Add steps
-    try @import("lambdabuild.zig").configureBuild(b, exe);
+    try @import("lambda_build.zig").configureBuild(b, cs);
-    try @import("standalone_server_build.zig").configureBuild(b, exe);
+    try @import("standalone_server_build.zig").configureBuild(b, cs);
-    try @import("flexilib_build.zig").configureBuild(b, exe, file_location);
+    try @import("flexilib_build.zig").configureBuild(b, cs, file_location);
 
     // Add options module so we can let our universal_lambda know what
     // type of interface is necessary
@@ -58,6 +60,7 @@ pub fn configureBuild(b: *std.Build, exe: *std.Build.Step.Compile) !void {
 /// for the example build.zig to simply import the file directly than it is
 /// to pull from a download location and update hashes every time we change
 fn findFileLocation(b: *std.Build) ![]const u8 {
+    if (module_root) |r| return b.pathJoin(&[_][]const u8{ r, "src" });
     const build_root = b.option([]const u8, "universal_lambda_build_root", "Build root for universal lambda (development of universal lambda only)");
     if (build_root) |br| {
         return b.pathJoin(&[_][]const u8{ br, "src" });
@@ -72,7 +75,7 @@ fn findFileLocation(b: *std.Build) ![]const u8 {
 /// Make our target platform visible to runtime through an import
 /// called "build_options". This will also be available to the consuming
 /// executable if needed
-fn createOptionsModule(b: *std.Build, exe: *std.Build.Step.Compile) !*std.Build.Module {
+pub fn createOptionsModule(b: *std.Build, cs: *std.Build.Step.Compile) !*std.Build.Module {
     // We need to go through the command line args, look for argument(s)
     // between "build" and anything prefixed with "-". First take, blow up
     // if there is more than one. That's the step we're rolling with
@@ -83,14 +86,13 @@ fn createOptionsModule(b: *std.Build, exe: *std.Build.Step.Compile) !*std.Build.
     defer b.allocator.free(args);
     const options = b.addOptions();
     options.addOption(BuildType, "build_type", findBuildType(args) orelse .exe_run);
-    exe.addOptions("build_options", options);
+    cs.addOptions("build_options", options);
-    return exe.modules.get("build_options").?;
+    return cs.modules.get("build_options").?;
 }
 
 fn findBuildType(build_args: [][:0]u8) ?BuildType {
     var rc: ?BuildType = null;
     for (build_args[1..]) |arg| {
-        if (std.mem.startsWith(u8, arg, "-")) break; // we're done as soon as we get to options
         inline for (std.meta.fields(BuildType)) |field| {
             if (std.mem.startsWith(u8, arg, field.name)) {
                 if (rc != null)