add build deploy step

README.md

@@ -1,16 +1,40 @@
 Upload worker to CloudFlare
 ===========================
 
-Until we're better
+`zig build run -- <worker name> <script file>`. Make sure that authentication
-------------------
+environment variables are set. An example index.js file is included in the
+root of the project
 
-1. Add `accountid.txt` to `src/` with the CloudFlare account id
+Environment Variables
-2. Add `worker_name.txt` to `src/` with CloudFlare worker name
+---------------------
-3. `zig build run`. Make sure that CF_X_AUTH_EMAIL and CF_X_AUTH_KEY environment
-   variables are set with the correct authentication parameters
 
-Getting new index.js
+The following environment variables are supported and match Wrangler behavior:
---------------------
+
+
+CLOUDFLARE_ACCOUNT_ID
+
+    The account ID for the Workers related account.
+
+CLOUDFLARE_API_TOKEN
+
+    The API token for your Cloudflare account, can be used for authentication for situations like CI/CD, and other automation.
+
+CLOUDFLARE_API_KEY
+
+    The API key for your Cloudflare account, usually used for older authentication method with CLOUDFLARE_EMAIL=.
+
+CLOUDFLARE_EMAIL
+
+    The email address associated with your Cloudflare account, usually used for older authentication method with CLOUDFLARE_API_KEY=.
+
+Note that either CLOUDFLARE_API_TOKEN or CLOUDFLARE_EMAIL/CLOUDFLARE_API_KEY
+environment variable pair are required
+
+Development notes
+=================
+
+Getting new src/script_harness.js
+---------------------------------
 
 * Run a real wrangler deploy, then go into the console and copy/paste
 
@@ -27,7 +51,8 @@ Getting new memfs.wasm
 
 Steps we take:
 --------------
-
+0. Get account id. CLOUDFLARE_ACCOUNT_ID environment variable will be checked first. If not,
+   GET https://api.cloudflare.com/client/v4/accounts/
 1. Check if the worker exists:
    GET https://api.cloudflare.com/client/v4/accounts/<account id>/workers/services/<worker_name>
    404 - does not exist

build.zig

@@ -1,4 +1,5 @@
 const std = @import("std");
+const CloudflareDeployStep = @import("src/CloudflareDeployStep.zig");
 
 // Although this function looks imperative, note that its job is to
 // declaratively construct a build graph that will be executed by an external
@@ -46,6 +47,13 @@ pub fn build(b: *std.Build) void {
         run_cmd.addArgs(args);
     }
 
+    const deploy_cmd = CloudflareDeployStep.create(b, "zigwasi", .{ .path = "index.js" }, .{});
+
+    // This creates a build step. It will be visible in the `zig build --help` menu,
+    // and can be selected like this: `zig build run`
+    // This will evaluate the `run` step rather than the default, which is "install".
+    const deploy_step = b.step("deploy", "Deploy test function to Cloudflare");
+    deploy_step.dependOn(&deploy_cmd.step);
     // This creates a build step. It will be visible in the `zig build --help` menu,
     // and can be selected like this: `zig build run`
     // This will evaluate the `run` step rather than the default, which is "install".

index.js

@@ -0,0 +1,31 @@
+import demoWasm from "demo.wasm";
+export default {
+  async fetch(request, _env2, ctx) {
+    const stdout = new TransformStream();
+    console.log(request);
+    console.log(_env2);
+    console.log(ctx);
+    let env = {};
+    request.headers.forEach((value, key) => {
+      env[key] = value;
+    });
+    const wasi = new WASI({
+      args: [
+        "./demo.wasm",
+        // In a CLI, the first arg is the name of the exe
+        "--url=" + request.url,
+        // this contains the target but is the full url, so we will use a different arg for this
+        "--method=" + request.method,
+        '-request="' + JSON.stringify(request) + '"'
+      ],
+      env,
+      stdin: request.body,
+      stdout: stdout.writable
+    });
+    const instance = new WebAssembly.Instance(demoWasm, {
+      wasi_snapshot_preview1: wasi.wasiImport
+    });
+    ctx.waitUntil(wasi.start(instance));
+    return new Response(stdout.readable);
+  }
+};

src/CloudflareDeployStep.zig

@@ -0,0 +1,61 @@
+const std = @import("std");
+const cloudflare = @import("main.zig");
+const CloudflareDeployStep = @This();
+
+pub const base_id: std.Build.Step.Id = .custom;
+
+step: std.Build.Step,
+primary_javascript_path: std.Build.LazyPath,
+worker_name: []const u8,
+
+pub const Options = struct {};
+
+pub fn create(
+    owner: *std.Build,
+    worker_name: []const u8,
+    primary_javascript_path: std.Build.LazyPath,
+    options: Options,
+) *CloudflareDeployStep {
+    _ = options;
+    const self = owner.allocator.create(CloudflareDeployStep) catch @panic("OOM");
+    self.* = CloudflareDeployStep{
+        .step = std.Build.Step.init(.{
+            .id = base_id,
+            .name = owner.fmt("cloudflare deploy {s}", .{primary_javascript_path.getDisplayName()}),
+            .owner = owner,
+            .makeFn = make,
+        }),
+        .primary_javascript_path = primary_javascript_path,
+        .worker_name = worker_name,
+    };
+    primary_javascript_path.addStepDependencies(&self.step);
+    return self;
+}
+
+fn make(step: *std.Build.Step, prog_node: *std.Progress.Node) !void {
+    _ = prog_node;
+    const b = step.owner;
+    const self = @fieldParentPtr(CloudflareDeployStep, "step", step);
+
+    var client = std.http.Client{ .allocator = b.allocator };
+    defer client.deinit();
+
+    const script = try std.fs.cwd().readFileAlloc(b.allocator, self.primary_javascript_path.path, std.math.maxInt(usize));
+
+    var al = std.ArrayList(u8).init(b.allocator);
+    defer al.deinit();
+    try cloudflare.pushWorker(
+        b.allocator,
+        &client,
+        self.worker_name,
+        script,
+        al.writer(),
+        std.io.getStdErr().writer(),
+    );
+    const start = std.mem.lastIndexOf(u8, al.items, "http").?;
+    step.name = try std.fmt.allocPrint(
+        b.allocator,
+        "cloudflare deploy {s} to {s}",
+        .{ self.primary_javascript_path.getDisplayName(), al.items[start .. al.items.len - 1] },
+    );
+}

src/main.zig

@@ -1,7 +1,9 @@
 const std = @import("std");
 
-var x_auth_email: [:0]const u8 = undefined;
+var x_auth_token: ?[:0]const u8 = undefined;
-var x_auth_key: [:0]const u8 = undefined;
+var x_auth_email: ?[:0]const u8 = undefined;
+var x_auth_key: ?[:0]const u8 = undefined;
+var initialized = false;
 
 const cf_api_base = "https://api.cloudflare.com/client/v4";
 
@@ -10,101 +12,50 @@ pub fn main() !u8 {
     defer arena.deinit();
     const allocator = arena.allocator();
     var client = std.http.Client{ .allocator = allocator };
+    defer client.deinit();
+    //     .allocator = allocator,
+    //     .proxy = .{
+    //         .protocol = .plain,
+    //         .host = "localhost",
+    //         .port = 8080,
+    //     },
+    // };
 
-    x_auth_email = std.os.getenv("CF_X_AUTH_EMAIL").?;
-    x_auth_key = std.os.getenv("CF_X_AUTH_KEY").?;
-    // TODO: All this stuff needs to be different
-    //
-    const worker_name = @embedFile("worker_name.txt");
-    // TODO: We need to break index.js into the wrangler-generated bundling thing
-    //       and the actual code we're using to run the wasm file.
-    //       We might actually want a "run this wasm" upload vs a "these are my
-    //       js files" upload. But for now we'll optimize for wasm
-    const script =
-        \\import demoWasm from "demo.wasm";
-        \\var src_default = {
-        \\  async fetch(request, _env2, ctx) {
-        \\    const stdout = new TransformStream();
-        \\    console.log(request);
-        \\    console.log(_env2);
-        \\    console.log(ctx);
-        \\    let env = {};
-        \\    request.headers.forEach((value, key) => {
-        \\      env[key] = value;
-        \\    });
-        \\    const wasi = new WASI({
-        \\      args: [
-        \\        "./demo.wasm",
-        \\        // In a CLI, the first arg is the name of the exe
-        \\        "--url=" + request.url,
-        \\        // this contains the target but is the full url, so we will use a different arg for this
-        \\        "--method=" + request.method,
-        \\        '-request="' + JSON.stringify(request) + '"'
-        \\      ],
-        \\      env,
-        \\      stdin: request.body,
-        \\      stdout: stdout.writable
-        \\    });
-        \\    const instance = new WebAssembly.Instance(demoWasm, {
-        \\      wasi_snapshot_preview1: wasi.wasiImport
-        \\    });
-        \\    ctx.waitUntil(wasi.start(instance));
-        \\    return new Response(stdout.readable);
-        \\  }
-        \\};
-        \\export {
-        \\  src_default as default
-        \\};
-    ;
-    var wasm = try loadWasm(allocator, script);
-    defer wasm.deinit();
-
-    // stdout is for the actual output of your application, for example if you
-    // are implementing gzip, then only the compressed bytes should be sent to
-    // stdout, not any debugging messages.
     const stdout_file = std.io.getStdOut().writer();
     var bw = std.io.bufferedWriter(stdout_file);
     const stdout = bw.writer();
 
-    var accountid = std.os.getenv("CLOUDFLARE_ACCOUNT_ID");
+    var argIterator = try std.process.argsWithAllocator(allocator);
-    const account_id_free = accountid == null;
+    defer argIterator.deinit();
-    if (accountid == null) accountid = try getAccountId(allocator, &client);
+    const exe_name = argIterator.next().?;
-    defer if (account_id_free) allocator.free(accountid.?);
+    var maybe_name = argIterator.next();
-
+    if (maybe_name == null) {
-    try stdout.print("Using Cloudflare account: {s}\n", .{accountid.?});
+        try usage(std.io.getStdErr().writer(), exe_name);
-
-    // Determine if worker exists. This lets us know if we need to enable it later
-    const worker_exists = try workerExists(allocator, &client, accountid.?, worker_name);
-    try stdout.print(
-        "{s}\n",
-        .{if (worker_exists) "Worker exists, will not re-enable" else "Worker is new. Will enable after code update"},
-    );
-
-    var worker = Worker{
-        .account_id = accountid.?,
-        .name = worker_name,
-        .wasm_file_data = wasm.data,
-        .main_module = script,
-    };
-    putNewWorker(allocator, &client, &worker) catch |err| {
-        if (worker.errors == null) return err;
-        const stderr = std.io.getStdErr().writer();
-        try stderr.print("{d} errors returned from CloudFlare:\n\n", .{worker.errors.?.len});
-        for (worker.errors.?) |cf_err| {
-            try stderr.print("{s}\n", .{cf_err});
-            allocator.free(cf_err);
-        }
         return 1;
+    }
+    const worker_name = maybe_name.?;
+    if (std.mem.eql(u8, worker_name, "-h")) {
+        try usage(stdout, exe_name);
+        return 0;
+    }
+    var maybe_script_name = argIterator.next();
+    if (maybe_script_name == null) {
+        try usage(std.io.getStdErr().writer(), exe_name);
+        return 1;
+    }
+    const script = std.fs.cwd().readFileAlloc(allocator, maybe_script_name.?, std.math.maxInt(usize)) catch |err| {
+        try usage(std.io.getStdErr().writer(), exe_name);
+        return err;
     };
-    const subdomain = try getSubdomain(allocator, &client, accountid.?);
+
-    defer allocator.free(subdomain);
+    pushWorker(allocator, &client, worker_name, script, stdout, std.io.getStdErr().writer()) catch return 1;
-    try stdout.print("Worker available at: https://{s}.{s}.workers.dev/\n", .{ worker_name, subdomain });
-    if (!worker_exists)
-        try enableWorker(allocator, &client, accountid.?, worker_name);
     try bw.flush(); // don't forget to flush!
     return 0;
 }
 
+fn usage(writer: anytype, this: []const u8) !void {
+    try writer.print("usage: {s} <worker name> <script file>\n", .{this});
+}
 const Wasm = struct {
     allocator: std.mem.Allocator,
     name: []const u8,
@@ -118,6 +69,53 @@ const Wasm = struct {
     }
 };
 
+pub fn pushWorker(
+    allocator: std.mem.Allocator,
+    client: *std.http.Client,
+    worker_name: []const u8,
+    script: []const u8,
+    writer: anytype,
+    err_writer: anytype,
+) !void {
+    var wasm = try loadWasm(allocator, script);
+    defer wasm.deinit();
+
+    var accountid = std.os.getenv("CLOUDFLARE_ACCOUNT_ID");
+    const account_id_free = accountid == null;
+    if (accountid == null) accountid = try getAccountId(allocator, client);
+    defer if (account_id_free) allocator.free(accountid.?);
+
+    try writer.print("Using Cloudflare account: {s}\n", .{accountid.?});
+
+    // Determine if worker exists. This lets us know if we need to enable it later
+    const worker_exists = try workerExists(allocator, client, accountid.?, worker_name);
+    try writer.print(
+        "{s}\n",
+        .{if (worker_exists) "Worker exists, will not re-enable" else "Worker is new. Will enable after code update"},
+    );
+
+    var worker = Worker{
+        .account_id = accountid.?,
+        .name = worker_name,
+        .wasm = wasm,
+        .main_module = script,
+    };
+    putNewWorker(allocator, client, &worker) catch |err| {
+        if (worker.errors == null) return err;
+        try err_writer.print("{d} errors returned from CloudFlare:\n\n", .{worker.errors.?.len});
+        for (worker.errors.?) |cf_err| {
+            try err_writer.print("{s}\n", .{cf_err});
+            allocator.free(cf_err);
+        }
+        return error.CloudFlareErrorResponse;
+    };
+    const subdomain = try getSubdomain(allocator, client, accountid.?);
+    defer allocator.free(subdomain);
+    try writer.print("Worker available at: https://{s}.{s}.workers.dev/\n", .{ worker_name, subdomain });
+    if (!worker_exists)
+        try enableWorker(allocator, client, accountid.?, worker_name);
+}
+
 fn loadWasm(allocator: std.mem.Allocator, script: []const u8) !Wasm {
     // Looking for a string like this: import demoWasm from "demo.wasm"
     // JavaScript may or may not have ; characters. We're not doing
@@ -240,7 +238,7 @@ const Worker = struct {
     account_id: []const u8,
     name: []const u8,
     main_module: []const u8,
-    wasm_file_data: []const u8,
+    wasm: Wasm,
     errors: ?[][]const u8 = null,
 };
 fn putNewWorker(allocator: std.mem.Allocator, client: *std.http.Client, worker: *Worker) !void {
@@ -248,7 +246,7 @@ fn putNewWorker(allocator: std.mem.Allocator, client: *std.http.Client, worker:
     const url = try std.fmt.allocPrint(allocator, put_script, .{ worker.account_id, worker.name });
     defer allocator.free(url);
     const memfs = @embedFile("dist/memfs.wasm");
-    const outer_script_shell = @embedFile("index.js");
+    const outer_script_shell = @embedFile("script_harness.js");
     const script = try std.fmt.allocPrint(allocator, "{s}{s}", .{ outer_script_shell, worker.main_module });
     defer allocator.free(script);
     const deploy_request =
@@ -261,7 +259,7 @@ fn putNewWorker(allocator: std.mem.Allocator, client: *std.http.Client, worker:
         "\r\n" ++
         "{[script]s}\r\n" ++
         "------formdata-undici-032998177938\r\n" ++
-        "Content-Disposition: form-data; name=\"./24526702f6c3ed7fb02b15125f614dd38804525f-demo.wasm\"; filename=\"./24526702f6c3ed7fb02b15125f614dd38804525f-demo.wasm\"\r\n" ++
+        "Content-Disposition: form-data; name=\"./{[wasm_name]s}\"; filename=\"./{[wasm_name]s}\"\r\n" ++
         "Content-Type: application/wasm\r\n" ++
         "\r\n" ++
         "{[wasm]s}\r\n" ++
@@ -279,7 +277,8 @@ fn putNewWorker(allocator: std.mem.Allocator, client: *std.http.Client, worker:
     try headers.append("Content-Type", "multipart/form-data; boundary=----formdata-undici-032998177938");
     const request_payload = try std.fmt.allocPrint(allocator, deploy_request, .{
         .script = script,
-        .wasm = worker.wasm_file_data,
+        .wasm_name = worker.wasm.name,
+        .wasm = worker.wasm.data,
         .memfs = memfs,
     });
     defer allocator.free(request_payload);
@@ -294,7 +293,14 @@ fn putNewWorker(allocator: std.mem.Allocator, client: *std.http.Client, worker:
 
     req.transfer_encoding = .{ .content_length = @as(u64, request_payload.len) };
     try req.start();
-    try req.writeAll(request_payload);
+
+    // Workaround for https://github.com/ziglang/zig/issues/15626
+    const max_bytes: usize = 1 << 14;
+    var inx: usize = 0;
+    while (request_payload.len > inx) {
+        try req.writeAll(request_payload[inx..@min(request_payload.len, inx + max_bytes)]);
+        inx += max_bytes;
+    }
     try req.finish();
     try req.wait();
     // std.debug.print("Status is {}\n", .{req.response.status});
@@ -338,9 +344,27 @@ fn workerExists(allocator: std.mem.Allocator, client: *std.http.Client, account_
     return req.response.status == .ok;
 }
 
+threadlocal var auth_buf: [1024]u8 = undefined;
+
 fn addAuthHeaders(headers: *std.http.Headers) !void {
-    try headers.append("X-Auth-Email", x_auth_email);
+    if (!initialized) {
-    try headers.append("X-Auth-Key", x_auth_key);
+        x_auth_email = std.os.getenv("CLOUDFLARE_EMAIL");
+        x_auth_key = std.os.getenv("CLOUDFLARE_API_KEY");
+        x_auth_token = std.os.getenv("CLOUDFLARE_API_TOKEN");
+        initialized = true;
+    }
+    if (x_auth_token) |tok| {
+        var auth = try std.fmt.bufPrint(auth_buf[0..], "Bearer {s}", .{tok});
+        try headers.append("Authorization", auth);
+        return;
+    }
+    if (x_auth_email) |email| {
+        if (x_auth_key == null)
+            return error.MissingCloudflareApiKeyEnvironmentVariable;
+        try headers.append("X-Auth-Email", email);
+        try headers.append("X-Auth-Key", x_auth_key.?);
+    }
+    return error.NoCfAuthenticationEnvironmentVariablesSet;
 }
 test "simple test" {
     var list = std.ArrayList(i32).init(std.testing.allocator);