better codegen but without caching

build.zig

@@ -35,14 +35,19 @@ pub fn build(b: *Builder) !void {
         .optimize = optimize,
     });
     exe.addModule("smithy", smithy_dep.module("smithy"));
-    // TODO: Smithy needs to be in a different repo
-    // https://github.com/ziglang/zig/issues/855
-    // exe.addModulePath("smithy", "smithy/src/smithy.zig");
 
-    if (target.getOs().tag != .macos) exe.linkage = .static;
-
-    // Strip is controlled by optimize options
-    // exe.strip = b.option(bool, "strip", "strip exe [true]") orelse true;
+    // TODO: This does not work correctly due to https://github.com/ziglang/zig/issues/16354
+    //
+    // We are working here with kind of a weird dependency though. So we can do this
+    // another way
+    //
+    // TODO: These target/optimize are not correct, as we need to run the thing
+    // const codegen = b.anonymousDependency("codegen/", @import("codegen/build.zig"), .{
+    //     .target = target,
+    //     .optimize = optimize,
+    // });
+    // const codegen_cmd = b.addRunArtifact(codegen.artifact("codegen"));
+    // exe.step.dependOn(&codegen_cmd.step);
 
     const run_cmd = b.addRunArtifact(exe);
     run_cmd.step.dependOn(b.getInstallStep());
@@ -53,14 +58,23 @@ pub fn build(b: *Builder) !void {
     const run_step = b.step("run", "Run the app");
     run_step.dependOn(&run_cmd.step);
 
-    // TODO: Proper testing
+    {
+        const cg = b.step("gen", "Generate zig service code from smithy models");
+
+        const cg_exe = b.addExecutable(.{
+            .name = "codegen",
+            .root_source_file = .{ .path = "codegen/src/main.zig" },
+            // We need this generated for the host, not the real target
+            // .target = target,
+            // .optimize = optimize,
+        });
+        cg_exe.addModule("smithy", smithy_dep.module("smithy"));
+        var cg_cmd = b.addRunArtifact(cg_exe);
+        cg_cmd.addArg("--models");
+        cg_cmd.addDirectoryArg(std.Build.FileSource.relative("codegen/models"));
+        cg_cmd.addArg("--output");
+        cg_cmd.addDirectoryArg(std.Build.FileSource.relative("src/models"));
 
-    var codegen: ?*std.build.Step = null;
-    if (target.getOs().tag == .linux and false) {
-        // TODO: Support > linux with RunStep
-        // std.build.RunStep.create(null,null).cwd(std.fs.path.resolve(b.build_root, "codegen")).addArgs(...)
-        codegen = b.step("gen", "Generate zig service code from smithy models");
-        const cg = codegen.?;
         // TODO: this should use zig_exe from std.Build
         // codegen should store a hash in a comment
         // this would be hash of the exe that created the file
@@ -78,21 +92,14 @@ pub fn build(b: *Builder) !void {
         // this scheme would permit cross plat codegen and maybe
         // we can have codegen added in a seperate repo,
         // though not sure how necessary that is
-        cg.dependOn(&b.addSystemCommand(&.{ "/bin/sh", "-c", "cd codegen && zig build" }).step);
+        // cg.dependOn(&b.addSystemCommand(&.{
+        //     b.zig_exe,
+        //     "build",
+        //     "run",
+        //     "-Doptimize=ReleaseSafe",
+        // }).step);
 
-        // triggering the re-gen
-        cg.dependOn(&b.addSystemCommand(&.{
-            "/bin/sh", "-c",
-            \\ [ ! -f src/models/service_manifest.zig ] || \
-            \\ [ $(find codegen -type f -newer src/models/service_manifest.zig -print -quit |wc -c) = '0' ] || \
-            \\ rm src/models/service_manifest.zig
-        }).step);
-        cg.dependOn(&b.addSystemCommand(&.{
-            "/bin/sh", "-c",
-            \\ mkdir -p src/models/ && \
-            \\ [ -f src/models/service_manifest.zig ] || \
-            \\ ( cd codegen/models && ../codegen *.json && mv *.zig ../../src/models )
-        }).step);
+        cg.dependOn(&cg_cmd.step);
         exe.step.dependOn(cg);
     }
 

codegen/build.zig

@@ -1,5 +1,8 @@
 const std = @import("std");
 
+// Although this function looks imperative, note that its job is to
+// declaratively construct a build graph that will be executed by an external
+// runner.
 pub fn build(b: *std.build.Builder) !void {
     // Standard target options allows the person running `zig build` to choose
     // what target to build for. Here we do not override the defaults, which
@@ -7,53 +10,65 @@ pub fn build(b: *std.build.Builder) !void {
     // for restricting supported target set are available.
     const target = b.standardTargetOptions(.{});
 
-    // Standard release options allow the person running `zig build` to select
-    // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall.
-    const mode = b.standardReleaseOptions();
+    // Standard optimization options allow the person running `zig build` to select
+    // between Debug, ReleaseSafe, ReleaseFast, and ReleaseSmall. Here we do not
+    // set a preferred release mode, allowing the user to decide how to optimize.
+    const optimize = b.standardOptimizeOption(.{});
 
-    const exe = b.addExecutable("codegen", "src/main.zig");
-    exe.addPackagePath("smithy", "../smithy/src/smithy.zig");
-    exe.setTarget(target);
-    exe.setBuildMode(mode);
-    // This line works as of c5d412268
-    // Earliest nightly is 05b5e49bc on 2021-06-12
-    // https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.113+05b5e49bc.tar.xz
-    // exe.override_dest_dir = .{ .Custom = ".." };
-    exe.override_dest_dir = .{ .custom = ".." };
+    const exe = b.addExecutable(.{
+        .name = "codegen",
+        .root_source_file = .{ .path = "src/main.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
 
-    // Static linkage flag was nonfunctional until 2b2efa24d0855
-    // Did not notice this until 2021-06-28, and that nightly is:
-    // https://ziglang.org/builds/zig-linux-x86_64-0.9.0-dev.321+15a030ef3.tar.xz
-    exe.linkage = .static;
+    const smithy_dep = b.dependency("smithy", .{
+        .target = target,
+        .optimize = optimize,
+    });
+    exe.addModule("smithy", smithy_dep.module("smithy"));
 
-    const is_strip = b.option(bool, "strip", "strip exe") orelse true;
-    exe.strip = !is_strip;
-    exe.install();
+    // This declares intent for the executable to be installed into the
+    // standard location when the user invokes the "install" step (the default
+    // step when running `zig build`).
+    b.installArtifact(exe);
 
-    const run_cmd = exe.run();
+    // This *creates* a Run step in the build graph, to be executed when another
+    // step is evaluated that depends on it. The next line below will establish
+    // such a dependency.
+    const run_cmd = b.addRunArtifact(exe);
+
+    // By making the run step depend on the install step, it will be run from the
+    // installation directory rather than directly from within the cache directory.
+    // This is not necessary, however, if the application depends on other installed
+    // files, this ensures they will be present and in the expected location.
     run_cmd.step.dependOn(b.getInstallStep());
+
+    // This allows the user to pass arguments to the application in the build
+    // command itself, like this: `zig build run -- arg1 arg2 etc`
     if (b.args) |args| {
         run_cmd.addArgs(args);
     }
 
+    // This creates a build step. It will be visible in the `zig build --help` menu,
+    // and can be selected like this: `zig build run`
+    // This will evaluate the `run` step rather than the default, which is "install".
     const run_step = b.step("run", "Run the app");
     run_step.dependOn(&run_cmd.step);
 
-    const test_step = b.step("test", "Run library tests");
-    var src_dir = try std.fs.openDirAbsolute(b.build_root, .{});
-    defer src_dir.close();
-    var iterable = try src_dir.openIterableDir(".", .{});
-    defer iterable.close();
-    var iterator = iterable.iterate();
-    while (try iterator.next()) |entry| {
-        if (std.mem.endsWith(u8, entry.name, ".zig") and
-            !std.mem.eql(u8, entry.name, "main.zig"))
-        {
-            const name = try std.fmt.allocPrint(b.allocator, "src/{s}", .{entry.name});
-            defer b.allocator.free(name);
-            const t = b.addTest(name);
-            t.setBuildMode(mode);
-            test_step.dependOn(&t.step);
-        }
-    }
+    // Creates a step for unit testing. This only builds the test executable
+    // but does not run it.
+    const unit_tests = b.addTest(.{
+        .root_source_file = .{ .path = "src/main.zig" },
+        .target = target,
+        .optimize = optimize,
+    });
+
+    const run_unit_tests = b.addRunArtifact(unit_tests);
+
+    // Similar to creating the run step earlier, this exposes a `test` step to
+    // the `zig build --help` menu, providing a way for the user to request
+    // running the unit tests.
+    const test_step = b.step("test", "Run unit tests");
+    test_step.dependOn(&run_unit_tests.step);
 }

codegen/build.zig.zon

@@ -0,0 +1,11 @@
+.{
+    .name = "aws-zig-codegen",
+    .version = "0.0.1",
+
+    .dependencies = .{
+        .smithy = .{
+            .url = "https://git.lerch.org/lobo/smithy/archive/41b61745d25a65817209dd5dddbb5f9b66896a99.tar.gz",
+            .hash = "122087deb0ae309b2258d59b40d82fe5921fdfc35b420bb59033244851f7f276fa34",
+        },
+    },
+}

codegen/src/json.zig

@@ -19,7 +19,7 @@ pub fn serializeMap(map: anytype, key: []const u8, options: anytype, out_stream:
     try out_stream.writeByte('{');
     if (options.whitespace) |_|
         try out_stream.writeByte('\n');
-    for (map) |tag, i| {
+    for (map, 0..) |tag, i| {
         if (tag.key == null or tag.value == null) continue;
         // TODO: Deal with escaping and general "json.stringify" the values...
         if (child_options.whitespace) |ws|
@@ -100,8 +100,8 @@ fn outputUnicodeEscape(
         assert(codepoint <= 0x10FFFF);
         // To escape an extended character that is not in the Basic Multilingual Plane,
         // the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair.
-        const high = @intCast(u16, (codepoint - 0x10000) >> 10) + 0xD800;
-        const low = @intCast(u16, codepoint & 0x3FF) + 0xDC00;
+        const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800;
+        const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00;
         try out_stream.writeAll("\\u");
         try std.fmt.formatIntValue(high, "x", std.fmt.FormatOptions{ .width = 4, .fill = '0' }, out_stream);
         try out_stream.writeAll("\\u");

codegen/src/main.zig

@@ -11,27 +11,74 @@ pub fn main() anyerror!void {
     const args = try std.process.argsAlloc(allocator);
     defer std.process.argsFree(allocator, args);
     const stdout = std.io.getStdOut().writer();
-    const json_file = try std.fs.cwd().createFile("json.zig", .{});
+
+    var output_dir = std.fs.cwd();
+    defer output_dir.close();
+    var models_dir: ?std.fs.IterableDir = null;
+    defer if (models_dir) |*m| m.close();
+    for (args, 0..) |arg, i| {
+        if (std.mem.eql(u8, "--help", arg) or
+            std.mem.eql(u8, "-h", arg))
+        {
+            try stdout.print("usage: {s} [--models dir] [--output dir] [file...]\n\n", .{args[0]});
+            try stdout.print(" --models specifies a directory with all model files (do not specify files if --models is used)\n", .{});
+            try stdout.print(" --output specifies an output directory, otherwise the current working directory will be used\n", .{});
+            std.process.exit(0);
+        }
+        if (std.mem.eql(u8, "--output", arg))
+            output_dir = try output_dir.openDir(args[i + 1], .{});
+        if (std.mem.eql(u8, "--models", arg))
+            models_dir = try std.fs.cwd().openIterableDir(args[i + 1], .{});
+    }
+    // TODO: Seems like we should remove this in favor of a package
+    const json_file = try output_dir.createFile("json.zig", .{});
     defer json_file.close();
     try json_file.writer().writeAll(json_zig);
-    const manifest_file = try std.fs.cwd().createFile("service_manifest.zig", .{});
+    const manifest_file = try output_dir.createFile("service_manifest.zig", .{});
     defer manifest_file.close();
     const manifest = manifest_file.writer();
-    var inx: u32 = 0;
+    var files_processed: usize = 0;
+    var skip_next = true;
     for (args) |arg| {
-        if (inx == 0) {
-            inx = inx + 1;
+        if (skip_next) {
+            skip_next = false;
             continue;
         }
-        try processFile(arg, stdout, manifest);
-        inx = inx + 1;
+        if (std.mem.eql(u8, "--models", arg) or
+            std.mem.eql(u8, "--output", arg))
+        {
+            skip_next = true;
+            continue;
+        }
+        try processFile(arg, stdout, output_dir, manifest);
+        files_processed += 1;
+    }
+    if (files_processed == 0) {
+        // no files specified, look for json files in models directory or cwd
+        if (models_dir) |m| {
+            var cwd = try std.fs.cwd().openDir(".", .{});
+            defer cwd.close();
+            defer cwd.setAsCwd() catch unreachable;
+
+            try stdout.print("orig cwd: {any}\n", .{cwd});
+            try m.dir.setAsCwd();
+            try stdout.print("cwd: {any}\n", .{m.dir});
+            // TODO: this is throwing an error?
+            // _ = cwd;
+            var mi = m.iterate();
+            while (try mi.next()) |e| {
+                if ((e.kind == .file or e.kind == .sym_link) and
+                    std.mem.endsWith(u8, e.name, ".json"))
+                    try processFile(e.name, stdout, output_dir, manifest);
+            }
+        }
     }
 
     if (args.len == 0)
         _ = try generateServices(allocator, ";", std.io.getStdIn(), stdout);
 }
 
-fn processFile(arg: []const u8, stdout: anytype, manifest: anytype) !void {
+fn processFile(file_name: []const u8, stdout: anytype, output_dir: std.fs.Dir, manifest: anytype) !void {
     // It's probably best to create our own allocator here so we can deint at the end and
     // toss all allocations related to the services in this file
     // I can't guarantee we're not leaking something, and at the end of the
@@ -41,17 +88,17 @@ fn processFile(arg: []const u8, stdout: anytype, manifest: anytype) !void {
     const allocator = arena.allocator();
     var writer = &stdout;
     var file: std.fs.File = undefined;
-    const filename = try std.fmt.allocPrint(allocator, "{s}.zig", .{arg});
-    defer allocator.free(filename);
-    file = try std.fs.cwd().createFile(filename, .{ .truncate = true });
+    const output_file_name = try std.fmt.allocPrint(allocator, "{s}.zig", .{file_name});
+    defer allocator.free(output_file_name);
+    file = try output_dir.createFile(output_file_name, .{ .truncate = true });
     errdefer file.close();
     writer = &file.writer();
     _ = try writer.write("const std = @import(\"std\");\n");
     _ = try writer.write("const serializeMap = @import(\"json.zig\").serializeMap;\n");
     _ = try writer.write("const smithy = @import(\"smithy\");\n\n");
-    std.log.info("Processing file: {s}", .{arg});
-    const service_names = generateServicesForFilePath(allocator, ";", arg, writer) catch |err| {
-        std.log.err("Error processing file: {s}", .{arg});
+    std.log.info("Processing file: {s}", .{file_name});
+    const service_names = generateServicesForFilePath(allocator, ";", file_name, writer) catch |err| {
+        std.log.err("Error processing file: {s}", .{file_name});
         return err;
     };
     defer {
@@ -60,11 +107,16 @@ fn processFile(arg: []const u8, stdout: anytype, manifest: anytype) !void {
     }
     file.close();
     for (service_names) |name| {
-        try manifest.print("pub const {s} = @import(\"{s}\");\n", .{ name, std.fs.path.basename(filename) });
+        try manifest.print("pub const {s} = @import(\"{s}\");\n", .{ name, std.fs.path.basename(output_file_name) });
     }
 }
 
-fn generateServicesForFilePath(allocator: std.mem.Allocator, comptime terminator: []const u8, path: []const u8, writer: anytype) ![][]const u8 {
+fn generateServicesForFilePath(
+    allocator: std.mem.Allocator,
+    comptime terminator: []const u8,
+    path: []const u8,
+    writer: anytype,
+) ![][]const u8 {
     const file = try std.fs.cwd().openFile(path, .{});
     defer file.close();
     return try generateServices(allocator, terminator, file, writer);

codegen/src/snake.zig

@@ -72,7 +72,7 @@ fn isAcronymChar(char: u8) bool {
 fn isAscii(codepoint: ?u21) !?u8 {
     if (codepoint) |cp| {
         if (cp > 0xff) return error.UnicodeNotSupported;
-        return @truncate(u8, cp);
+        return @as(u8, @truncate(cp));
     }
     return null;
 }