introduce ddb.zig to serve as core DB operations/move core into ddb.zig

src/batchgetitem.zig

@@ -14,4 +14,5 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
     var parsed = try std.json.parseFromSlice(std.json.Value, allocator, request.event_data, .{});
     defer parsed.deinit();
     // const request_params = try parseRequest(request, parsed, writer);
+    return "hi";
 }

src/batchwriteitem.zig

@@ -505,6 +505,7 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
     //    3. Delete any existing records with that hash value (for delete requests, we're done here)
     //    4. If put request, put the new item in the table (with encrypted values, using table encryption)
     // TODO: Capacity limiting and metrics
+    return "hi";
 }
 
 test "basic request parsing failure" {

src/createtable.zig

@@ -4,6 +4,7 @@ const AuthenticatedRequest = @import("AuthenticatedRequest.zig");
 const Account = @import("Account.zig");
 const encryption = @import("encryption.zig");
 const returnException = @import("main.zig").returnException;
+const ddb = @import("ddb.zig");
 const ddb_types = @import("ddb_types.zig");
 
 // These are in the original casing so as to make the error messages nice
@@ -15,17 +16,9 @@ const RequiredFields = enum(u3) {
     // zig fmt: on
 };
 
-const TableInfo = struct {
-    attribute_definitions: []*ddb_types.AttributeDefinition,
-    // gsi_list: []const u8, // Not sure how this is used
-    // gsi_description_list: []const u8, // Not sure how this is used
-    // sqlite_index: []const u8, // Not sure how this is used
-    table_key: [encryption.encoded_key_length]u8,
-};
-
 const Params = struct {
     table_name: []const u8,
-    table_info: TableInfo,
+    table_info: ddb_types.TableInfo,
     read_capacity_units: ?i64 = null,
     write_capacity_units: ?i64 = null,
     billing_mode_pay_per_request: bool = false,
@@ -47,23 +40,18 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
     var db = try Account.dbForAccount(allocator, account_id);
     const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed
     defer account.deinit();
-    // TODO: better to do all encryption when request params are parsed?
-    const table_name = try encryption.encryptAndEncode(allocator, account.root_account_key.*, request_params.table_name);
-    defer allocator.free(table_name);
-    // We'll json serialize our table_info structure, encrypt, encode, and plow in
-    const table_info_string = try std.json.stringifyAlloc(allocator, request_params.table_info, .{ .whitespace = .indent_2 });
-    defer allocator.free(table_info_string);
-    const table_info = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_info_string);
-    defer allocator.free(table_info);
 
-    try insertIntoDm(
+    try ddb.createDdbTable(
+        allocator,
         &db,
-        table_name,
+        account,
-        table_info,
+        request_params.table_name,
+        request_params.table_info,
         request_params.read_capacity_units orelse 0,
         request_params.write_capacity_units orelse 0,
         request_params.billing_mode_pay_per_request,
     );
+
     // Server side Input validation error on live DDB results in this for a 2 char table name
     // 400 - bad request
     // {"__type":"com.amazon.coral.validate#ValidationException","message":"TableName must be at least 3 characters long and at most 255 characters long"}
@@ -127,39 +115,6 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
     //   "UniqueGSIIndexes": []
     // }
     //
-    var diags = sqlite.Diagnostics{};
-
-    // It doesn't seem that I can bind a variable here. But it actually doesn't matter as we're
-    // encoding the name...
-    // IF NOT EXISTS doesn't apply - we want this to bounce if the table exists
-    const create_stmt = try std.fmt.allocPrint(allocator,
-        \\CREATE TABLE '{s}' (
-        \\    hashKey TEXT DEFAULT NULL,
-        \\    rangeKey TEXT DEFAULT NULL,
-        \\    hashValue BLOB NOT NULL,
-        \\    rangeValue BLOB NOT NULL,
-        \\    itemSize INTEGER DEFAULT 0,
-        \\    ObjectJSON BLOB NOT NULL,
-        \\    PRIMARY KEY(hashKey, rangeKey)
-        \\)
-    , .{table_name});
-    defer allocator.free(create_stmt);
-    // db.exec requires a comptime statement. execDynamic does not
-    db.execDynamic(
-        create_stmt,
-        .{ .diags = &diags },
-        .{},
-    ) catch |e| {
-        std.log.debug("SqlLite Diags: {}", .{diags});
-        return e;
-    };
-    const create_index_stmt = try std.fmt.allocPrint(
-        allocator,
-        "CREATE INDEX \"{s}*HVI\" ON \"{s}\" (hashValue)",
-        .{ table_name, table_name },
-    );
-    defer allocator.free(create_index_stmt);
-    try db.execDynamic(create_index_stmt, .{}, .{});
 
     var al = std.ArrayList(u8).init(allocator);
     var response_writer = al.writer();
@@ -167,54 +122,6 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
     return al.toOwnedSlice();
 }
 
-fn insertIntoDm(
-    db: *sqlite.Db,
-    table_name: []const u8,
-    table_info: []const u8,
-    read_capacity_units: i64,
-    write_capacity_units: i64,
-    billing_mode_pay_per_request: bool,
-) !void {
-    // const current_time = std.time.nanotimestamp();
-    const current_time = std.time.microTimestamp(); // SQLlite integers are only 64bit max
-    try db.exec(
-        \\INSERT INTO dm(
-        \\  TableName,
-        \\  CreationDateTime,
-        \\  LastDecreaseDate,
-        \\  LastIncreaseDate,
-        \\  NumberOfDecreasesToday,
-        \\  ReadCapacityUnits,
-        \\  WriteCapacityUnits,
-        \\  TableInfo,
-        \\  BillingMode,
-        \\  PayPerRequestDateTime
-        \\  ) VALUES (
-        \\  $tablename{[]const u8},
-        \\  $createdate{i64},
-        \\  $lastdecreasedate{usize},
-        \\  $lastincreasedate{usize},
-        \\  $numberofdecreasestoday{usize},
-        \\  $readcapacityunits{i64},
-        \\  $writecapacityunits{i64},
-        \\  $tableinfo{[]const u8},
-        \\  $billingmode{usize},
-        \\  $payperrequestdatetime{usize}
-        \\  )
-    , .{}, .{
-        table_name,
-        current_time,
-        @as(usize, 0),
-        @as(usize, 0),
-        @as(usize, 0),
-        read_capacity_units,
-        write_capacity_units,
-        table_info,
-        if (billing_mode_pay_per_request) @as(usize, 1) else @as(usize, 0),
-        @as(usize, 0),
-    });
-}
-
 fn parseRequest(
     request: *AuthenticatedRequest,
     parsed: std.json.Parsed(std.json.Value),
@@ -273,7 +180,7 @@ fn parseRequest(
     errdefer {
         if (attribute_definitions_assigned) {
             for (request_params.table_info.attribute_definitions) |d| {
-                request.allocator.free(d.*.name);
+                request.allocator.free(d.name);
                 request.allocator.destroy(d);
             }
             request.allocator.free(request_params.table_info.attribute_definitions);

src/ddb.zig

@@ -0,0 +1,273 @@
+const std = @import("std");
+const sqlite = @import("sqlite");
+const AuthenticatedRequest = @import("AuthenticatedRequest.zig");
+const Account = @import("Account.zig");
+const ddb_types = @import("ddb_types.zig");
+const encryption = @import("encryption.zig");
+const builtin = @import("builtin");
+
+pub const TableArray = struct {
+    items: []Table,
+    allocator: std.mem.Allocator,
+
+    pub fn init(allocator: std.mem.Allocator, length: usize) !TableArray {
+        return .{
+            .allocator = allocator,
+            .items = try allocator.alloc(Table, length),
+        };
+    }
+
+    pub fn deinit(self: *TableArray) void {
+        for (self.items) |*item|
+            item.deinit();
+        self.allocator.free(self.items);
+    }
+};
+pub const Table = struct {
+    table_name: []const u8,
+    table_key: [encryption.key_length]u8,
+    allocator: std.mem.Allocator,
+
+    pub fn deinit(self: *Table) void {
+        std.crypto.utils.secureZero(u8, &self.table_key);
+        self.allocator.free(self.table_name);
+    }
+};
+
+// Gets all table names/keys for the account. Caller owns returned array
+pub fn tablesForAccount(allocator: std.mem.Allocator, account_id: []const u8) !TableArray {
+    var db = try Account.dbForAccount(allocator, account_id);
+    defer if (!builtin.is_test) db.deinit();
+    const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed
+    defer account.deinit();
+
+    const query =
+        \\SELECT TableName as table_name, TableInfo as table_info FROM dm
+    ;
+
+    var stmt = try db.prepare(query);
+    defer stmt.deinit();
+
+    const rows = try stmt.all(struct {
+        table_name: []const u8,
+        table_info: []const u8,
+    }, allocator, .{}, .{});
+    defer allocator.free(rows);
+    var rc = try TableArray.init(allocator, rows.len);
+    errdefer rc.deinit();
+
+    // std.debug.print(" \n===\nRow count: {d}\n===\n", .{rows.len});
+    for (rows, 0..) |row, inx| {
+        defer allocator.free(row.table_name);
+        defer allocator.free(row.table_info);
+        const table_name = try encryption.decodeAndDecrypt(
+            allocator,
+            account.root_account_key.*,
+            row.table_name,
+        );
+        errdefer allocator.free(table_name);
+        const table_info_str = try encryption.decodeAndDecrypt(
+            allocator,
+            account.root_account_key.*,
+            row.table_info,
+        );
+        defer allocator.free(table_info_str);
+        // std.debug.print(" \n===TableInfo: {s}\n===\n", .{table_info_str});
+        const table_info = try std.json.parseFromSlice(ddb_types.TableInfo, allocator, table_info_str, .{});
+        defer table_info.deinit();
+        // errdefer allocator.free(table_info.table_key);
+        // defer {
+        //     // we don't even really need to defer this...
+        //     for (table_info.value.attribute_definitions) |*def| {
+        //         allocator.free(def.*.name);
+        //         allocator.destroy(def);
+        //     }
+        //     allocator.free(table_info.table_key);
+        // }
+
+        rc.items[inx] = .{
+            .allocator = allocator,
+            .table_name = table_name,
+            .table_key = undefined,
+        };
+        try encryption.decodeKey(&rc.items[inx].table_key, table_info.value.table_key);
+    }
+    return rc;
+}
+
+/// creates a table in the underlying storage
+pub fn createDdbTable(
+    allocator: std.mem.Allocator,
+    db: *sqlite.Db,
+    account: Account,
+    table_name: []const u8,
+    table_info: ddb_types.TableInfo,
+    read_capacity_units: i64,
+    write_capacity_units: i64,
+    billing_mode_pay_per_request: bool,
+) !void {
+    const encrypted_table_name = try insertIntoDatabaseMetadata(
+        allocator,
+        db,
+        account,
+        table_name,
+        table_info,
+        read_capacity_units,
+        write_capacity_units,
+        billing_mode_pay_per_request,
+    );
+    defer allocator.free(encrypted_table_name);
+
+    var diags = sqlite.Diagnostics{};
+
+    // It doesn't seem that I can bind a variable here. But it actually doesn't matter as we're
+    // encoding the name...
+    // IF NOT EXISTS doesn't apply - we want this to bounce if the table exists
+    const create_stmt = try std.fmt.allocPrint(allocator,
+        \\CREATE TABLE '{s}' (
+        \\    hashKey TEXT DEFAULT NULL,
+        \\    rangeKey TEXT DEFAULT NULL,
+        \\    hashValue BLOB NOT NULL,
+        \\    rangeValue BLOB NOT NULL,
+        \\    itemSize INTEGER DEFAULT 0,
+        \\    ObjectJSON BLOB NOT NULL,
+        \\    PRIMARY KEY(hashKey, rangeKey)
+        \\)
+    , .{encrypted_table_name});
+    defer allocator.free(create_stmt);
+    // db.exec requires a comptime statement. execDynamic does not
+    db.execDynamic(
+        create_stmt,
+        .{ .diags = &diags },
+        .{},
+    ) catch |e| {
+        std.log.debug("SqlLite Diags: {}", .{diags});
+        return e;
+    };
+    const create_index_stmt = try std.fmt.allocPrint(
+        allocator,
+        "CREATE INDEX \"{s}*HVI\" ON \"{s}\" (hashValue)",
+        .{ encrypted_table_name, encrypted_table_name },
+    );
+    defer allocator.free(create_index_stmt);
+    try db.execDynamic(create_index_stmt, .{}, .{});
+}
+
+/// Inserts a new table into the database metadata (dm) table. Handles encryption
+/// Returns encrypted table name
+fn insertIntoDatabaseMetadata(
+    allocator: std.mem.Allocator,
+    db: *sqlite.Db,
+    account: Account,
+    table_name: []const u8,
+    table_info: ddb_types.TableInfo,
+    read_capacity_units: i64,
+    write_capacity_units: i64,
+    billing_mode_pay_per_request: bool,
+) ![]const u8 {
+    // TODO: better to do all encryption when request params are parsed?
+    const encrypted_table_name = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_name);
+    errdefer allocator.free(encrypted_table_name);
+    // We'll json serialize our table_info structure, encrypt, encode, and plow in
+    const table_info_string = try std.json.stringifyAlloc(allocator, table_info, .{ .whitespace = .indent_2 });
+    defer allocator.free(table_info_string);
+    const encrypted_table_info = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_info_string);
+    defer allocator.free(encrypted_table_info);
+    try insertIntoDm(db, encrypted_table_name, encrypted_table_info, read_capacity_units, write_capacity_units, billing_mode_pay_per_request);
+    return encrypted_table_name;
+}
+
+fn insertIntoDm(
+    db: *sqlite.Db,
+    table_name: []const u8,
+    table_info: []const u8,
+    read_capacity_units: i64,
+    write_capacity_units: i64,
+    billing_mode_pay_per_request: bool,
+) !void {
+    // const current_time = std.time.nanotimestamp();
+    const current_time = std.time.microTimestamp(); // SQLlite integers are only 64bit max
+    try db.exec(
+        \\INSERT INTO dm(
+        \\  TableName,
+        \\  CreationDateTime,
+        \\  LastDecreaseDate,
+        \\  LastIncreaseDate,
+        \\  NumberOfDecreasesToday,
+        \\  ReadCapacityUnits,
+        \\  WriteCapacityUnits,
+        \\  TableInfo,
+        \\  BillingMode,
+        \\  PayPerRequestDateTime
+        \\  ) VALUES (
+        \\  $tablename{[]const u8},
+        \\  $createdate{i64},
+        \\  $lastdecreasedate{usize},
+        \\  $lastincreasedate{usize},
+        \\  $numberofdecreasestoday{usize},
+        \\  $readcapacityunits{i64},
+        \\  $writecapacityunits{i64},
+        \\  $tableinfo{[]const u8},
+        \\  $billingmode{usize},
+        \\  $payperrequestdatetime{usize}
+        \\  )
+    , .{}, .{
+        table_name,
+        current_time,
+        @as(usize, 0),
+        @as(usize, 0),
+        @as(usize, 0),
+        read_capacity_units,
+        write_capacity_units,
+        table_info,
+        if (billing_mode_pay_per_request) @as(usize, 1) else @as(usize, 0),
+        @as(usize, 0),
+    });
+}
+
+fn testCreateTable(allocator: std.mem.Allocator, account_id: []const u8) !sqlite.Db {
+    var db = try Account.dbForAccount(allocator, account_id);
+    const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed
+    defer account.deinit();
+    var hash = ddb_types.AttributeDefinition{ .name = "Artist", .type = .S };
+    var range = ddb_types.AttributeDefinition{ .name = "SongTitle", .type = .S };
+    var definitions = @constCast(&[_]*ddb_types.AttributeDefinition{
+        &hash,
+        &range,
+    });
+    var table_info: ddb_types.TableInfo = .{
+        .table_key = undefined,
+        .attribute_definitions = definitions[0..],
+    };
+    encryption.randomEncodedKey(&table_info.table_key);
+    try createDdbTable(
+        allocator,
+        &db,
+        account,
+        "MusicCollection",
+        table_info,
+        5,
+        5,
+        false,
+    );
+    return db;
+}
+test "can create a table" {
+    const allocator = std.testing.allocator;
+    const account_id = "1234";
+    var db = try testCreateTable(allocator, account_id);
+    defer db.deinit();
+}
+test "can list tables in an account" {
+    Account.test_retain_db = true;
+    defer Account.test_retain_db = false;
+    const allocator = std.testing.allocator;
+    const account_id = "1234";
+    var db = try testCreateTable(allocator, account_id);
+    defer db.deinit();
+    var table_list = try tablesForAccount(allocator, account_id);
+    defer table_list.deinit();
+    try std.testing.expectEqual(@as(usize, 1), table_list.items.len);
+    try std.testing.expectEqualStrings("MusicCollection", table_list.items[0].table_name);
+    // std.debug.print(" \n===\nKey: {s}\n===\n", .{std.fmt.fmtSliceHexLower(&table_list.items[0].table_key)});
+}