diff --git a/src/batchgetitem.zig b/src/batchgetitem.zig index 6ce9b86..72635ba 100644 --- a/src/batchgetitem.zig +++ b/src/batchgetitem.zig @@ -14,4 +14,5 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 { var parsed = try std.json.parseFromSlice(std.json.Value, allocator, request.event_data, .{}); defer parsed.deinit(); // const request_params = try parseRequest(request, parsed, writer); + return "hi"; } diff --git a/src/batchwriteitem.zig b/src/batchwriteitem.zig index 587ed01..a45e854 100644 --- a/src/batchwriteitem.zig +++ b/src/batchwriteitem.zig @@ -505,6 +505,7 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 { // 3. Delete any existing records with that hash value (for delete requests, we're done here) // 4. If put request, put the new item in the table (with encrypted values, using table encryption) // TODO: Capacity limiting and metrics + return "hi"; } test "basic request parsing failure" { diff --git a/src/createtable.zig b/src/createtable.zig index ab2c543..91bbd75 100644 --- a/src/createtable.zig +++ b/src/createtable.zig @@ -4,6 +4,7 @@ const AuthenticatedRequest = @import("AuthenticatedRequest.zig"); const Account = @import("Account.zig"); const encryption = @import("encryption.zig"); const returnException = @import("main.zig").returnException; +const ddb = @import("ddb.zig"); const ddb_types = @import("ddb_types.zig"); // These are in the original casing so as to make the error messages nice @@ -15,17 +16,9 @@ const RequiredFields = enum(u3) { // zig fmt: on }; -const TableInfo = struct { - attribute_definitions: []*ddb_types.AttributeDefinition, - // gsi_list: []const u8, // Not sure how this is used - // gsi_description_list: []const u8, // Not sure how this is used - // sqlite_index: []const u8, // Not sure how this is used - table_key: [encryption.encoded_key_length]u8, -}; - const Params = struct { table_name: []const u8, - table_info: TableInfo, + table_info: ddb_types.TableInfo, read_capacity_units: ?i64 = null, write_capacity_units: ?i64 = null, billing_mode_pay_per_request: bool = false, @@ -47,23 +40,18 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 { var db = try Account.dbForAccount(allocator, account_id); const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed defer account.deinit(); - // TODO: better to do all encryption when request params are parsed? - const table_name = try encryption.encryptAndEncode(allocator, account.root_account_key.*, request_params.table_name); - defer allocator.free(table_name); - // We'll json serialize our table_info structure, encrypt, encode, and plow in - const table_info_string = try std.json.stringifyAlloc(allocator, request_params.table_info, .{ .whitespace = .indent_2 }); - defer allocator.free(table_info_string); - const table_info = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_info_string); - defer allocator.free(table_info); - try insertIntoDm( + try ddb.createDdbTable( + allocator, &db, - table_name, - table_info, + account, + request_params.table_name, + request_params.table_info, request_params.read_capacity_units orelse 0, request_params.write_capacity_units orelse 0, request_params.billing_mode_pay_per_request, ); + // Server side Input validation error on live DDB results in this for a 2 char table name // 400 - bad request // {"__type":"com.amazon.coral.validate#ValidationException","message":"TableName must be at least 3 characters long and at most 255 characters long"} @@ -127,39 +115,6 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 { // "UniqueGSIIndexes": [] // } // - var diags = sqlite.Diagnostics{}; - - // It doesn't seem that I can bind a variable here. But it actually doesn't matter as we're - // encoding the name... - // IF NOT EXISTS doesn't apply - we want this to bounce if the table exists - const create_stmt = try std.fmt.allocPrint(allocator, - \\CREATE TABLE '{s}' ( - \\ hashKey TEXT DEFAULT NULL, - \\ rangeKey TEXT DEFAULT NULL, - \\ hashValue BLOB NOT NULL, - \\ rangeValue BLOB NOT NULL, - \\ itemSize INTEGER DEFAULT 0, - \\ ObjectJSON BLOB NOT NULL, - \\ PRIMARY KEY(hashKey, rangeKey) - \\) - , .{table_name}); - defer allocator.free(create_stmt); - // db.exec requires a comptime statement. execDynamic does not - db.execDynamic( - create_stmt, - .{ .diags = &diags }, - .{}, - ) catch |e| { - std.log.debug("SqlLite Diags: {}", .{diags}); - return e; - }; - const create_index_stmt = try std.fmt.allocPrint( - allocator, - "CREATE INDEX \"{s}*HVI\" ON \"{s}\" (hashValue)", - .{ table_name, table_name }, - ); - defer allocator.free(create_index_stmt); - try db.execDynamic(create_index_stmt, .{}, .{}); var al = std.ArrayList(u8).init(allocator); var response_writer = al.writer(); @@ -167,54 +122,6 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 { return al.toOwnedSlice(); } -fn insertIntoDm( - db: *sqlite.Db, - table_name: []const u8, - table_info: []const u8, - read_capacity_units: i64, - write_capacity_units: i64, - billing_mode_pay_per_request: bool, -) !void { - // const current_time = std.time.nanotimestamp(); - const current_time = std.time.microTimestamp(); // SQLlite integers are only 64bit max - try db.exec( - \\INSERT INTO dm( - \\ TableName, - \\ CreationDateTime, - \\ LastDecreaseDate, - \\ LastIncreaseDate, - \\ NumberOfDecreasesToday, - \\ ReadCapacityUnits, - \\ WriteCapacityUnits, - \\ TableInfo, - \\ BillingMode, - \\ PayPerRequestDateTime - \\ ) VALUES ( - \\ $tablename{[]const u8}, - \\ $createdate{i64}, - \\ $lastdecreasedate{usize}, - \\ $lastincreasedate{usize}, - \\ $numberofdecreasestoday{usize}, - \\ $readcapacityunits{i64}, - \\ $writecapacityunits{i64}, - \\ $tableinfo{[]const u8}, - \\ $billingmode{usize}, - \\ $payperrequestdatetime{usize} - \\ ) - , .{}, .{ - table_name, - current_time, - @as(usize, 0), - @as(usize, 0), - @as(usize, 0), - read_capacity_units, - write_capacity_units, - table_info, - if (billing_mode_pay_per_request) @as(usize, 1) else @as(usize, 0), - @as(usize, 0), - }); -} - fn parseRequest( request: *AuthenticatedRequest, parsed: std.json.Parsed(std.json.Value), @@ -273,7 +180,7 @@ fn parseRequest( errdefer { if (attribute_definitions_assigned) { for (request_params.table_info.attribute_definitions) |d| { - request.allocator.free(d.*.name); + request.allocator.free(d.name); request.allocator.destroy(d); } request.allocator.free(request_params.table_info.attribute_definitions); diff --git a/src/ddb.zig b/src/ddb.zig new file mode 100644 index 0000000..c5731e8 --- /dev/null +++ b/src/ddb.zig @@ -0,0 +1,273 @@ +const std = @import("std"); +const sqlite = @import("sqlite"); +const AuthenticatedRequest = @import("AuthenticatedRequest.zig"); +const Account = @import("Account.zig"); +const ddb_types = @import("ddb_types.zig"); +const encryption = @import("encryption.zig"); +const builtin = @import("builtin"); + +pub const TableArray = struct { + items: []Table, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator, length: usize) !TableArray { + return .{ + .allocator = allocator, + .items = try allocator.alloc(Table, length), + }; + } + + pub fn deinit(self: *TableArray) void { + for (self.items) |*item| + item.deinit(); + self.allocator.free(self.items); + } +}; +pub const Table = struct { + table_name: []const u8, + table_key: [encryption.key_length]u8, + allocator: std.mem.Allocator, + + pub fn deinit(self: *Table) void { + std.crypto.utils.secureZero(u8, &self.table_key); + self.allocator.free(self.table_name); + } +}; + +// Gets all table names/keys for the account. Caller owns returned array +pub fn tablesForAccount(allocator: std.mem.Allocator, account_id: []const u8) !TableArray { + var db = try Account.dbForAccount(allocator, account_id); + defer if (!builtin.is_test) db.deinit(); + const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed + defer account.deinit(); + + const query = + \\SELECT TableName as table_name, TableInfo as table_info FROM dm + ; + + var stmt = try db.prepare(query); + defer stmt.deinit(); + + const rows = try stmt.all(struct { + table_name: []const u8, + table_info: []const u8, + }, allocator, .{}, .{}); + defer allocator.free(rows); + var rc = try TableArray.init(allocator, rows.len); + errdefer rc.deinit(); + + // std.debug.print(" \n===\nRow count: {d}\n===\n", .{rows.len}); + for (rows, 0..) |row, inx| { + defer allocator.free(row.table_name); + defer allocator.free(row.table_info); + const table_name = try encryption.decodeAndDecrypt( + allocator, + account.root_account_key.*, + row.table_name, + ); + errdefer allocator.free(table_name); + const table_info_str = try encryption.decodeAndDecrypt( + allocator, + account.root_account_key.*, + row.table_info, + ); + defer allocator.free(table_info_str); + // std.debug.print(" \n===TableInfo: {s}\n===\n", .{table_info_str}); + const table_info = try std.json.parseFromSlice(ddb_types.TableInfo, allocator, table_info_str, .{}); + defer table_info.deinit(); + // errdefer allocator.free(table_info.table_key); + // defer { + // // we don't even really need to defer this... + // for (table_info.value.attribute_definitions) |*def| { + // allocator.free(def.*.name); + // allocator.destroy(def); + // } + // allocator.free(table_info.table_key); + // } + + rc.items[inx] = .{ + .allocator = allocator, + .table_name = table_name, + .table_key = undefined, + }; + try encryption.decodeKey(&rc.items[inx].table_key, table_info.value.table_key); + } + return rc; +} + +/// creates a table in the underlying storage +pub fn createDdbTable( + allocator: std.mem.Allocator, + db: *sqlite.Db, + account: Account, + table_name: []const u8, + table_info: ddb_types.TableInfo, + read_capacity_units: i64, + write_capacity_units: i64, + billing_mode_pay_per_request: bool, +) !void { + const encrypted_table_name = try insertIntoDatabaseMetadata( + allocator, + db, + account, + table_name, + table_info, + read_capacity_units, + write_capacity_units, + billing_mode_pay_per_request, + ); + defer allocator.free(encrypted_table_name); + + var diags = sqlite.Diagnostics{}; + + // It doesn't seem that I can bind a variable here. But it actually doesn't matter as we're + // encoding the name... + // IF NOT EXISTS doesn't apply - we want this to bounce if the table exists + const create_stmt = try std.fmt.allocPrint(allocator, + \\CREATE TABLE '{s}' ( + \\ hashKey TEXT DEFAULT NULL, + \\ rangeKey TEXT DEFAULT NULL, + \\ hashValue BLOB NOT NULL, + \\ rangeValue BLOB NOT NULL, + \\ itemSize INTEGER DEFAULT 0, + \\ ObjectJSON BLOB NOT NULL, + \\ PRIMARY KEY(hashKey, rangeKey) + \\) + , .{encrypted_table_name}); + defer allocator.free(create_stmt); + // db.exec requires a comptime statement. execDynamic does not + db.execDynamic( + create_stmt, + .{ .diags = &diags }, + .{}, + ) catch |e| { + std.log.debug("SqlLite Diags: {}", .{diags}); + return e; + }; + const create_index_stmt = try std.fmt.allocPrint( + allocator, + "CREATE INDEX \"{s}*HVI\" ON \"{s}\" (hashValue)", + .{ encrypted_table_name, encrypted_table_name }, + ); + defer allocator.free(create_index_stmt); + try db.execDynamic(create_index_stmt, .{}, .{}); +} + +/// Inserts a new table into the database metadata (dm) table. Handles encryption +/// Returns encrypted table name +fn insertIntoDatabaseMetadata( + allocator: std.mem.Allocator, + db: *sqlite.Db, + account: Account, + table_name: []const u8, + table_info: ddb_types.TableInfo, + read_capacity_units: i64, + write_capacity_units: i64, + billing_mode_pay_per_request: bool, +) ![]const u8 { + // TODO: better to do all encryption when request params are parsed? + const encrypted_table_name = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_name); + errdefer allocator.free(encrypted_table_name); + // We'll json serialize our table_info structure, encrypt, encode, and plow in + const table_info_string = try std.json.stringifyAlloc(allocator, table_info, .{ .whitespace = .indent_2 }); + defer allocator.free(table_info_string); + const encrypted_table_info = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_info_string); + defer allocator.free(encrypted_table_info); + try insertIntoDm(db, encrypted_table_name, encrypted_table_info, read_capacity_units, write_capacity_units, billing_mode_pay_per_request); + return encrypted_table_name; +} + +fn insertIntoDm( + db: *sqlite.Db, + table_name: []const u8, + table_info: []const u8, + read_capacity_units: i64, + write_capacity_units: i64, + billing_mode_pay_per_request: bool, +) !void { + // const current_time = std.time.nanotimestamp(); + const current_time = std.time.microTimestamp(); // SQLlite integers are only 64bit max + try db.exec( + \\INSERT INTO dm( + \\ TableName, + \\ CreationDateTime, + \\ LastDecreaseDate, + \\ LastIncreaseDate, + \\ NumberOfDecreasesToday, + \\ ReadCapacityUnits, + \\ WriteCapacityUnits, + \\ TableInfo, + \\ BillingMode, + \\ PayPerRequestDateTime + \\ ) VALUES ( + \\ $tablename{[]const u8}, + \\ $createdate{i64}, + \\ $lastdecreasedate{usize}, + \\ $lastincreasedate{usize}, + \\ $numberofdecreasestoday{usize}, + \\ $readcapacityunits{i64}, + \\ $writecapacityunits{i64}, + \\ $tableinfo{[]const u8}, + \\ $billingmode{usize}, + \\ $payperrequestdatetime{usize} + \\ ) + , .{}, .{ + table_name, + current_time, + @as(usize, 0), + @as(usize, 0), + @as(usize, 0), + read_capacity_units, + write_capacity_units, + table_info, + if (billing_mode_pay_per_request) @as(usize, 1) else @as(usize, 0), + @as(usize, 0), + }); +} + +fn testCreateTable(allocator: std.mem.Allocator, account_id: []const u8) !sqlite.Db { + var db = try Account.dbForAccount(allocator, account_id); + const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed + defer account.deinit(); + var hash = ddb_types.AttributeDefinition{ .name = "Artist", .type = .S }; + var range = ddb_types.AttributeDefinition{ .name = "SongTitle", .type = .S }; + var definitions = @constCast(&[_]*ddb_types.AttributeDefinition{ + &hash, + &range, + }); + var table_info: ddb_types.TableInfo = .{ + .table_key = undefined, + .attribute_definitions = definitions[0..], + }; + encryption.randomEncodedKey(&table_info.table_key); + try createDdbTable( + allocator, + &db, + account, + "MusicCollection", + table_info, + 5, + 5, + false, + ); + return db; +} +test "can create a table" { + const allocator = std.testing.allocator; + const account_id = "1234"; + var db = try testCreateTable(allocator, account_id); + defer db.deinit(); +} +test "can list tables in an account" { + Account.test_retain_db = true; + defer Account.test_retain_db = false; + const allocator = std.testing.allocator; + const account_id = "1234"; + var db = try testCreateTable(allocator, account_id); + defer db.deinit(); + var table_list = try tablesForAccount(allocator, account_id); + defer table_list.deinit(); + try std.testing.expectEqual(@as(usize, 1), table_list.items.len); + try std.testing.expectEqualStrings("MusicCollection", table_list.items[0].table_name); + // std.debug.print(" \n===\nKey: {s}\n===\n", .{std.fmt.fmtSliceHexLower(&table_list.items[0].table_key)}); +}