lobo/ddblocal
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

introduce ddb.zig to serve as core DB operations/move core into ddb.zig

Browse source
This commit is contained in:
Emil Lerch 2024-02-01 08:44:56 -08:00
parent e94184419e
commit 796eed8de0
Signed by: lobo
GPG key ID: A7B62D657EF764F8
4 changed files with 284 additions and 102 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
src/batchgetitem.zig
View file

@ -14,4 +14,5 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
var parsed = try std.json.parseFromSlice(std.json.Value, allocator, request.event_data, .{});
defer parsed.deinit();
// const request_params = try parseRequest(request, parsed, writer);
return "hi";
}

1
src/batchwriteitem.zig
View file

@ -505,6 +505,7 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
// 3. Delete any existing records with that hash value (for delete requests, we're done here)
// 4. If put request, put the new item in the table (with encrypted values, using table encryption)
// TODO: Capacity limiting and metrics
return "hi";
}
test "basic request parsing failure" {

111
src/createtable.zig
View file

@ -4,6 +4,7 @@ const AuthenticatedRequest = @import("AuthenticatedRequest.zig");
const Account = @import("Account.zig");
const encryption = @import("encryption.zig");
const returnException = @import("main.zig").returnException;
const ddb = @import("ddb.zig");
const ddb_types = @import("ddb_types.zig");
// These are in the original casing so as to make the error messages nice
@ -15,17 +16,9 @@ const RequiredFields = enum(u3) {
// zig fmt: on
};
const TableInfo = struct {
attribute_definitions: []*ddb_types.AttributeDefinition,
// gsi_list: []const u8, // Not sure how this is used
// gsi_description_list: []const u8, // Not sure how this is used
// sqlite_index: []const u8, // Not sure how this is used
table_key: [encryption.encoded_key_length]u8,
};
const Params = struct {
table_name: []const u8,
table_info: TableInfo,
table_info: ddb_types.TableInfo,
read_capacity_units: ?i64 = null,
write_capacity_units: ?i64 = null,
billing_mode_pay_per_request: bool = false,
@ -47,23 +40,18 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
var db = try Account.dbForAccount(allocator, account_id);
const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed
defer account.deinit();
// TODO: better to do all encryption when request params are parsed?
const table_name = try encryption.encryptAndEncode(allocator, account.root_account_key.*, request_params.table_name);
defer allocator.free(table_name);
// We'll json serialize our table_info structure, encrypt, encode, and plow in
const table_info_string = try std.json.stringifyAlloc(allocator, request_params.table_info, .{ .whitespace = .indent_2 });
defer allocator.free(table_info_string);
const table_info = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_info_string);
defer allocator.free(table_info);
try insertIntoDm(
try ddb.createDdbTable(
allocator,
&db,
table_name,
table_info,
account,
request_params.table_name,
request_params.table_info,
request_params.read_capacity_units orelse 0,
request_params.write_capacity_units orelse 0,
request_params.billing_mode_pay_per_request,
);
// Server side Input validation error on live DDB results in this for a 2 char table name
// 400 - bad request
// {"__type":"com.amazon.coral.validate#ValidationException","message":"TableName must be at least 3 characters long and at most 255 characters long"}
@ -127,39 +115,6 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
// "UniqueGSIIndexes": []
// }
//
var diags = sqlite.Diagnostics{};
// It doesn't seem that I can bind a variable here. But it actually doesn't matter as we're
// encoding the name...
// IF NOT EXISTS doesn't apply - we want this to bounce if the table exists
const create_stmt = try std.fmt.allocPrint(allocator,
\\CREATE TABLE '{s}' (
\\ hashKey TEXT DEFAULT NULL,
\\ rangeKey TEXT DEFAULT NULL,
\\ hashValue BLOB NOT NULL,
\\ rangeValue BLOB NOT NULL,
\\ itemSize INTEGER DEFAULT 0,
\\ ObjectJSON BLOB NOT NULL,
\\ PRIMARY KEY(hashKey, rangeKey)
\\)
, .{table_name});
defer allocator.free(create_stmt);
// db.exec requires a comptime statement. execDynamic does not
db.execDynamic(
create_stmt,
.{ .diags = &diags },
.{},
) catch |e| {
std.log.debug("SqlLite Diags: {}", .{diags});
return e;
};
const create_index_stmt = try std.fmt.allocPrint(
allocator,
"CREATE INDEX \"{s}*HVI\" ON \"{s}\" (hashValue)",
.{ table_name, table_name },
);
defer allocator.free(create_index_stmt);
try db.execDynamic(create_index_stmt, .{}, .{});
var al = std.ArrayList(u8).init(allocator);
var response_writer = al.writer();
@ -167,54 +122,6 @@ pub fn handler(request: *AuthenticatedRequest, writer: anytype) ![]const u8 {
return al.toOwnedSlice();
}
fn insertIntoDm(
db: *sqlite.Db,
table_name: []const u8,
table_info: []const u8,
read_capacity_units: i64,
write_capacity_units: i64,
billing_mode_pay_per_request: bool,
) !void {
// const current_time = std.time.nanotimestamp();
const current_time = std.time.microTimestamp(); // SQLlite integers are only 64bit max
try db.exec(
\\INSERT INTO dm(
\\ TableName,
\\ CreationDateTime,
\\ LastDecreaseDate,
\\ LastIncreaseDate,
\\ NumberOfDecreasesToday,
\\ ReadCapacityUnits,
\\ WriteCapacityUnits,
\\ TableInfo,
\\ BillingMode,
\\ PayPerRequestDateTime
\\ ) VALUES (
\\ $tablename{[]const u8},
\\ $createdate{i64},
\\ $lastdecreasedate{usize},
\\ $lastincreasedate{usize},
\\ $numberofdecreasestoday{usize},
\\ $readcapacityunits{i64},
\\ $writecapacityunits{i64},
\\ $tableinfo{[]const u8},
\\ $billingmode{usize},
\\ $payperrequestdatetime{usize}
\\ )
, .{}, .{
table_name,
current_time,
@as(usize, 0),
@as(usize, 0),
@as(usize, 0),
read_capacity_units,
write_capacity_units,
table_info,
if (billing_mode_pay_per_request) @as(usize, 1) else @as(usize, 0),
@as(usize, 0),
});
}
fn parseRequest(
request: *AuthenticatedRequest,
parsed: std.json.Parsed(std.json.Value),
@ -273,7 +180,7 @@ fn parseRequest(
errdefer {
if (attribute_definitions_assigned) {
for (request_params.table_info.attribute_definitions) |d| {
request.allocator.free(d.*.name);
request.allocator.free(d.name);
request.allocator.destroy(d);
}
request.allocator.free(request_params.table_info.attribute_definitions);

273
src/ddb.zig Normal file
View file

@ -0,0 +1,273 @@
const std = @import("std");
const sqlite = @import("sqlite");
const AuthenticatedRequest = @import("AuthenticatedRequest.zig");
const Account = @import("Account.zig");
const ddb_types = @import("ddb_types.zig");
const encryption = @import("encryption.zig");
const builtin = @import("builtin");
pub const TableArray = struct {
items: []Table,
allocator: std.mem.Allocator,
pub fn init(allocator: std.mem.Allocator, length: usize) !TableArray {
return .{
.allocator = allocator,
.items = try allocator.alloc(Table, length),
};
}
pub fn deinit(self: *TableArray) void {
for (self.items) |*item|
item.deinit();
self.allocator.free(self.items);
}
};
pub const Table = struct {
table_name: []const u8,
table_key: [encryption.key_length]u8,
allocator: std.mem.Allocator,
pub fn deinit(self: *Table) void {
std.crypto.utils.secureZero(u8, &self.table_key);
self.allocator.free(self.table_name);
}
};
// Gets all table names/keys for the account. Caller owns returned array
pub fn tablesForAccount(allocator: std.mem.Allocator, account_id: []const u8) !TableArray {
var db = try Account.dbForAccount(allocator, account_id);
defer if (!builtin.is_test) db.deinit();
const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed
defer account.deinit();
const query =
\\SELECT TableName as table_name, TableInfo as table_info FROM dm
;
var stmt = try db.prepare(query);
defer stmt.deinit();
const rows = try stmt.all(struct {
table_name: []const u8,
table_info: []const u8,
}, allocator, .{}, .{});
defer allocator.free(rows);
var rc = try TableArray.init(allocator, rows.len);
errdefer rc.deinit();
// std.debug.print(" \n===\nRow count: {d}\n===\n", .{rows.len});
for (rows, 0..) |row, inx| {
defer allocator.free(row.table_name);
defer allocator.free(row.table_info);
const table_name = try encryption.decodeAndDecrypt(
allocator,
account.root_account_key.*,
row.table_name,
);
errdefer allocator.free(table_name);
const table_info_str = try encryption.decodeAndDecrypt(
allocator,
account.root_account_key.*,
row.table_info,
);
defer allocator.free(table_info_str);
// std.debug.print(" \n===TableInfo: {s}\n===\n", .{table_info_str});
const table_info = try std.json.parseFromSlice(ddb_types.TableInfo, allocator, table_info_str, .{});
defer table_info.deinit();
// errdefer allocator.free(table_info.table_key);
// defer {
// // we don't even really need to defer this...
// for (table_info.value.attribute_definitions) |*def| {
// allocator.free(def.*.name);
// allocator.destroy(def);
// }
// allocator.free(table_info.table_key);
// }
rc.items[inx] = .{
.allocator = allocator,
.table_name = table_name,
.table_key = undefined,
};
try encryption.decodeKey(&rc.items[inx].table_key, table_info.value.table_key);
}
return rc;
}
/// creates a table in the underlying storage
pub fn createDdbTable(
allocator: std.mem.Allocator,
db: *sqlite.Db,
account: Account,
table_name: []const u8,
table_info: ddb_types.TableInfo,
read_capacity_units: i64,
write_capacity_units: i64,
billing_mode_pay_per_request: bool,
) !void {
const encrypted_table_name = try insertIntoDatabaseMetadata(
allocator,
db,
account,
table_name,
table_info,
read_capacity_units,
write_capacity_units,
billing_mode_pay_per_request,
);
defer allocator.free(encrypted_table_name);
var diags = sqlite.Diagnostics{};
// It doesn't seem that I can bind a variable here. But it actually doesn't matter as we're
// encoding the name...
// IF NOT EXISTS doesn't apply - we want this to bounce if the table exists
const create_stmt = try std.fmt.allocPrint(allocator,
\\CREATE TABLE '{s}' (
\\ hashKey TEXT DEFAULT NULL,
\\ rangeKey TEXT DEFAULT NULL,
\\ hashValue BLOB NOT NULL,
\\ rangeValue BLOB NOT NULL,
\\ itemSize INTEGER DEFAULT 0,
\\ ObjectJSON BLOB NOT NULL,
\\ PRIMARY KEY(hashKey, rangeKey)
\\)
, .{encrypted_table_name});
defer allocator.free(create_stmt);
// db.exec requires a comptime statement. execDynamic does not
db.execDynamic(
create_stmt,
.{ .diags = &diags },
.{},
) catch |e| {
std.log.debug("SqlLite Diags: {}", .{diags});
return e;
};
const create_index_stmt = try std.fmt.allocPrint(
allocator,
"CREATE INDEX \"{s}*HVI\" ON \"{s}\" (hashValue)",
.{ encrypted_table_name, encrypted_table_name },
);
defer allocator.free(create_index_stmt);
try db.execDynamic(create_index_stmt, .{}, .{});
}
/// Inserts a new table into the database metadata (dm) table. Handles encryption
/// Returns encrypted table name
fn insertIntoDatabaseMetadata(
allocator: std.mem.Allocator,
db: *sqlite.Db,
account: Account,
table_name: []const u8,
table_info: ddb_types.TableInfo,
read_capacity_units: i64,
write_capacity_units: i64,
billing_mode_pay_per_request: bool,
) ![]const u8 {
// TODO: better to do all encryption when request params are parsed?
const encrypted_table_name = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_name);
errdefer allocator.free(encrypted_table_name);
// We'll json serialize our table_info structure, encrypt, encode, and plow in
const table_info_string = try std.json.stringifyAlloc(allocator, table_info, .{ .whitespace = .indent_2 });
defer allocator.free(table_info_string);
const encrypted_table_info = try encryption.encryptAndEncode(allocator, account.root_account_key.*, table_info_string);
defer allocator.free(encrypted_table_info);
try insertIntoDm(db, encrypted_table_name, encrypted_table_info, read_capacity_units, write_capacity_units, billing_mode_pay_per_request);
return encrypted_table_name;
}
fn insertIntoDm(
db: *sqlite.Db,
table_name: []const u8,
table_info: []const u8,
read_capacity_units: i64,
write_capacity_units: i64,
billing_mode_pay_per_request: bool,
) !void {
// const current_time = std.time.nanotimestamp();
const current_time = std.time.microTimestamp(); // SQLlite integers are only 64bit max
try db.exec(
\\INSERT INTO dm(
\\ TableName,
\\ CreationDateTime,
\\ LastDecreaseDate,
\\ LastIncreaseDate,
\\ NumberOfDecreasesToday,
\\ ReadCapacityUnits,
\\ WriteCapacityUnits,
\\ TableInfo,
\\ BillingMode,
\\ PayPerRequestDateTime
\\ ) VALUES (
\\ $tablename{[]const u8},
\\ $createdate{i64},
\\ $lastdecreasedate{usize},
\\ $lastincreasedate{usize},
\\ $numberofdecreasestoday{usize},
\\ $readcapacityunits{i64},
\\ $writecapacityunits{i64},
\\ $tableinfo{[]const u8},
\\ $billingmode{usize},
\\ $payperrequestdatetime{usize}
\\ )
, .{}, .{
table_name,
current_time,
@as(usize, 0),
@as(usize, 0),
@as(usize, 0),
read_capacity_units,
write_capacity_units,
table_info,
if (billing_mode_pay_per_request) @as(usize, 1) else @as(usize, 0),
@as(usize, 0),
});
}
fn testCreateTable(allocator: std.mem.Allocator, account_id: []const u8) !sqlite.Db {
var db = try Account.dbForAccount(allocator, account_id);
const account = try Account.accountForId(allocator, account_id); // This will get us the encryption key needed
defer account.deinit();
var hash = ddb_types.AttributeDefinition{ .name = "Artist", .type = .S };
var range = ddb_types.AttributeDefinition{ .name = "SongTitle", .type = .S };
var definitions = @constCast(&[_]*ddb_types.AttributeDefinition{
&hash,
&range,
});
var table_info: ddb_types.TableInfo = .{
.table_key = undefined,
.attribute_definitions = definitions[0..],
};
encryption.randomEncodedKey(&table_info.table_key);
try createDdbTable(
allocator,
&db,
account,
"MusicCollection",
table_info,
5,
5,
false,
);
return db;
}
test "can create a table" {
const allocator = std.testing.allocator;
const account_id = "1234";
var db = try testCreateTable(allocator, account_id);
defer db.deinit();
}
test "can list tables in an account" {
Account.test_retain_db = true;
defer Account.test_retain_db = false;
const allocator = std.testing.allocator;
const account_id = "1234";
var db = try testCreateTable(allocator, account_id);
defer db.deinit();
var table_list = try tablesForAccount(allocator, account_id);
defer table_list.deinit();
try std.testing.expectEqual(@as(usize, 1), table_list.items.len);
try std.testing.expectEqualStrings("MusicCollection", table_list.items[0].table_name);
// std.debug.print(" \n===\nKey: {s}\n===\n", .{std.fmt.fmtSliceHexLower(&table_list.items[0].table_key)});
}