From 55303c13f407518cfb24e085639d4f3c5e24a207 Mon Sep 17 00:00:00 2001 From: Maas-Maarten Zeeman Date: Thu, 26 May 2022 16:59:10 +0200 Subject: [PATCH] Added dirty versions of the backup api --- c_src/esqlite3_nif.c | 476 +++++++++++++++--------------------------- rebar.config.script | 2 +- src/esqlite3.erl | 281 +++---------------------- src/esqlite3_nif.erl | 68 +++--- test/esqlite_test.erl | 210 +++++-------------- 5 files changed, 270 insertions(+), 767 deletions(-) diff --git a/c_src/esqlite3_nif.c b/c_src/esqlite3_nif.c index efa0b48..a91de62 100644 --- a/c_src/esqlite3_nif.c +++ b/c_src/esqlite3_nif.c @@ -32,26 +32,29 @@ static ErlNifResourceType *esqlite3_type = NULL; static ErlNifResourceType *esqlite3_stmt_type = NULL; static ErlNifResourceType *esqlite3_backup_type = NULL; -/* database connection context */ +/* Database connection context */ typedef struct { sqlite3 *db; ErlNifPid update_hook_pid; } esqlite3; -/* prepared statement */ +/* Prepared statement */ typedef struct { - sqlite3_stmt *statement; + esqlite3 *connection; + sqlite3_stmt *statement; int column_count; } esqlite3_stmt; -/* data associated with ongoing backup */ +/* Data associated with an ongoing backup */ typedef struct { + esqlite3 *source; + esqlite3 *destination; + sqlite3_backup *backup; } esqlite3_backup; -static ERL_NIF_TERM atom_esqlite3; static ERL_NIF_TERM make_atom(ErlNifEnv *env, const char *atom_name) @@ -104,19 +107,34 @@ destruct_esqlite3_stmt(ErlNifEnv *env, void *arg) esqlite3_stmt *stmt = (esqlite3_stmt *) arg; sqlite3_finalize(stmt->statement); stmt->statement = NULL; + stmt->column_count = 0; + + if(stmt->connection) { + enif_release_resource(stmt->connection); + stmt->connection = NULL; + } } static void destruct_esqlite3_backup(ErlNifEnv *env, void *arg) { esqlite3_backup *backup = (esqlite3_backup *) arg; - + if(backup->backup) { sqlite3_backup_finish(backup->backup); } - backup->backup = NULL; + + if(backup->destination) { + enif_release_resource(backup->destination); + backup->destination = NULL; + } + + if(backup->source) { + enif_release_resource(backup->source); + backup->source = NULL; + } } static ERL_NIF_TERM @@ -159,168 +177,6 @@ make_cell(ErlNifEnv *env, sqlite3_stmt *statement, unsigned int i) return enif_raise_exception(env, make_atom(env, "internal_error")); } -/* -static ERL_NIF_TERM -do_backup_init(ErlNifEnv *env, sqlite3 *db, const ERL_NIF_TERM arg) -{ - int tuple_arity; - const ERL_NIF_TERM *elements; - sqlite3_backup *backup; - unsigned int size; - char dst_name[MAX_SQLITE_NAME_LENGTH]; - char src_name[MAX_SQLITE_NAME_LENGTH]; - esqlite3 *src; - esqlite3_backup *esqlite3_backup; - ERL_NIF_TERM erl_backup_term; - - if(db == NULL) { - return make_error_tuple(env, "dst_closed"); - } - - if(!enif_get_tuple(env, arg, &tuple_arity, &elements)) { - return make_error_tuple(env, "no_tuple"); - } - if(tuple_arity != 3) { - return make_error_tuple(env, "invalid_tuple"); - } - - size = enif_get_string(env, elements[0], dst_name, MAX_PATHNAME, ERL_NIF_LATIN1); - if(size <= 0) - return make_error_tuple(env, "invalid_dst_name"); - - if(!enif_get_resource(env, elements[1], esqlite3_type, (void **) &src)) { - return make_error_tuple(env, "invalid_src_db"); - } - if(!src->db) { - return make_error_tuple(env, "src_closed"); - } - - size = enif_get_string(env, elements[2], src_name, MAX_PATHNAME, ERL_NIF_LATIN1); - if(size <= 0) - return make_error_tuple(env, "invalid_src_name"); - - backup = sqlite3_backup_init(db, dst_name, src->db, src_name); - if(backup == NULL) { - return make_sqlite3_error_tuple(env, sqlite3_errcode(db), db); - } - - esqlite3_backup = enif_alloc_resource(esqlite3_backup_type, sizeof(esqlite3_backup)); - if(!esqlite3_backup) { - // Release backup resouces - (void) sqlite3_backup_finish(backup); - return make_error_tuple(env, "no_memory"); - } - - esqlite3_backup->backup = backup; - erl_backup_term = enif_make_resource(env, esqlite3_backup); - enif_release_resource(esqlite3_backup); - - return make_ok_tuple(env, erl_backup_term); -} -*/ - -/* -static ERL_NIF_TERM -do_backup_step(ErlNifEnv *env, sqlite3 *db, const ERL_NIF_TERM arg) -{ - int tuple_arity; - const ERL_NIF_TERM *elements; - esqlite3_backup *esqlite3_backup; - int n_page = 0; - int rc; - - if(db == NULL) { - return make_error_tuple(env, "closed"); - } - - if(!enif_get_tuple(env, arg, &tuple_arity, &elements)) { - return make_error_tuple(env, "no_tuple"); - } - if(tuple_arity != 2) { - return make_error_tuple(env, "invalid_tuple"); - } - - if(!enif_get_resource(env, elements[0], esqlite3_backup_type, (void **) &esqlite3_backup)) { - return make_error_tuple(env, "invalid"); - } - if(!esqlite3_backup->backup) { - return make_error_tuple(env, "backup"); - } - - if(!enif_get_int(env, elements[1], &n_page)) { - return make_error_tuple(env, "n_page"); - } - - rc = sqlite3_backup_step(esqlite3_backup->backup, n_page); - if(rc == SQLITE_DONE) { - return make_atom(env, "done"); - } - - if(rc != SQLITE_OK) { - return make_sqlite3_error_tuple(env, rc, db); - } - - return make_atom(env, "ok"); -} -*/ - -/* -static ERL_NIF_TERM -do_backup_remaining(ErlNifEnv *env, const ERL_NIF_TERM arg) -{ - esqlite3_backup *esqlite3_backup; - int remaining; - ERL_NIF_TERM remaining_term; - - if(!enif_get_resource(env, arg, esqlite3_backup_type, (void **) &esqlite3_backup)) { - return make_error_tuple(env, "invalid"); - } - - remaining = sqlite3_backup_remaining(esqlite3_backup->backup); - remaining_term = enif_make_int64(env, remaining); - - return make_ok_tuple(env, remaining_term); -} -*/ - -/* -static ERL_NIF_TERM -do_backup_pagecount(ErlNifEnv *env, const ERL_NIF_TERM arg) -{ - esqlite3_backup *esqlite3_backup; - int pagecount; - ERL_NIF_TERM pagecount_term; - - if(!enif_get_resource(env, arg, esqlite3_backup_type, (void **) &esqlite3_backup)) { - return make_error_tuple(env, "invalid"); - } - - pagecount = sqlite3_backup_pagecount(esqlite3_backup->backup); - pagecount_term = enif_make_int64(env, pagecount); - - return make_ok_tuple(env, pagecount_term); -} -*/ - -/* -static ERL_NIF_TERM -do_backup_finish(ErlNifEnv *env, const ERL_NIF_TERM arg) -{ - esqlite3_backup *esqlite3_backup; - - if(!enif_get_resource(env, arg, esqlite3_backup_type, (void **) &esqlite3_backup)) { - return make_error_tuple(env, "invalid"); - } - - if(esqlite3_backup->backup) { - (void) sqlite3_backup_finish(esqlite3_backup->backup); - esqlite3_backup->backup = NULL; - } - - return make_atom(env, "ok"); -} -*/ - /* * Open the database */ @@ -588,6 +444,11 @@ esqlite_prepare(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) if(!stmt) { return enif_raise_exception(env, make_atom(env, "no_memory")); } + /* Keep a reference to the connection to prevent it from being garbage collected + * before the statement. + */ + enif_keep_resource((void *) conn); + stmt->connection = conn; rc = sqlite3_prepare_v3(conn->db, (char *) bin.data, bin.size, prep_flags, &(stmt->statement), &tail); if(rc != SQLITE_OK) { @@ -971,183 +832,177 @@ esqlite_reset(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) * */ -/* static ERL_NIF_TERM esqlite_backup_init(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { esqlite3 *destination; - esqlite_command *cmd = NULL; - ErlNifPid pid; + ErlNifBinary destination_name; + esqlite3 *source; + ErlNifBinary source_name; + ERL_NIF_TERM eos = enif_make_int(env, 0); - if(argc != 6) + if(argc != 4) { return enif_make_badarg(env); + } - if(!enif_get_resource(env, argv[0], esqlite3_type, (void **) &destination)) + if(!enif_get_resource(env, argv[0], esqlite3_type, (void **) &destination)) { return enif_make_badarg(env); - // 1 destination name - // 2 source connection with database - // 3 source name - if(!enif_is_ref(env, argv[4])) - return make_error_tuple(env, "invalid_ref"); - if(!enif_get_local_pid(env, argv[5], &pid)) - return make_error_tuple(env, "invalid_pid"); + } - cmd = command_create(); - if(!cmd) - return make_error_tuple(env, "command_create_failed"); + if(!enif_inspect_iolist_as_binary(env, enif_make_list2(env, argv[1], eos), &destination_name)) { + return enif_make_badarg(env); + } - cmd->type = cmd_backup_init; - cmd->ref = enif_make_copy(cmd->env, argv[4]); - cmd->pid = pid; - cmd->arg = enif_make_tuple3(cmd->env, argv[1], argv[2], argv[3]); + if(!enif_get_resource(env, argv[2], esqlite3_type, (void **) &source)) { + return enif_make_badarg(env); + } - // Use the connection of the destination database - return push_command(env, destination, cmd); + if(!enif_inspect_iolist_as_binary(env, enif_make_list2(env, argv[3], eos), &source_name)) { + return enif_make_badarg(env); + } + + sqlite3_backup *backup = sqlite3_backup_init(destination->db, (const char *) destination_name.data, source->db, (const char *) source_name.data); + if(backup == NULL) { + return make_sqlite3_error_tuple(env, sqlite3_errcode(destination->db)); + } + + esqlite3_backup *ebackup = enif_alloc_resource(esqlite3_backup_type, sizeof(esqlite3_backup)); + if(!ebackup) { + (void) sqlite3_backup_finish(backup); + return enif_raise_exception(env, make_atom(env, "no_memory")); + } + + ebackup->backup = backup; + + /** + * Keep references to both database connections to prevent + * them from being garbage collected during the operation. + */ + enif_keep_resource((void *)destination); + ebackup->destination = destination; + enif_keep_resource((void *)source); + ebackup->source = source; + + ERL_NIF_TERM erl_backup_term = enif_make_resource(env, ebackup); + enif_release_resource(ebackup); + + return make_ok_tuple(env, erl_backup_term); } -*/ - -/* -static ERL_NIF_TERM -esqlite_backup_finish(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - esqlite3 *conn; - esqlite3_backup *backup; - esqlite_command *cmd = NULL; - ErlNifPid pid; - - if(argc != 4) - return enif_make_badarg(env); - - if(!enif_get_resource(env, argv[0], esqlite3_type, (void **) &conn)) - return enif_make_badarg(env); - - if(!enif_is_ref(env, argv[2])) - return make_error_tuple(env, "invalid_ref"); - if(!enif_get_local_pid(env, argv[3], &pid)) - return make_error_tuple(env, "invalid_pid"); - - cmd = command_create(); - if(!cmd) - return make_error_tuple(env, "command_create_failed"); - - cmd->type = cmd_backup_finish; - cmd->ref = enif_make_copy(cmd->env, argv[2]); - cmd->pid = pid; - cmd->arg = enif_make_copy(cmd->env, argv[1]); - - return push_command(env, conn, cmd); -} -*/ - -/* -static ERL_NIF_TERM -esqlite_backup_step(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) -{ - esqlite3 *conn; - esqlite3_backup *backup; - esqlite_command *cmd = NULL; - ErlNifPid pid; - - if(argc != 5) - return enif_make_badarg(env); - - if(!enif_get_resource(env, argv[0], esqlite3_type, (void **) &conn)) - return enif_make_badarg(env); - // 1 backup - if(!enif_is_number(env, argv[2])) - return make_error_tuple(env, "invalid_count"); - if(!enif_is_ref(env, argv[3])) - return make_error_tuple(env, "invalid_ref"); - if(!enif_get_local_pid(env, argv[4], &pid)) - return make_error_tuple(env, "invalid_pid"); - - cmd = command_create(); - if(!cmd) - return make_error_tuple(env, "command_create_failed"); - - cmd->type = cmd_backup_step; - cmd->ref = enif_make_copy(cmd->env, argv[3]); - cmd->pid = pid; - cmd->arg = enif_make_tuple2(cmd->env, argv[1], argv[2]); - - return push_command(env, conn, cmd); -} -*/ /* * Get the remaining pagecount of the backup. - + */ static ERL_NIF_TERM esqlite_backup_remaining(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - esqlite3 *conn; esqlite3_backup *backup; - esqlite_command *cmd = NULL; - ErlNifPid pid; - if(argc != 4) + if(argc != 1) { return enif_make_badarg(env); + } - if(!enif_get_resource(env, argv[0], esqlite3_type, (void **) &conn)) + if(!enif_get_resource(env, argv[0], esqlite3_backup_type, (void **) &backup)) { return enif_make_badarg(env); - // backup 1 - if(!enif_is_ref(env, argv[2])) - return make_error_tuple(env, "invalid_ref"); - if(!enif_get_local_pid(env, argv[3], &pid)) - return make_error_tuple(env, "invalid_pid"); + } - cmd = command_create(); - if(!cmd) - return make_error_tuple(env, "command_create_failed"); + sqlite3_int64 remaining = sqlite3_backup_remaining(backup->backup); - cmd->type = cmd_backup_remaining; - cmd->ref = enif_make_copy(cmd->env, argv[2]); - cmd->pid = pid; - cmd->arg = enif_make_copy(cmd->env, argv[1]); - - return push_command(env, conn, cmd); + return enif_make_int64(env, remaining); } - */ /* * Get the total pagecount of the backup - + */ static ERL_NIF_TERM esqlite_backup_pagecount(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { - esqlite3 *conn; esqlite3_backup *backup; - esqlite_command *cmd = NULL; - ErlNifPid pid; - if(argc != 4) + if(argc != 1) { return enif_make_badarg(env); + } - if(!enif_get_resource(env, argv[0], esqlite3_type, (void **) &conn)) + if(!enif_get_resource(env, argv[0], esqlite3_backup_type, (void **) &backup)) { return enif_make_badarg(env); - // backup 1 - if(!enif_is_ref(env, argv[2])) - return make_error_tuple(env, "invalid_ref"); - if(!enif_get_local_pid(env, argv[3], &pid)) - return make_error_tuple(env, "invalid_pid"); + } - cmd = command_create(); - if(!cmd) - return make_error_tuple(env, "command_create_failed"); + sqlite3_int64 pagecount = sqlite3_backup_pagecount(backup->backup); - cmd->type = cmd_backup_pagecount; - cmd->ref = enif_make_copy(cmd->env, argv[2]); - cmd->pid = pid; - cmd->arg = enif_make_copy(cmd->env, argv[1]); - - return push_command(env, conn, cmd); + return enif_make_int64(env, pagecount); +} + +static ERL_NIF_TERM +esqlite_backup_step(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + esqlite3_backup *backup; + int n_page; + + if(argc != 2) { + return enif_make_badarg(env); + } + + if(!enif_get_resource(env, argv[0], esqlite3_backup_type, (void **) &backup)) { + return enif_make_badarg(env); + } + + if(!enif_get_int(env, argv[1], &n_page)) { + return enif_make_badarg(env); + } + + int rc = sqlite3_backup_step(backup->backup, n_page); + if(rc == SQLITE_DONE) { + return make_atom(env, "done"); + } + + if(rc != SQLITE_OK) { + return make_sqlite3_error_tuple(env, rc); + } + + return make_atom(env, "ok"); } - */ /* - * Interrupt currently active query. + * Explicitly finish the backup. */ +static ERL_NIF_TERM +esqlite_backup_finish(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) +{ + esqlite3_backup *backup; + if(argc != 1) { + return enif_make_badarg(env); + } + + if(!enif_get_resource(env, argv[0], esqlite3_backup_type, (void **) &backup)) { + return enif_make_badarg(env); + } + + int rc = SQLITE_OK; + if(backup->backup) { + rc = sqlite3_backup_finish(backup->backup); + backup->backup = NULL; + } + + if(backup->source) { + enif_release_resource(backup->source); + backup->source = NULL; + } + + if(backup->destination) { + enif_release_resource(backup->destination); + backup->destination = NULL; + } + + if(rc != SQLITE_OK) { + return make_sqlite3_error_tuple(env, rc); + } + + return make_atom(env, "ok"); +} + +/* + * Interrupt any currently active query. + */ static ERL_NIF_TERM esqlite_interrupt(ErlNifEnv *env, int argc, const ERL_NIF_TERM argv[]) { @@ -1311,8 +1166,6 @@ on_load(ErlNifEnv* env, void** priv, ERL_NIF_TERM info) if(!rt) return -1; esqlite3_backup_type = rt; - atom_esqlite3 = make_atom(env, "esqlite3"); - if(SQLITE_OK != sqlite3_initialize()) { return -1; } @@ -1356,7 +1209,7 @@ static ErlNifFunc nif_funcs[] = { {"bind_blob", 3, esqlite_bind_blob}, {"bind_null", 2, esqlite_bind_null}, - {"step", 1, esqlite_step}, + {"step", 1, esqlite_step, ERL_NIF_DIRTY_JOB_IO_BOUND}, {"reset", 1, esqlite_reset}, {"interrupt", 1, esqlite_interrupt, ERL_NIF_DIRTY_JOB_IO_BOUND}, @@ -1364,17 +1217,14 @@ static ErlNifFunc nif_funcs[] = { {"get_autocommit", 1, esqlite_get_autocommit}, {"changes", 1, esqlite_changes}, - /* - {"backup_init", 6, esqlite_backup_init}, - {"backup_step", 5, esqlite_backup_step}, - {"backup_remaining", 4, esqlite_backup_remaining}, - {"backup_pagecount", 4, esqlite_backup_pagecount}, - {"backup_finish", 4, esqlite_backup_finish}, - */ + {"backup_init", 4, esqlite_backup_init, ERL_NIF_DIRTY_JOB_IO_BOUND}, + {"backup_remaining", 1, esqlite_backup_remaining}, + {"backup_pagecount", 1, esqlite_backup_pagecount}, + {"backup_step", 2, esqlite_backup_step}, + {"backup_finish", 1, esqlite_backup_finish}, {"memory_stats", 1, esqlite_memory_stats}, {"status", 2, esqlite_status} - }; ERL_NIF_INIT(esqlite3_nif, nif_funcs, on_load, on_reload, on_upgrade, NULL); diff --git a/rebar.config.script b/rebar.config.script index d997ecb..8615bf0 100644 --- a/rebar.config.script +++ b/rebar.config.script @@ -25,7 +25,7 @@ CFlags = [ {minimum_otp_vsn, "22.0"}, - {erl_opts, [debug_info, warnings_as_errors]}, + {erl_opts, [debug_info]}, {xref_checks, [undefined_function_calls, undefined_functions, diff --git a/src/esqlite3.erl b/src/esqlite3.erl index e89abd2..fafa4e7 100644 --- a/src/esqlite3.erl +++ b/src/esqlite3.erl @@ -55,16 +55,14 @@ q/2, q/3, fetchone/1, - fetchall/1 -% -% backup_init/4, backup_init/5, -% backup_finish/1, backup_finish/2, -% backup_remaining/1, backup_remaining/2, -% backup_pagecount/1, backup_pagecount/2, -% backup_step/2, backup_step/3, -]). + fetchall/1, -% -export([q/2, q/3, q/4, map/3, map/4, foreach/3, foreach/4]). + backup_init/4, + backup_remaining/1, + backup_pagecount/1, + backup_step/2, + backup_finish/1 +]). -define(DEFAULT_TIMEOUT, infinity). -define(DEFAULT_CHUNK_SIZE, 5000). @@ -81,14 +79,13 @@ stmt :: esqlite3_nif:esqlite3_stmt() }). -%-%record(esqlite3_backup, { -% db :: esqlite3_nif:esqlite3(), -% backup :: esqlite3_nif:esqlite3_backup() -%}). +-record(esqlite3_backup, { + backup :: esqlite3_nif:esqlite3_backup() +}). -type esqlite3() :: #esqlite3{}. -type esqlite3_stmt() :: #esqlite3_stmt{}. -%-type esqlite3_backup() :: #esqlite3_backup{}. +-type esqlite3_backup() :: #esqlite3_backup{}. -type sql() :: esqlite3_nif:sql(). -type prepare_flags() :: persistent | no_vtab. @@ -107,7 +104,7 @@ -type row() :: tuple(). % tuple of cell_type -type cell_type() :: undefined | integer() | binary() | float(). --export_type([esqlite3/0, esqlite3_stmt/0, prepare_flags/0, sql/0, row/0, rowid/0, cell_type/0]). +-export_type([esqlite3/0, esqlite3_stmt/0, esqlite3_backup/0, prepare_flags/0, sql/0, row/0, rowid/0, cell_type/0]). %% @doc Opens a sqlite3 database mentioned in Filename. %% @@ -195,81 +192,6 @@ q(Connection, Sql, Args) -> Error end. -%%% -%% map -%%% -% -%%% @doc Execute statement and return a list with the result of F for each row. -%-spec map(Fun, sql(), connection()) -> list(Type) when -% Fun :: fun((Row) -> Type) | fun((ColumnNames, Row) -> Type), -% Row :: row(), -% ColumnNames :: tuple(), -% Type :: any(). -%map(Fun, Sql, Connection) -> -% case prepare(Sql, Connection) of -% {ok, Statement} -> -% map_s(Fun, Statement); -% {error, _Msg}=Error -> -% Error -% end. -% -%%% @doc Execute statement, bind args and return a list with the result of F for each row. -%-spec map(F, sql(), list(), connection()) -> list(Type) when -% F :: fun((Row) -> Type) | fun((ColumnNames, Row) -> Type), -% Row :: tuple(), -% ColumnNames :: tuple(), -% Type :: any(). -%map(Fun, Sql, [], Connection) -> -% map(Fun, Sql, Connection); -%map(Fun, Sql, Args, Connection) -> -% case prepare(Sql, Connection) of -% {ok, Statement} -> -% case bind(Statement, Args) of -% ok -> -% map_s(Fun, Statement); -%% {error, _}=Error -> -% Error -% end; -% {error, _Msg}=Error -> -%% Error -% end. - -%% -%% foreach -%% - -%% @doc Execute statement and call F with each row. -%-spec foreach(Fun, sql(), connection()) -> ok when -% Fun :: fun((Row) -> any()) | fun((ColumnNames, Row) -> any()), -% Row :: tuple(), -% ColumnNames :: tuple(). -%foreach(Fun, Sql, Connection) -> -% case prepare(Sql, Connection) of -% {ok, Statement} -> -% foreach_s(Fun, Statement); -% {error, _Msg}=Error -> -% Error -% end. - -%% @doc Execute statement, bind args and call F with each row. -%-spec foreach(Fun, sql(), list(), connection()) -> ok when -% Fun :: fun((Row) -> any()) | fun((ColumnNames, Row) -> any()), -% Row :: row(), -% ColumnNames :: tuple(). -%foreach(F, Sql, [], Connection) -> -% foreach(F, Sql, Connection); -%foreach(F, Sql, Args, Connection) -> -% case prepare(Sql, Connection) of -% {ok, Statement} -> -% case bind(Statement, Args) of -%% ok -> -% foreach_s(F, Statement); -% {error, _Msg}=Error -> -% Error -% end; -% {error, _Msg}=Error -> -% Error -% end. %% %% fetchall @@ -342,7 +264,6 @@ bind_arg(Statement, Column, {blob, Value}) -> bind_blob(Statement, Column, Value). - %% @doc Get the last insert rowid. %% -spec last_insert_rowid(Connection) -> RowidResult when @@ -486,172 +407,34 @@ column_decltypes(#esqlite3_stmt{stmt=Stmt}) -> % @doc Initialize a backup procedure. %% -%-spec backup_init(connection(), string(), connection(), string()) -> {ok, backup()} | {error, _}. -%backup_init(Dest, DestName, Src, SrcName) -> -% backup_init(Dest, DestName, Src, SrcName, ?DEFAULT_TIMEOUT). -% -%% @doc Like backup_init/4, but with an extra timeout value. -%% -%-spec backup_init(connection(), string(), connection(), string(), timeout()) -> {ok, backup()} | {error, _}. -%backup_init(#connection{raw_connection=Dest}, DestName, #connection{raw_connection=Src}, SrcName, Timeout) -> -% Ref = make_ref(), -% ok = esqlite3_nif:backup_init(Dest, DestName, Src, SrcName, Ref, self()), -% case receive_answer(Dest, Ref, Timeout) of -% {ok, RawBackup} when is_reference(RawBackup) -> -% {ok, #backup{raw_connection=Dest, raw_backup=RawBackup}}; -% {error, _} = Error -> -% Error -% end. - +-spec backup_init(esqlite3(), iodata(), esqlite3(), iodata()) -> {ok, esqlite3_backup()} | {error, _}. +backup_init(#esqlite3{db=Dest}, DestName, #esqlite3{db=Src}, SrcName) -> + case esqlite3_nif:backup_init(Dest, DestName, Src, SrcName) of + {ok, BackupRef} -> + {ok, #esqlite3_backup{backup=BackupRef}}; + {error, _}=Error -> + Error + end. %% @doc Release the resources held by the backup. -%-spec backup_finish(backup()) -> ok | {error, _}. -%backup_finish(Backup) -> -% backup_finish(Backup, ?DEFAULT_TIMEOUT). -%%% @doc Like backup_finish/1, but with an extra timeout. -%-spec backup_finish(backup(), timeout()) -> ok | {error, _}. -%backup_finish(#backup{raw_connection=Conn, raw_backup=Back}, Timeout) -> -% Ref = make_ref(), -% ok = esqlite3_nif:backup_finish(Conn, Back, Ref, self()), -% receive_answer(Conn, Ref, Timeout). +-spec backup_finish(esqlite3_backup()) -> ok | {error, _}. +backup_finish(#esqlite3_backup{backup=Backup}) -> + esqlite3_nif:backup_finish(Backup). %% @doc Do a backup step. -%-spec backup_step(backup(), integer()) -> ok | {error, _}. -%backup_step(Backup, NPage) -> -% backup_step(Backup, NPage, ?DEFAULT_TIMEOUT). - -%% @doc Do a backup step. -%-spec backup_step(backup(), integer(), timeout()) -> ok | {error, _}. -%backup_step(#backup{raw_connection=Conn, raw_backup=Back}, NPage, Timeout) -> -% Ref = make_ref(), -% ok = esqlite3_nif:backup_step(Conn, Back, NPage, Ref, self()), -% receive_answer(Conn, Ref, Timeout). - +-spec backup_step(esqlite3_backup(), integer()) -> ok | {error, _}. +backup_step(#esqlite3_backup{backup=Backup}, NPage) -> + esqlite3_nif:backup_step(Backup, NPage). %% @doc Get the remaining number of pages which need to be backed up. -%-spec backup_remaining(backup()) -> {ok, pos_integer()} | {error, _}. -%backup_remaining(Backup) -> -% backup_remaining(Backup, ?DEFAULT_TIMEOUT). +-spec backup_remaining(esqlite3_backup()) -> pos_integer(). +backup_remaining(#esqlite3_backup{backup=Backup}) -> + esqlite3_nif:backup_remaining(Backup). %% @doc Get the remaining number of pages which need to be backed up. -%-spec backup_remaining(backup(), timeout()) -> {ok, pos_integer()} | {error, _}. -%backup_remaining(#backup{raw_connection=Conn, raw_backup=Back}, Timeout) -> -% Ref = make_ref(), -% ok = esqlite3_nif:backup_remaining(Conn, Back, Ref, self()), -% case receive_answer(Conn, Ref, Timeout) of -% {ok, R} when is_integer(R) -> -% {ok, R}; -% {error, _}=E -> -% E -% end. -%% -%% @doc Get the remaining number of pages which need to be backed up. -%-spec backup_pagecount(backup()) -> {ok, pos_integer()} | {error, _}. -%backup_pagecount(Backup) -> -% backup_pagecount(Backup, ?DEFAULT_TIMEOUT). -% -%% @doc Get the remaining number of pages which need to be backed up. -%-spec backup_pagecount(backup(), timeout()) -> {ok, pos_integer()} | {error, _}. -%backup_pagecount(#backup{raw_connection=Conn, raw_backup=Back}, Timeout) -> -% Ref = make_ref(), -% ok = esqlite3_nif:backup_pagecount(Conn, Back, Ref, self()), -% case receive_answer(Conn, Ref, Timeout) of -% {ok, R} when is_integer(R) -> -% {ok, R}; -% {error, _}=E -> -% E -% end. - -%% -%% Helpers -%% - -%-spec foreach_s(Fun, statement()) -> ok when -% Fun :: fun((Row) -> any()) | fun((ColumnNames, Row) -> any()), -% Row :: row(), -% ColumnNames :: tuple(). -%foreach_s(Fun, Statement) when is_function(Fun, 1) -> -% case try_multi_step(Statement, 1, [], 0) of -% {'$done', []} -> -%% ok; -% {error, _} = Error -> -% Error; -% {rows, [Row | []]} -> -% Fun(Row), -% foreach_s(Fun, Statement) -% end; -%foreach_s(Fun, Statement) when is_function(Fun, 2) -> -% ColumnNames = column_names(Statement), -% case try_multi_step(Statement, 1, [], 0) of -% {'$done', []} -> -% ok; -% {error, _} = Error -> -% Error; -% {rows, [Row | []]} -> -% Fun(ColumnNames, Row), -% foreach_s(Fun, Statement) -% end. - -%-spec map_s(Fun, statement()) -> list(Type) when -%% Fun :: fun((Row) -> Type) | fun((ColumnNames, Row) -> Type), -% Row :: row(), -% ColumnNames :: tuple(), -% Type :: term(). -%map_s(Fun, Statement) when is_function(Fun, 1) -> -% case try_multi_step(Statement, 1, [], 0) of -% {'$done', []} -> -% []; -% {error, _} = Error -> -% Error; -% {rows, [Row | []]} -> -% [Fun(Row) | map_s(Fun, Statement)] -% end; -%map_s(Fun, Statement) when is_function(Fun, 2) -> -% ColumnNames = column_names(Statement), -% case try_multi_step(Statement, 1, [], 0) of -% {'$done', []} -> -% []; -% {error, _} = Error -> -% Error; -% {rows, [Row | []]} -> -% [Fun(ColumnNames, Row) | map_s(Fun, Statement)] -% end. - -%% return rows in reverse order -%-spec fetchall_internal(statement(), pos_integer(), list(row()), timeout()) -> -% {'$done', list(row())} | -% {error, _}. -%fetchall_internal(Statement, ChunkSize, Rest, Timeout) -> -% case try_multi_step(Statement, ChunkSize, Rest, 0, Timeout) of -% {rows, Rows} -> fetchall_internal(Statement, ChunkSize, Rows, Timeout); -% Else -> Else -% end. - -%% Try a number of steps, when the database is busy, -%%% return rows in revers order -%try_multi_step(Statement, ChunkSize, Rest, Tries) -> -% try_multi_step(Statement, ChunkSize, Rest, Tries, ?DEFAULT_TIMEOUT). - -%% Try a number of steps, when the database is busy, -%% return rows in revers order -%-spec try_multi_step(statement(), pos_integer(), list(tuple()), non_neg_integer(), timeout()) -> -% {rows, list(tuple())} | -% {'$done', list(tuple())} | -% {error, term()}. -%try_multi_step(_Statement, _ChunkSize, _Rest, Tries, _Timeout) when Tries > 5 -> -% throw(too_many_tries); -%try_multi_step(Statement, ChunkSize, Rest, Tries, Timeout) -> -% case multi_step(Statement, ChunkSize, Timeout) of -% {'$busy', Rows} -> %% core can fetch a number of rows (rows < ChunkSize) per 'multi_step' call and then get busy... -% erlang:display({"busy", Tries}), -% timer:sleep(100 * Tries), -% try_multi_step(Statement, ChunkSize, Rows ++ Rest, Tries + 1, Timeout); -% {rows, Rows} -> -% {rows, Rows ++ Rest}; -% {'$done', Rows} -> -% {'$done', Rows ++ Rest}; -% Else -> Else -% end. +-spec backup_pagecount(esqlite3_backup()) -> pos_integer(). +backup_pagecount(#esqlite3_backup{backup=Backup}) -> + esqlite3_nif:backup_pagecount(Backup). %% %% Helpers diff --git a/src/esqlite3_nif.erl b/src/esqlite3_nif.erl index c49dabf..ff012d9 100644 --- a/src/esqlite3_nif.erl +++ b/src/esqlite3_nif.erl @@ -48,11 +48,11 @@ interrupt/1, -% backup_init/6, -% backup_step/5, -% backup_remaining/4, -% backup_pagecount/4, -% backup_finish/4, + backup_init/4, + backup_remaining/1, + backup_pagecount/1, + backup_step/2, + backup_finish/1, memory_stats/1, status/2 @@ -60,11 +60,10 @@ -type esqlite3() :: reference(). -type esqlite3_stmt() :: reference(). -%-type esqlite3_backup() :: reference(). +-type esqlite3_backup() :: reference(). -type sql() :: iodata(). --export_type([esqlite3/0, esqlite3_stmt/0, sql/0]). -%-export_type([esqlite3_backup/0]). +-export_type([esqlite3/0, esqlite3_stmt/0, esqlite3_backup/0, sql/0]). -on_load(init/0). @@ -156,24 +155,6 @@ step(_Statement) -> reset(_Statement) -> erlang:nif_error(nif_library_not_loaded). -%% @doc -%% -%-spec multi_step(esqlite3(), esqlite3_stmt(), pos_integer(), reference(), pid()) -> ok | {error, _}. -%multi_step(_Db, _Stmt, _Chunk_Size, _Ref, _Dest) -> -% erlang:nif_error(nif_library_not_loaded). - -%% @doc -%% -%-spec reset(esqlite3(), esqlite3_stmt(), reference(), pid()) -> ok | {error, _}. -%reset(_Db, _Stmt, _Ref, _Dest) -> -% erlang:nif_error(nif_library_not_loaded). - -%% @doc -%% -%-spec finalize(esqlite3(), esqlite3_stmt(), reference(), pid()) -> ok | {error, _}. -%finalize(_Db, _Stmt, _Ref, _Dest) -> -% erlang:nif_error(nif_library_not_loaded). - %% @doc Retrieve the column names of the prepared statement %% -spec column_names(esqlite3_stmt()) -> list(binary()) | {error, _}. @@ -188,29 +169,28 @@ column_decltypes(_Stmt) -> %% @doc Initialize a backup procedure of a database. -%-spec backup_init(esqlite3(), string(), esqlite3_stmt(), string(), reference(), pid()) -> ok | {error, _}. -%backup_init(_DestDb, _DestName, _SourceDb, _SourceName, _Ref, _Dest) -> % erlang:nif_error(nif_library_not_loaded). +-spec backup_init(Destination, DestinationName, Source, SourceName) -> InitResult when + Destination :: esqlite3(), + DestinationName :: iodata(), + Source :: esqlite3(), + SourceName :: iodata(), + InitResult :: {ok, esqlite3_backup()} | {error, _}. +backup_init(_Dest, _DestName, _Src, _SrcName) -> + erlang:nif_error(nif_library_not_loaded). -%% @doc Do a backup step. -%-spec backup_step(esqlite3(), esqlite3_backup(), integer(), reference(), pid()) -> ok | {error, _}. -%backup_step(_Db, _Backup, _NPages, _Ref, _Dest) -> -% erlang:nif_error(nif_library_not_loaded). -%% @doc Get the amount of remaining pages which need to be backed up. -%-spec backup_remaining(esqlite3(), esqlite3_backup(), reference(), pid()) -> ok | {error, _}. -%backup_remaining(_Db, _Backup, _Ref, _Dest) -> -% erlang:nif_error(nif_library_not_loaded). +backup_remaining(_Backup) -> + erlang:nif_error(nif_library_not_loaded). -%% @doc Get the total number of pages which need to be backed up. -%-spec backup_pagecount(esqlite3(), esqlite3_backup(), reference(), pid()) -> ok | {error, _}. -%backup_pagecount(_Db, _Backup, _Ref, _Dest) -> -% erlang:nif_error(nif_library_not_loaded). +backup_pagecount(_Backup) -> + erlang:nif_error(nif_library_not_loaded). -%% @doc Finish the backup. -%-spec backup_finish(esqlite3(), esqlite3_backup(), reference(), pid()) -> ok | {error, _}. -%backup_finish(_Db, _Backup, _Ref, _Dest) -> -% erlang:nif_error(nif_library_not_loaded). +backup_step(_Backup, _PageCount) -> + erlang:nif_error(nif_library_not_loaded). + +backup_finish(_Backup) -> + erlang:nif_error(nif_library_not_loaded). %% @doc Interrupt all active queries. -spec interrupt(esqlite3()) -> ok. diff --git a/test/esqlite_test.erl b/test/esqlite_test.erl index bc22e5d..3c65466 100644 --- a/test/esqlite_test.erl +++ b/test/esqlite_test.erl @@ -383,119 +383,6 @@ reset_test() -> [1] = esqlite3:step(Stmt), ok. -% -%foreach_test() -> -% {ok, Db} = esqlite3:open(":memory:"), - -% ok = esqlite3:exec("begin;", Db), -% ok = esqlite3:exec("create table test_table(one varchar(10), two int);", Db), -% -% ok = esqlite3:exec("insert into test_table values('hello1', 10);", Db), -% ok = esqlite3:exec("insert into test_table values('hello2', 11);", Db), -% ok = esqlite3:exec("insert into test_table values('hello3', 12);", Db), -% ok = esqlite3:exec("insert into test_table values('hello4', 13);", Db), -% -% ok = esqlite3:exec("commit;", Db), - -% F = fun(Row) -> -% case Row of -% {Key, Value} -> -% put(Key, Value); -% _ -> -% ok -% end -% end, -% -% esqlite3:foreach(F, "select * from test_table;", Db), -% -% 10 = get(<<"hello1">>), -% 11 = get(<<"hello2">>), -% 12 = get(<<"hello3">>), -% 13 = get(<<"hello4">>), -% -% ok. -% -%bind_for_foreach_test() -> -% {ok, Db} = esqlite3:open(":memory:"), -% -% ok = esqlite3:exec("begin;", Db), -% ok = esqlite3:exec("create table test_table(one varchar(10), two int);", Db), -% ok = esqlite3:exec("insert into test_table values('hello1', 10);", Db), -% ok = esqlite3:exec("insert into test_table values('hello2', 11);", Db), -% ok = esqlite3:exec("insert into test_table values('hello3', 12);", Db), -% ok = esqlite3:exec("insert into test_table values('hello4', 13);", Db), -% ok = esqlite3:exec("commit;", Db), - -% F = fun(Row) -> -% case Row of -% {Key, Value} -> -% put(Key, Value); -%% _ -> -% ok -% end -% end, - -% esqlite3:foreach(F, "select * from test_table where one = ?;", ["hello1"], Db), -% -% 10 = get(<<"hello1">>), -% -% ok. - -%map_test() -> -% {ok, Db} = esqlite3:open(":memory:"), -% -% ok = esqlite3:exec("begin;", Db), -% ok = esqlite3:exec("create table test_table(one varchar(10), two int);", Db), -%% ok = esqlite3:exec("insert into test_table values('hello1', 10);", Db), -% ok = esqlite3:exec("insert into test_table values('hello2', 11);", Db), -% ok = esqlite3:exec("insert into test_table values('hello3', 12);", Db), -% ok = esqlite3:exec("insert into test_table values('hello4', 13);", Db), -% ok = esqlite3:exec("commit;", Db), - -% F = fun(Row) -> Row end, -% -% [{<<"hello1">>,10}, -% {<<"hello2">>,11}, -% {<<"hello3">>,12}, -% {<<"hello4">>,13}] = esqlite3:map(F, "select * from test_table", Db), - - %% Test that when the row-names are added.. -% Assoc = fun(Names, Row) -> -% lists:zip(tuple_to_list(Names), tuple_to_list(Row)) -% end, -% -% [[{one,<<"hello1">>},{two,10}], -% [{one,<<"hello2">>},{two,11}], -% [{one,<<"hello3">>},{two,12}], -% [{one,<<"hello4">>},{two,13}]] = esqlite3:map(Assoc, "select * from test_table", Db), -% -% ok. - -%bind_for_map_test() -> -% {ok, Db} = esqlite3:open(":memory:"), -% -% ok = esqlite3:exec("begin;", Db), -% ok = esqlite3:exec("create table test_table(one varchar(10), two int);", Db), -% ok = esqlite3:exec("insert into test_table values('hello1', 10);", Db), -% ok = esqlite3:exec("insert into test_table values('hello2', 11);", Db), -% ok = esqlite3:exec("insert into test_table values('hello3', 12);", Db), -% ok = esqlite3:exec("insert into test_table values('hello4', 13);", Db), -% ok = esqlite3:exec("commit;", Db), - -% F = fun(Row) -> Row end, -% -% [{<<"hello1">>,10}] -% = esqlite3:map(F, "select * from test_table where one = ?", ["hello1"], Db), -% -% %% Test that when the row-names are added.. -% Assoc = fun(Names, Row) -> -% lists:zip(tuple_to_list(Names), tuple_to_list(Row)) -% end, -% -% [[{one,<<"hello1">>},{two,10}]] = esqlite3:map(Assoc, "select * from test_table where one = ?", ["hello1"], Db), -% -% ok. - %error1_msg_test() -> % {ok, Db} = esqlite3:open(":memory:"), @@ -533,63 +420,66 @@ reset_test() -> % ok. -%backup_test() -> -% cleanup(), -% -% {ok, Dest} = esqlite3:open(?DB1), -% {ok, Source} = esqlite3:open(?DB2), -% -% {ok, Backup} = esqlite3:backup_init(Dest, "main", Source, "main"), -% {ok, 0} = esqlite3:backup_remaining(Backup), -% {ok, 0} = esqlite3:backup_pagecount(Backup), -% done = esqlite3:backup_step(Backup, 1), -% -% cleanup(), -% -% ok. +backup_test() -> + cleanup(), -%backup1_test() -> -% cleanup(), -% -% {ok, Dest} = esqlite3:open(?DB1), -% {ok, Source} = esqlite3:open(?DB2), + {ok, Dest} = esqlite3:open(?DB1), + {ok, Source} = esqlite3:open(?DB2), -% [] = esqlite3:q("create table test(one, two)", Source), -% [] = esqlite3:q("begin;", Source), -% [] = esqlite3:q("insert into test values(randomblob(10000), randomblob(10000));", Source), -% [] = esqlite3:q("insert into test values(randomblob(10000), randomblob(10000));", Source), -% [] = esqlite3:q("insert into test values(randomblob(10000), randomblob(10000));", Source), -% [] = esqlite3:q("insert into test values(randomblob(10000), randomblob(10000));", Source), -% [] = esqlite3:q("insert into test values(randomblob(10000), randomblob(10000));", Source), -% [] = esqlite3:q("commit;", Source), -% -% [{5}] = esqlite3:q("select count(*) from test", Source), -% {error, {sqlite_error, "no such table: test"}} = esqlite3:q("select count(*) from test", Dest), + {ok, Backup} = esqlite3:backup_init(Dest, <<"main">>, Source, <<"main">>), -% {ok, Backup} = esqlite3:backup_init(Dest, "main", Source, "main"), + 0 = esqlite3:backup_remaining(Backup), + 0 = esqlite3:backup_pagecount(Backup), -% {ok, 0} = esqlite3:backup_remaining(Backup), -% {ok, 0} = esqlite3:backup_pagecount(Backup), + done = esqlite3:backup_step(Backup, 1), + + cleanup(), + + ok. + +backup1_test() -> + cleanup(), + + {ok, Dest} = esqlite3:open(?DB1), + {ok, Source} = esqlite3:open(?DB2), + + [] = esqlite3:q(Source, "create table test(one, two)"), + [] = esqlite3:q(Source, "begin;"), + [] = esqlite3:q(Source, "insert into test values(randomblob(10000), randomblob(10000));"), + [] = esqlite3:q(Source, "insert into test values(randomblob(10000), randomblob(10000));"), + [] = esqlite3:q(Source, "insert into test values(randomblob(10000), randomblob(10000));"), + [] = esqlite3:q(Source, "insert into test values(randomblob(10000), randomblob(10000));"), + [] = esqlite3:q(Source, "insert into test values(randomblob(10000), randomblob(10000));"), + [] = esqlite3:q(Source, "commit;"), + + [[5]] = esqlite3:q(Source, "select count(*) from test"), + {error, 1} = esqlite3:q(Dest, "select count(*) from test"), + #{ errmsg := <<"no such table: test">> } = esqlite3:error_info(Dest), + + {ok, Backup} = esqlite3:backup_init(Dest, "main", Source, "main"), + + 0 = esqlite3:backup_remaining(Backup), + 0 = esqlite3:backup_pagecount(Backup), %% Backup 1 page. -% ok = esqlite3:backup_step(Backup, 1), -% -% {ok, 26} = esqlite3:backup_remaining(Backup), -% {ok, 27} = esqlite3:backup_pagecount(Backup), + ok = esqlite3:backup_step(Backup, 1), + + 26 = esqlite3:backup_remaining(Backup), + 27 = esqlite3:backup_pagecount(Backup), %% Do all the remaining pages. -%% done = esqlite3:backup_step(Backup, -1), + done = esqlite3:backup_step(Backup, -1), -% {ok, 0} = esqlite3:backup_remaining(Backup), -% {ok, 27} = esqlite3:backup_pagecount(Backup), + 0 = esqlite3:backup_remaining(Backup), + 27 = esqlite3:backup_pagecount(Backup), -% ok = esqlite3:backup_finish(Backup), -% -% [{5}] = esqlite3:q("select count(*) from test", Dest), -% -% cleanup(), -% -% ok. + ok = esqlite3:backup_finish(Backup), + + [[5]] = esqlite3:q(Dest, "select count(*) from test"), + + cleanup(), + + ok. sqlite_version_test() ->