mirror of https://github.com/qTox/qTox.git
tux3
10 years ago
31 changed files with 1045 additions and 455 deletions
@ -0,0 +1,466 @@ |
|||||||
|
#include "rawdatabase.h" |
||||||
|
#include <QDebug> |
||||||
|
#include <QMetaObject> |
||||||
|
#include <QMutexLocker> |
||||||
|
#include <QCoreApplication> |
||||||
|
#include <QFile> |
||||||
|
#include <cassert> |
||||||
|
#include <tox/toxencryptsave.h> |
||||||
|
|
||||||
|
/// The two following defines are required to use SQLCipher
|
||||||
|
/// They are used by the sqlite3.h header
|
||||||
|
#define SQLITE_HAS_CODEC |
||||||
|
#define SQLITE_TEMP_STORE 2 |
||||||
|
|
||||||
|
#include <sqlcipher/sqlite3.h> |
||||||
|
|
||||||
|
RawDatabase::RawDatabase(const QString &path, const QString& password) |
||||||
|
: workerThread{new QThread}, path{path}, currentHexKey{deriveKey(password)} |
||||||
|
{ |
||||||
|
workerThread->setObjectName("qTox Database"); |
||||||
|
moveToThread(workerThread.get()); |
||||||
|
workerThread->start(); |
||||||
|
|
||||||
|
if (!open(path, currentHexKey)) |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
RawDatabase::~RawDatabase() |
||||||
|
{ |
||||||
|
close(); |
||||||
|
workerThread->exit(0); |
||||||
|
while (workerThread->isRunning()) |
||||||
|
workerThread->wait(50); |
||||||
|
} |
||||||
|
|
||||||
|
bool RawDatabase::open(const QString& path, const QString &hexKey) |
||||||
|
{ |
||||||
|
if (QThread::currentThread() != workerThread.get()) |
||||||
|
{ |
||||||
|
bool ret; |
||||||
|
QMetaObject::invokeMethod(this, "open", Qt::BlockingQueuedConnection, Q_RETURN_ARG(bool, ret), |
||||||
|
Q_ARG(const QString&, path), Q_ARG(const QString&, hexKey)); |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
if (!QFile::exists(path) && QFile::exists(path+".tmp")) |
||||||
|
{ |
||||||
|
qWarning() << "Restoring database from temporary export file! Did we crash while changing the password?"; |
||||||
|
QFile::rename(path+".tmp", path); |
||||||
|
} |
||||||
|
|
||||||
|
if (sqlite3_open_v2(path.toUtf8().data(), &sqlite, |
||||||
|
SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_NOMUTEX, nullptr) != SQLITE_OK) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to open database"<<path<<"with error:"<<sqlite3_errmsg(sqlite); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (!hexKey.isEmpty()) |
||||||
|
{ |
||||||
|
if (!execNow("PRAGMA key = \"x'"+hexKey+"'\"")) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to set encryption key"; |
||||||
|
close(); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (!execNow("SELECT count(*) FROM sqlite_master")) |
||||||
|
{ |
||||||
|
qWarning() << "Database is unusable, check that the password is correct"; |
||||||
|
close(); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
void RawDatabase::close() |
||||||
|
{ |
||||||
|
if (QThread::currentThread() != workerThread.get()) |
||||||
|
return (void)QMetaObject::invokeMethod(this, "close", Qt::BlockingQueuedConnection); |
||||||
|
|
||||||
|
// We assume we're in the ctor or dtor, so we just need to finish processing our transactions
|
||||||
|
process(); |
||||||
|
|
||||||
|
if (sqlite3_close(sqlite) == SQLITE_OK) |
||||||
|
sqlite = nullptr; |
||||||
|
else |
||||||
|
qWarning() << "Error closing database:"<<sqlite3_errmsg(sqlite); |
||||||
|
} |
||||||
|
|
||||||
|
bool RawDatabase::isOpen() |
||||||
|
{ |
||||||
|
// We don't need thread safety since only the ctor/dtor can write this pointer
|
||||||
|
return sqlite != nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
bool RawDatabase::execNow(const QString& statement) |
||||||
|
{ |
||||||
|
return execNow(Query{statement}); |
||||||
|
} |
||||||
|
|
||||||
|
bool RawDatabase::execNow(const RawDatabase::Query &statement) |
||||||
|
{ |
||||||
|
return execNow(QVector<Query>{statement}); |
||||||
|
} |
||||||
|
|
||||||
|
bool RawDatabase::execNow(const QVector<RawDatabase::Query> &statements) |
||||||
|
{ |
||||||
|
if (!sqlite) |
||||||
|
{ |
||||||
|
qWarning() << "Trying to exec, but the database is not open"; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
std::atomic_bool done{false}; |
||||||
|
std::atomic_bool success{false}; |
||||||
|
|
||||||
|
Transaction trans; |
||||||
|
trans.queries = statements; |
||||||
|
trans.done = &done; |
||||||
|
trans.success = &success; |
||||||
|
{ |
||||||
|
QMutexLocker locker{&transactionsMutex}; |
||||||
|
pendingTransactions.enqueue(trans); |
||||||
|
} |
||||||
|
|
||||||
|
// We can't use blocking queued here, otherwise we might process future transactions
|
||||||
|
// before returning, but we only want to wait until this transaction is done.
|
||||||
|
QMetaObject::invokeMethod(this, "process"); |
||||||
|
while (!done.load(std::memory_order_acquire)) |
||||||
|
QThread::msleep(10); |
||||||
|
|
||||||
|
return success.load(std::memory_order_acquire); |
||||||
|
} |
||||||
|
|
||||||
|
void RawDatabase::execLater(const QString &statement) |
||||||
|
{ |
||||||
|
execLater(Query{statement}); |
||||||
|
} |
||||||
|
|
||||||
|
void RawDatabase::execLater(const RawDatabase::Query &statement) |
||||||
|
{ |
||||||
|
execLater(QVector<Query>{statement}); |
||||||
|
} |
||||||
|
|
||||||
|
void RawDatabase::execLater(const QVector<RawDatabase::Query> &statements) |
||||||
|
{ |
||||||
|
if (!sqlite) |
||||||
|
{ |
||||||
|
qWarning() << "Trying to exec, but the database is not open"; |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
Transaction trans; |
||||||
|
trans.queries = statements; |
||||||
|
{ |
||||||
|
QMutexLocker locker{&transactionsMutex}; |
||||||
|
pendingTransactions.enqueue(trans); |
||||||
|
} |
||||||
|
|
||||||
|
QMetaObject::invokeMethod(this, "process"); |
||||||
|
} |
||||||
|
|
||||||
|
void RawDatabase::sync() |
||||||
|
{ |
||||||
|
QMetaObject::invokeMethod(this, "process", Qt::BlockingQueuedConnection); |
||||||
|
} |
||||||
|
|
||||||
|
bool RawDatabase::setPassword(const QString& password) |
||||||
|
{ |
||||||
|
if (!sqlite) |
||||||
|
{ |
||||||
|
qWarning() << "Trying to change the password, but the database is not open"; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (QThread::currentThread() != workerThread.get()) |
||||||
|
{ |
||||||
|
bool ret; |
||||||
|
QMetaObject::invokeMethod(this, "setPassword", Qt::BlockingQueuedConnection, |
||||||
|
Q_RETURN_ARG(bool, ret), Q_ARG(const QString&, password)); |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
// If we need to decrypt or encrypt, we'll need to sync and close,
|
||||||
|
// so we always process the pending queue before rekeying for consistency
|
||||||
|
process(); |
||||||
|
|
||||||
|
if (QFile::exists(path+".tmp")) |
||||||
|
{ |
||||||
|
qWarning() << "Found old temporary export file while rekeying, deleting it"; |
||||||
|
QFile::remove(path+".tmp"); |
||||||
|
} |
||||||
|
|
||||||
|
if (!password.isEmpty()) |
||||||
|
{ |
||||||
|
QString newHexKey = deriveKey(password); |
||||||
|
if (!currentHexKey.isEmpty()) |
||||||
|
{ |
||||||
|
if (!execNow("PRAGMA rekey = \"x'"+newHexKey+"'\"")) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to change encryption key"; |
||||||
|
close(); |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
// Need to encrypt the database
|
||||||
|
if (!execNow("ATTACH DATABASE '"+path+".tmp' AS encrypted KEY \"x'"+newHexKey+"'\";" |
||||||
|
"SELECT sqlcipher_export('encrypted');" |
||||||
|
"DETACH DATABASE encrypted;")) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to export encrypted database"; |
||||||
|
close(); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
// This is racy as hell, but nobody will race with us since we hold the profile lock
|
||||||
|
// If we crash or die here, the rename should be atomic, so we can recover no matter what
|
||||||
|
close(); |
||||||
|
QFile::remove(path); |
||||||
|
QFile::rename(path+".tmp", path); |
||||||
|
currentHexKey = newHexKey; |
||||||
|
if (!open(path, currentHexKey)) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to open encrypted database"; |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
if (currentHexKey.isEmpty()) |
||||||
|
return true; |
||||||
|
|
||||||
|
// Need to decrypt the database
|
||||||
|
if (!execNow("ATTACH DATABASE '"+path+".tmp' AS plaintext KEY '';" |
||||||
|
"SELECT sqlcipher_export('plaintext');" |
||||||
|
"DETACH DATABASE plaintext;")) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to export decrypted database"; |
||||||
|
close(); |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
// This is racy as hell, but nobody will race with us since we hold the profile lock
|
||||||
|
// If we crash or die here, the rename should be atomic, so we can recover no matter what
|
||||||
|
close(); |
||||||
|
QFile::remove(path); |
||||||
|
QFile::rename(path+".tmp", path); |
||||||
|
currentHexKey.clear(); |
||||||
|
if (!open(path)) |
||||||
|
{ |
||||||
|
qCritical() << "Failed to open decrypted database"; |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
bool RawDatabase::rename(const QString &newPath) |
||||||
|
{ |
||||||
|
if (!sqlite) |
||||||
|
{ |
||||||
|
qWarning() << "Trying to change the password, but the database is not open"; |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
if (QThread::currentThread() != workerThread.get()) |
||||||
|
{ |
||||||
|
bool ret; |
||||||
|
QMetaObject::invokeMethod(this, "rename", Qt::BlockingQueuedConnection, |
||||||
|
Q_RETURN_ARG(bool, ret), Q_ARG(const QString&, newPath)); |
||||||
|
return ret; |
||||||
|
} |
||||||
|
|
||||||
|
process(); |
||||||
|
|
||||||
|
if (path == newPath) |
||||||
|
return true; |
||||||
|
|
||||||
|
if (QFile::exists(newPath)) |
||||||
|
return false; |
||||||
|
|
||||||
|
close(); |
||||||
|
if (!QFile::rename(path, newPath)) |
||||||
|
return false; |
||||||
|
path = newPath; |
||||||
|
return open(path, currentHexKey); |
||||||
|
} |
||||||
|
|
||||||
|
|
||||||
|
QString RawDatabase::deriveKey(QString password) |
||||||
|
{ |
||||||
|
if (password.isEmpty()) |
||||||
|
return {}; |
||||||
|
|
||||||
|
QByteArray passData = password.toUtf8(); |
||||||
|
|
||||||
|
static_assert(TOX_PASS_KEY_LENGTH >= 32, "toxcore must provide 256bit or longer keys"); |
||||||
|
|
||||||
|
static const uint8_t expandConstant[TOX_PASS_SALT_LENGTH+1] = "L'ignorance est le pire des maux"; |
||||||
|
TOX_PASS_KEY key; |
||||||
|
tox_derive_key_with_salt((uint8_t*)passData.data(), passData.size(), expandConstant, &key, nullptr); |
||||||
|
return QByteArray((char*)key.key, 32).toHex(); |
||||||
|
} |
||||||
|
|
||||||
|
void RawDatabase::process() |
||||||
|
{ |
||||||
|
assert(QThread::currentThread() == workerThread.get()); |
||||||
|
|
||||||
|
if (!sqlite) |
||||||
|
return; |
||||||
|
|
||||||
|
forever |
||||||
|
{ |
||||||
|
// Fetch the next transaction
|
||||||
|
Transaction trans; |
||||||
|
{ |
||||||
|
QMutexLocker locker{&transactionsMutex}; |
||||||
|
if (pendingTransactions.isEmpty()) |
||||||
|
return; |
||||||
|
trans = pendingTransactions.dequeue(); |
||||||
|
} |
||||||
|
|
||||||
|
// In case we exit early, prepare to signal errors
|
||||||
|
if (trans.success != nullptr) |
||||||
|
trans.success->store(false, std::memory_order_release); |
||||||
|
|
||||||
|
// Add transaction commands if necessary
|
||||||
|
if (trans.queries.size() > 1) |
||||||
|
{ |
||||||
|
trans.queries.prepend(Query{"BEGIN;"}); |
||||||
|
trans.queries.append({"COMMIT;"}); |
||||||
|
} |
||||||
|
|
||||||
|
// Compile queries
|
||||||
|
for (Query& query : trans.queries) |
||||||
|
{ |
||||||
|
assert(query.statements.isEmpty()); |
||||||
|
// sqlite3_prepare_v2 only compiles one statement at a time in the query, we need to loop over them all
|
||||||
|
int curParam=0; |
||||||
|
const char* compileTail = query.query.data(); |
||||||
|
do { |
||||||
|
// Compile the next statement
|
||||||
|
sqlite3_stmt* stmt; |
||||||
|
int r; |
||||||
|
if ((r = sqlite3_prepare_v2(sqlite, compileTail, |
||||||
|
query.query.size() - static_cast<int>(compileTail - query.query.data()), |
||||||
|
&stmt, &compileTail)) != SQLITE_OK) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to prepare statement"<<query.query<<"with error"<<r; |
||||||
|
goto cleanupStatements; |
||||||
|
} |
||||||
|
query.statements += stmt; |
||||||
|
|
||||||
|
// Now we can bind our params to this statement
|
||||||
|
int nParams = sqlite3_bind_parameter_count(stmt); |
||||||
|
if (query.blobs.size() < curParam+nParams) |
||||||
|
{ |
||||||
|
qWarning() << "Not enough parameters to bind to query "<<query.query; |
||||||
|
goto cleanupStatements; |
||||||
|
} |
||||||
|
for (int i=0; i<nParams; ++i) |
||||||
|
{ |
||||||
|
const QByteArray& blob = query.blobs[curParam+i]; |
||||||
|
if (sqlite3_bind_blob(stmt, i+1, blob.data(), blob.size(), SQLITE_STATIC) != SQLITE_OK) |
||||||
|
{ |
||||||
|
qWarning() << "Failed to bind param"<<curParam+i<<"to query "<<query.query; |
||||||
|
goto cleanupStatements; |
||||||
|
} |
||||||
|
} |
||||||
|
curParam += nParams; |
||||||
|
} while (compileTail != query.query.data()+query.query.size()); |
||||||
|
} |
||||||
|
|
||||||
|
// Execute each statement of each query of our transaction
|
||||||
|
for (Query& query : trans.queries) |
||||||
|
{ |
||||||
|
for (sqlite3_stmt* stmt : query.statements) |
||||||
|
{ |
||||||
|
int column_count = sqlite3_column_count(stmt); |
||||||
|
int result; |
||||||
|
do { |
||||||
|
result = sqlite3_step(stmt); |
||||||
|
|
||||||
|
// Execute our row callback
|
||||||
|
if (result == SQLITE_ROW && query.rowCallback) |
||||||
|
{ |
||||||
|
QVector<QVariant> row; |
||||||
|
for (int i=0; i<column_count; ++i) |
||||||
|
row += extractData(stmt, i); |
||||||
|
|
||||||
|
query.rowCallback(row); |
||||||
|
} |
||||||
|
} while (result == SQLITE_ROW); |
||||||
|
if (result == SQLITE_ERROR) |
||||||
|
{ |
||||||
|
qWarning() << "Error executing query "<<query.query; |
||||||
|
goto cleanupStatements; |
||||||
|
} |
||||||
|
else if (result == SQLITE_MISUSE) |
||||||
|
{ |
||||||
|
qWarning() << "Misuse executing query "<<query.query; |
||||||
|
goto cleanupStatements; |
||||||
|
} |
||||||
|
else if (result == SQLITE_CONSTRAINT) |
||||||
|
{ |
||||||
|
qWarning() << "Constraint error executing query "<<query.query; |
||||||
|
goto cleanupStatements; |
||||||
|
} |
||||||
|
else if (result != SQLITE_DONE) |
||||||
|
{ |
||||||
|
qWarning() << "Unknown error"<<result<<"executing query "<<query.query; |
||||||
|
goto cleanupStatements; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
if (query.insertCallback) |
||||||
|
query.insertCallback(sqlite3_last_insert_rowid(sqlite)); |
||||||
|
} |
||||||
|
|
||||||
|
if (trans.success != nullptr) |
||||||
|
trans.success->store(true, std::memory_order_release); |
||||||
|
|
||||||
|
// Free our statements
|
||||||
|
cleanupStatements: |
||||||
|
for (Query& query : trans.queries) |
||||||
|
{ |
||||||
|
for (sqlite3_stmt* stmt : query.statements) |
||||||
|
sqlite3_finalize(stmt); |
||||||
|
query.statements.clear(); |
||||||
|
} |
||||||
|
|
||||||
|
// Signal transaction results
|
||||||
|
if (trans.done != nullptr) |
||||||
|
trans.done->store(true, std::memory_order_release); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
QVariant RawDatabase::extractData(sqlite3_stmt *stmt, int col) |
||||||
|
{ |
||||||
|
int type = sqlite3_column_type(stmt, col); |
||||||
|
if (type == SQLITE_INTEGER) |
||||||
|
{ |
||||||
|
return sqlite3_column_int64(stmt, col); |
||||||
|
} |
||||||
|
else if (type == SQLITE_TEXT) |
||||||
|
{ |
||||||
|
const char* str = reinterpret_cast<const char*>(sqlite3_column_text(stmt, col)); |
||||||
|
int len = sqlite3_column_bytes(stmt, col); |
||||||
|
return QString::fromUtf8(str, len); |
||||||
|
} |
||||||
|
else if (type == SQLITE_NULL) |
||||||
|
{ |
||||||
|
return QVariant{}; |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
const char* data = reinterpret_cast<const char*>(sqlite3_column_blob(stmt, col)); |
||||||
|
int len = sqlite3_column_bytes(stmt, col); |
||||||
|
return QByteArray::fromRawData(data, len); |
||||||
|
} |
||||||
|
} |
||||||
@ -0,0 +1,115 @@ |
|||||||
|
#ifndef RAWDATABASE_H |
||||||
|
#define RAWDATABASE_H |
||||||
|
|
||||||
|
#include <QString> |
||||||
|
#include <QByteArray> |
||||||
|
#include <QThread> |
||||||
|
#include <QQueue> |
||||||
|
#include <QVector> |
||||||
|
#include <QPair> |
||||||
|
#include <QMutex> |
||||||
|
#include <QVariant> |
||||||
|
#include <memory> |
||||||
|
#include <atomic> |
||||||
|
|
||||||
|
struct sqlite3; |
||||||
|
struct sqlite3_stmt; |
||||||
|
|
||||||
|
/// Implements a low level RAII interface to a SQLCipher (SQlite3) database
|
||||||
|
/// Thread-safe, does all database operations on a worker thread
|
||||||
|
/// The queries must not contain transaction commands (BEGIN/COMMIT/...) or the behavior is undefined
|
||||||
|
class RawDatabase : QObject |
||||||
|
{ |
||||||
|
Q_OBJECT |
||||||
|
|
||||||
|
public: |
||||||
|
/// A query to be executed by the database. Can be composed of one or more SQL statements in the query,
|
||||||
|
/// optional BLOB parameters to be bound, and callbacks fired when the query is executed
|
||||||
|
/// Calling any database method from a query callback is undefined behavior
|
||||||
|
class Query |
||||||
|
{ |
||||||
|
public: |
||||||
|
Query(QString query, QVector<QByteArray> blobs = {}, std::function<void(int64_t)> insertCallback={}) |
||||||
|
: query{query.toUtf8()}, blobs{blobs}, insertCallback{insertCallback} {} |
||||||
|
Query(QString query, std::function<void(int64_t)> insertCallback) |
||||||
|
: query{query.toUtf8()}, insertCallback{insertCallback} {} |
||||||
|
Query(QString query, std::function<void(const QVector<QVariant>&)> rowCallback) |
||||||
|
: query{query.toUtf8()}, rowCallback{rowCallback} {} |
||||||
|
Query() = default; |
||||||
|
private: |
||||||
|
QByteArray query; ///< UTF-8 query string
|
||||||
|
QVector<QByteArray> blobs; ///< Bound data blobs
|
||||||
|
std::function<void(int64_t)> insertCallback; ///< Called after execution with the last insert rowid
|
||||||
|
std::function<void(const QVector<QVariant>&)> rowCallback; ///< Called during execution for each row
|
||||||
|
QVector<sqlite3_stmt*> statements; ///< Statements to be compiled from the query
|
||||||
|
|
||||||
|
friend class RawDatabase; |
||||||
|
}; |
||||||
|
|
||||||
|
public: |
||||||
|
/// Tries to open a database
|
||||||
|
/// If password is empty, the database will be opened unencrypted
|
||||||
|
/// Otherwise we will use toxencryptsave to derive a key and encrypt the database
|
||||||
|
RawDatabase(const QString& path, const QString& password); |
||||||
|
~RawDatabase(); |
||||||
|
bool isOpen(); ///< Returns true if the database was opened successfully
|
||||||
|
/// Executes a SQL transaction synchronously.
|
||||||
|
/// Returns whether the transaction was successful.
|
||||||
|
bool execNow(const QString& statement); |
||||||
|
bool execNow(const Query& statement); |
||||||
|
bool execNow(const QVector<Query>& statements); |
||||||
|
/// Executes a SQL transaction asynchronously.
|
||||||
|
void execLater(const QString& statement); |
||||||
|
void execLater(const Query& statement); |
||||||
|
void execLater(const QVector<Query>& statements); |
||||||
|
/// Waits until all the pending transactions are executed
|
||||||
|
void sync(); |
||||||
|
|
||||||
|
public slots: |
||||||
|
/// Changes the database password, encrypting or decrypting if necessary
|
||||||
|
/// If password is empty, the database will be decrypted
|
||||||
|
/// Will process all transactions before changing the password
|
||||||
|
bool setPassword(const QString& password); |
||||||
|
/// Moves the database file on disk to match the new path
|
||||||
|
/// /// Will process all transactions before renaming
|
||||||
|
bool rename(const QString& newPath); |
||||||
|
|
||||||
|
protected slots: |
||||||
|
/// Should only be called from the constructor, runs on the caller's thread
|
||||||
|
bool open(const QString& path, const QString& hexKey = {}); |
||||||
|
/// Should only be called from the destructor, runs on the caller's thread
|
||||||
|
void close(); |
||||||
|
/// Implements the actual processing of pending transactions
|
||||||
|
/// Unqueues, compiles, binds and executes queries, then notifies of results
|
||||||
|
/// MUST only be called from the worker thread
|
||||||
|
void process(); |
||||||
|
/// Extracts a variant from one column of a result row depending on the column type
|
||||||
|
QVariant extractData(sqlite3_stmt* stmt, int col); |
||||||
|
|
||||||
|
protected: |
||||||
|
/// Derives a 256bit key from the password and returns it hex-encoded
|
||||||
|
static QString deriveKey(QString password); |
||||||
|
|
||||||
|
private: |
||||||
|
/// SQL transactions to be processed
|
||||||
|
/// A transaction is made of queries, which can have bound BLOBs
|
||||||
|
struct Transaction |
||||||
|
{ |
||||||
|
QVector<Query> queries; |
||||||
|
/// If not a nullptr, the result of the transaction will be set
|
||||||
|
std::atomic_bool* success = nullptr; |
||||||
|
/// If not a nullptr, will be set to true when the transaction has been executed
|
||||||
|
std::atomic_bool* done = nullptr; |
||||||
|
}; |
||||||
|
|
||||||
|
private: |
||||||
|
sqlite3* sqlite; |
||||||
|
std::unique_ptr<QThread> workerThread; |
||||||
|
QQueue<Transaction> pendingTransactions; |
||||||
|
/// Protects pendingTransactions
|
||||||
|
QMutex transactionsMutex; |
||||||
|
QString path; |
||||||
|
QString currentHexKey; |
||||||
|
}; |
||||||
|
|
||||||
|
#endif // RAWDATABASE_H
|
||||||
@ -0,0 +1,231 @@ |
|||||||
|
#include "history.h" |
||||||
|
#include "src/persistence/profile.h" |
||||||
|
#include "src/persistence/settings.h" |
||||||
|
#include "src/persistence/db/rawdatabase.h" |
||||||
|
#include "src/persistence/historykeeper.h" |
||||||
|
#include <QDebug> |
||||||
|
#include <cassert> |
||||||
|
|
||||||
|
using namespace std; |
||||||
|
|
||||||
|
History::History(const QString &profileName, const QString &password) |
||||||
|
: db{getDbPath(profileName), password} |
||||||
|
{ |
||||||
|
init(); |
||||||
|
} |
||||||
|
|
||||||
|
History::History(const QString &profileName, const QString &password, const HistoryKeeper &oldHistory) |
||||||
|
: History{profileName, password} |
||||||
|
{ |
||||||
|
import(oldHistory); |
||||||
|
} |
||||||
|
|
||||||
|
History::~History() |
||||||
|
{ |
||||||
|
// We could have execLater requests pending with a lambda attached,
|
||||||
|
// so clear the pending transactions first
|
||||||
|
db.sync(); |
||||||
|
} |
||||||
|
|
||||||
|
bool History::isValid() |
||||||
|
{ |
||||||
|
return db.isOpen(); |
||||||
|
} |
||||||
|
|
||||||
|
void History::setPassword(const QString& password) |
||||||
|
{ |
||||||
|
db.setPassword(password); |
||||||
|
} |
||||||
|
|
||||||
|
void History::rename(const QString &newName) |
||||||
|
{ |
||||||
|
db.rename(getDbPath(newName)); |
||||||
|
} |
||||||
|
|
||||||
|
void History::eraseHistory() |
||||||
|
{ |
||||||
|
db.execNow("DELETE FROM faux_offline_pending;" |
||||||
|
"DELETE FROM history;" |
||||||
|
"DELETE FROM aliases;" |
||||||
|
"DELETE FROM peers;" |
||||||
|
"VACUUM;"); |
||||||
|
} |
||||||
|
|
||||||
|
void History::removeFriendHistory(const QString &friendPk) |
||||||
|
{ |
||||||
|
if (!peers.contains(friendPk)) |
||||||
|
return; |
||||||
|
int64_t id = peers[friendPk]; |
||||||
|
|
||||||
|
if (db.execNow(QString("DELETE FROM faux_offline_pending " |
||||||
|
"WHERE faux_offline_pending.id IN ( " |
||||||
|
"SELECT faux_offline_pending.id FROM faux_offline_pending " |
||||||
|
"LEFT JOIN history ON faux_offline_pending.id = history.id " |
||||||
|
"WHERE chat_id=%1 " |
||||||
|
"); " |
||||||
|
"DELETE FROM history WHERE chat_id=%1; " |
||||||
|
"DELETE FROM aliases WHERE owner=%1; " |
||||||
|
"DELETE FROM peers WHERE id=%1; " |
||||||
|
"VACUUM;").arg(id))) |
||||||
|
{ |
||||||
|
peers.remove(friendPk); |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
qWarning() << "Failed to remove friend's history"; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
QVector<RawDatabase::Query> History::generateNewMessageQueries(const QString &friendPk, const QString &message, |
||||||
|
const QString &sender, const QDateTime &time, bool isSent, QString dispName, |
||||||
|
std::function<void(int64_t)> insertIdCallback) |
||||||
|
{ |
||||||
|
QVector<RawDatabase::Query> queries; |
||||||
|
|
||||||
|
// Get the db id of the peer we're chatting with
|
||||||
|
int64_t peerId; |
||||||
|
if (peers.contains(friendPk)) |
||||||
|
{ |
||||||
|
peerId = peers[friendPk]; |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
if (peers.isEmpty()) |
||||||
|
peerId = 0; |
||||||
|
else |
||||||
|
peerId = *max_element(begin(peers), end(peers))+1; |
||||||
|
peers[friendPk] = peerId; |
||||||
|
queries += RawDatabase::Query{("INSERT INTO peers (id, public_key) VALUES (%1, '"+friendPk+"');").arg(peerId)}; |
||||||
|
} |
||||||
|
|
||||||
|
// Get the db id of the sender of the message
|
||||||
|
int64_t senderId; |
||||||
|
if (peers.contains(sender)) |
||||||
|
{ |
||||||
|
senderId = peers[sender]; |
||||||
|
} |
||||||
|
else |
||||||
|
{ |
||||||
|
if (peers.isEmpty()) |
||||||
|
senderId = 0; |
||||||
|
else |
||||||
|
senderId = *max_element(begin(peers), end(peers))+1; |
||||||
|
peers[sender] = senderId; |
||||||
|
queries += RawDatabase::Query{("INSERT INTO peers (id, public_key) VALUES (%1, '"+sender+"');").arg(senderId)}; |
||||||
|
} |
||||||
|
|
||||||
|
queries += RawDatabase::Query(QString("INSERT OR IGNORE INTO aliases (owner, display_name) VALUES (%1, ?);") |
||||||
|
.arg(senderId), {dispName.toUtf8()}); |
||||||
|
|
||||||
|
// If the alias already existed, the insert will ignore the conflict and last_insert_rowid() will return garbage,
|
||||||
|
// so we have to check changes() and manually fetch the row ID in this case
|
||||||
|
queries += RawDatabase::Query(QString("INSERT INTO history (timestamp, chat_id, message, sender_alias) " |
||||||
|
"VALUES (%1, %2, ?, (" |
||||||
|
" CASE WHEN changes() IS 0 THEN (" |
||||||
|
" SELECT id FROM aliases WHERE owner=%3 AND display_name=?)" |
||||||
|
" ELSE last_insert_rowid() END" |
||||||
|
"));") |
||||||
|
.arg(time.toMSecsSinceEpoch()).arg(peerId).arg(senderId), |
||||||
|
{message.toUtf8(), dispName.toUtf8()}, insertIdCallback); |
||||||
|
|
||||||
|
if (!isSent) |
||||||
|
queries += RawDatabase::Query{"INSERT INTO faux_offline_pending (id) VALUES (last_insert_rowid());"}; |
||||||
|
|
||||||
|
return queries; |
||||||
|
} |
||||||
|
|
||||||
|
void History::addNewMessage(const QString &friendPk, const QString &message, const QString &sender, |
||||||
|
const QDateTime &time, bool isSent, QString dispName, std::function<void(int64_t)> insertIdCallback) |
||||||
|
{ |
||||||
|
db.execLater(generateNewMessageQueries(friendPk, message, sender, time, isSent, dispName, insertIdCallback)); |
||||||
|
} |
||||||
|
|
||||||
|
QList<History::HistMessage> History::getChatHistory(const QString &friendPk, const QDateTime &from, const QDateTime &to) |
||||||
|
{ |
||||||
|
QList<HistMessage> messages; |
||||||
|
|
||||||
|
auto rowCallback = [&messages](const QVector<QVariant>& row) |
||||||
|
{ |
||||||
|
messages += {row[0].toLongLong(), |
||||||
|
row[1].isNull(), |
||||||
|
QDateTime::fromMSecsSinceEpoch(row[2].toLongLong()), |
||||||
|
row[3].toString(), |
||||||
|
row[4].toString(), |
||||||
|
row[5].toString(), |
||||||
|
row[6].toString()}; |
||||||
|
}; |
||||||
|
|
||||||
|
// Don't forget to update the rowCallback if you change the selected columns!
|
||||||
|
db.execNow({QString("SELECT history.id, faux_offline_pending.id, timestamp, chat.public_key, " |
||||||
|
"aliases.display_name, sender.public_key, message FROM history " |
||||||
|
"LEFT JOIN faux_offline_pending ON history.id = faux_offline_pending.id " |
||||||
|
"JOIN peers chat ON chat_id = chat.id " |
||||||
|
"JOIN aliases ON sender_alias = aliases.id " |
||||||
|
"JOIN peers sender ON aliases.owner = sender.id " |
||||||
|
"WHERE timestamp BETWEEN %1 AND %2 AND chat.public_key='%3';") |
||||||
|
.arg(from.toMSecsSinceEpoch()).arg(to.toMSecsSinceEpoch()).arg(friendPk), rowCallback}); |
||||||
|
|
||||||
|
return messages; |
||||||
|
} |
||||||
|
|
||||||
|
void History::markAsSent(qint64 id) |
||||||
|
{ |
||||||
|
db.execLater(QString("DELETE FROM faux_offline_pending WHERE id=%1;").arg(id)); |
||||||
|
} |
||||||
|
|
||||||
|
QString History::getDbPath(const QString &profileName) |
||||||
|
{ |
||||||
|
return Settings::getInstance().getSettingsDirPath() + profileName + ".db"; |
||||||
|
} |
||||||
|
|
||||||
|
void History::init() |
||||||
|
{ |
||||||
|
if (!isValid()) |
||||||
|
{ |
||||||
|
qWarning() << "Database not open, init failed"; |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
db.execLater("CREATE TABLE IF NOT EXISTS peers (id INTEGER PRIMARY KEY, public_key TEXT NOT NULL UNIQUE);" |
||||||
|
"CREATE TABLE IF NOT EXISTS aliases (id INTEGER PRIMARY KEY, owner INTEGER," |
||||||
|
"display_name BLOB NOT NULL, UNIQUE(owner, display_name));" |
||||||
|
"CREATE TABLE IF NOT EXISTS history (id INTEGER PRIMARY KEY, timestamp INTEGER NOT NULL, " |
||||||
|
"chat_id INTEGER NOT NULL, sender_alias INTEGER NOT NULL, " |
||||||
|
"message BLOB NOT NULL);" |
||||||
|
"CREATE TABLE IF NOT EXISTS faux_offline_pending (id INTEGER PRIMARY KEY);"); |
||||||
|
|
||||||
|
// Cache our current peers
|
||||||
|
db.execLater(RawDatabase::Query{"SELECT id, public_key FROM peers;", [this](const QVector<QVariant>& row) |
||||||
|
{ |
||||||
|
peers[row[1].toString()] = row[0].toInt(); |
||||||
|
}}); |
||||||
|
} |
||||||
|
|
||||||
|
void History::import(const HistoryKeeper &oldHistory) |
||||||
|
{ |
||||||
|
if (!isValid()) |
||||||
|
{ |
||||||
|
qWarning() << "New database not open, import failed"; |
||||||
|
return; |
||||||
|
} |
||||||
|
|
||||||
|
qDebug() << "Importing old database..."; |
||||||
|
QTime t=QTime::currentTime(); |
||||||
|
t.start(); |
||||||
|
QVector<RawDatabase::Query> queries; |
||||||
|
constexpr int batchSize = 1000; |
||||||
|
queries.reserve(batchSize); |
||||||
|
QList<HistoryKeeper::HistMessage> oldMessages = oldHistory.exportMessagesDeleteFile(); |
||||||
|
for (const HistoryKeeper::HistMessage& msg : oldMessages) |
||||||
|
{ |
||||||
|
queries += generateNewMessageQueries(msg.chat, msg.message, msg.sender, msg.timestamp, true, msg.dispName); |
||||||
|
if (queries.size() == batchSize) |
||||||
|
{ |
||||||
|
db.execLater(queries); |
||||||
|
queries.clear(); |
||||||
|
} |
||||||
|
} |
||||||
|
db.execLater(queries); |
||||||
|
db.sync(); |
||||||
|
qDebug() << "Imported old database in"<<t.elapsed()<<"ms"; |
||||||
|
} |
||||||
@ -0,0 +1,77 @@ |
|||||||
|
#ifndef HISTORY_H |
||||||
|
#define HISTORY_H |
||||||
|
|
||||||
|
#include <tox/toxencryptsave.h> |
||||||
|
#include <QDateTime> |
||||||
|
#include <QVector> |
||||||
|
#include <QHash> |
||||||
|
#include <cstdint> |
||||||
|
#include "src/persistence/db/rawdatabase.h" |
||||||
|
|
||||||
|
class Profile; |
||||||
|
class HistoryKeeper; |
||||||
|
class RawDatabase; |
||||||
|
|
||||||
|
/// Interacts with the profile database to save the chat history
|
||||||
|
class History |
||||||
|
{ |
||||||
|
public: |
||||||
|
struct HistMessage |
||||||
|
{ |
||||||
|
HistMessage(qint64 id, bool isSent, QDateTime timestamp, QString chat, QString dispName, QString sender, QString message) : |
||||||
|
chat{chat}, sender{sender}, message{message}, dispName{dispName}, timestamp{timestamp}, id{id}, isSent{isSent} {} |
||||||
|
|
||||||
|
QString chat; |
||||||
|
QString sender; |
||||||
|
QString message; |
||||||
|
QString dispName; |
||||||
|
QDateTime timestamp; |
||||||
|
qint64 id; |
||||||
|
bool isSent; |
||||||
|
}; |
||||||
|
|
||||||
|
public: |
||||||
|
/// Opens the profile database and prepares to work with the history
|
||||||
|
/// If password is empty, the database will be opened unencrypted
|
||||||
|
History(const QString& profileName, const QString& password); |
||||||
|
/// Opens the profile database, and import from the old database
|
||||||
|
/// If password is empty, the database will be opened unencrypted
|
||||||
|
History(const QString& profileName, const QString& password, const HistoryKeeper& oldHistory); |
||||||
|
~History(); |
||||||
|
/// Checks if the database was opened successfully
|
||||||
|
bool isValid(); |
||||||
|
/// Imports messages from the old history file
|
||||||
|
void import(const HistoryKeeper& oldHistory); |
||||||
|
/// Changes the database password, will encrypt or decrypt if necessary
|
||||||
|
void setPassword(const QString& password); |
||||||
|
/// Moves the database file on disk to match the new name
|
||||||
|
void rename(const QString& newName); |
||||||
|
|
||||||
|
/// Erases all the chat history from the database
|
||||||
|
void eraseHistory(); |
||||||
|
/// Erases the chat history with one friend
|
||||||
|
void removeFriendHistory(const QString& friendPk); |
||||||
|
/// Saves a chat message in the database
|
||||||
|
void addNewMessage(const QString& friendPk, const QString& message, const QString& sender, |
||||||
|
const QDateTime &time, bool isSent, QString dispName, |
||||||
|
std::function<void(int64_t)> insertIdCallback={}); |
||||||
|
/// Fetches chat messages from the database
|
||||||
|
QList<HistMessage> getChatHistory(const QString& friendPk, const QDateTime &from, const QDateTime &to); |
||||||
|
/// Marks a message as sent, removing it from the faux-offline pending messages list
|
||||||
|
void markAsSent(qint64 id); |
||||||
|
|
||||||
|
protected: |
||||||
|
/// Makes sure the history tables are created
|
||||||
|
void init(); |
||||||
|
static QString getDbPath(const QString& profileName); |
||||||
|
QVector<RawDatabase::Query> generateNewMessageQueries(const QString& friendPk, const QString& message, |
||||||
|
const QString& sender, const QDateTime &time, bool isSent, QString dispName, |
||||||
|
std::function<void(int64_t)> insertIdCallback={}); |
||||||
|
|
||||||
|
private: |
||||||
|
RawDatabase db; |
||||||
|
// Cached mappings to speed up message saving
|
||||||
|
QHash<QString, int64_t> peers; ///< Maps friend public keys to unique IDs by index
|
||||||
|
}; |
||||||
|
|
||||||
|
#endif // HISTORY_H
|
||||||
Loading…
Reference in new issue