7#include "querybuilder.h"
8#include "akonadiserver_debug.h"
9#include "dbexception.h"
11#include "storage/query.h"
14#include <qsqldatabase.h>
16#ifndef QUERYBUILDER_UNITTEST
17#include "storage/datastore.h"
18#include "storage/querycache.h"
19#include "storage/storagedebugger.h"
22#include "shared/akranges.h"
24#include <QElapsedTimer>
30using namespace Akonadi::Server;
31using namespace AkRanges;
38#ifdef QUERYBUILDER_UNITTEST
52 case Query::NotEquals:
60 case Query::LessOrEqual:
64 case Query::GreaterOrEqual:
73 Q_ASSERT_X(
false,
"QueryBuilder::compareOperatorToString()",
"Unknown compare operator.");
85 Q_ASSERT_X(
false,
"QueryBuilder::logicOperatorToString()",
"Unknown logic operator.");
92 case Query::Ascending:
94 case Query::Descending:
97 Q_ASSERT_X(
false,
"QueryBuilder::sortOrderToString()",
"Unknown sort order.");
103 for (
int i = 0, c = strings.
size(); i < c; ++i) {
104 *statement += strings.
at(i);
118#ifndef QUERYBUILDER_UNITTEST
120 , mDatabaseType(
DbType::type(store->database()))
122 , mDatabaseType(
DbType::Unknown)
129#ifdef QUERYBUILDER_UNITTEST
132 static const QString defaultIdColumn = QStringLiteral(
"id");
133 mIdentificationColumn = defaultIdColumn;
137 :
QueryBuilder(defaultDataStore(), tableQuery, tableQueryAlias)
142 : mTable(tableQueryAlias)
143 , mTableSubQuery(std::ref(tableQuery))
144#ifndef QUERYBUILDER_UNITTEST
146 , mDatabaseType(
DbType::type(store->database()))
148 , mDatabaseType(
DbType::Unknown)
150 , mType(QueryType::Select)
155#ifdef QUERYBUILDER_UNITTEST
158 static const QString defaultIdColumn = QStringLiteral(
"id");
159 mIdentificationColumn = defaultIdColumn;
163 : mTable(other.mTable)
164 , mTableSubQuery(std::move(other.mTableSubQuery))
165 , mDataStore(other.mDataStore)
166 , mDatabaseType(other.mDatabaseType)
167 , mRootCondition{std::move(other.mRootCondition[0]), std::move(other.mRootCondition[1])}
168 , mQuery(std::move(other.mQuery))
170 , mColumns(std::move(other.mColumns))
171 , mBindValues(std::move(other.mBindValues))
172 , mSortColumns(std::move(other.mSortColumns))
173 , mGroupColumns(std::move(other.mGroupColumns))
174 , mColumnValues(std::move(other.mColumnValues))
175 , mColumnMultiValues(std::move(other.mColumnMultiValues))
176 , mIdentificationColumn(std::move(other.mIdentificationColumn))
177 , mJoinedTables(std::move(other.mJoinedTables))
178 , mJoins(std::move(other.mJoins))
179 , mLimit(other.mLimit)
180 , mOffset(other.mOffset)
181 , mDistinct(other.mDistinct)
182 , mForUpdate(other.mForUpdate)
191 if (
this != &other) {
192 mTable = other.mTable;
193 mTableSubQuery = std::move(other.mTableSubQuery);
194 mDataStore = other.mDataStore;
195 mDatabaseType = other.mDatabaseType;
196 mRootCondition[0] = std::move(other.mRootCondition[0]);
197 mRootCondition[1] = std::move(other.mRootCondition[1]);
198 mQuery = std::move(other.mQuery);
200 mColumns = std::move(other.mColumns);
201 mBindValues = std::move(other.mBindValues);
202 mSortColumns = std::move(other.mSortColumns);
203 mGroupColumns = std::move(other.mGroupColumns);
204 mColumnValues = std::move(other.mColumnValues);
205 mColumnMultiValues = std::move(other.mColumnMultiValues);
206 mIdentificationColumn = std::move(other.mIdentificationColumn);
207 mJoinedTables = std::move(other.mJoinedTables);
208 mJoins = std::move(other.mJoins);
209 mLimit = other.mLimit;
210 mOffset = other.mOffset;
211 mDistinct = other.mDistinct;
212 mForUpdate = other.mForUpdate;
214 other.mQuery = QSqlQuery();
219QueryBuilder::~QueryBuilder()
221 if (mQuery.isActive()) {
223#ifndef QUERYBUILDER_UNITTEST
225 const auto stmt = mQuery.executedQuery();
233 mDatabaseType = type;
238 Q_ASSERT((joinType ==
InnerJoin && (mType == Select || mType == Update)) || (joinType ==
LeftJoin && mType == Select)
241 if (mJoinedTables.contains(table)) {
243 mJoins[table].first = qMin(joinType, mJoins.value(table).first);
244 mJoins[table].second.addCondition(condition);
246 mJoins[table] = qMakePair(joinType, condition);
247 mJoinedTables << table;
255 addJoin(joinType, table, condition);
261 mRootCondition[type].addValueCondition(column, op, value);
267 mRootCondition[type].addValueCondition(column, op, value);
273 mRootCondition[type].addValueCondition(column, op, value);
279 mRootCondition[type].addColumnCondition(column, op, column2);
288 if (!condition.mSubConditions.
isEmpty()) {
289 for (
int i = condition.mSubConditions.
count() - 1; i >= 0; --i) {
290 sqliteAdaptUpdateJoin(condition.mSubConditions[i]);
302 if (!mJoinedTables.contains(table)) {
306 const auto &[
type, joinCondition] = mJoins.value(table);
309 qb.addColumn(condition.mColumn);
310 qb.addCondition(joinCondition);
313 condition.mColumn.
reserve(1024);
314 condition.mColumn.
resize(0);
315 condition.mColumn += QLatin1StringView(
"( ");
316 qb.buildQuery(&condition.mColumn);
317 condition.mColumn += QLatin1StringView(
" )");
320void QueryBuilder::buildInsertColumns(QString *statement)
322 const auto &vals = mColumnMultiValues.empty() ? mColumnValues : mColumnMultiValues;
324 for (qsizetype i = 0; i < vals.size(); ++i) {
325 *statement += vals.
at(i).first;
326 if (i + 1 < vals.size()) {
333void QueryBuilder::buildInsertValues(QString *statement)
335 if (mColumnMultiValues.empty()) {
337 for (
int col = 0, columnCount = mColumnValues.size(); col < columnCount; ++col) {
338 bindValue(statement, mColumnValues.at(col).second);
339 if (col + 1 < columnCount) {
345 const auto rows = mColumnMultiValues.front().second.toList().size();
346 for (qsizetype row = 0; row < rows; ++row) {
348 for (
int col = 0, columnCount = mColumnMultiValues.size(); col < columnCount; ++col) {
349 const auto &[_, values] = mColumnMultiValues.at(col);
352 bindValue(statement, values.toList().at(row));
353 if (col + 1 < columnCount) {
358 if (row + 1 < rows) {
365void QueryBuilder::buildQuery(QString *statement)
375 mQuery.setForwardOnly(
true);
376 *statement += QLatin1StringView(
"SELECT ");
378 *statement += QLatin1StringView(
"DISTINCT ");
380 Q_ASSERT_X(mColumns.count() > 0,
"QueryBuilder::exec()",
"No columns specified");
381 appendJoined(statement, mColumns);
382 *statement += QLatin1StringView(
" FROM ");
383 *statement += mTableSubQuery.has_value() ? getTableQuery(*mTableSubQuery, mTable) : mTable;
384 for (
const QString &joinedTable : std::as_const(mJoinedTables)) {
385 const auto &[joinType, joinCond] = mJoins.value(joinedTable);
388 *statement += QLatin1StringView(
" LEFT JOIN ");
391 *statement += QLatin1StringView(
" LEFT OUTER JOIN ");
394 *statement += QLatin1StringView(
" INNER JOIN ");
397 *statement += joinedTable;
398 *statement += QLatin1StringView(
" ON ");
399 buildWhereCondition(statement, joinCond);
403 *statement += QLatin1StringView(
"INSERT INTO ");
404 *statement += mTable + u
" ";
405 buildInsertColumns(statement);
406 *statement += u
" VALUES ";
407 buildInsertValues(statement);
408 if (mDatabaseType == DbType::PostgreSQL && !mIdentificationColumn.isEmpty()) {
409 *statement += QLatin1StringView(
" RETURNING ") + mIdentificationColumn;
415 if (mDatabaseType != DbType::Sqlite) {
416 for (
const QString &table : std::as_const(mJoinedTables)) {
417 const auto &[joinType, joinCond] = mJoins.value(table);
423 sqliteAdaptUpdateJoin(whereCondition);
426 *statement += QLatin1StringView(
"UPDATE ");
427 *statement += mTable;
429 if (mDatabaseType == DbType::MySQL && !mJoinedTables.isEmpty()) {
431 *statement += QLatin1StringView(
", ");
432 appendJoined(statement, mJoinedTables);
435 *statement += QLatin1StringView(
" SET ");
436 Q_ASSERT_X(mColumnValues.count() >= 1,
"QueryBuilder::exec()",
"At least one column needs to be changed");
437 for (
int i = 0, c = mColumnValues.size(); i < c; ++i) {
438 const auto &[column, value] = mColumnValues.at(i);
439 *statement += column;
440 *statement += QLatin1StringView(
" = ");
441 bindValue(statement, value);
443 *statement += QLatin1StringView(
", ");
447 if (mDatabaseType == DbType::PostgreSQL && !mJoinedTables.isEmpty()) {
450 *statement += QLatin1StringView(
" FROM ");
451 appendJoined(statement, mJoinedTables, QLatin1StringView(
" JOIN "));
456 *statement += QLatin1StringView(
"DELETE FROM ");
457 *statement += mTable;
460 Q_ASSERT_X(
false,
"QueryBuilder::exec()",
"Unknown enum value");
463 if (!whereCondition.
isEmpty()) {
464 *statement += QLatin1StringView(
" WHERE ");
465 buildWhereCondition(statement, whereCondition);
468 if (!mGroupColumns.isEmpty()) {
469 *statement += QLatin1StringView(
" GROUP BY ");
470 appendJoined(statement, mGroupColumns);
474 *statement += QLatin1StringView(
" HAVING ");
478 if (!mSortColumns.isEmpty()) {
479 Q_ASSERT_X(mType == Select,
"QueryBuilder::exec()",
"Order statements are only valid for SELECT queries");
480 *statement += QLatin1StringView(
" ORDER BY ");
481 for (
int i = 0, c = mSortColumns.size(); i < c; ++i) {
482 const auto &[column, order] = mSortColumns.at(i);
483 *statement += column;
484 *statement += sortOrderToString(order);
486 *statement += QLatin1StringView(
", ");
494 *statement += QLatin1StringView(
" OFFSET ") +
QString::number(mOffset);
498 if (mType == Select && mForUpdate) {
499 if (mDatabaseType == DbType::Sqlite) {
503 *statement += QLatin1StringView(
" FOR UPDATE");
512 buildQuery(&statement);
514#ifndef QUERYBUILDER_UNITTEST
517 mQuery = std::move(*
query);
519 mQuery =
QSqlQuery(mDataStore->database());
520 if (!mQuery.prepare(statement)) {
521 qCCritical(AKONADISERVER_LOG) <<
"DATABASE ERROR while PREPARING QUERY:";
522 qCCritical(AKONADISERVER_LOG) <<
" Error code:" << mQuery.lastError().nativeErrorCode();
523 qCCritical(AKONADISERVER_LOG) <<
" DB error: " << mQuery.lastError().databaseText();
524 qCCritical(AKONADISERVER_LOG) <<
" Error text:" << mQuery.lastError().text();
525 qCCritical(AKONADISERVER_LOG) <<
" Query:" << statement;
532 bool isBatch =
false;
533 for (
int i = 0; i < mBindValues.count(); ++i) {
535 if (!isBatch &&
static_cast<QMetaType::Type>(mBindValues[i].typeId()) == qMetaTypeId<QVariantList>()) {
543 if (StorageDebugger::instance()->isSQLDebuggingEnabled()) {
547 ret = mQuery.execBatch();
551 StorageDebugger::instance()->queryExecuted(
reinterpret_cast<qint64
>(mDataStore), mQuery, t.
elapsed());
553 StorageDebugger::instance()->incSequence();
555 ret = mQuery.execBatch();
562 bool needsRetry =
false;
564 if (mDatabaseType == DbType::PostgreSQL) {
565 const QString dbError = mQuery.lastError().databaseText();
567 qCWarning(AKONADISERVER_LOG) <<
"QueryBuilder::exec(): database reported transaction deadlock, retrying transaction";
568 qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text();
571 }
else if (mDatabaseType == DbType::MySQL) {
572 const QString lastErrorStr = mQuery.lastError().nativeErrorCode();
573 const int error = lastErrorStr.
isEmpty() ? -1 : lastErrorStr.
toInt();
574 if (error == 1213 ) {
575 qCWarning(AKONADISERVER_LOG) <<
"QueryBuilder::exec(): database reported transaction deadlock, retrying transaction";
576 qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text();
578 }
else if (error == 1205 ) {
579 qCWarning(AKONADISERVER_LOG) <<
"QueryBuilder::exec(): database reported transaction timeout, retrying transaction";
580 qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text();
584 }
else if (mDatabaseType == DbType::Sqlite) {
585 const QString lastErrorStr = mQuery.lastError().nativeErrorCode();
586 const int error = lastErrorStr.
isEmpty() ? -1 : lastErrorStr.
toInt();
588 qCWarning(AKONADISERVER_LOG) <<
"QueryBuilder::exec(): database reported transaction deadlock, retrying transaction";
589 qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text();
590 mDataStore->doRollback();
592 }
else if (error == 5 ) {
593 qCWarning(AKONADISERVER_LOG) <<
"QueryBuilder::exec(): database reported transaction timeout, retrying transaction";
594 qCWarning(AKONADISERVER_LOG) << mQuery.lastError().text();
595 mDataStore->doRollback();
601 mDataStore->transactionKilledByDB();
602 throw DbDeadlockException(mQuery);
605 qCCritical(AKONADISERVER_LOG) <<
"DATABASE ERROR:";
606 qCCritical(AKONADISERVER_LOG) <<
" Error code:" << mQuery.lastError().nativeErrorCode();
607 qCCritical(AKONADISERVER_LOG) <<
" DB error: " << mQuery.lastError().databaseText();
608 qCCritical(AKONADISERVER_LOG) <<
" Error text:" << mQuery.lastError().text();
609 qCCritical(AKONADISERVER_LOG) <<
" Values:" << mQuery.boundValues();
610 qCCritical(AKONADISERVER_LOG) <<
" Query:" << statement;
614 mStatement = statement;
632 buildCaseStatement(&
query, caseStmt);
633 mColumns.append(
query);
644 buildCaseStatement(&
query, caseStmt);
647 mColumns.append(
query);
652 if (value.
metaType().
id() == qMetaTypeId<QDateTime>()) {
663bool isType(
const QVariant &value, QByteArrayView typeName)
678bool isList(
const QVariant &value)
680 if (value.
typeId() == qMetaTypeId<QVariantList>() || value.
typeId() == qMetaTypeId<QStringList>() || value.
typeId() == qMetaTypeId<QByteArrayList>()) {
684 return isType(value,
"QList<");
687bool isSet(
const QVariant &value)
689 return isType(value,
"QSet<qlonglong>");
694void QueryBuilder::buildWhereCondition(QString *query,
const Query::Condition &cond)
696 constexpr auto buildWhereInContainerCondition = [](
QueryBuilder *self, QString *
query,
const auto &values) {
698 if (values.empty()) {
699 qCWarning(AKONADISERVER_LOG) <<
"Empty list given for IN condition.";
701 for (
const auto &[i, entry] : values | Views::enumerate()) {
703 *
query += QLatin1StringView(
", ");
705 self->bindValue(
query, entry);
711 *
query += QLatin1StringView(
"( ");
712 const QLatin1StringView glue = logicOperatorToString(cond.mCombineOp);
714 for (qsizetype i = 0, c = subConditions.
size(); i < c; ++i) {
715 buildWhereCondition(
query, subConditions.
at(i));
720 *
query += QLatin1StringView(
" )");
722 *
query += cond.mColumn;
723 *
query += compareOperatorToString(cond.mCompareOp);
724 if (!cond.mComparedColumn.
isEmpty()) {
725 *
query += cond.mComparedColumn;
726 }
else if (!cond.mComparedValue.
isValid()) {
728 }
else if (isList(cond.mComparedValue)) {
731 buildWhereInContainerCondition(
this,
query, cond.mComparedValue.
toList());
732 }
else if (isSet(cond.mComparedValue)) {
733 buildWhereInContainerCondition(
this,
query, cond.mComparedValue.
value<QSet<qint64>>());
735 bindValue(
query, cond.mComparedValue);
740void QueryBuilder::buildCaseStatement(QString *query,
const Query::Case &caseStmt)
742 *
query += QLatin1StringView(
"CASE ");
743 for (
const auto &whenThen : caseStmt.mWhenThen) {
744 *
query += QLatin1StringView(
"WHEN ");
745 buildWhereCondition(
query, whenThen.first);
746 *
query += QLatin1StringView(
" THEN ") + whenThen.second;
748 if (!caseStmt.mElse.
isEmpty()) {
749 *
query += QLatin1StringView(
" ELSE ") + caseStmt.mElse;
751 *
query += QLatin1StringView(
" END");
757 mRootCondition[type].setSubQueryMode(op);
763 mRootCondition[type].addCondition(condition);
768 mSortColumns << qMakePair(column, order);
773 Q_ASSERT(mType == Select);
774 mGroupColumns << column;
779 Q_ASSERT(mType == Select);
780 mGroupColumns += columns;
785 mColumnMultiValues.
clear();
786 mColumnValues.push_back(qMakePair(column, value));
791 Q_ASSERT(mType == Insert);
793 mColumnValues.
clear();
794 mColumnMultiValues.
push_back(qMakePair(column, values));
799 mDistinct = distinct;
810 mIdentificationColumn = column;
815 if (mDatabaseType == DbType::PostgreSQL) {
817 if (mIdentificationColumn.isEmpty()) {
838 mForUpdate = forUpdate;
853 Q_ASSERT_X(
query.
isValid() &&
query.
isSelect(),
"QueryBuilder::getTableQuery",
"Table subquery use only for valid SELECT queries");
857 qCWarning(AKONADISERVER_LOG) <<
"Table subquery is empty";
864 for (qsizetype pos = boundValues.
size() - 1; pos >= 0; --pos) {
871 field.setValue(value);
874 const QString formattedValue =
query.driver()->formatValue(field);
875 tableQuery.
replace(key, formattedValue);
878 tableQuery.
append(QLatin1StringView(
" ) AS %1").arg(alias));
This class handles all the database access.
static DataStore * self()
Per thread singleton.
Helper class to construct arbitrary SQL queries.
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
void setDatabaseType(DbType::Type type)
Sets the database which should execute the query.
void addGroupColumn(const QString &column)
Add a GROUP BY column.
void addSortColumn(const QString &column, Query::SortOrder order=Query::Ascending)
Add sort column.
QString getTableWithColumn(const QString &column) const
Returns concatenated table name with column name.
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
bool exec()
Executes the query, returns true on success.
void addColumns(const QStringList &cols)
Adds the given columns to a select query.
void addCondition(const Query::Condition &condition, ConditionType type=WhereCondition)
Add a WHERE condition.
void setColumnValue(const QString &column, const QVariant &value)
Sets a column to the given value (only valid for INSERT and UPDATE queries).
void setColumnValues(const QString &column, const QList< T > &values)
Set column to given values (only valid for INSERT query).
void setDistinct(bool distinct)
Specify whether duplicates should be included in the result.
void addColumnCondition(const QString &column, Query::CompareOperator op, const QString &column2, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with another column.
void addGroupColumns(const QStringList &columns)
Add list of columns to GROUP BY.
void setForUpdate(bool forUpdate=true)
Indicate to the database to acquire an exclusive lock on the rows already during SELECT statement.
ConditionType
Defines the place at which a condition should be evaluated.
@ HavingCondition
add condition to HAVING part of the query NOTE: only supported for SELECT queries
@ WhereCondition
add condition to WHERE part of the query
QSqlQuery & query()
Returns the query, only valid after exec().
void setIdentificationColumn(const QString &column)
Sets the column used for identification in an INSERT statement.
qint64 insertId()
Returns the ID of the newly created record (only valid for INSERT queries)
void setSubQueryMode(Query::LogicOperator op, ConditionType type=WhereCondition)
Define how WHERE or HAVING conditions are combined.
void addAggregation(const QString &col, const QString &aggregate)
Adds an aggregation statement.
void addColumn(const QString &col)
Adds the given column to a select query.
QueryBuilder(const QString &table, QueryType type=Select)
Creates a new query builder.
JoinType
When the same table gets joined as both, Inner- and LeftJoin, it will be merged into a single InnerJo...
@ LeftOuterJoin
NOTE: only supported for SELECT queries.
@ InnerJoin
NOTE: only supported for UPDATE and SELECT queries.
@ LeftJoin
NOTE: only supported for SELECT queries.
QString getTable() const
Returns the name of the main table or subquery.
void setLimit(int limit, int offset=-1)
Limits the amount of retrieved rows.
Represents a WHERE condition tree.
void addColumnCondition(const QString &column, CompareOperator op, const QString &column2)
Add a WHERE condition which compares a column with another column.
Condition::List subConditions() const
Returns the list of sub-conditions.
QList< Condition > List
A list of conditions.
bool isEmpty() const
Returns if there are sub conditions.
void addCondition(const Condition &condition)
Add a WHERE condition.
Helper methods for checking the database system we are dealing with.
Type
Supported database types.
void insert(const QSqlDatabase &db, const QString &queryStatement, QSqlQuery query)
Insert query into the cache for queryStatement.
std::optional< QSqlQuery > query(const QString &queryStatement)
Return a cached QSqlQuery for given queryStatement.
CompareOperator
Compare operators to be used in query conditions.
LogicOperator
Logic operations used to combine multiple query conditions.
VehicleSection::Type type(QStringView coachNumber, QStringView coachClassification)
QString name(StandardAction id)
KOSM_EXPORT const char * typeName(Type type)
qint64 elapsed() const const
const_reference at(qsizetype i) const const
qsizetype count() const const
reference emplace_back(Args &&... args)
bool isEmpty() const const
void push_back(parameter_type value)
qsizetype size() const const
QVariantList boundValues() const const
bool isSelect() const const
bool isValid() const const
QVariant lastInsertId() const const
QString lastQuery() const const
QSqlRecord record() const const
QVariant value(const QString &name) const const
QString & append(QChar ch)
const QChar at(qsizetype position) const const
bool contains(QChar ch, Qt::CaseSensitivity cs) const const
qsizetype indexOf(QChar ch, qsizetype from, Qt::CaseSensitivity cs) const const
bool isEmpty() const const
QString left(qsizetype n) const const
QString number(double n, char format, int precision)
QString & prepend(QChar ch)
QString & replace(QChar before, QChar after, Qt::CaseSensitivity cs)
void reserve(qsizetype size)
void resize(qsizetype newSize, QChar fillChar)
int toInt(bool *ok, int base) const const
bool isNull() const const
bool isValid() const const
QDateTime toDateTime() const const
QList< QVariant > toList() const const
qlonglong toLongLong(bool *ok) const const
const char * typeName() const const