Akonadi

itemfetchhelper.cpp
1
2/***************************************************************************
3 * SPDX-FileCopyrightText: 2006-2009 Tobias Koenig <tokoe@kde.org> *
4 * *
5 * SPDX-License-Identifier: LGPL-2.0-or-later *
6 ***************************************************************************/
7
8#include "itemfetchhelper.h"
9
10#include "akonadi.h"
11#include "connection.h"
12#include "handler.h"
13#include "handlerhelper.h"
14#include "shared/akranges.h"
15#include "storage/itemqueryhelper.h"
16#include "storage/itemretrievalmanager.h"
17#include "storage/parttypehelper.h"
18#include "storage/selectquerybuilder.h"
19#include "storage/transaction.h"
20
21#include "agentmanagerinterface.h"
22#include "akonadiserver_debug.h"
23#include "intervalcheck.h"
24#include "utils.h"
25
26#include "private/dbus_p.h"
27
28#include <QDateTime>
29#include <QSqlQuery>
30#include <QStringList>
31#include <QVariant>
32
33#include <QElapsedTimer>
34
35using namespace Akonadi;
36using namespace Akonadi::Server;
37using namespace AkRanges;
38
39#define ENABLE_FETCH_PROFILING 0
40#if ENABLE_FETCH_PROFILING
41#define BEGIN_TIMER(name) \
42 QElapsedTimer name##Timer; \
43 name##Timer.start();
44
45#define END_TIMER(name) const double name##Elapsed = name##Timer.nsecsElapsed() / 1000000.0;
46#define PROF_INC(name) ++name;
47#else
48#define BEGIN_TIMER(name)
49#define END_TIMER(name)
50#define PROF_INC(name)
51#endif
52
53ItemFetchHelper::ItemFetchHelper(Connection *connection,
54 const Scope &scope,
55 const Protocol::ItemFetchScope &itemFetchScope,
56 const Protocol::TagFetchScope &tagFetchScope,
57 AkonadiServer &akonadi,
58 const Protocol::FetchLimit &itemsLimit)
59 : ItemFetchHelper(connection, connection->context(), scope, itemFetchScope, tagFetchScope, akonadi, itemsLimit)
60{
61}
62
63ItemFetchHelper::ItemFetchHelper(Connection *connection,
64 const CommandContext &context,
65 const Scope &scope,
66 const Protocol::ItemFetchScope &itemFetchScope,
67 const Protocol::TagFetchScope &tagFetchScope,
68 AkonadiServer &akonadi,
69 const Protocol::FetchLimit &itemsLimit)
70 : mConnection(connection)
71 , mContext(context)
72 , mScope(scope)
73 , mItemFetchScope(itemFetchScope)
74 , mTagFetchScope(tagFetchScope)
75 , mAkonadi(akonadi)
76 , mItemsLimit(itemsLimit)
77 , mPimItemQueryAlias(QLatin1StringView("pimItem_alias"))
78{
79 std::fill(mItemQueryColumnMap, mItemQueryColumnMap + ItemQueryColumnCount, -1);
80}
81
82void ItemFetchHelper::disableATimeUpdates()
83{
84 mUpdateATimeEnabled = false;
85}
86
87enum PartQueryColumns {
88 PartQueryPimIdColumn,
89 PartQueryTypeIdColumn,
90 PartQueryDataColumn,
91 PartQueryStorageColumn,
92 PartQueryVersionColumn,
93 PartQueryDataSizeColumn
94};
95
96QueryBuilder ItemFetchHelper::buildPartQuery(QSqlQuery &itemQuery, const QList<QByteArray> &partList, bool allPayload, bool allAttrs)
97{
98 /// TODO: merge with ItemQuery
99 QueryBuilder partQuery = mItemsLimit.limit() > 0 ? QueryBuilder(itemQuery, mPimItemQueryAlias) : QueryBuilder(PimItem::tableName());
100
101 if (!partList.isEmpty() || allPayload || allAttrs) {
102 partQuery.addJoin(QueryBuilder::InnerJoin, Part::tableName(), partQuery.getTableWithColumn(PimItem::idColumn()), Part::pimItemIdFullColumnName());
103 partQuery.addColumn(partQuery.getTableWithColumn(PimItem::idColumn()));
104 partQuery.addColumn(Part::partTypeIdFullColumnName());
105 partQuery.addColumn(Part::dataFullColumnName());
106 partQuery.addColumn(Part::storageFullColumnName());
107 partQuery.addColumn(Part::versionFullColumnName());
108 partQuery.addColumn(Part::datasizeFullColumnName());
109
110 partQuery.addSortColumn(partQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
111
112 if (!partList.isEmpty() || allPayload || allAttrs) {
113 Query::Condition cond(Query::Or);
114 for (const QByteArray &b : std::as_const(partList)) {
115 if (b.startsWith("PLD") || b.startsWith("ATR")) {
116 cond.addValueCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartTypeHelper::fromFqName(b).id());
117 }
118 }
119 if (allPayload || allAttrs) {
120 partQuery.addJoin(QueryBuilder::InnerJoin, PartType::tableName(), Part::partTypeIdFullColumnName(), PartType::idFullColumnName());
121 if (allPayload) {
122 cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
123 }
124 if (allAttrs) {
125 cond.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("ATR"));
126 }
127 }
128
129 partQuery.addCondition(cond);
130 }
131
132 ItemQueryHelper::scopeToQuery(mScope, mContext, partQuery);
133
134 if (!partQuery.exec()) {
135 throw HandlerException("Unable to list item parts");
136 }
137 partQuery.query().next();
138 }
139
140 return partQuery;
141}
142
143QueryBuilder ItemFetchHelper::buildItemQuery()
144{
145 int column = 0;
146 QueryBuilder itemQuery(PimItem::tableName());
147#define ADD_COLUMN(colName, colId) \
148 { \
149 itemQuery.addColumn(colName); \
150 mItemQueryColumnMap[colId] = column++; \
151 }
152 ADD_COLUMN(PimItem::idFullColumnName(), ItemQueryPimItemIdColumn);
153 if (mItemFetchScope.fetchRemoteId()) {
154 ADD_COLUMN(PimItem::remoteIdFullColumnName(), ItemQueryPimItemRidColumn)
155 }
156 ADD_COLUMN(PimItem::mimeTypeIdFullColumnName(), ItemQueryMimeTypeIdColumn)
157 ADD_COLUMN(PimItem::revFullColumnName(), ItemQueryRevColumn)
158 if (mItemFetchScope.fetchRemoteRevision()) {
159 ADD_COLUMN(PimItem::remoteRevisionFullColumnName(), ItemQueryRemoteRevisionColumn)
160 }
161 if (mItemFetchScope.fetchSize()) {
162 ADD_COLUMN(PimItem::sizeFullColumnName(), ItemQuerySizeColumn)
163 }
164 if (mItemFetchScope.fetchMTime()) {
165 ADD_COLUMN(PimItem::datetimeFullColumnName(), ItemQueryDatetimeColumn)
166 }
167 ADD_COLUMN(PimItem::collectionIdFullColumnName(), ItemQueryCollectionIdColumn)
168 if (mItemFetchScope.fetchGID()) {
169 ADD_COLUMN(PimItem::gidFullColumnName(), ItemQueryPimItemGidColumn)
170 }
171#undef ADD_COLUMN
172
173 itemQuery.addSortColumn(PimItem::idFullColumnName(), static_cast<Query::SortOrder>(mItemsLimit.sortOrder()));
174 if (mItemsLimit.limit() > 0) {
175 itemQuery.setLimit(mItemsLimit.limit(), mItemsLimit.limitOffset());
176 }
177
178 ItemQueryHelper::scopeToQuery(mScope, mContext, itemQuery);
179
180 if (mItemFetchScope.changedSince().isValid()) {
181 itemQuery.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual, mItemFetchScope.changedSince().toUTC());
182 }
183
184 if (!itemQuery.exec()) {
185 throw HandlerException("Unable to list items");
186 }
187
188 itemQuery.query().next();
189
190 return itemQuery;
191}
192
193enum FlagQueryColumns {
194 FlagQueryPimItemIdColumn,
195 FlagQueryFlagIdColumn,
196};
197
198QueryBuilder ItemFetchHelper::buildFlagQuery(QSqlQuery &itemQuery)
199{
200 QueryBuilder flagQuery = mItemsLimit.limit() > 0 ? QueryBuilder(itemQuery, mPimItemQueryAlias) : QueryBuilder(PimItem::tableName());
201
202 flagQuery.addJoin(QueryBuilder::InnerJoin,
203 PimItemFlagRelation::tableName(),
204 flagQuery.getTableWithColumn(PimItem::idColumn()),
205 PimItemFlagRelation::leftFullColumnName());
206
207 flagQuery.addColumn(flagQuery.getTableWithColumn(PimItem::idColumn()));
208 flagQuery.addColumn(PimItemFlagRelation::rightFullColumnName());
209
210 ItemQueryHelper::scopeToQuery(mScope, mContext, flagQuery);
211 flagQuery.addSortColumn(flagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
212
213 if (!flagQuery.exec()) {
214 throw HandlerException("Unable to retrieve item flags");
215 }
216
217 flagQuery.query().next();
218
219 return flagQuery;
220}
221
222enum TagQueryColumns {
223 TagQueryItemIdColumn,
224 TagQueryTagIdColumn,
225};
226
227QueryBuilder ItemFetchHelper::buildTagQuery(QSqlQuery &itemQuery)
228{
229 QueryBuilder tagQuery(PimItem::tableName());
230 if (mItemsLimit.limit() > 0) {
231 tagQuery = QueryBuilder(itemQuery, mPimItemQueryAlias);
232 }
233
234 tagQuery.addJoin(QueryBuilder::InnerJoin,
235 PimItemTagRelation::tableName(),
236 tagQuery.getTableWithColumn(PimItem::idColumn()),
237 PimItemTagRelation::leftFullColumnName());
238 tagQuery.addJoin(QueryBuilder::InnerJoin, Tag::tableName(), Tag::idFullColumnName(), PimItemTagRelation::rightFullColumnName());
239 tagQuery.addColumn(tagQuery.getTableWithColumn(PimItem::idColumn()));
240 tagQuery.addColumn(Tag::idFullColumnName());
241
242 ItemQueryHelper::scopeToQuery(mScope, mContext, tagQuery);
243 tagQuery.addSortColumn(tagQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
244
245 if (!tagQuery.exec()) {
246 throw HandlerException("Unable to retrieve item tags");
247 }
248
249 tagQuery.query().next();
250
251 return tagQuery;
252}
253
254enum VRefQueryColumns {
255 VRefQueryCollectionIdColumn,
256 VRefQueryItemIdColumn,
257};
258
259QueryBuilder ItemFetchHelper::buildVRefQuery(QSqlQuery &itemQuery)
260{
261 QueryBuilder vRefQuery(PimItem::tableName());
262 if (mItemsLimit.limit() > 0) {
263 vRefQuery = QueryBuilder(itemQuery, mPimItemQueryAlias);
264 }
265
266 vRefQuery.addJoin(QueryBuilder::LeftJoin,
267 CollectionPimItemRelation::tableName(),
268 CollectionPimItemRelation::rightFullColumnName(),
269 vRefQuery.getTableWithColumn(PimItem::idColumn()));
270 vRefQuery.addColumn(CollectionPimItemRelation::leftFullColumnName());
271 vRefQuery.addColumn(CollectionPimItemRelation::rightFullColumnName());
272 ItemQueryHelper::scopeToQuery(mScope, mContext, vRefQuery);
273 vRefQuery.addSortColumn(vRefQuery.getTableWithColumn(PimItem::idColumn()), Query::Descending);
274
275 if (!vRefQuery.exec()) {
276 throw HandlerException("Unable to retrieve virtual references");
277 }
278
279 vRefQuery.query().next();
280
281 return vRefQuery;
282}
283
284bool ItemFetchHelper::isScopeLocal(const Scope &scope)
285{
286 // The only agent allowed to override local scope is the Baloo Indexer
287 if (!mConnection->sessionId().startsWith("akonadi_indexing_agent")) {
288 return false;
289 }
290
291 // Get list of all resources that own all items in the scope
292 QueryBuilder qb(PimItem::tableName(), QueryBuilder::Select);
293 qb.setDistinct(true);
294 qb.addColumn(Resource::nameFullColumnName());
295 qb.addJoin(QueryBuilder::LeftJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
296 qb.addJoin(QueryBuilder::LeftJoin, Resource::tableName(), Collection::resourceIdFullColumnName(), Resource::idFullColumnName());
297 ItemQueryHelper::scopeToQuery(scope, mContext, qb);
298 if (mContext.resource().isValid()) {
299 qb.addValueCondition(Resource::nameFullColumnName(), Query::NotEquals, mContext.resource().name());
300 }
301
302 if (!qb.exec()) {
303 throw HandlerException("Failed to query database");
304 return false;
305 }
306
307 // If there is more than one resource, i.e. this is a fetch from multiple
308 // collections, then don't bother and just return FALSE. This case is aimed
309 // specifically on Baloo, which fetches items from each collection independently,
310 // so it will pass this check.
311 QSqlQuery &query = qb.query();
312 if (query.size() != 1) {
313 return false;
314 }
315
316 query.next();
317 const QString resourceName = query.value(0).toString();
318 query.finish();
319
320 org::freedesktop::Akonadi::AgentManager manager(DBus::serviceName(DBus::Control), QStringLiteral("/AgentManager"), QDBusConnection::sessionBus());
321 const QString typeIdentifier = manager.agentInstanceType(resourceName);
322 const QVariantMap properties = manager.agentCustomProperties(typeIdentifier);
323 return properties.value(QStringLiteral("HasLocalStorage"), false).toBool();
324}
325
326DataStore *ItemFetchHelper::storageBackend() const
327{
328 if (mConnection) {
329 if (auto store = mConnection->storageBackend()) {
330 return store;
331 }
332 }
333
334 return DataStore::self();
335}
336
337bool ItemFetchHelper::fetchItems(std::function<void(Protocol::FetchItemsResponse &&)> &&itemCallback)
338{
339 BEGIN_TIMER(fetch)
340
341 // retrieve missing parts
342 // HACK: isScopeLocal() is a workaround for resources that have cache expiration
343 // because when the cache expires, Baloo is not able to content of the items. So
344 // we allow fetch of items that belong to local resources (like maildir) to ignore
345 // cacheOnly and retrieve missing parts from the resource. However ItemRetriever
346 // is painfully slow with many items and is generally designed to fetch a few
347 // messages, not all of them. In the long term, we need a better way to do this.
348 BEGIN_TIMER(itemRetriever)
349 BEGIN_TIMER(scopeLocal)
350#if ENABLE_FETCH_PROFILING
351 double scopeLocalElapsed = 0;
352#endif
353 if (!mItemFetchScope.cacheOnly() || isScopeLocal(mScope)) {
354#if ENABLE_FETCH_PROFILING
355 scopeLocalElapsed = scopeLocalTimer.elapsed();
356#endif
357
358 // trigger a collection sync if configured to do so
359 triggerOnDemandFetch();
360
361 // Prepare for a call to ItemRetriever::exec();
362 // From a resource perspective the only parts that can be fetched are payloads.
363 ItemRetriever retriever(mAkonadi.itemRetrievalManager(), mConnection, mContext);
364 retriever.setScope(mScope);
365 retriever.setRetrieveParts(mItemFetchScope.requestedPayloads());
366 retriever.setRetrieveFullPayload(mItemFetchScope.fullPayload());
367 retriever.setChangedSince(mItemFetchScope.changedSince());
368 if (!retriever.exec() && !mItemFetchScope.ignoreErrors()) { // There we go, retrieve the missing parts from the resource.
369 if (mContext.resource().isValid()) {
370 throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1, resource %2) : %3")
371 .arg(mContext.collectionId())
372 .arg(mContext.resource().id())
373 .arg(QString::fromLatin1(retriever.lastError())));
374 } else {
375 throw HandlerException(QStringLiteral("Unable to fetch item from backend (collection %1) : %2")
376 .arg(mContext.collectionId())
377 .arg(QString::fromLatin1(retriever.lastError())));
378 }
379 }
380 }
381 END_TIMER(itemRetriever)
382
383 BEGIN_TIMER(items)
384 std::optional<QueryBuilder> itemQb = buildItemQuery();
385 auto &itemQuery = itemQb->query();
386 END_TIMER(items)
387
388 // error if query did not find any item and scope is not listing items but
389 // a request for a specific item
390 if (!itemQuery.isValid()) {
391 if (mItemFetchScope.ignoreErrors()) {
392 return true;
393 }
394 switch (mScope.scope()) {
395 case Scope::Uid: // fall through
396 case Scope::Rid: // fall through
397 case Scope::HierarchicalRid: // fall through
398 case Scope::Gid:
399 throw HandlerException("Item query returned empty result set");
400 break;
401 default:
402 break;
403 }
404 }
405 // build part query if needed
406 BEGIN_TIMER(parts)
407 std::optional<QueryBuilder> partQb;
408 if (!mItemFetchScope.requestedParts().isEmpty() || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
409 partQb = buildPartQuery(itemQuery, mItemFetchScope.requestedParts(), mItemFetchScope.fullPayload(), mItemFetchScope.allAttributes());
410 }
411 END_TIMER(parts)
412
413 // build flag query if needed
414 BEGIN_TIMER(flags)
415 std::optional<QueryBuilder> flagQb;
416 if (mItemFetchScope.fetchFlags()) {
417 flagQb = buildFlagQuery(itemQuery);
418 }
419 END_TIMER(flags)
420
421 // build tag query if needed
422 BEGIN_TIMER(tags)
423 std::optional<QueryBuilder> tagQb;
424 if (mItemFetchScope.fetchTags()) {
425 tagQb = buildTagQuery(itemQuery);
426 }
427 END_TIMER(tags)
428
429 BEGIN_TIMER(vRefs)
430 std::optional<QueryBuilder> vRefQb;
431 if (mItemFetchScope.fetchVirtualReferences()) {
432 vRefQb = buildVRefQuery(itemQuery);
433 }
434 END_TIMER(vRefs)
435
436#if ENABLE_FETCH_PROFILING
437 int itemsCount = 0;
438 int flagsCount = 0;
439 int partsCount = 0;
440 int tagsCount = 0;
441 int vRefsCount = 0;
442#endif
443
444 BEGIN_TIMER(processing)
445 QHash<qint64, QByteArray> flagIdNameCache;
446 QHash<qint64, QString> mimeTypeIdNameCache;
447 QHash<qint64, QByteArray> partTypeIdNameCache;
448 while (itemQuery.isValid()) {
449 PROF_INC(itemsCount)
450
451 const qint64 pimItemId = extractQueryResult(itemQuery, ItemQueryPimItemIdColumn).toLongLong();
452 const int pimItemRev = extractQueryResult(itemQuery, ItemQueryRevColumn).toInt();
453
454 Protocol::FetchItemsResponse response;
455 response.setId(pimItemId);
456 response.setRevision(pimItemRev);
457 const qint64 mimeTypeId = extractQueryResult(itemQuery, ItemQueryMimeTypeIdColumn).toLongLong();
458 auto mtIter = mimeTypeIdNameCache.find(mimeTypeId);
459 if (mtIter == mimeTypeIdNameCache.end()) {
460 mtIter = mimeTypeIdNameCache.insert(mimeTypeId, MimeType::retrieveById(mimeTypeId).name());
461 }
462 response.setMimeType(mtIter.value());
463 if (mItemFetchScope.fetchRemoteId()) {
464 response.setRemoteId(extractQueryResult(itemQuery, ItemQueryPimItemRidColumn).toString());
465 }
466 response.setParentId(extractQueryResult(itemQuery, ItemQueryCollectionIdColumn).toLongLong());
467
468 if (mItemFetchScope.fetchSize()) {
469 response.setSize(extractQueryResult(itemQuery, ItemQuerySizeColumn).toLongLong());
470 }
471 if (mItemFetchScope.fetchMTime()) {
472 response.setMTime(extractQueryResult(itemQuery, ItemQueryDatetimeColumn).toDateTime());
473 }
474 if (mItemFetchScope.fetchRemoteRevision()) {
475 response.setRemoteRevision(extractQueryResult(itemQuery, ItemQueryRemoteRevisionColumn).toString());
476 }
477 if (mItemFetchScope.fetchGID()) {
478 response.setGid(extractQueryResult(itemQuery, ItemQueryPimItemGidColumn).toString());
479 }
480
481 if (mItemFetchScope.fetchFlags() && flagQb) {
482 QList<QByteArray> flags;
483 auto &flagQuery = flagQb->query();
484 while (flagQuery.isValid()) {
485 const qint64 id = flagQuery.value(FlagQueryPimItemIdColumn).toLongLong();
486 if (id > pimItemId) {
487 flagQuery.next();
488 continue;
489 } else if (id < pimItemId) {
490 break;
491 }
492 const qint64 flagId = flagQuery.value(FlagQueryFlagIdColumn).toLongLong();
493 auto flagNameIter = flagIdNameCache.find(flagId);
494 if (flagNameIter == flagIdNameCache.end()) {
495 flagNameIter = flagIdNameCache.insert(flagId, Flag::retrieveById(flagId).name().toUtf8());
496 }
497 flags << flagNameIter.value();
498 flagQuery.next();
499 }
500 response.setFlags(flags);
501 }
502
503 if (mItemFetchScope.fetchTags() && tagQb) {
504 QList<qint64> tagIds;
505 QList<Protocol::FetchTagsResponse> tags;
506 auto &tagQuery = tagQb->query();
507 while (tagQuery.isValid()) {
508 PROF_INC(tagsCount)
509 const qint64 id = tagQuery.value(TagQueryItemIdColumn).toLongLong();
510 if (id > pimItemId) {
511 tagQuery.next();
512 continue;
513 } else if (id < pimItemId) {
514 break;
515 }
516 tagIds << tagQuery.value(TagQueryTagIdColumn).toLongLong();
517 tagQuery.next();
518 }
519
520 if (mTagFetchScope.fetchIdOnly()) {
521 tags = tagIds | Views::transform([](const auto tagId) {
522 Protocol::FetchTagsResponse resp;
523 resp.setId(tagId);
524 return resp;
525 })
526 | Actions::toQVector;
527 } else {
528 tags = tagIds | Views::transform([this](const auto tagId) {
529 return HandlerHelper::fetchTagsResponse(Tag::retrieveById(tagId), mTagFetchScope, mConnection);
530 })
531 | Actions::toQVector;
532 }
533 response.setTags(tags);
534 }
535
536 if (mItemFetchScope.fetchVirtualReferences() && vRefQb) {
537 QList<qint64> vRefs;
538 auto &vRefQuery = vRefQb->query();
539 while (vRefQuery.isValid()) {
540 PROF_INC(vRefsCount)
541 const qint64 id = vRefQuery.value(VRefQueryItemIdColumn).toLongLong();
542 if (id > pimItemId) {
543 vRefQuery.next();
544 continue;
545 } else if (id < pimItemId) {
546 break;
547 }
548 vRefs << vRefQuery.value(VRefQueryCollectionIdColumn).toLongLong();
549 vRefQuery.next();
550 }
551 response.setVirtualReferences(vRefs);
552 }
553
554 if (mItemFetchScope.ancestorDepth() != Protocol::ItemFetchScope::NoAncestor) {
555 response.setAncestors(ancestorsForItem(response.parentId()));
556 }
557
558 bool skipItem = false;
559
560 QList<QByteArray> cachedParts;
561 if (partQb) {
562 QList<Protocol::StreamPayloadResponse> parts;
563 auto &partQuery = partQb->query();
564 while (partQuery.isValid()) {
565 PROF_INC(partsCount)
566 const qint64 id = partQuery.value(PartQueryPimIdColumn).toLongLong();
567 if (id > pimItemId) {
568 partQuery.next();
569 continue;
570 } else if (id < pimItemId) {
571 break;
572 }
573
574 const qint64 partTypeId = partQuery.value(PartQueryTypeIdColumn).toLongLong();
575 auto ptIter = partTypeIdNameCache.find(partTypeId);
576 if (ptIter == partTypeIdNameCache.end()) {
577 ptIter = partTypeIdNameCache.insert(partTypeId, PartTypeHelper::fullName(PartType::retrieveById(partTypeId)).toUtf8());
578 }
579 Protocol::PartMetaData metaPart;
580 Protocol::StreamPayloadResponse partData;
581 partData.setPayloadName(ptIter.value());
582 metaPart.setName(ptIter.value());
583 metaPart.setVersion(partQuery.value(PartQueryVersionColumn).toInt());
584 metaPart.setSize(partQuery.value(PartQueryDataSizeColumn).toLongLong());
585
586 const QByteArray data = Utils::variantToByteArray(partQuery.value(PartQueryDataColumn));
587 if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
588 if (!data.isEmpty()) {
589 cachedParts << ptIter.value();
590 }
591 partQuery.next();
592 } else {
593 if (mItemFetchScope.ignoreErrors() && data.isEmpty()) {
594 // We wanted the payload, couldn't get it, and are ignoring errors. Skip the item.
595 // This is not an error though, it's fine to have empty payload parts (to denote existing but not cached parts)
596 qCDebug(AKONADISERVER_LOG) << "item" << id << "has an empty payload part in parttable for part" << metaPart.name();
597 skipItem = true;
598 break;
599 }
600 metaPart.setStorageType(static_cast<Protocol::PartMetaData::StorageType>(partQuery.value(PartQueryStorageColumn).toInt()));
601 if (data.isEmpty()) {
602 partData.setData(QByteArray(""));
603 } else {
604 partData.setData(data);
605 }
606 partData.setMetaData(metaPart);
607
608 if (mItemFetchScope.requestedParts().contains(ptIter.value()) || mItemFetchScope.fullPayload() || mItemFetchScope.allAttributes()) {
609 parts.append(partData);
610 }
611
612 partQuery.next();
613 }
614 }
615 response.setParts(parts);
616 }
617
618 if (skipItem) {
619 itemQuery.next();
620 continue;
621 }
622
623 if (mItemFetchScope.checkCachedPayloadPartsOnly()) {
624 response.setCachedParts(cachedParts);
625 }
626
627 if (itemCallback) {
628 itemCallback(std::move(response));
629 } else {
630 mConnection->sendResponse(std::move(response));
631 }
632
633 itemQuery.next();
634 }
635 // Destroy the query builders in order to finalize and cache the prepared statements
636 // before doing any more SQL queries.
637 tagQb.reset();
638 flagQb.reset();
639 partQb.reset();
640 vRefQb.reset();
641 itemQb.reset();
642 END_TIMER(processing)
643
644 // update atime (only if the payload was actually requested, otherwise a simple resource sync prevents cache clearing)
645 BEGIN_TIMER(aTime)
646 if (mUpdateATimeEnabled && (needsAccessTimeUpdate(mItemFetchScope.requestedParts()) || mItemFetchScope.fullPayload())) {
647 updateItemAccessTime();
648 }
649 END_TIMER(aTime)
650
651 END_TIMER(fetch)
652#if ENABLE_FETCH_PROFILING
653 qCDebug(AKONADISERVER_LOG) << "ItemFetchHelper execution stats:";
654 qCDebug(AKONADISERVER_LOG) << "\tItems query:" << itemsElapsed << "ms," << itemsCount << " items in total";
655 qCDebug(AKONADISERVER_LOG) << "\tFlags query:" << flagsElapsed << "ms, " << flagsCount << " flags in total";
656 qCDebug(AKONADISERVER_LOG) << "\tParts query:" << partsElapsed << "ms, " << partsCount << " parts in total";
657 qCDebug(AKONADISERVER_LOG) << "\tTags query: " << tagsElapsed << "ms, " << tagsCount << " tags in total";
658 qCDebug(AKONADISERVER_LOG) << "\tVRefs query:" << vRefsElapsed << "ms, " << vRefsCount << " vRefs in total";
659 qCDebug(AKONADISERVER_LOG) << "\t------------";
660 qCDebug(AKONADISERVER_LOG) << "\tItem retriever:" << itemRetrieverElapsed << "ms (scope local:" << scopeLocalElapsed << "ms)";
661 qCDebug(AKONADISERVER_LOG) << "\tTotal query:" << (itemsElapsed + flagsElapsed + partsElapsed + tagsElapsed + vRefsElapsed) << "ms";
662 qCDebug(AKONADISERVER_LOG) << "\tTotal processing: " << processingElapsed << "ms";
663 qCDebug(AKONADISERVER_LOG) << "\tATime update:" << aTimeElapsed << "ms";
664 qCDebug(AKONADISERVER_LOG) << "\t============";
665 qCDebug(AKONADISERVER_LOG) << "\tTotal FETCH:" << fetchElapsed << "ms";
666 qCDebug(AKONADISERVER_LOG);
667 qCDebug(AKONADISERVER_LOG);
668#endif
669
670 return true;
671}
672
673bool ItemFetchHelper::needsAccessTimeUpdate(const QList<QByteArray> &parts)
674{
675 // TODO technically we should compare the part list with the cache policy of
676 // the parent collection of the retrieved items, but that's kinda expensive
677 // Only updating the atime if the full payload was requested is a good
678 // approximation though.
679 return parts.contains(AKONADI_PARAM_PLD_RFC822);
680}
681
682void ItemFetchHelper::updateItemAccessTime()
683{
684 Transaction transaction(storageBackend(), QStringLiteral("update atime"));
685 QueryBuilder qb(PimItem::tableName(), QueryBuilder::Update);
686 qb.setColumnValue(PimItem::atimeColumn(), QDateTime::currentDateTimeUtc());
687 ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
688
689 if (!qb.exec()) {
690 qCWarning(AKONADISERVER_LOG) << "Unable to update item access time";
691 } else {
692 transaction.commit();
693 }
694}
695
696void ItemFetchHelper::triggerOnDemandFetch()
697{
698 if (mContext.collectionId() <= 0 || mItemFetchScope.cacheOnly()) {
699 return;
700 }
701
702 Collection collection = mContext.collection();
703
704 // HACK: don't trigger on-demand syncing if the resource is the one triggering it
705 if (mConnection->sessionId() == collection.resource().name().toLatin1()) {
706 return;
707 }
708
709 storageBackend()->activeCachePolicy(collection);
710 if (!collection.cachePolicySyncOnDemand()) {
711 return;
712 }
713
714 mConnection->akonadi().intervalChecker().requestCollectionSync(collection);
715}
716
717QList<Protocol::Ancestor> ItemFetchHelper::ancestorsForItem(Collection::Id parentColId)
718{
719 if (mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::NoAncestor || parentColId == 0) {
720 return QList<Protocol::Ancestor>();
721 }
722 const auto it = mAncestorCache.constFind(parentColId);
723 if (it != mAncestorCache.cend()) {
724 return *it;
725 }
726
727 QList<Protocol::Ancestor> ancestors;
728 Collection col = Collection::retrieveById(parentColId);
729 const int depthNum = mItemFetchScope.ancestorDepth() == Protocol::ItemFetchScope::ParentAncestor ? 1 : INT_MAX;
730 for (int i = 0; i < depthNum; ++i) {
731 if (!col.isValid()) {
732 Protocol::Ancestor ancestor;
733 ancestor.setId(0);
734 ancestors << ancestor;
735 break;
736 }
737 Protocol::Ancestor ancestor;
738 ancestor.setId(col.id());
739 ancestor.setRemoteId(col.remoteId());
740 ancestors << ancestor;
741 col = col.parent();
742 }
743 mAncestorCache.insert(parentColId, ancestors);
744 return ancestors;
745}
746
747QVariant ItemFetchHelper::extractQueryResult(const QSqlQuery &query, ItemFetchHelper::ItemQueryColumns column) const
748{
749 const int colId = mItemQueryColumnMap[column];
750 Q_ASSERT(colId >= 0);
751 return query.value(colId);
752}
Akonadi::Collection
Represents a collection of PIM items.
Definition collection.h:62
Akonadi::Collection::Id
qint64 Id
Describes the unique id type.
Definition collection.h:79
Akonadi::Collection::remoteId
QString remoteId() const
Returns the remote id of the collection.
Definition collection.cpp:106
Akonadi::Server::Connection
An Connection represents one connection of a client to the server.
Definition connection.h:39
Akonadi::Server::DataStore
This class handles all the database access.
Definition datastore.h:95
Akonadi::Server::DataStore::self
static DataStore * self()
Per thread singleton.
Definition datastore.cpp:248
Akonadi::Server::DataStore::activeCachePolicy
virtual void activeCachePolicy(Collection &col)
Determines the active cache policy for this Collection.
Definition datastore.cpp:985
Akonadi::Server::IntervalCheck::requestCollectionSync
void requestCollectionSync(const Collection &collection)
Requests the given collection to be synced.
Definition intervalcheck.cpp:29
Akonadi::Server::ItemRetriever
Helper class for retrieving missing items parts from remote resources.
Definition itemretriever.h:39
Akonadi::Server::QueryBuilder
Helper class to construct arbitrary SQL queries.
Definition querybuilder.h:33
Akonadi::Server::QueryBuilder::addSortColumn
void addSortColumn(const QString &column, Query::SortOrder order=Query::Ascending)
Add sort column.
Definition querybuilder.cpp:766
Akonadi::Server::QueryBuilder::getTableWithColumn
QString getTableWithColumn(const QString &column) const
Returns concatenated table name with column name.
Definition querybuilder.cpp:846
Akonadi::Server::QueryBuilder::addJoin
void addJoin(JoinType joinType, const QString &table, const Query::Condition &condition)
Join a table to the query.
Definition querybuilder.cpp:236
Akonadi::Server::QueryBuilder::exec
bool exec()
Executes the query, returns true on success.
Definition querybuilder.cpp:508
Akonadi::Server::QueryBuilder::query
QSqlQuery & query()
Returns the query, only valid after exec().
Definition querybuilder.h:291
Akonadi::Server::QueryBuilder::addColumn
void addColumn(const QString &col)
Adds the given column to a select query.
Definition querybuilder.cpp:624
Akonadi::Server::QueryBuilder::InnerJoin
@ InnerJoin
NOTE: only supported for UPDATE and SELECT queries.
Definition querybuilder.h:51
Akonadi::Server::QueryBuilder::LeftJoin
@ LeftJoin
NOTE: only supported for SELECT queries.
Definition querybuilder.h:53
Akonadi::Server::Transaction
Helper class for DataStore transaction handling.
Definition transaction.h:23
Resource::name
QString name() const
Akonadi::Server::ItemQueryHelper::scopeToQuery
void scopeToQuery(const Scope &scope, const CommandContext &context, QueryBuilder &qb)
Add conditions to qb for the given item operation scope scope.
Definition itemqueryhelper.cpp:109
Akonadi::Server::PartTypeHelper::fullName
QString fullName(const PartType &type)
Returns full part name.
Definition parttypehelper.cpp:66
Akonadi::Server::Query::SortOrder
SortOrder
Sort orders.
Definition query.h:53
Akonadi
Helper integration between Akonadi and Qt.
Baloo::toString
char * toString(const EngineQuery &query)
KApplicationTrader::query
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
KStandardAction::name
QString name(StandardAction id)
KStandardGuiItem::properties
KGuiItem properties()
QByteArray::isEmpty
bool isEmpty() const const
QByteArray::startsWith
bool startsWith(QByteArrayView bv) const const
QDateTime::currentDateTimeUtc
QDateTime currentDateTimeUtc()
QDBusConnection::sessionBus
QDBusConnection sessionBus()
QHash::cend
const_iterator cend() const const
QHash::constFind
const_iterator constFind(const Key &key) const const
QHash::end
iterator end()
QHash::find
iterator find(const Key &key)
QHash::insert
iterator insert(const Key &key, const T &value)
QList::append
void append(QList< T > &&value)
QList::contains
bool contains(const AT &value) const const
QList::isEmpty
bool isEmpty() const const
QList::size
qsizetype size() const const
QList::value
T value(qsizetype i) const const
QSqlQuery::exec
bool exec()
QSqlQuery::isValid
bool isValid() const const
QSqlQuery::next
bool next()
QString::fromLatin1
QString fromLatin1(QByteArrayView str)
QString::toLatin1
QByteArray toLatin1() const const
QVariant::toInt
int toInt(bool *ok) const const
QVariant::toLongLong
qlonglong toLongLong(bool *ok) const const
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Fri Oct 11 2024 12:11:39 by doxygen 1.12.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.