Akonadi

itemretriever.cpp
1/*
2 SPDX-FileCopyrightText: 2009 Volker Krause <vkrause@kde.org>
3 SPDX-FileCopyrightText: 2010 Milian Wolff <mail@milianw.de>
4
5 SPDX-License-Identifier: LGPL-2.0-or-later
6*/
7
8#include "itemretriever.h"
9
10#include "connection.h"
11#include "storage/itemqueryhelper.h"
12#include "storage/itemretrievalmanager.h"
13#include "storage/itemretrievalrequest.h"
14#include "storage/parthelper.h"
15#include "storage/parttypehelper.h"
16#include "storage/querybuilder.h"
17#include "storage/selectquerybuilder.h"
18#include "utils.h"
19
20#include "private/protocol_p.h"
21#include "shared/akranges.h"
22
23#include <QEventLoop>
24
25#include "akonadiserver_debug.h"
26
27using namespace Akonadi;
28using namespace Akonadi::Server;
29using namespace AkRanges;
30
31Q_DECLARE_METATYPE(ItemRetrievalResult)
32
33ItemRetriever::ItemRetriever(ItemRetrievalManager &manager, Connection *connection, const CommandContext &context)
34 : mItemRetrievalManager(manager)
35 , mConnection(connection)
36 , mContext(context)
37 , mFullPayload(false)
38 , mRecursive(false)
39 , mCanceled(false)
40{
41 qRegisterMetaType<ItemRetrievalResult>("Akonadi::Server::ItemRetrievalResult");
42 if (mConnection) {
43 connect(mConnection, &Connection::disconnected, this, [this]() {
44 mCanceled = true;
45 });
46 }
47}
48
49Connection *ItemRetriever::connection() const
50{
51 return mConnection;
52}
53
54void ItemRetriever::setRetrieveParts(const QList<QByteArray> &parts)
55{
56 mParts = parts;
57 std::sort(mParts.begin(), mParts.end());
58 mParts.erase(std::unique(mParts.begin(), mParts.end()), mParts.end());
59
60 // HACK, we need a full payload available flag in PimItem
61 if (mFullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
63 }
64}
65
66void ItemRetriever::setItemSet(const QList<PimItem::Id> &set, const Collection &collection)
67{
68 mItemSet = set;
69 mCollection = collection;
70}
71
72void ItemRetriever::setItem(PimItem::Id id)
73{
74 setItemSet({id});
75}
76
77void ItemRetriever::setRetrieveFullPayload(bool fullPayload)
78{
79 mFullPayload = fullPayload;
80 // HACK, we need a full payload available flag in PimItem
81 if (fullPayload && !mParts.contains(AKONADI_PARAM_PLD_RFC822)) {
83 }
84}
85
86void ItemRetriever::setCollection(const Collection &collection, bool recursive)
87{
88 mCollection = collection;
89 mItemSet.clear();
90 mRecursive = recursive;
91}
92
93void ItemRetriever::setScope(const Scope &scope)
94{
95 mScope = scope;
96}
97
98Scope ItemRetriever::scope() const
99{
100 return mScope;
101}
102
103void ItemRetriever::setChangedSince(const QDateTime &changedSince)
104{
105 mChangedSince = changedSince;
106}
107
108QList<QByteArray> ItemRetriever::retrieveParts() const
109{
110 return mParts;
111}
112
113enum QueryColumns {
114 PimItemIdColumn,
115
116 CollectionIdColumn,
117 ResourceIdColumn,
118
119 PartTypeNameColumn,
120 PartDatasizeColumn
121};
122
123QSqlQuery ItemRetriever::buildQuery() const
124{
125 QueryBuilder qb(PimItem::tableName());
126
127 qb.addJoin(QueryBuilder::InnerJoin, Collection::tableName(), PimItem::collectionIdFullColumnName(), Collection::idFullColumnName());
128
129 qb.addJoin(QueryBuilder::LeftJoin, Part::tableName(), PimItem::idFullColumnName(), Part::pimItemIdFullColumnName());
130
132 partTypeJoinCondition.addColumnCondition(Part::partTypeIdFullColumnName(), Query::Equals, PartType::idFullColumnName());
133 if (!mFullPayload && !mParts.isEmpty()) {
135 }
136 partTypeJoinCondition.addValueCondition(PartType::nsFullColumnName(), Query::Equals, QStringLiteral("PLD"));
137 qb.addJoin(QueryBuilder::LeftJoin, PartType::tableName(), partTypeJoinCondition);
138
139 qb.addColumn(PimItem::idFullColumnName());
140 qb.addColumn(PimItem::collectionIdFullColumnName());
141 qb.addColumn(Collection::resourceIdFullColumnName());
142 qb.addColumn(PartType::nameFullColumnName());
143 qb.addColumn(Part::datasizeFullColumnName());
144
145 if (!mItemSet.isEmpty() || mCollection.isValid()) {
146 ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
147 } else {
148 ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
149 }
150
151 // prevent a resource to trigger item retrieval from itself
152 if (mConnection) {
153 const Resource res = Resource::retrieveByName(QString::fromUtf8(mConnection->sessionId()));
154 if (res.isValid()) {
155 qb.addValueCondition(Collection::resourceIdFullColumnName(), Query::NotEquals, res.id());
156 }
157 }
158
159 if (mChangedSince.isValid()) {
160 qb.addValueCondition(PimItem::datetimeFullColumnName(), Query::GreaterOrEqual, mChangedSince.toUTC());
161 }
162
163 qb.addSortColumn(PimItem::idFullColumnName(), Query::Ascending);
164
165 if (!qb.exec()) {
166 mLastError = "Unable to retrieve items";
167 throw ItemRetrieverException(mLastError);
168 }
169
170 qb.query().next();
171
172 return qb.query();
173}
174
175namespace
176{
177bool hasAllParts(const ItemRetrievalRequest &req, const QSet<QByteArray> &availableParts)
178{
179 return std::all_of(req.parts.begin(), req.parts.end(), [&availableParts](const auto &part) {
180 return availableParts.contains(part);
181 });
182}
183}
184
185bool ItemRetriever::runItemRetrievalRequests(std::list<ItemRetrievalRequest> requests) // clazy:exclude=function-args-by-ref
186{
188 std::vector<ItemRetrievalRequest::Id> pendingRequests;
189 connect(&mItemRetrievalManager,
190 &ItemRetrievalManager::requestFinished,
191 &eventLoop,
192 [this, &eventLoop, &pendingRequests](const ItemRetrievalResult &result) { // clazy:exclude=lambda-in-connect
193 const auto requestId = std::find(pendingRequests.begin(), pendingRequests.end(), result.request.id);
194 if (requestId != pendingRequests.end()) {
195 if (mCanceled) {
196 eventLoop.exit(1);
197 } else if (result.errorMsg.has_value()) {
198 mLastError = result.errorMsg->toUtf8();
199 eventLoop.exit(1);
200 } else {
201 Q_EMIT itemsRetrieved(result.request.ids);
203 if (pendingRequests.empty()) {
204 eventLoop.quit();
205 }
206 }
207 }
208 });
209
210 if (mConnection) {
211 connect(mConnection, &Connection::connectionClosing, &eventLoop, [&eventLoop]() {
212 eventLoop.exit(1);
213 });
214 }
215
216 for (auto &&request : requests) {
217 if ((!mFullPayload && request.parts.isEmpty()) || request.ids.isEmpty()) {
218 continue;
219 }
220
221 // TODO: how should we handle retrieval errors here? so far they have been ignored,
222 // which makes sense in some cases, do we need a command parameter for this?
223 try {
224 // Request is deleted inside ItemRetrievalManager, so we need to take
225 // a copy here
226 // const auto ids = request->ids;
227 pendingRequests.push_back(request.id);
228 mItemRetrievalManager.requestItemDelivery(std::move(request));
229 } catch (const ItemRetrieverException &e) {
230 qCCritical(AKONADISERVER_LOG) << e.type() << ": " << e.what();
231 mLastError = e.what();
232 return false;
233 }
234 }
235
236 if (!pendingRequests.empty()) {
237 if (eventLoop.exec()) {
238 return false;
239 }
240 }
241
242 return true;
243}
244
245std::optional<ItemRetriever::PreparedRequests> ItemRetriever::prepareRequests(QSqlQuery &query, const QByteArrayList &parts)
246{
248 std::list<ItemRetrievalRequest> requests;
249 QHash<qint64 /* collection */, decltype(requests)::iterator> colRequests;
250 QHash<qint64 /* item */, decltype(requests)::iterator> itemRequests;
251 QList<qint64> readyItems;
252 qint64 prevPimItemId = -1;
253 QSet<QByteArray> availableParts;
254 auto lastRequest = requests.end();
255 while (query.isValid()) {
256 const qint64 pimItemId = query.value(PimItemIdColumn).toLongLong();
257 const qint64 collectionId = query.value(CollectionIdColumn).toLongLong();
258 const qint64 resourceId = query.value(ResourceIdColumn).toLongLong();
259 const auto itemIter = itemRequests.constFind(pimItemId);
260
261 if (Q_UNLIKELY(mCanceled)) {
262 return std::nullopt;
263 }
264
265 if (pimItemId == prevPimItemId) {
266 if (query.value(PartTypeNameColumn).isNull()) {
267 // This is not the first part of the Item we saw, but LEFT JOIN PartTable
268 // returned a null row - that means the row is an ATR part
269 // which we don't care about
270 query.next();
271 continue;
272 }
273 } else {
274 if (lastRequest != requests.end()) {
275 if (hasAllParts(*lastRequest, availableParts)) {
276 // We went through all parts of a single item, if we have all
277 // parts available in the DB and they are not expired, then
278 // exclude this item from the retrieval
279 lastRequest->ids.removeOne(prevPimItemId);
281 readyItems.push_back(prevPimItemId);
282 }
283 }
284 availableParts.clear();
286 }
287
288 if (itemIter != itemRequests.constEnd()) {
289 lastRequest = itemIter.value();
290 } else {
291 const auto colIt = colRequests.find(collectionId);
292 lastRequest = (colIt == colRequests.end()) ? requests.end() : colIt.value();
293 if (lastRequest == requests.end() || lastRequest->ids.size() > 100) {
294 requests.emplace_front(ItemRetrievalRequest{});
295 lastRequest = requests.begin();
296 lastRequest->ids.push_back(pimItemId);
297 auto resIter = resourceIdNameCache.find(resourceId);
298 if (resIter == resourceIdNameCache.end()) {
299 resIter = resourceIdNameCache.insert(resourceId, Resource::retrieveById(resourceId).name());
300 }
301 lastRequest->resourceId = *resIter;
302 lastRequest->parts = parts;
303 colRequests.insert(collectionId, lastRequest);
305 } else {
306 lastRequest->ids.push_back(pimItemId);
308 colRequests.insert(collectionId, lastRequest);
309 }
310 }
311 Q_ASSERT(lastRequest != requests.end());
312
313 if (query.value(PartTypeNameColumn).isNull()) {
314 // LEFT JOIN did not find anything, retrieve all parts
315 query.next();
316 continue;
317 }
318
319 qint64 datasize = query.value(PartDatasizeColumn).toLongLong();
320 const QByteArray partName = Utils::variantToByteArray(query.value(PartTypeNameColumn));
322 if (datasize <= 0) {
323 // request update for this part
324 if (mFullPayload && !lastRequest->parts.contains(partName)) {
325 lastRequest->parts.push_back(partName);
326 }
327 } else {
328 // add the part to list of available parts, we will compare it with
329 // the list of request parts once we handle all parts of this item
330 availableParts.insert(partName);
331 }
332 query.next();
333 }
334 query.finish();
335
336 // Post-check in case we only queried one item thus did not reach the check
337 // at the beginning of the while() loop above
338 if (lastRequest != requests.end() && hasAllParts(*lastRequest, availableParts)) {
339 lastRequest->ids.removeOne(prevPimItemId);
340 readyItems.push_back(prevPimItemId);
341 // No need to update the hashtable at this point
342 }
343
344 return PreparedRequests{std::move(requests), std::move(readyItems)};
345}
346
347bool ItemRetriever::exec()
348{
349 if (mParts.isEmpty() && !mFullPayload) {
350 return true;
351 }
352
353 verifyCache();
354
355 QSqlQuery query = buildQuery();
356 const auto parts = mParts | Views::filter([](const auto &part) {
357 return part.startsWith(AKONADI_PARAM_PLD);
358 })
359 | Views::transform([](const auto &part) {
360 return part.mid(4);
361 })
362 | Actions::toQList;
363
364 auto requests = prepareRequests(query, parts);
365 if (!requests.has_value()) {
366 return false;
367 }
368
369 if (!requests->readyItems.isEmpty()) {
370 Q_EMIT itemsRetrieved(requests->readyItems);
371 }
372
373 if (!runItemRetrievalRequests(std::move(requests->requests))) {
374 return false;
375 }
376
377 // retrieve items in child collections if requested
378 bool result = true;
379 if (mRecursive && mCollection.isValid()) {
380 const auto children = mCollection.children();
381 for (const Collection &col : children) {
382 ItemRetriever retriever(mItemRetrievalManager, mConnection, mContext);
383 retriever.setCollection(col, mRecursive);
384 retriever.setRetrieveParts(mParts);
385 retriever.setRetrieveFullPayload(mFullPayload);
386 connect(&retriever, &ItemRetriever::itemsRetrieved, this, &ItemRetriever::itemsRetrieved);
387 result = retriever.exec();
388 if (!result) {
389 break;
390 }
391 }
392 }
393
394 return result;
395}
396
397void ItemRetriever::verifyCache()
398{
399 if (!connection() || !connection()->verifyCacheOnRetrieval()) {
400 return;
401 }
402
404 qb.addJoin(QueryBuilder::InnerJoin, PimItem::tableName(), Part::pimItemIdFullColumnName(), PimItem::idFullColumnName());
405 qb.addValueCondition(Part::storageFullColumnName(), Query::Equals, Part::External);
406 qb.addValueCondition(Part::dataFullColumnName(), Query::IsNot, QVariant());
407 if (mScope.scope() != Scope::Invalid) {
408 ItemQueryHelper::scopeToQuery(mScope, mContext, qb);
409 } else {
410 ItemQueryHelper::itemSetToQuery(mItemSet, qb, mCollection);
411 }
412
413 if (!qb.exec()) {
414 mLastError = QByteArrayLiteral("Unable to query parts.");
415 throw ItemRetrieverException(mLastError);
416 }
417
418 const Part::List externalParts = qb.result();
419 for (Part part : externalParts) {
420 PartHelper::verify(part);
421 }
422}
423
424QByteArray ItemRetriever::lastError() const
425{
426 return mLastError;
427}
428
429#include "moc_itemretriever.cpp"
Represents a collection of PIM items.
Definition collection.h:62
An Connection represents one connection of a client to the server.
Definition connection.h:39
Manages and processes item retrieval requests.
virtual void requestItemDelivery(ItemRetrievalRequest request)
Added for convenience.
Details of a single item retrieval request.
Helper class for retrieving missing items parts from remote resources.
void setScope(const Scope &scope)
Retrieve all items matching the given item scope.
void setCollection(const Collection &collection, bool recursive=true)
Retrieve all items in the given collection.
Helper class to construct arbitrary SQL queries.
@ InnerJoin
NOTE: only supported for UPDATE and SELECT queries.
@ LeftJoin
NOTE: only supported for SELECT queries.
Represents a WHERE condition tree.
Definition query.h:62
Helper class for creating and executing database SELECT queries.
void scopeToQuery(const Scope &scope, const CommandContext &context, QueryBuilder &qb)
Add conditions to qb for the given item operation scope scope.
void itemSetToQuery(const QList< PimItem::Id > &set, QueryBuilder &qb, const Collection &collection=Collection())
Add conditions to qb for the given item set set.
bool verify(Part &part)
Verifies and if necessary fixes the external reference of this part.
Query::Condition conditionFromFqNames(const QStringList &fqNames)
Returns a query condition that matches the given part type list.
Helper integration between Akonadi and Qt.
KSERVICE_EXPORT KService::List query(FilterFunc filterFunc)
bool startsWith(QByteArrayView bv) const const
bool isValid() const const
QDateTime toUTC() const const
void append(QList< T > &&value)
iterator begin()
void clear()
bool contains(const AT &value) const const
iterator end()
iterator erase(const_iterator begin, const_iterator end)
bool isEmpty() const const
T value(qsizetype i) const const
Q_EMITQ_EMIT
const QObjectList & children() const const
QMetaObject::Connection connect(const QObject *sender, PointerToMemberFunction signal, Functor functor)
T qobject_cast(QObject *object)
void clear()
iterator insert(const T &value)
QString fromUtf8(QByteArrayView str)
QFuture< ArgsType< Signal > > connect(Sender *sender, Signal signal)
This file is part of the KDE documentation.
Documentation copyright © 1996-2024 The KDE developers.
Generated on Tue Mar 26 2024 11:13:38 by doxygen 1.10.0 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.