Akonadi

searchmanager.cpp
1 /*
2  SPDX-FileCopyrightText: 2010 Volker Krause <[email protected]>
3  SPDX-FileCopyrightText: 2013 Daniel Vr├ítil <[email protected]>
4 
5  SPDX-License-Identifier: LGPL-2.0-or-later
6 */
7 
8 #include "searchmanager.h"
9 #include "abstractsearchplugin.h"
10 #include "akonadiserver_search_debug.h"
11 
12 #include "agentsearchengine.h"
13 #include "akonadi.h"
14 #include "handler/searchhelper.h"
15 #include "notificationmanager.h"
16 #include "searchrequest.h"
17 #include "searchtaskmanager.h"
18 #include "storage/datastore.h"
19 #include "storage/querybuilder.h"
20 #include "storage/selectquerybuilder.h"
21 #include "storage/transaction.h"
22 
23 #include <private/protocol_p.h>
24 
25 #include <QDBusConnection>
26 #include <QDir>
27 #include <QPluginLoader>
28 #include <QTimer>
29 
30 #include <memory>
31 
32 Q_DECLARE_METATYPE(Akonadi::Server::NotificationCollector *)
33 
34 using namespace Akonadi;
35 using namespace Akonadi::Server;
36 
37 Q_DECLARE_METATYPE(Collection)
38 
39 SearchManager::SearchManager(const QStringList &searchEngines, SearchTaskManager &agentSearchManager)
40  : AkThread(QStringLiteral("SearchManager"), AkThread::ManualStart, QThread::InheritPriority)
41  , mAgentSearchManager(agentSearchManager)
42  , mEngineNames(searchEngines)
43  , mSearchUpdateTimer(nullptr)
44 {
45  qRegisterMetaType<Collection>();
46 
47  // We load search plugins (as in QLibrary::load()) in the main thread so that
48  // static initialization happens in the QApplication thread
49  loadSearchPlugins();
50 
51  // Register to DBus on the main thread connection - otherwise we don't appear
52  // on the service.
54  conn.registerObject(QStringLiteral("/SearchManager"), this, QDBusConnection::ExportAllSlots);
55 
56  // Delay-call init()
57  startThread();
58 }
59 
60 void SearchManager::init()
61 {
62  AkThread::init();
63 
64  mEngines.reserve(mEngineNames.size());
65  for (const QString &engineName : std::as_const(mEngineNames)) {
66  if (engineName == QLatin1String("Agent")) {
67  mEngines.append(new AgentSearchEngine);
68  } else {
69  qCCritical(AKONADISERVER_SEARCH_LOG) << "Unknown search engine type: " << engineName;
70  }
71  }
72 
73  initSearchPlugins();
74 
75  // The timer will tick 15 seconds after last change notification. If a new notification
76  // is delivered in the meantime, the timer is reset
77  mSearchUpdateTimer = new QTimer(this);
78  mSearchUpdateTimer->setInterval(15 * 1000);
79  mSearchUpdateTimer->setSingleShot(true);
80  connect(mSearchUpdateTimer, &QTimer::timeout, this, &SearchManager::searchUpdateTimeout);
81 }
82 
83 void SearchManager::quit()
84 {
86  conn.unregisterObject(QStringLiteral("/SearchManager"), QDBusConnection::UnregisterTree);
87  conn.disconnectFromBus(conn.name());
88 
89  // Make sure all children are deleted within context of this thread
90  qDeleteAll(children());
91 
92  qDeleteAll(mEngines);
93  qDeleteAll(mPlugins);
94  /*
95  * FIXME: Unloading plugin messes up some global statics from client libs
96  * and causes crash on Akonadi shutdown (below main). Keeping the plugins
97  * loaded is not really a big issue as this is only invoked on server shutdown
98  * anyway, so we are not leaking any memory.
99  Q_FOREACH (QPluginLoader *loader, mPluginLoaders) {
100  loader->unload();
101  delete loader;
102  }
103  */
104 
105  AkThread::quit();
106 }
107 
108 SearchManager::~SearchManager()
109 {
110  quitThread();
111 }
112 
114 {
115  mAgentSearchManager.registerInstance(id);
116 }
117 
119 {
120  mAgentSearchManager.unregisterInstance(id);
121 }
122 
124 {
125  return mPlugins;
126 }
127 
128 void SearchManager::loadSearchPlugins()
129 {
130  QStringList loadedPlugins;
131  const QString pluginOverride = QString::fromLatin1(qgetenv("AKONADI_OVERRIDE_SEARCHPLUGIN"));
132  if (!pluginOverride.isEmpty()) {
133  qCInfo(AKONADISERVER_SEARCH_LOG) << "Overriding the search plugins with: " << pluginOverride;
134  }
135 
137  for (const QString &pluginDir : dirs) {
138  QDir dir(pluginDir + QLatin1String("/akonadi"));
139  const QStringList fileNames = dir.entryList(QDir::Files);
140  qCDebug(AKONADISERVER_SEARCH_LOG) << "SEARCH MANAGER: searching in " << pluginDir + QLatin1String("/akonadi") << ":" << fileNames;
141  for (const QString &fileName : fileNames) {
142  const QString filePath = pluginDir % QLatin1String("/akonadi/") % fileName;
143  std::unique_ptr<QPluginLoader> loader(new QPluginLoader(filePath));
144  const QVariantMap metadata = loader->metaData().value(QStringLiteral("MetaData")).toVariant().toMap();
145  if (metadata.value(QStringLiteral("X-Akonadi-PluginType")).toString() != QLatin1String("SearchPlugin")) {
146  continue;
147  }
148 
149  const QString libraryName = metadata.value(QStringLiteral("X-Akonadi-Library")).toString();
150  if (loadedPlugins.contains(libraryName)) {
151  qCDebug(AKONADISERVER_SEARCH_LOG) << "Already loaded one version of this plugin, skipping: " << libraryName;
152  continue;
153  }
154 
155  // When search plugin override is active, ignore all plugins except for the override
156  if (!pluginOverride.isEmpty()) {
157  if (libraryName != pluginOverride) {
158  qCDebug(AKONADISERVER_SEARCH_LOG) << libraryName << "skipped because of AKONADI_OVERRIDE_SEARCHPLUGIN";
159  continue;
160  }
161 
162  // When there's no override, only load plugins enabled by default
163  } else if (!metadata.value(QStringLiteral("X-Akonadi-LoadByDefault"), true).toBool()) {
164  continue;
165  }
166 
167  if (!loader->load()) {
168  qCCritical(AKONADISERVER_SEARCH_LOG) << "Failed to load search plugin" << libraryName << ":" << loader->errorString();
169  continue;
170  }
171 
172  mPluginLoaders << loader.release();
173  loadedPlugins << libraryName;
174  }
175  }
176 }
177 
178 void SearchManager::initSearchPlugins()
179 {
180  for (QPluginLoader *loader : std::as_const(mPluginLoaders)) {
181  if (!loader->load()) {
182  qCCritical(AKONADISERVER_SEARCH_LOG) << "Failed to load search plugin" << loader->fileName() << ":" << loader->errorString();
183  continue;
184  }
185 
186  AbstractSearchPlugin *plugin = qobject_cast<AbstractSearchPlugin *>(loader->instance());
187  if (!plugin) {
188  qCCritical(AKONADISERVER_SEARCH_LOG) << loader->fileName() << "is not a valid Akonadi search plugin";
189  continue;
190  }
191 
192  qCDebug(AKONADISERVER_SEARCH_LOG) << "SearchManager: loaded search plugin" << loader->fileName();
193  mPlugins << plugin;
194  }
195 }
196 
197 void SearchManager::scheduleSearchUpdate()
198 {
199  // Reset if the timer is active (use QueuedConnection to invoke start() from
200  // the thread the QTimer lives in instead of caller's thread, otherwise crashes
201  // and weird things can happen.
202  QMetaObject::invokeMethod(mSearchUpdateTimer, qOverload<>(&QTimer::start), Qt::QueuedConnection);
203 }
204 
205 void SearchManager::searchUpdateTimeout()
206 {
207  // Get all search collections, that is subcollections of "Search", which always has ID 1
208  const Collection::List collections = Collection::retrieveFiltered(Collection::parentIdFullColumnName(), 1);
209  for (const Collection &collection : collections) {
210  updateSearchAsync(collection);
211  }
212 }
213 
215 {
217  this,
218  [this, collection]() {
219  updateSearchImpl(collection);
220  },
222 }
223 
225 {
226  mLock.lock();
227  if (mUpdatingCollections.contains(collection.id())) {
228  mLock.unlock();
229  return;
230  // FIXME: If another thread already requested an update, we return to the caller before the
231  // search update is performed; this contradicts the docs
232  }
233  mUpdatingCollections.insert(collection.id());
234  mLock.unlock();
236  this,
237  [this, collection]() {
238  updateSearchImpl(collection);
239  },
241  mLock.lock();
242  mUpdatingCollections.remove(collection.id());
243  mLock.unlock();
244 }
245 
246 void SearchManager::updateSearchImpl(const Collection &collection)
247 {
248  if (collection.queryString().size() >= 32768) {
249  qCWarning(AKONADISERVER_SEARCH_LOG) << "The query is at least 32768 chars long, which is the maximum size supported by the akonadi db schema. The "
250  "query is therefore most likely truncated and will not be executed.";
251  return;
252  }
253  if (collection.queryString().isEmpty()) {
254  return;
255  }
256 
257  const QStringList queryAttributes = collection.queryAttributes().split(QLatin1Char(' '));
258  const bool remoteSearch = queryAttributes.contains(QLatin1String(AKONADI_PARAM_REMOTE));
259  bool recursive = queryAttributes.contains(QLatin1String(AKONADI_PARAM_RECURSIVE));
260 
261  QStringList queryMimeTypes;
262  const QVector<MimeType> mimeTypes = collection.mimeTypes();
263  queryMimeTypes.reserve(mimeTypes.count());
264 
265  for (const MimeType &mt : mimeTypes) {
266  queryMimeTypes << mt.name();
267  }
268 
269  QVector<qint64> queryAncestors;
270  if (collection.queryCollections().isEmpty()) {
271  queryAncestors << 0;
272  recursive = true;
273  } else {
274  const QStringList collectionIds = collection.queryCollections().split(QLatin1Char(' '));
275  queryAncestors.reserve(collectionIds.count());
276  for (const QString &colId : collectionIds) {
277  queryAncestors << colId.toLongLong();
278  }
279  }
280 
281  // Always query the given collections
282  QVector<qint64> queryCollections = queryAncestors;
283 
284  if (recursive) {
285  // Resolve subcollections if necessary
286  queryCollections += SearchHelper::matchSubcollectionsByMimeType(queryAncestors, queryMimeTypes);
287  }
288 
289  // This happens if we try to search a virtual collection in recursive mode (because virtual collections are excluded from listCollectionsRecursive)
290  if (queryCollections.isEmpty()) {
291  qCDebug(AKONADISERVER_SEARCH_LOG) << "No collections to search, you're probably trying to search a virtual collection.";
292  return;
293  }
294 
295  // Query all plugins for search results
296  const QByteArray id = "searchUpdate-" + QByteArray::number(QDateTime::currentDateTimeUtc().toSecsSinceEpoch());
297  SearchRequest request(id, *this, mAgentSearchManager);
298  request.setCollections(queryCollections);
299  request.setMimeTypes(queryMimeTypes);
300  request.setQuery(collection.queryString());
301  request.setRemoteSearch(remoteSearch);
302  request.setStoreResults(true);
303  request.setProperty("SearchCollection", QVariant::fromValue(collection));
304  connect(&request, &SearchRequest::resultsAvailable, this, &SearchManager::searchUpdateResultsAvailable);
305  request.exec(); // blocks until all searches are done
306 
307  const QSet<qint64> results = request.results();
308 
309  // Get all items in the collection
310  QueryBuilder qb(CollectionPimItemRelation::tableName());
311  qb.addColumn(CollectionPimItemRelation::rightColumn());
312  qb.addValueCondition(CollectionPimItemRelation::leftColumn(), Query::Equals, collection.id());
313  if (!qb.exec()) {
314  return;
315  }
316 
317  Transaction transaction(DataStore::self(), QStringLiteral("UPDATE SEARCH"));
318 
319  // Unlink all items that were not in search results from the collection
320  QVariantList toRemove;
321  while (qb.query().next()) {
322  const qint64 id = qb.query().value(0).toLongLong();
323  if (!results.contains(id)) {
324  toRemove << id;
325  Collection::removePimItem(collection.id(), id);
326  }
327  }
328 
329  if (!transaction.commit()) {
330  return;
331  }
332 
333  if (!toRemove.isEmpty()) {
335  qb.addValueCondition(PimItem::idFullColumnName(), Query::In, toRemove);
336  if (!qb.exec()) {
337  return;
338  }
339 
340  const QVector<PimItem> removedItems = qb.result();
341  DataStore::self()->notificationCollector()->itemsUnlinked(removedItems, collection);
342  }
343 
344  qCInfo(AKONADISERVER_SEARCH_LOG) << "Search update for collection" << collection.name() << "(" << collection.id() << ") finished:"
345  << "all results: " << results.count() << ", removed results:" << toRemove.count();
346 }
347 
348 void SearchManager::searchUpdateResultsAvailable(const QSet<qint64> &results)
349 {
350  const auto collection = sender()->property("SearchCollection").value<Collection>();
351  qCDebug(AKONADISERVER_SEARCH_LOG) << "searchUpdateResultsAvailable" << collection.id() << results.count() << "results";
352 
353  QSet<qint64> newMatches = results;
354  QSet<qint64> existingMatches;
355  {
356  QueryBuilder qb(CollectionPimItemRelation::tableName());
357  qb.addColumn(CollectionPimItemRelation::rightColumn());
358  qb.addValueCondition(CollectionPimItemRelation::leftColumn(), Query::Equals, collection.id());
359  if (!qb.exec()) {
360  return;
361  }
362 
363  while (qb.query().next()) {
364  const qint64 id = qb.query().value(0).toLongLong();
365  if (newMatches.contains(id)) {
366  existingMatches << id;
367  }
368  }
369  }
370 
371  qCDebug(AKONADISERVER_SEARCH_LOG) << "Got" << newMatches.count() << "results, out of which" << existingMatches.count() << "are already in the collection";
372 
373  newMatches = newMatches - existingMatches;
374  if (newMatches.isEmpty()) {
375  qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results: 0 (fast path)";
376  return;
377  }
378 
379  Transaction transaction(DataStore::self(), QStringLiteral("PUSH SEARCH RESULTS"), !DataStore::self()->inTransaction());
380 
381  // First query all the IDs we got from search plugin/agent against the DB.
382  // This will remove IDs that no longer exist in the DB.
383  QVariantList newMatchesVariant;
384  newMatchesVariant.reserve(newMatches.count());
385  for (qint64 id : std::as_const(newMatches)) {
386  newMatchesVariant << id;
387  }
388 
390  qb.addValueCondition(PimItem::idFullColumnName(), Query::In, newMatchesVariant);
391  if (!qb.exec()) {
392  return;
393  }
394 
395  const auto items = qb.result();
396  if (items.count() != newMatches.count()) {
397  qCDebug(AKONADISERVER_SEARCH_LOG) << "Search backend returned" << (newMatches.count() - items.count()) << "results that no longer exist in Akonadi.";
398  qCDebug(AKONADISERVER_SEARCH_LOG) << "Please reindex collection" << collection.id();
399  // TODO: Request the reindexing directly from here
400  }
401 
402  if (items.isEmpty()) {
403  qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results: 0 (no existing result)";
404  return;
405  }
406 
407  for (const auto &item : items) {
408  Collection::addPimItem(collection.id(), item.id());
409  }
410 
411  if (!transaction.commit()) {
412  qCWarning(AKONADISERVER_SEARCH_LOG) << "Failed to commit search results transaction";
413  return;
414  }
415 
416  DataStore::self()->notificationCollector()->itemsLinked(items, collection);
417  // Force collector to dispatch the notification now
419 
420  qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results:" << items.count();
421 }
NotificationCollector * notificationCollector()
Returns the notification collector of this DataStore object.
Definition: datastore.cpp:206
void lock()
QVariant value(int index) const const
bool isEmpty() const const
Helper class for DataStore transaction handling.
Definition: transaction.h:22
bool remove(const T &value)
static DataStore * self()
Per thread singleton.
Definition: datastore.cpp:215
void start()
QVariant fromValue(const T &value)
Part of the DataStore, collects change notifications and emits them after the current transaction has...
void unlock()
int count() const const
const T value(const Key &key, const T &defaultValue) const const
void setSingleShot(bool singleShot)
int count(const T &value) const const
T value() const const
bool contains(const QString &str, Qt::CaseSensitivity cs) const const
virtual void updateSearch(const Collection &collection)
Updates the search query synchronously.
QByteArray number(int n, int base)
Represents a collection of PIM items.
Definition: collection.h:61
QObject * sender() const const
QDateTime currentDateTimeUtc()
bool registerObject(const QString &path, QObject *object, QDBusConnection::RegisterOptions options)
qlonglong toLongLong(bool *ok) const const
QMetaObject::Connection connect(const QObject *sender, const char *signal, const QObject *receiver, const char *method, Qt::ConnectionType type)
virtual void updateSearchAsync(const Collection &collection)
Updates the search query asynchronously.
virtual void unregisterInstance(const QString &id)
This is called via D-Bus from AgentManager to unregister an agent with search interface.
QStringList libraryPaths()
void reserve(int alloc)
int size() const const
QDBusConnection sessionBus()
virtual QVector< AbstractSearchPlugin * > searchPlugins() const
Returns currently available search plugins.
void timeout()
bool isEmpty() const const
QVector< T > result()
Returns the result of this SELECT query.
bool next()
QueuedConnection
void disconnectFromBus(const QString &name)
void addColumn(const QString &col)
Adds the given column to a select query.
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
bool dispatchNotifications()
Trigger sending of collected notifications.
void reserve(int size)
bool contains(const T &value) const const
bool exec()
Executes the query, returns true on success.
void unregisterObject(const QString &path, QDBusConnection::UnregisterMode mode)
virtual void registerInstance(const QString &id)
This is called via D-Bus from AgentManager to register an agent with search interface.
QString name() const const
void itemsLinked(const PimItem::List &items, const Collection &collection)
Notify about linked items.
SearchManager creates and deletes persistent searches for all currently active search engines.
Definition: searchmanager.h:33
Helper class for creating and executing database SELECT queries.
QString fromLatin1(const char *str, int size)
Search engine for distributing searches to agents.
bool invokeMethod(QObject *obj, const char *member, Qt::ConnectionType type, QGenericReturnArgument ret, QGenericArgument val0, QGenericArgument val1, QGenericArgument val2, QGenericArgument val3, QGenericArgument val4, QGenericArgument val5, QGenericArgument val6, QGenericArgument val7, QGenericArgument val8, QGenericArgument val9)
QSet::iterator insert(const T &value)
QSqlQuery & query()
Returns the query, only valid after exec().
void itemsUnlinked(const PimItem::List &items, const Collection &collection)
Notify about unlinked items.
void setInterval(int msec)
bool isEmpty() const const
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:31
const QObjectList & children() const const
QVariant property(const char *name) const const
Helper integration between Akonadi and Qt.
This file is part of the KDE documentation.
Documentation copyright © 1996-2022 The KDE developers.
Generated on Sat Jul 2 2022 06:41:49 by doxygen 1.8.17 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.