Akonadi

searchmanager.cpp
1 /*
2  Copyright (c) 2010 Volker Krause <[email protected]>
3  Copyright (c) 2013 Daniel Vrátil <[email protected]>
4 
5  This library is free software; you can redistribute it and/or modify it
6  under the terms of the GNU Library General Public License as published by
7  the Free Software Foundation; either version 2 of the License, or (at your
8  option) any later version.
9 
10  This library is distributed in the hope that it will be useful, but WITHOUT
11  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12  FITNESS FOR A PARTICULAR PURPOSE. See the GNU Library General Public
13  License for more details.
14 
15  You should have received a copy of the GNU Library General Public License
16  along with this library; see the file COPYING.LIB. If not, write to the
17  Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
18  02110-1301, USA.
19 */
20 
21 #include "searchmanager.h"
22 #include "abstractsearchplugin.h"
23 #include "akonadiserver_search_debug.h"
24 
25 #include "akonadi.h"
26 #include "agentsearchengine.h"
27 #include "notificationmanager.h"
28 #include "searchrequest.h"
29 #include "searchtaskmanager.h"
30 #include "storage/datastore.h"
31 #include "storage/querybuilder.h"
32 #include "storage/transaction.h"
33 #include "storage/selectquerybuilder.h"
34 #include "handler/searchhelper.h"
35 
36 #include <private/protocol_p.h>
37 
38 #include <QDir>
39 #include <QPluginLoader>
40 #include <QDBusConnection>
41 #include <QTimer>
42 
43 #include <memory>
44 
45 Q_DECLARE_METATYPE(Akonadi::Server::NotificationCollector *)
46 
47 using namespace Akonadi;
48 using namespace Akonadi::Server;
49 
50 Q_DECLARE_METATYPE(Collection)
51 
52 SearchManager::SearchManager(const QStringList &searchEngines, SearchTaskManager &agentSearchManager)
53  : AkThread(QStringLiteral("SearchManager"), AkThread::ManualStart, QThread::InheritPriority)
54  , mAgentSearchManager(agentSearchManager)
55  , mEngineNames(searchEngines),
56  mSearchUpdateTimer(nullptr)
57 {
58  qRegisterMetaType<Collection>();
59 
60  // We load search plugins (as in QLibrary::load()) in the main thread so that
61  // static initialization happens in the QApplication thread
62  loadSearchPlugins();
63 
64  // Register to DBus on the main thread connection - otherwise we don't appear
65  // on the service.
67  conn.registerObject(QStringLiteral("/SearchManager"), this,
68  QDBusConnection::ExportAllSlots);
69 
70  // Delay-call init()
71  startThread();
72 }
73 
74 void SearchManager::init()
75 {
76  AkThread::init();
77 
78  mEngines.reserve(mEngineNames.size());
79  for (const QString &engineName : qAsConst(mEngineNames)) {
80  if (engineName == QLatin1String("Agent")) {
81  mEngines.append(new AgentSearchEngine);
82  } else {
83  qCCritical(AKONADISERVER_SEARCH_LOG) << "Unknown search engine type: " << engineName;
84  }
85  }
86 
87  initSearchPlugins();
88 
89  // The timer will tick 15 seconds after last change notification. If a new notification
90  // is delivered in the meantime, the timer is reset
91  mSearchUpdateTimer = new QTimer(this);
92  mSearchUpdateTimer->setInterval(15 * 1000);
93  mSearchUpdateTimer->setSingleShot(true);
94  connect(mSearchUpdateTimer, &QTimer::timeout,
95  this, &SearchManager::searchUpdateTimeout);
96 }
97 
98 void SearchManager::quit()
99 {
101  conn.unregisterObject(QStringLiteral("/SearchManager"), QDBusConnection::UnregisterTree);
102  conn.disconnectFromBus(conn.name());
103 
104  // Make sure all children are deleted within context of this thread
105  qDeleteAll(children());
106 
107  qDeleteAll(mEngines);
108  qDeleteAll(mPlugins);
109  /*
110  * FIXME: Unloading plugin messes up some global statics from client libs
111  * and causes crash on Akonadi shutdown (below main). Keeping the plugins
112  * loaded is not really a big issue as this is only invoked on server shutdown
113  * anyway, so we are not leaking any memory.
114  Q_FOREACH (QPluginLoader *loader, mPluginLoaders) {
115  loader->unload();
116  delete loader;
117  }
118  */
119 
120  AkThread::quit();
121 }
122 
123 SearchManager::~SearchManager()
124 {
125  quitThread();
126 }
127 
129 {
130  mAgentSearchManager.registerInstance(id);
131 }
132 
134 {
135  mAgentSearchManager.unregisterInstance(id);
136 }
137 
139 {
140  return mPlugins;
141 }
142 
143 void SearchManager::loadSearchPlugins()
144 {
145  QStringList loadedPlugins;
146  const QString pluginOverride = QString::fromLatin1(qgetenv("AKONADI_OVERRIDE_SEARCHPLUGIN"));
147  if (!pluginOverride.isEmpty()) {
148  qCInfo(AKONADISERVER_SEARCH_LOG) << "Overriding the search plugins with: " << pluginOverride;
149  }
150 
152  for (const QString &pluginDir : dirs) {
153  QDir dir(pluginDir + QLatin1String("/akonadi"));
154  const QStringList fileNames = dir.entryList(QDir::Files);
155  qCDebug(AKONADISERVER_SEARCH_LOG) << "SEARCH MANAGER: searching in " << pluginDir + QLatin1String("/akonadi") << ":" << fileNames;
156  for (const QString &fileName : fileNames) {
157  const QString filePath = pluginDir % QLatin1String("/akonadi/") % fileName;
158  std::unique_ptr<QPluginLoader> loader(new QPluginLoader(filePath));
159  const QVariantMap metadata = loader->metaData().value(QStringLiteral("MetaData")).toVariant().toMap();
160  if (metadata.value(QStringLiteral("X-Akonadi-PluginType")).toString() != QLatin1String("SearchPlugin")) {
161  continue;
162  }
163 
164  const QString libraryName = metadata.value(QStringLiteral("X-Akonadi-Library")).toString();
165  if (loadedPlugins.contains(libraryName)) {
166  qCDebug(AKONADISERVER_SEARCH_LOG) << "Already loaded one version of this plugin, skipping: " << libraryName;
167  continue;
168  }
169 
170  // When search plugin override is active, ignore all plugins except for the override
171  if (!pluginOverride.isEmpty()) {
172  if (libraryName != pluginOverride) {
173  qCDebug(AKONADISERVER_SEARCH_LOG) << libraryName << "skipped because of AKONADI_OVERRIDE_SEARCHPLUGIN";
174  continue;
175  }
176 
177  // When there's no override, only load plugins enabled by default
178  } else if (metadata.value(QStringLiteral("X-Akonadi-LoadByDefault"), true).toBool() == false) {
179  continue;
180  }
181 
182  if (!loader->load()) {
183  qCCritical(AKONADISERVER_SEARCH_LOG) << "Failed to load search plugin" << libraryName << ":" << loader->errorString();
184  continue;
185  }
186 
187  mPluginLoaders << loader.release();
188  loadedPlugins << libraryName;
189  }
190  }
191 }
192 
193 void SearchManager::initSearchPlugins()
194 {
195  for (QPluginLoader *loader : qAsConst(mPluginLoaders)) {
196  if (!loader->load()) {
197  qCCritical(AKONADISERVER_SEARCH_LOG) << "Failed to load search plugin" << loader->fileName() << ":" << loader->errorString();
198  continue;
199  }
200 
201  AbstractSearchPlugin *plugin = qobject_cast<AbstractSearchPlugin *>(loader->instance());
202  if (!plugin) {
203  qCCritical(AKONADISERVER_SEARCH_LOG) << loader->fileName() << "is not a valid Akonadi search plugin";
204  continue;
205  }
206 
207  qCDebug(AKONADISERVER_SEARCH_LOG) << "SearchManager: loaded search plugin" << loader->fileName();
208  mPlugins << plugin;
209  }
210 }
211 
212 void SearchManager::scheduleSearchUpdate()
213 {
214  // Reset if the timer is active (use QueuedConnection to invoke start() from
215  // the thread the QTimer lives in instead of caller's thread, otherwise crashes
216  // and weird things can happen.
217  QMetaObject::invokeMethod(mSearchUpdateTimer, QOverload<>::of(&QTimer::start), Qt::QueuedConnection);
218 }
219 
220 void SearchManager::searchUpdateTimeout()
221 {
222  // Get all search collections, that is subcollections of "Search", which always has ID 1
223  const Collection::List collections = Collection::retrieveFiltered(Collection::parentIdFullColumnName(), 1);
224  for (const Collection &collection : collections) {
225  updateSearchAsync(collection);
226  }
227 }
228 
230 {
231  QMetaObject::invokeMethod(this, [this, collection]() { updateSearchImpl(collection); }, Qt::QueuedConnection);
232 }
233 
235 {
236  mLock.lock();
237  if (mUpdatingCollections.contains(collection.id())) {
238  mLock.unlock();
239  return;
240  // FIXME: If another thread already requested an update, we return to the caller before the
241  // search update is performed; this contradicts the docs
242  }
243  mUpdatingCollections.insert(collection.id());
244  mLock.unlock();
245  QMetaObject::invokeMethod(this, [this, collection]() { updateSearchImpl(collection); }, Qt::BlockingQueuedConnection);
246  mLock.lock();
247  mUpdatingCollections.remove(collection.id());
248  mLock.unlock();
249 }
250 
251 void SearchManager::updateSearchImpl(const Collection &collection)
252 {
253  if (collection.queryString().size() >= 32768) {
254  qCWarning(AKONADISERVER_SEARCH_LOG) << "The query is at least 32768 chars long, which is the maximum size supported by the akonadi db schema. The query is therefore most likely truncated and will not be executed.";
255  return;
256  }
257  if (collection.queryString().isEmpty()) {
258  return;
259  }
260 
261  const QStringList queryAttributes = collection.queryAttributes().split(QLatin1Char(' '));
262  const bool remoteSearch = queryAttributes.contains(QLatin1String(AKONADI_PARAM_REMOTE));
263  bool recursive = queryAttributes.contains(QLatin1String(AKONADI_PARAM_RECURSIVE));
264 
265  QStringList queryMimeTypes;
266  const QVector<MimeType> mimeTypes = collection.mimeTypes();
267  queryMimeTypes.reserve(mimeTypes.count());
268 
269  for (const MimeType &mt : mimeTypes) {
270  queryMimeTypes << mt.name();
271  }
272 
273  QVector<qint64> queryAncestors;
274  if (collection.queryCollections().isEmpty()) {
275  queryAncestors << 0;
276  recursive = true;
277  } else {
278  const QStringList collectionIds = collection.queryCollections().split(QLatin1Char(' '));
279  queryAncestors.reserve(collectionIds.count());
280  for (const QString &colId : collectionIds) {
281  queryAncestors << colId.toLongLong();
282  }
283  }
284 
285  // Always query the given collections
286  QVector<qint64> queryCollections = queryAncestors;
287 
288  if (recursive) {
289  // Resolve subcollections if necessary
290  queryCollections += SearchHelper::matchSubcollectionsByMimeType(queryAncestors, queryMimeTypes);
291  }
292 
293  //This happens if we try to search a virtual collection in recursive mode (because virtual collections are excluded from listCollectionsRecursive)
294  if (queryCollections.isEmpty()) {
295  qCDebug(AKONADISERVER_SEARCH_LOG) << "No collections to search, you're probably trying to search a virtual collection.";
296  return;
297  }
298 
299  // Query all plugins for search results
300  const QByteArray id = "searchUpdate-" + QByteArray::number(QDateTime::currentDateTimeUtc().toTime_t());
301  SearchRequest request(id, *this, mAgentSearchManager);
302  request.setCollections(queryCollections);
303  request.setMimeTypes(queryMimeTypes);
304  request.setQuery(collection.queryString());
305  request.setRemoteSearch(remoteSearch);
306  request.setStoreResults(true);
307  request.setProperty("SearchCollection", QVariant::fromValue(collection));
308  connect(&request, &SearchRequest::resultsAvailable,
309  this, &SearchManager::searchUpdateResultsAvailable);
310  request.exec(); // blocks until all searches are done
311 
312  const QSet<qint64> results = request.results();
313 
314  // Get all items in the collection
315  QueryBuilder qb(CollectionPimItemRelation::tableName());
316  qb.addColumn(CollectionPimItemRelation::rightColumn());
317  qb.addValueCondition(CollectionPimItemRelation::leftColumn(), Query::Equals, collection.id());
318  if (!qb.exec()) {
319  return;
320  }
321 
322  Transaction transaction(DataStore::self(), QStringLiteral("UPDATE SEARCH"));
323 
324  // Unlink all items that were not in search results from the collection
325  QVariantList toRemove;
326  while (qb.query().next()) {
327  const qint64 id = qb.query().value(0).toLongLong();
328  if (!results.contains(id)) {
329  toRemove << id;
330  Collection::removePimItem(collection.id(), id);
331  }
332  }
333 
334  if (!transaction.commit()) {
335  return;
336  }
337 
338  if (!toRemove.isEmpty()) {
340  qb.addValueCondition(PimItem::idFullColumnName(), Query::In, toRemove);
341  if (!qb.exec()) {
342  return;
343  }
344 
345  const QVector<PimItem> removedItems = qb.result();
346  DataStore::self()->notificationCollector()->itemsUnlinked(removedItems, collection);
347  }
348 
349  qCInfo(AKONADISERVER_SEARCH_LOG) << "Search update for collection" << collection.name()
350  << "(" << collection.id() << ") finished:"
351  << "all results: " << results.count() << ", removed results:" << toRemove.count();
352 }
353 
354 void SearchManager::searchUpdateResultsAvailable(const QSet<qint64> &results)
355 {
356  const Collection collection = sender()->property("SearchCollection").value<Collection>();
357  qCDebug(AKONADISERVER_SEARCH_LOG) << "searchUpdateResultsAvailable" << collection.id() << results.count() << "results";
358 
359  QSet<qint64> newMatches = results;
360  QSet<qint64> existingMatches;
361  {
362  QueryBuilder qb(CollectionPimItemRelation::tableName());
363  qb.addColumn(CollectionPimItemRelation::rightColumn());
364  qb.addValueCondition(CollectionPimItemRelation::leftColumn(), Query::Equals, collection.id());
365  if (!qb.exec()) {
366  return;
367  }
368 
369  while (qb.query().next()) {
370  const qint64 id = qb.query().value(0).toLongLong();
371  if (newMatches.contains(id)) {
372  existingMatches << id;
373  }
374  }
375  }
376 
377  qCDebug(AKONADISERVER_SEARCH_LOG) << "Got" << newMatches.count() << "results, out of which" << existingMatches.count() << "are already in the collection";
378 
379  newMatches = newMatches - existingMatches;
380  if (newMatches.isEmpty()) {
381  qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results: 0 (fast path)";
382  return;
383  }
384 
385  Transaction transaction(DataStore::self(), QStringLiteral("PUSH SEARCH RESULTS"),
386  !DataStore::self()->inTransaction());
387 
388  // First query all the IDs we got from search plugin/agent against the DB.
389  // This will remove IDs that no longer exist in the DB.
390  QVariantList newMatchesVariant;
391  newMatchesVariant.reserve(newMatches.count());
392  for (qint64 id : qAsConst(newMatches)) {
393  newMatchesVariant << id;
394  }
395 
397  qb.addValueCondition(PimItem::idFullColumnName(), Query::In, newMatchesVariant);
398  if (!qb.exec()) {
399  return;
400  }
401 
402  const auto items = qb.result();
403  if (items.count() != newMatches.count()) {
404  qCDebug(AKONADISERVER_SEARCH_LOG) << "Search backend returned" << (newMatches.count() - items.count()) << "results that no longer exist in Akonadi.";
405  qCDebug(AKONADISERVER_SEARCH_LOG) << "Please reindex collection" << collection.id();
406  // TODO: Request the reindexing directly from here
407  }
408 
409  if (items.isEmpty()) {
410  qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results: 0 (no existing result)";
411  return;
412  }
413 
414  for (const auto &item : items) {
415  Collection::addPimItem(collection.id(), item.id());
416  }
417 
418  if (!transaction.commit()) {
419  qCWarning(AKONADISERVER_SEARCH_LOG) << "Failed to commit search results transaction";
420  return;
421  }
422 
423  DataStore::self()->notificationCollector()->itemsLinked(items, collection);
424  // Force collector to dispatch the notification now
426 
427  qCDebug(AKONADISERVER_SEARCH_LOG) << "Added results:" << items.count();
428 }
void setInterval(int msec)
qlonglong toLongLong(bool *ok) const const
void addValueCondition(const QString &column, Query::CompareOperator op, const QVariant &value, ConditionType type=WhereCondition)
Add a WHERE or HAVING condition which compares a column with a given value.
NotificationCollector * notificationCollector()
Returns the notification collector of this DataStore object.
Definition: datastore.cpp:226
bool dispatchNotifications()
Trigger sending of collected notifications.
QSqlQuery & query()
Returns the query, only valid after exec().
SearchManager creates and deletes persistent searches for all currently active search engines...
Definition: searchmanager.h:49
Search engine for distributing searches to agents.
QObject * sender() const const
QStringList split(const QString &sep, QString::SplitBehavior behavior, Qt::CaseSensitivity cs) const const
void reserve(int alloc)
Representation of a record in the MimeType table.
Definition: entities.h:1011
void itemsUnlinked(const PimItem::List &items, const Collection &collection)
Notify about unlinked items.
Represents a collection of PIM items.
Definition: collection.h:76
void itemsLinked(const PimItem::List &items, const Collection &collection)
Notify about linked items.
const QObjectList & children() const const
int size() const const
bool registerObject(const QString &path, QObject *object, QDBusConnection::RegisterOptions options)
bool contains(const QString &str, Qt::CaseSensitivity cs) const const
T value() const const
virtual void registerInstance(const QString &id)
This is called via D-Bus from AgentManager to register an agent with search interface.
QDBusConnection sessionBus()
void unlock()
QSet::iterator insert(const T &value)
int size() const const
virtual QVector< AbstractSearchPlugin * > searchPlugins() const
Returns currently available search plugins.
void addColumn(const QString &col)
Adds the given column to a select query.
void timeout()
static Collection::List retrieveFiltered(const QString &key, const QVariant &value)
Retrieve all records with value value in column key.
Definition: entities.cpp:2529
int count(const T &value) const const
Helper class for DataStore transaction handling.
Definition: transaction.h:37
QVariant property(const char *name) const const
QVariant value(int index) const const
void disconnectFromBus(const QString &name)
bool isEmpty() const const
QByteArray number(int n, int base)
bool next()
QVector< T > result()
Returns the result of this SELECT query.
QString queryAttributes() const
Returns the value of the queryAttributes column of this record.
Definition: entities.cpp:1598
Part of the DataStore, collects change notifications and emits them after the current transaction has...
bool commit()
Commits the transaction.
Definition: transaction.cpp:42
int count() const const
void unregisterObject(const QString &path, QDBusConnection::UnregisterMode mode)
3rd party applications can install a search plugin for Akonadi server to provide access to their sear...
void reserve(int size)
void lock()
QString name() const const
QVariant fromValue(const T &value)
QStringList libraryPaths()
bool invokeMethod(QObject *obj, const char *member, Qt::ConnectionType type, QGenericReturnArgument ret, QGenericArgument val0, QGenericArgument val1, QGenericArgument val2, QGenericArgument val3, QGenericArgument val4, QGenericArgument val5, QGenericArgument val6, QGenericArgument val7, QGenericArgument val8, QGenericArgument val9)
bool contains(const T &value) const const
static DataStore * self()
Per thread singleton.
Definition: datastore.cpp:235
Helper class for creating and executing database SELECT queries.
QString queryString() const
Returns the value of the queryString column of this record.
Definition: entities.cpp:1585
bool remove(const T &value)
Definition: item.h:44
QString name() const
Returns the value of the name column of this record.
bool isEmpty() const const
virtual void updateSearch(const Collection &collection)
Updates the search query synchronously.
QStringList entryList(QDir::Filters filters, QDir::SortFlags sort) const const
Helper integration between Akonadi and Qt.
int count(const T &value) const const
virtual void unregisterInstance(const QString &id)
This is called via D-Bus from AgentManager to unregister an agent with search interface.
QString queryCollections() const
Returns the value of the queryCollections column of this record.
Definition: entities.cpp:1611
bool isEmpty() const const
QString fromLatin1(const char *str, int size)
bool removePimItem(const PimItem &value) const
Removes a n:m relation between this record and the PimItem value.
virtual void updateSearchAsync(const Collection &collection)
Updates the search query asynchronously.
QMetaObject::Connection connect(const QObject *sender, const char *signal, const QObject *receiver, const char *method, Qt::ConnectionType type)
bool addPimItem(const PimItem &value) const
Adds a n:m relation between this record and the PimItem value.
T qobject_cast(QObject *object)
Helper class to construct arbitrary SQL queries.
Definition: querybuilder.h:45
QDateTime currentDateTimeUtc()
void start()
const T value(const Key &key, const T &defaultValue) const const
bool exec()
Executes the query, returns true on success.
void setSingleShot(bool singleShot)
Representation of a record in the Collection table.
Definition: entities.h:451
This file is part of the KDE documentation.
Documentation copyright © 1996-2020 The KDE developers.
Generated on Tue May 26 2020 22:46:21 by doxygen 1.8.11 written by Dimitri van Heesch, © 1997-2006

KDE's Doxygen guidelines are available online.