collectionsync.cpp

/*
    Copyright (c) 2007, 2009 Volker Krause <vkrause@kde.org>

    This library is free software; you can redistribute it and/or modify it
    under the terms of the GNU Library General Public License as published by
    the Free Software Foundation; either version 2 of the License, or (at your
    option) any later version.

    This library is distributed in the hope that it will be useful, but WITHOUT
    ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Library General Public
    License for more details.

    You should have received a copy of the GNU Library General Public License
    along with this library; see the file COPYING.LIB.  If not, write to the
    Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
    02110-1301, USA.
*/

#include "collectionsync_p.h"
#include "collection.h"

#include "collectioncreatejob.h"
#include "collectiondeletejob.h"
#include "collectionfetchjob.h"
#include "collectionmodifyjob.h"
#include "collectionfetchscope.h"
#include "collectionmovejob.h"
#include "entitydisplayattribute.h"

#include "cachepolicy.h"

#include <kdebug.h>
#include <KLocalizedString>
#include <QtCore/QVariant>

using namespace Akonadi;

struct RemoteNode;

struct LocalNode
{
  LocalNode( const Collection &col ) :
    collection( col ),
    processed( false )
  {}

  ~LocalNode()
  {
    qDeleteAll( childNodes );
    qDeleteAll( pendingRemoteNodes );
  }

  Collection collection;
  QList<LocalNode*> childNodes;
  QHash<QString, LocalNode*> childRidMap;
  QList<RemoteNode*> pendingRemoteNodes;
  bool processed;
};

Q_DECLARE_METATYPE( LocalNode* )
static const char LOCAL_NODE[] = "LocalNode";

struct RemoteNode
{
  RemoteNode( const Collection &col ) :
    collection( col )
  {}

  Collection collection;
};

Q_DECLARE_METATYPE( RemoteNode* )
static const char REMOTE_NODE[] = "RemoteNode";

class CollectionSync::Private
{
  public:
    Private( CollectionSync *parent ) :
      q( parent ),
      pendingJobs( 0 ),
      progress( 0 ),
      localRoot( 0 ),
      currentTransaction( 0 ),
      knownLocalCollections( 0 ),
      incremental( false ),
      streaming( false ),
      hierarchicalRIDs( false ),
      localListDone( false ),
      deliveryDone( false )
    {
    }

    ~Private()
    {
      delete localRoot;
      qDeleteAll( rootRemoteNodes );
    }

    void resetNodeTree()
    {
      delete localRoot;
      localRoot = new LocalNode( Collection::root() );
      localRoot->processed = true; // never try to delete that one
      if ( currentTransaction ) {
        // we are running the update transaction, initialize pending remote nodes
        localRoot->pendingRemoteNodes.swap( rootRemoteNodes );
      }

      localUidMap.clear();
      localRidMap.clear();
      localUidMap.insert( localRoot->collection.id(), localRoot );
      if ( !hierarchicalRIDs ) {
        localRidMap.insert( QString(), localRoot );
      }
    }

    LocalNode* createLocalNode( const Collection &col )
    {
      LocalNode *node = new LocalNode( col );
      Q_ASSERT( !localUidMap.contains( col.id() ) );
      localUidMap.insert( node->collection.id(), node );
      if ( !hierarchicalRIDs && !col.remoteId().isEmpty() ) {
        localRidMap.insert( node->collection.remoteId(), node );
      }

      // add already existing children
      if ( localPendingCollections.contains( col.id() ) ) {
        QVector<Collection::Id> childIds = localPendingCollections.take( col.id() );
        foreach ( Collection::Id childId, childIds ) {
          Q_ASSERT( localUidMap.contains( childId ) );
          LocalNode *childNode = localUidMap.value( childId );
          node->childNodes.append( childNode );
          if ( !childNode->collection.remoteId().isEmpty() ) {
            node->childRidMap.insert( childNode->collection.remoteId(), childNode );
          }
        }
      }

      // set our parent and add ourselves as child
      if ( localUidMap.contains( col.parentCollection().id() ) ) {
        LocalNode* parentNode = localUidMap.value( col.parentCollection().id() );
        parentNode->childNodes.append( node );
        if ( !node->collection.remoteId().isEmpty() ) {
          parentNode->childRidMap.insert( node->collection.remoteId(), node );
        }
      } else {
        localPendingCollections[ col.parentCollection().id() ].append( col.id() );
      }

      return node;
    }

    void createRemoteNode( const Collection &col )
    {
      if ( col.remoteId().isEmpty() ) {
        kWarning() << "Collection '" << col.name() << "' does not have a remote identifier - skipping";
        return;
      }
      RemoteNode *node = new RemoteNode( col );
      rootRemoteNodes.append( node );
    }

    void localCollectionsReceived( const Akonadi::Collection::List &localCols )
    {
      foreach ( const Collection &c, localCols ) {
        createLocalNode( c );
        knownLocalCollections++;
      }
    }

    void localCollectionFetchResult( KJob *job )
    {
      if ( job->error() ) {
        return; // handled by the base class
      }

      // safety check: the local tree has to be connected
      if ( !localPendingCollections.isEmpty() ) {
        q->setError( Unknown );
        q->setErrorText( i18n( "Inconsistent local collection tree detected." ) );
        q->emitResult();
        return;
      }

      localListDone = true;
      execute();
    }

    LocalNode* findLocalChildNodeByName( LocalNode *localParentNode, const QString &name ) const
    {
      if ( name.isEmpty() ) { // shouldn't happen...
        return 0;
      }

      if ( localParentNode == localRoot ) { // possibly non-unique names on top-level
        return 0;
      }

      foreach ( LocalNode *childNode, localParentNode->childNodes ) {
        // the restriction on empty RIDs can possibly removed, but for now I only understand the implication for this case
        if ( childNode->collection.name() == name && childNode->collection.remoteId().isEmpty() ) {
          return childNode;
        }
      }
      return 0;
    }

    LocalNode* findMatchingLocalNode( const Collection &collection ) const
    {
      if ( !hierarchicalRIDs ) {
        if ( localRidMap.contains( collection.remoteId() ) ) {
          return localRidMap.value( collection.remoteId() );
        }
        return 0;
      } else {
        if ( collection.id() == Collection::root().id() || collection.remoteId() == Collection::root().remoteId() ) {
          return localRoot;
        }
        LocalNode *localParent = 0;
        if ( collection.parentCollection().id() < 0 && collection.parentCollection().remoteId().isEmpty() ) {
          kWarning() << "Remote collection without valid parent found: " << collection;
          return 0;
        }
        if ( collection.parentCollection().id() == Collection::root().id() || collection.parentCollection().remoteId() == Collection::root().remoteId() ) {
          localParent = localRoot;
        } else {
          localParent = findMatchingLocalNode( collection.parentCollection() );
        }

        if ( localParent ) {
          if ( localParent->childRidMap.contains( collection.remoteId() ) ) {
            return localParent->childRidMap.value( collection.remoteId() );
          }
          // check if we have a local folder with a matching name and no RID, if so let's use that one
          // we would get an error if we don't do this anyway, as we'd try to create two sibling nodes with the same name
          if ( LocalNode *recoveredLocalNode = findLocalChildNodeByName( localParent, collection.name() ) ) {
            kDebug() << "Recovering collection with lost RID:" << collection << recoveredLocalNode->collection;
            return recoveredLocalNode;
          }
        }
        return 0;
      }
    }

    LocalNode* findBestLocalAncestor( const Collection &collection, bool *exactMatch = 0 )
    {
      if ( !hierarchicalRIDs ) {
        return localRoot;
      }
      if ( collection == Collection::root() ) {
        if ( exactMatch ) {
          *exactMatch = true;
        }
        return localRoot;
      }
      if ( collection.parentCollection().id() < 0 && collection.parentCollection().remoteId().isEmpty() ) {
        kWarning() << "Remote collection without valid parent found: " << collection;
        return 0;
      }
      bool parentIsExact = false;
      LocalNode *localParent = findBestLocalAncestor( collection.parentCollection(), &parentIsExact );
      if ( !parentIsExact ) {
        if ( exactMatch ) {
          *exactMatch = false;
        }
        return localParent;
      }
      if ( localParent->childRidMap.contains( collection.remoteId() ) ) {
        if ( exactMatch ) {
          *exactMatch = true;
        }
        return localParent->childRidMap.value( collection.remoteId() );
      }
      if ( exactMatch ) {
        *exactMatch = false;
      }
      return localParent;
    }

    bool checkPendingRemoteNodes() const
    {
      if ( rootRemoteNodes.size() != knownLocalCollections ) {
        return true;
      }

      foreach ( RemoteNode *remoteNode, rootRemoteNodes ) {
        // every remote note should have a local node already
        LocalNode *localNode = findMatchingLocalNode( remoteNode->collection );
        if ( localNode ) {
          if ( checkLocalCollection( localNode, remoteNode ) ) {
            return true;
          }
        } else {
          return true;
        }
      }
      return false;
    }

    void processPendingRemoteNodes( LocalNode *_localRoot )
    {
      QList<RemoteNode*> pendingRemoteNodes( _localRoot->pendingRemoteNodes );
      _localRoot->pendingRemoteNodes.clear();
      QHash<LocalNode*, QList<RemoteNode*> > pendingCreations;
      foreach ( RemoteNode *remoteNode, pendingRemoteNodes ) {
        // step 1: see if we have a matching local node already
        LocalNode *localNode = findMatchingLocalNode( remoteNode->collection );
        if ( localNode ) {
          Q_ASSERT( !localNode->processed );
          updateLocalCollection( localNode, remoteNode );
          continue;
        }
        // step 2: check if we have the parent at least, then we can create it
        localNode = findMatchingLocalNode( remoteNode->collection.parentCollection() );
        if ( localNode ) {
          pendingCreations[localNode].append( remoteNode );
          continue;
        }
        // step 3: find the best matching ancestor and enqueue it for later processing
        localNode = findBestLocalAncestor( remoteNode->collection );
        if ( !localNode ) {
          q->setError( Unknown );
          q->setErrorText( i18n( "Remote collection without root-terminated ancestor chain provided, resource is broken." ) );
          q->emitResult();
          return;
        }
        localNode->pendingRemoteNodes.append( remoteNode );
      }

      // process the now possible collection creations
      for ( QHash<LocalNode*, QList<RemoteNode*> >::const_iterator it = pendingCreations.constBegin();
            it != pendingCreations.constEnd(); ++it ) {
        createLocalCollections( it.key(), it.value() );
      }
    }

    bool checkLocalCollection( LocalNode *localNode, RemoteNode *remoteNode ) const
    {
      const Collection &localCollection = localNode->collection;
      const Collection &remoteCollection = remoteNode->collection;

      if ( localCollection.contentMimeTypes().size() != remoteCollection.contentMimeTypes().size() ) {
        return true;
      } else {
        for ( int i = 0; i < remoteCollection.contentMimeTypes().size(); i++ ) {
          const QString &m = remoteCollection.contentMimeTypes().at( i );
          if( !localCollection.contentMimeTypes().contains( m ) ) {
            return true;
          }
        }
      }

      if ( localCollection.parentCollection().remoteId() != remoteCollection.parentCollection().remoteId() ) {
        return true;
      }
      if ( localCollection.name() != remoteCollection.name() ) {
        return true;
      }
      if ( localCollection.remoteId() != remoteCollection.remoteId() ) {
        return true;
      }
      if ( localCollection.remoteRevision() != remoteCollection.remoteRevision() ) {
        return true;
      }
      if ( !(localCollection.cachePolicy() == remoteCollection.cachePolicy()) ) {
        return true;
      }

      // CollectionModifyJob adds the remote attributes to the local collection
      foreach ( const Attribute* attr, remoteCollection.attributes() ) {
        const Attribute* localAttr = localCollection.attribute( attr->type() );
        // The attribute must both exist and have equal contents
        if ( !localAttr || localAttr->serialized() != attr->serialized() ) {
          return true;
        }
      }

      return false;
    }

    void updateLocalCollection( LocalNode *localNode, RemoteNode *remoteNode )
    {
      Collection upd( remoteNode->collection );
      Q_ASSERT( !upd.remoteId().isEmpty() );
      Q_ASSERT( currentTransaction );
      upd.setId( localNode->collection.id() );
      if ( localNode->collection.attribute<EntityDisplayAttribute>() ) {
        upd.removeAttribute<EntityDisplayAttribute>();
        upd.addAttribute( localNode->collection.attribute<EntityDisplayAttribute>()->clone() );
      }

      {
        // ### HACK to work around the implicit move attempts of CollectionModifyJob
        // which we do explicitly below
        Collection c( upd );
        c.setParentCollection( localNode->collection.parentCollection() );
        ++pendingJobs;
        CollectionModifyJob *mod = new CollectionModifyJob( c, currentTransaction );
        connect( mod, SIGNAL(result(KJob*)), q, SLOT(updateLocalCollectionResult(KJob*)) );
      }

      // detecting moves is only possible with global RIDs
      if ( !hierarchicalRIDs ) {
        LocalNode *oldParent = localUidMap.value( localNode->collection.parentCollection().id() );
        LocalNode *newParent = findMatchingLocalNode( remoteNode->collection.parentCollection() );
        // TODO: handle the newParent == 0 case correctly, ie. defer the move until the new
        // local parent has been created
        if ( newParent && oldParent != newParent ) {
          ++pendingJobs;
          CollectionMoveJob *move = new CollectionMoveJob( upd, newParent->collection, currentTransaction );
          connect( move, SIGNAL(result(KJob*)), q, SLOT(updateLocalCollectionResult(KJob*)) );
        }
      }

      localNode->processed = true;
      delete remoteNode;
    }

    void updateLocalCollectionResult( KJob* job )
    {
      --pendingJobs;
      if ( job->error() ) {
        return; // handled by the base class
      }
      if ( qobject_cast<CollectionModifyJob*>( job ) ) {
        ++progress;
      }
      checkDone();
    }

    void createLocalCollections( LocalNode* localParent, QList<RemoteNode*> remoteNodes )
    {
      foreach ( RemoteNode *remoteNode, remoteNodes ) {
        ++pendingJobs;
        Collection col( remoteNode->collection );
        Q_ASSERT( !col.remoteId().isEmpty() );
        col.setParentCollection( localParent->collection );
        CollectionCreateJob *create = new CollectionCreateJob( col, currentTransaction );
        create->setProperty( LOCAL_NODE, QVariant::fromValue( localParent ) );
        create->setProperty( REMOTE_NODE, QVariant::fromValue( remoteNode ) );
        connect( create, SIGNAL(result(KJob*)), q, SLOT(createLocalCollectionResult(KJob*)) );
      }
    }

    void createLocalCollectionResult( KJob* job )
    {
      --pendingJobs;
      if ( job->error() ) {
        return; // handled by the base class
      }

      const Collection newLocal = static_cast<CollectionCreateJob*>( job )->collection();
      LocalNode *localNode = createLocalNode( newLocal );
      localNode->processed = true;

      LocalNode *localParent = job->property( LOCAL_NODE ).value<LocalNode*>();
      Q_ASSERT( localParent->childNodes.contains( localNode ) );
      RemoteNode *remoteNode = job->property( REMOTE_NODE ).value<RemoteNode*>();
      delete remoteNode;
      ++progress;

      processPendingRemoteNodes( localParent );
      if ( !hierarchicalRIDs ) {
        processPendingRemoteNodes( localRoot );
      }

      checkDone();
    }

    bool hasProcessedChildren( LocalNode *localNode ) const
    {
      if ( localNode->processed ) {
        return true;
      }
      foreach ( LocalNode *child, localNode->childNodes ) {
        if ( hasProcessedChildren( child ) ) {
          return true;
        }
      }
      return false;
    }

    Collection::List findUnprocessedLocalCollections( LocalNode *localNode ) const
    {
      Collection::List rv;
      if ( !localNode->processed ) {
        if ( hasProcessedChildren( localNode ) ) {
          kWarning() << "Found unprocessed local node with processed children, excluding from deletion";
          kWarning() << localNode->collection;
          return rv;
        }
        if ( localNode->collection.remoteId().isEmpty() ) {
          kWarning() << "Found unprocessed local node without remoteId, excluding from deletion";
          kWarning() << localNode->collection;
          return rv;
        }
        rv.append( localNode->collection );
        return rv;
      }

      foreach ( LocalNode *child, localNode->childNodes ) {
        rv.append( findUnprocessedLocalCollections( child ) );
      }
      return rv;
    }

    void deleteUnprocessedLocalNodes()
    {
      if ( incremental ) {
        return;
      }
      const Collection::List cols = findUnprocessedLocalCollections( localRoot );
      deleteLocalCollections( cols );
    }

    void deleteLocalCollections( const Collection::List &cols )
    {
      q->setTotalAmount( KJob::Bytes, q->totalAmount( KJob::Bytes ) + cols.size() );
      foreach ( const Collection &col, cols ) {
        Q_ASSERT( !col.remoteId().isEmpty() ); // empty RID -> stuff we haven't even written to the remote side yet

        ++pendingJobs;
        Q_ASSERT( currentTransaction );
        CollectionDeleteJob *job = new CollectionDeleteJob( col, currentTransaction );
        connect( job, SIGNAL(result(KJob*)), q, SLOT(deleteLocalCollectionsResult(KJob*)) );

        // It can happen that the groupware servers report us deleted collections
        // twice, in this case this collection delete job will fail on the second try.
        // To avoid a rollback of the complete transaction we gracefully allow the job
        // to fail :)
        currentTransaction->setIgnoreJobFailure( job );
      }
    }

    void deleteLocalCollectionsResult( KJob* )
    {
      --pendingJobs;

      ++progress;
      checkDone();
    }

    void checkUpdateNecessity()
    {
      bool updateNeeded = checkPendingRemoteNodes();
      if ( !updateNeeded ) {
        // We can end right now
        q->emitResult();
        return;
      }

      // Since there are differences with the remote collections we need to sync. Start a transaction here.
      Q_ASSERT( !currentTransaction );
      currentTransaction = new TransactionSequence( q );
      currentTransaction->setAutomaticCommittingEnabled( false );
      q->connect( currentTransaction, SIGNAL(result(KJob*)), SLOT(transactionSequenceResult(KJob*)) );

      // Now that a transaction is started we need to fetch local collections again and do the update
      q->doStart();
    }

    void transactionSequenceResult( KJob *job )
    {
      if ( job->error() ) {
        return; // handled by the base class
      }

      q->emitResult();
    }

    void execute()
    {
      kDebug() << Q_FUNC_INFO << "localListDone: " << localListDone << " deliveryDone: " << deliveryDone;
      if ( !localListDone ) {
        return;
      }

      // If a transaction is not started yet we are still checking if the update is
      // actually needed.
      if ( !currentTransaction ) {
        checkUpdateNecessity();
        return;
      }

      // Since the transaction is already running we need to execute the update.
      processPendingRemoteNodes( localRoot );

      if ( !incremental && deliveryDone ) {
        deleteUnprocessedLocalNodes();
      }

      if ( !hierarchicalRIDs ) {
        deleteLocalCollections( removedRemoteCollections );
      } else {
        Collection::List localCols;
        foreach ( const Collection &c, removedRemoteCollections ) {
          LocalNode *node = findMatchingLocalNode( c );
          if ( node ) {
            localCols.append( node->collection );
          }
        }
        deleteLocalCollections( localCols );
      }
      removedRemoteCollections.clear();

      checkDone();
    }

    QList<RemoteNode*> findPendingRemoteNodes( LocalNode *localNode )
    {
      QList<RemoteNode*> rv;
      rv.append( localNode->pendingRemoteNodes );
      foreach ( LocalNode *child, localNode->childNodes ) {
        rv.append( findPendingRemoteNodes( child ) );
      }
      return rv;
    }

    void checkDone()
    {
      q->setProcessedAmount( KJob::Bytes, progress );

      // still running jobs or not fully delivered local/remote state
      if ( !deliveryDone || pendingJobs > 0 || !localListDone ) {
        return;
      }

      // safety check: there must be no pending remote nodes anymore
      QList<RemoteNode*> orphans = findPendingRemoteNodes( localRoot );
      if ( !orphans.isEmpty() ) {
        q->setError( Unknown );
        q->setErrorText( i18n( "Found unresolved orphan collections" ) );
        foreach ( RemoteNode* orphan, orphans ) {
          kDebug() << "found orphan collection:" << orphan->collection;
        }
        q->emitResult();
        return;
      }

      kDebug() << Q_FUNC_INFO << "q->commit()";
      Q_ASSERT( currentTransaction );
      currentTransaction->commit();
    }

    CollectionSync *q;

    QString resourceId;

    int pendingJobs;
    int progress;

    LocalNode* localRoot;
    TransactionSequence* currentTransaction;
    QHash<Collection::Id, LocalNode*> localUidMap;
    QHash<QString, LocalNode*> localRidMap;

    // temporary during build-up of the local node tree, must be empty afterwards
    QHash<Collection::Id, QVector<Collection::Id> > localPendingCollections;

    // removed remote collections in incremental mode
    Collection::List removedRemoteCollections;

    // used to store the list of remote nodes passed by the user
    QList<RemoteNode*> rootRemoteNodes;

    // keep track of the total number of local collections that are known
    // only used during the preliminary check to see if updating is needed
    int knownLocalCollections;

    bool incremental;
    bool streaming;
    bool hierarchicalRIDs;

    bool localListDone;
    bool deliveryDone;
};

CollectionSync::CollectionSync( const QString &resourceId, QObject *parent ) :
    Job( parent ),
    d( new Private( this ) )
{
  d->resourceId = resourceId;
  setTotalAmount( KJob::Bytes, 0 );
}

CollectionSync::~CollectionSync()
{
  delete d;
}

void CollectionSync::setRemoteCollections(const Collection::List & remoteCollections)
{
  setTotalAmount( KJob::Bytes, totalAmount( KJob::Bytes ) + remoteCollections.count() );
  foreach ( const Collection &c, remoteCollections ) {
    d->createRemoteNode( c );
  }

  if ( !d->streaming ) {
    d->deliveryDone = true;
  }
  d->execute();
}

void CollectionSync::setRemoteCollections(const Collection::List & changedCollections, const Collection::List & removedCollections)
{
  setTotalAmount( KJob::Bytes, totalAmount( KJob::Bytes ) + changedCollections.count() );
  d->incremental = true;
  foreach ( const Collection &c, changedCollections ) {
    d->createRemoteNode( c );
  }
  d->removedRemoteCollections += removedCollections;

  if ( !d->streaming ) {
    d->deliveryDone = true;
  }
  d->execute();
}

void CollectionSync::doStart()
{
  d->resetNodeTree();
  Job *parent = ( d->currentTransaction ? static_cast<Job*>( d->currentTransaction ) : static_cast<Job*>( this ) );
  CollectionFetchJob *job = new CollectionFetchJob( Collection::root(), CollectionFetchJob::Recursive, parent );
  job->fetchScope().setResource( d->resourceId );
  job->fetchScope().setIncludeUnsubscribed( true );
  job->fetchScope().setAncestorRetrieval( CollectionFetchScope::Parent );
  connect( job, SIGNAL(collectionsReceived(Akonadi::Collection::List)),
           SLOT(localCollectionsReceived(Akonadi::Collection::List)) );
  connect( job, SIGNAL(result(KJob*)), SLOT(localCollectionFetchResult(KJob*)) );
}

void CollectionSync::setStreamingEnabled( bool streaming )
{
  d->streaming = streaming;
}

void CollectionSync::retrievalDone()
{
  d->deliveryDone = true;
  d->execute();
}

void CollectionSync::setHierarchicalRemoteIds( bool hierarchical )
{
  d->hierarchicalRIDs = hierarchical;
}

void CollectionSync::rollback()
{
  if ( d->currentTransaction ) {
    d->currentTransaction->rollback();
  }
}

#include "moc_collectionsync_p.cpp"