Fix kolab cache sync issues

Summary: Use QRESYNC, get rid of "scheduled" cache reset, other small improvements

Reviewers: #roundcube_kolab_plugins_developers, vanmeeuwen

Reviewed By: #roundcube_kolab_plugins_developers, vanmeeuwen

Subscribers: vanmeeuwen, #roundcube_kolab_plugins_developers

Differential Revision: https://git.kolab.org/D1726

plugins/libkolab/config.inc.php.dist

@@ -5,10 +5,6 @@
 // Enable caching of Kolab objects in local database
 $config['kolab_cache'] = true;
 
-// Cache refresh interval (default is 12 hours)
-// after this period, cache is forced to synchronize with IMAP
-$config['kolab_cache_refresh'] = '12h';
-
 // Specify format version to write Kolab objects (must be a string value!)
 $config['kolab_format_version']  = '3.0';
 

plugins/libkolab/lib/kolab_storage_cache.php

@@ -27,8 +27,6 @@ class kolab_storage_cache
     const DB_DATE_FORMAT = 'Y-m-d H:i:s';
     const MAX_RECORDS    = 500;
 
-    public $sync_complete = false;
-
     protected $db;
     protected $imap;
     protected $folder;
@@ -42,7 +40,6 @@ class kolab_storage_cache
     protected $synclock = false;
     protected $ready = false;
     protected $cache_table;
-    protected $cache_refresh = 3600;
     protected $folders_table;
     protected $max_sql_packet;
     protected $max_sync_lock_time = 600;
@@ -85,7 +82,6 @@ class kolab_storage_cache
         $this->imap = $rcmail->get_storage();
         $this->enabled = $rcmail->config->get('kolab_cache', false);
         $this->folders_table = $this->db->table_name('kolab_folders');
-        $this->cache_refresh = get_offset_sec($rcmail->config->get('kolab_cache_refresh', '12h'));
         $this->server_timezone = new DateTimeZone(date_default_timezone_get());
 
         if ($this->enabled) {
@@ -174,16 +170,6 @@ class kolab_storage_cache
         if ($this->synched)
             return;
 
-        // increase time limit
-        @set_time_limit($this->max_sync_lock_time - 60);
-
-        // get effective time limit we have for synchronization (~70% of the execution time)
-        $time_limit = ini_get('max_execution_time') * 0.7;
-        $sync_start = time();
-
-        // assume sync will be completed
-        $this->sync_complete = true;
-
         if (!$this->ready) {
             // kolab cache is disabled, synchronize IMAP mailbox cache only
             $this->imap_mode(true);
@@ -191,24 +177,72 @@ class kolab_storage_cache
             $this->imap_mode(false);
         }
         else {
+            $this->sync_start = time();
+
             // read cached folder metadata
             $this->_read_folder_data();
 
+            // Read folder data from IMAP
+            $ctag = $this->folder->get_ctag();
+
+            // Validate current ctag
+            list($uidvalidity, $highestmodseq, $uidnext) = explode('-', $ctag);
+
+            if (empty($uidvalidity) || empty($highestmodseq)) {
+                rcube::raise_error(array(
+                    'code' => 900,
+                    'message' => "Failed to sync the kolab cache (Invalid ctag)"
+                ), true);
+            }
             // check cache status ($this->metadata is set in _read_folder_data())
-            if (  empty($this->metadata['ctag']) ||
+            else if (
-                  empty($this->metadata['changed']) ||
+                empty($this->metadata['ctag'])
-                  $this->metadata['objectcount'] === null ||
+                || empty($this->metadata['changed'])
-                  $this->metadata['changed'] < date(self::DB_DATE_FORMAT, time() - $this->cache_refresh) ||
+                || $this->metadata['ctag'] !== $ctag
-                  $this->metadata['ctag'] != $this->folder->get_ctag() ||
-                  intval($this->metadata['objectcount']) !== $this->count()
             ) {
                 // lock synchronization for this folder or wait if locked
                 $this->_sync_lock();
 
+                // Run a full-sync (initial sync or continue the aborted sync)
+                if (empty($this->metadata['changed']) || empty($this->metadata['ctag'])) {
+                    $result = $this->synchronize_full();
+                }
+                // Synchronize only the changes since last sync
+                else {
+                    $result = $this->synchronize_update($ctag);
+                }
+
+                // update ctag value (will be written to database in _sync_unlock())
+                if ($result) {
+                    $this->metadata['ctag']    = $ctag;
+                    $this->metadata['changed'] = date(self::DB_DATE_FORMAT, time());
+                }
+
+                // remove lock
+                $this->_sync_unlock();
+            }
+        }
+
+        $this->check_error();
+        $this->synched = time();
+    }
+
+    /**
+     * Perform full cache synchronization
+     */
+    protected function synchronize_full()
+    {
+        // get effective time limit we have for synchronization (~70% of the execution time)
+        $time_limit = $this->_max_sync_lock_time() * 0.7;
+
+        if (time() - $this->sync_start > $time_limit) {
+            return false;
+        }
+
         // disable messages cache if configured to do so
         $this->imap_mode(true);
 
-                // synchronize IMAP mailbox cache
+        // synchronize IMAP mailbox cache, does nothing if messages cache is disabled
         $this->imap->folder_sync($this->folder->name);
 
         // compare IMAP index with object cache index
@@ -216,32 +250,160 @@ class kolab_storage_cache
 
         $this->imap_mode(false);
 
-                // determine objects to fetch or to invalidate
+        if ($imap_index->is_error()) {
-                if (!$imap_index->is_error()) {
+            rcube::raise_error(array(
-                    $imap_index = $imap_index->get();
+                    'code' => 900,
-                    $old_index  = array();
+                    'message' => "Failed to sync the kolab cache (SEARCH failed)"
-                    $del_index  = array();
+                ), true);
+            return false;
+        }
 
-                    // read cache index
+        // determine objects to fetch or to invalidate
-                    $sql_result = $this->db->query(
+        $imap_index = $imap_index->get();
-                        "SELECT `msguid`, `uid` FROM `{$this->cache_table}` WHERE `folder_id` = ?"
+        $del_index  = array();
-                            . " ORDER BY `msguid` DESC", $this->folder_id
+        $old_index  = $this->current_index($del_index);
+
+        // Fetch objects and store in DB
+        $result = $this->synchronize_fetch($imap_index, $old_index, $del_index);
+
+        if ($result) {
+            // Remove redundant entries from IMAP and cache
+            $rem_index = array_intersect($del_index, $imap_index);
+            $del_index = array_merge(array_unique($del_index), array_diff($old_index, $imap_index));
+
+            $this->synchronize_delete($rem_index, $del_index);
+        }
+
+        return $result;
+    }
+
+    /**
+     * Perform partial cache synchronization, based on QRESYNC
+     */
+    protected function synchronize_update()
+    {
+        if (!$this->imap->get_capability('QRESYNC')) {
+            rcube::raise_error(array(
+                    'code' => 900,
+                    'message' => "Failed to sync the kolab cache (no QRESYNC capability)"
+                ), true);
+
+            return $this->synchronize_full();
+        }
+
+        // Handle the previous ctag
+        list($uidvalidity, $highestmodseq, $uidnext) = explode('-', $this->metadata['ctag']);
+
+        if (empty($uidvalidity) || empty($highestmodseq)) {
+            rcube::raise_error(array(
+                    'code' => 900,
+                    'message' => "Failed to sync the kolab cache (Invalid old ctag)"
+                ), true);
+            return false;
+        }
+
+        // Enable QRESYNC
+        $res = $this->imap->conn->enable('QRESYNC');
+        if ($res === false) {
+            rcube::raise_error(array(
+                    'code' => 900,
+                    'message' => "Failed to sync the kolab cache (failed to enable QRESYNC/CONDSTORE)"
+                ), true);
+
+            return false;
+        }
+
+        $mbox_data = $this->imap->folder_data($this->folder->name);
+        if (empty($mbox_data)) {
+            rcube::raise_error(array(
+                    'code' => 900,
+                    'message' => "Failed to sync the kolab cache (failed to get folder state)"
+                ), true);
+
+             return false;
+        }
+
+        // Check UIDVALIDITY
+        if ($uidvalidity != $mbox_data['UIDVALIDITY']) {
+            return $this->synchronize_full();
+        }
+
+        // QRESYNC not supported on specified mailbox
+        if (!empty($mbox_data['NOMODSEQ']) || empty($mbox_data['HIGHESTMODSEQ'])) {
+            rcube::raise_error(array(
+                    'code' => 900,
+                    'message' => "Failed to sync the kolab cache (QRESYNC not supported on the folder)"
+                ), true);
+
+             return $this->synchronize_full();
+        }
+
+        // Get modified flags and vanished messages
+        // UID FETCH 1:* (FLAGS) (CHANGEDSINCE 0123456789 VANISHED)
+        $result = $this->imap->conn->fetch(
+            $this->folder->name, '1:*', true, array('FLAGS'), $highestmodseq, true
         );
 
-                    while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+        $removed  = array();
-                        // Mark all duplicates for removal (note sorting order above)
+        $modified = array();
-                        // Duplicates here should not happen, but they do sometimes
+        $existing = $this->current_index($removed);
-                        if (isset($old_index[$sql_arr['uid']])) {
+
-                            $del_index[] = $sql_arr['msguid'];
+        if (!empty($result)) {
+            foreach ($result as $msg) {
+                $uid = $msg->uid;
+
+                // Message marked as deleted
+                if (!empty($msg->flags['DELETED'])) {
+                    $removed[] = $uid;
+                    continue;
                 }
-                        else {
+
-                            $old_index[$sql_arr['uid']] = $sql_arr['msguid'];
+                // Flags changed or new
+                $modified[] = $uid;
             }
         }
 
-                    // fetch new objects from imap
+        $new    = array_diff($modified, $existing, $removed);
+        $result = true;
+
+        if (!empty($new)) {
+            $result = $this->synchronize_fetch($new, $existing, $removed);
+
+            if (!$result) {
+                return false;
+            }
+        }
+
+        // VANISHED found?
+        $mbox_data = $this->imap->folder_data($this->folder->name);
+
+        // Removed vanished messages from the database
+        $vanished = (array) rcube_imap_generic::uncompressMessageSet($mbox_data['VANISHED']);
+
+        // Remove redundant entries from IMAP and DB
+        $vanished = array_merge($removed, array_intersect($vanished, $existing));
+        $this->synchronize_delete($removed, $vanished);
+
+        return $result;
+    }
+
+    /**
+     * Fetch objects from IMAP and save into the database
+     */
+    protected function synchronize_fetch($new_index, &$old_index, &$del_index)
+    {
+        // get effective time limit we have for synchronization (~70% of the execution time)
+        $time_limit = $this->_max_sync_lock_time() * 0.7;
+
+        if (time() - $this->sync_start > $time_limit) {
+            return false;
+        }
+
         $i = 0;
-                    foreach (array_diff($imap_index, $old_index) as $msguid) {
+        $aborted = false;
+
+        // fetch new objects from imap
+        foreach (array_diff($new_index, $old_index) as $msguid) {
             // Note: We'll store only objects matching the folder type
             // anything else will be silently ignored
             if ($object = $this->folder->read_object($msguid)) {
@@ -263,52 +425,64 @@ class kolab_storage_cache
                 $this->_extended_insert($msguid, $object);
 
                 // check time limit and abort sync if running too long
-                            if (++$i % 50 == 0 && time() - $sync_start > $time_limit) {
+                if (++$i % 50 == 0 && time() - $this->sync_start > $time_limit) {
-                                $this->sync_complete = false;
+                    $aborted = true;
                     break;
                 }
             }
         }
+
         $this->_extended_insert(0, null);
 
-                    $del_index = array_unique($del_index);
+        return $aborted === false;
+    }
 
-                    // delete duplicate entries from IMAP
+    /**
-                    $rem_index = array_intersect($del_index, $imap_index);
+     * Remove specified objects from the database and IMAP
-                    if (!empty($rem_index)) {
+     */
+    protected function synchronize_delete($imap_delete, $db_delete)
+    {
+        if (!empty($imap_delete)) {
             $this->imap_mode(true);
-                        $this->imap->delete_message($rem_index, $this->folder->name);
+            $this->imap->delete_message($imap_delete, $this->folder->name);
             $this->imap_mode(false);
         }
 
-                    // delete old/invalid entries from the cache
+        if (!empty($db_delete)) {
-                    $del_index += array_diff($old_index, $imap_index);
+            $quoted_ids = join(',', array_map(array($this->db, 'quote'), $db_delete));
-                    if (!empty($del_index)) {
-                        $quoted_ids = join(',', array_map(array($this->db, 'quote'), $del_index));
             $this->db->query(
                 "DELETE FROM `{$this->cache_table}` WHERE `folder_id` = ? AND `msguid` IN ($quoted_ids)",
                 $this->folder_id
             );
         }
+    }
 
-                    // update ctag value (will be written to database in _sync_unlock())
+    /**
-                    if ($this->sync_complete) {
+     * Return current use->msguid index
-                        $this->metadata['ctag'] = $this->folder->get_ctag();
+     */
-                        $this->metadata['changed'] = date(self::DB_DATE_FORMAT, time());
+    protected function current_index(&$duplicates = array())
-                        // remember the number of cache entries linked to this folder
+    {
-                        $this->metadata['objectcount'] = $this->count();
+        // read cache index
+        $sql_result = $this->db->query(
+            "SELECT `msguid`, `uid` FROM `{$this->cache_table}` WHERE `folder_id` = ?"
+                . " ORDER BY `msguid` DESC", $this->folder_id
+        );
+
+        $index = $del_index = array();
+
+        while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+            // Mark all duplicates for removal (note sorting order above)
+            // Duplicates here should not happen, but they do sometimes
+            if (isset($index[$sql_arr['uid']])) {
+                $duplicates[] = $sql_arr['msguid'];
+            }
+            else {
+                $index[$sql_arr['uid']] = $sql_arr['msguid'];
             }
         }
 
-                // remove lock
+        return $index;
-                $this->_sync_unlock();
     }
-        }
-
-        $this->check_error();
-        $this->synched = time();
-    }
-
 
     /**
      * Read a single entry from cache or from IMAP directly
@@ -1044,7 +1218,7 @@ class kolab_storage_cache
             return;
 
         $sql_arr = $this->db->fetch_assoc($this->db->query(
-                "SELECT `folder_id`, `synclock`, `ctag`, `changed`, `objectcount`"
+                "SELECT `folder_id`, `synclock`, `ctag`, `changed`"
                 . " FROM `{$this->folders_table}` WHERE `resource` = ?",
                 $this->resource_uri
         ));
@@ -1082,10 +1256,12 @@ class kolab_storage_cache
         $read_query  = "SELECT `synclock`, `ctag` FROM `{$this->folders_table}` WHERE `folder_id` = ?";
         $write_query = "UPDATE `{$this->folders_table}` SET `synclock` = ? WHERE `folder_id` = ? AND `synclock` = ?";
 
+        $max_lock_time = $this->_max_sync_lock_time();
+
         // wait if locked (expire locks after 10 minutes) ...
         // ... or if setting lock fails (another process meanwhile set it)
         while (
-            (intval($this->metadata['synclock']) + $this->max_sync_lock_time > time()) ||
+            (intval($this->metadata['synclock']) + $max_lock_time > time()) ||
             (($res = $this->db->query($write_query, time(), $this->folder_id, intval($this->metadata['synclock']))) &&
                 !($affected = $this->db->affected_rows($res)))
         ) {
@@ -1105,16 +1281,26 @@ class kolab_storage_cache
             return;
 
         $this->db->query(
-            "UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ?, `changed` = ?, `objectcount` = ? WHERE `folder_id` = ?",
+            "UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ?, `changed` = ? WHERE `folder_id` = ?",
             $this->metadata['ctag'],
             $this->metadata['changed'],
-            $this->metadata['objectcount'],
             $this->folder_id
         );
 
         $this->synclock = false;
     }
 
+    protected function _max_sync_lock_time()
+    {
+        $limit = get_offset_sec(ini_get('max_execution_time'));
+
+        if ($limit <= 0 || $limit > $this->max_sync_lock_time) {
+            $limit = $this->max_sync_lock_time;
+        }
+
+        return $limit;
+    }
+
     /**
      * Check IMAP connection error state
      */