Improve groupware object cache synchronization (#3933):

- store and compare the number of objects stored after last sync
- use a (configurable) refresh interval and periodically force re-sync

ATTENTION: requires database schema updates

plugins/libkolab/SQL/mysql.initial.sql

@@ -15,6 +15,8 @@ CREATE TABLE `kolab_folders` (
   `type` VARCHAR(32) NOT NULL,
   `synclock` INT(10) NOT NULL DEFAULT '0',
   `ctag` VARCHAR(40) DEFAULT NULL,
+  `changed` DATETIME DEFAULT NULL,
+  `objectcount` BIGINT DEFAULT NULL,
   PRIMARY KEY(`folder_id`),
   INDEX `resource_type` (`resource`, `type`)
 ) /*!40000 ENGINE=INNODB */ /*!40101 CHARACTER SET utf8 COLLATE utf8_general_ci */;

plugins/libkolab/SQL/mysql/2015020600.sql

@@ -0,0 +1,4 @@
+-- improve cache synchronization (#3933)
+ALTER TABLE `kolab_folders`
+  ADD `changed` DATETIME DEFAULT NULL,
+  ADD `objectcount` BIGINT DEFAULT NULL;

plugins/libkolab/SQL/oracle.initial.sql

@@ -12,7 +12,9 @@ CREATE TABLE "kolab_folders" (
     "resource" VARCHAR(255) NOT NULL,
     "type" VARCHAR(32) NOT NULL,
     "synclock" integer DEFAULT 0 NOT NULL,
-    "ctag" VARCHAR(40) DEFAULT NULL
+    "ctag" VARCHAR(40) DEFAULT NULL,
+    "changed" timestamp DEFAULT NULL,
+    "objectcount" number DEFAULT NULL
 );
 
 CREATE INDEX "kolab_folders_resource_idx" ON "kolab_folders" ("resource", "type");

plugins/libkolab/SQL/oracle/2015020600.sql

@@ -0,0 +1,4 @@
+-- improve cache synchronization (#3933)
+ALTER TABLE "kolab_folders"
+  ADD "changed" timestamp DEFAULT NULL,
+  ADD "objectcount" number DEFAULT NULL;

plugins/libkolab/config.inc.php.dist

@@ -5,6 +5,10 @@
 // Enable caching of Kolab objects in local database
 $config['kolab_cache'] = true;
 
+// Cache refresh interval (default is 12 hours)
+// after this period, cache is forced to synchronize with IMAP
+$config['kolab_cache_refresh'] = '12h';
+
 // Specify format version to write Kolab objects (must be a string value!)
 $config['kolab_format_version']  = '3.0';
 

plugins/libkolab/lib/kolab_storage_cache.php

@@ -41,6 +41,7 @@ class kolab_storage_cache
     protected $synclock = false;
     protected $ready = false;
     protected $cache_table;
+    protected $cache_refresh = 3600;
     protected $folders_table;
     protected $max_sql_packet;
     protected $max_sync_lock_time = 600;
@@ -82,6 +83,7 @@ class kolab_storage_cache
         $this->imap = $rcmail->get_storage();
         $this->enabled = $rcmail->config->get('kolab_cache', false);
         $this->folders_table = $this->db->table_name('kolab_folders');
+        $this->cache_refresh = get_offset_sec($rcmail->config->get('kolab_cache_refresh', '12h'));
 
         if ($this->enabled) {
             // always read folder cache and lock state from DB master
@@ -187,8 +189,14 @@ class kolab_storage_cache
             // read cached folder metadata
             $this->_read_folder_data();
 
-            // check cache status hash first ($this->metadata is set in _read_folder_data())
-            if ($this->metadata['ctag'] != $this->folder->get_ctag()) {
+            // check cache status ($this->metadata is set in _read_folder_data())
+            if (  empty($this->metadata['ctag']) ||
+                  empty($this->metadata['changed']) ||
+                  $this->metadata['objectcount'] === null ||
+                  $this->metadata['changed'] < date(self::DB_DATE_FORMAT, time() - $this->cache_refresh) ||
+                  $this->metadata['ctag'] != $this->folder->get_ctag() ||
+                  intval($this->metadata['objectcount']) !== $this->count()
+                ) {
                 // lock synchronization for this folder or wait if locked
                 $this->_sync_lock();
 
@@ -244,6 +252,9 @@ class kolab_storage_cache
                     // update ctag value (will be written to database in _sync_unlock())
                     if ($this->sync_complete) {
                         $this->metadata['ctag'] = $this->folder->get_ctag();
+                        $this->metadata['changed'] = date(self::DB_DATE_FORMAT, time());
+                        // remember the number of cache entries linked to this folder
+                        $this->metadata['objectcount'] = $this->count();
                     }
                 }
 
@@ -749,7 +760,7 @@ class kolab_storage_cache
         $sql_data = array('changed' => null, 'xml' => '', 'tags' => '', 'words' => '');
 
         if ($object['changed']) {
-            $sql_data['changed'] = date('Y-m-d H:i:s', is_object($object['changed']) ? $object['changed']->format('U') : $object['changed']);
+            $sql_data['changed'] = date(self::DB_DATE_FORMAT, is_object($object['changed']) ? $object['changed']->format('U') : $object['changed']);
         }
 
         if ($object['_formatobj']) {
@@ -943,7 +954,7 @@ class kolab_storage_cache
             return;
 
         $sql_arr = $this->db->fetch_assoc($this->db->query(
-                "SELECT `folder_id`, `synclock`, `ctag`"
+                "SELECT `folder_id`, `synclock`, `ctag`, `changed`, `objectcount`"
                 . " FROM `{$this->folders_table}` WHERE `resource` = ?",
                 $this->resource_uri
         ));
@@ -1004,8 +1015,10 @@ class kolab_storage_cache
             return;
 
         $this->db->query(
-            "UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ? WHERE `folder_id` = ?",
+            "UPDATE `{$this->folders_table}` SET `synclock` = 0, `ctag` = ?, `changed` = ?, `objectcount` = ? WHERE `folder_id` = ?",
             $this->metadata['ctag'],
+            $this->metadata['changed'],
+            $this->metadata['objectcount'],
             $this->folder_id
         );