From 7d1b199034a3d8d79fda1234e5dc992aef8d2276 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 25 Feb 2016 18:16:54 +0100
Subject: [PATCH 01/30] Performance: Load tags UI (and initialize configuration
 folder/cache) on when it's needed

---
 plugins/kolab_tags/kolab_tags.php            |  4 --
 plugins/kolab_tags/lib/kolab_tags_engine.php | 44 +++++++++++---------
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/plugins/kolab_tags/kolab_tags.php b/plugins/kolab_tags/kolab_tags.php
index b9be9b86..12fbb01d 100644
--- a/plugins/kolab_tags/kolab_tags.php
+++ b/plugins/kolab_tags/kolab_tags.php
@@ -79,10 +79,6 @@ class kolab_tags extends rcube_plugin
             return;
         }
 
-        if ($this->rc->action == 'print') {
-            return;
-        }
-
         if ($engine = $this->engine()) {
             $engine->ui();
         }
diff --git a/plugins/kolab_tags/lib/kolab_tags_engine.php b/plugins/kolab_tags/lib/kolab_tags_engine.php
index 8f881851..4e15069b 100644
--- a/plugins/kolab_tags/lib/kolab_tags_engine.php
+++ b/plugins/kolab_tags/lib/kolab_tags_engine.php
@@ -47,26 +47,32 @@ class kolab_tags_engine
     public function ui()
     {
         // set templates of Files UI and widgets
-        if ($this->rc->task == 'mail') {
-            $this->plugin->add_texts('localization/');
-
-            $this->plugin->include_stylesheet($this->plugin->local_skin_path().'/style.css');
-            $this->plugin->include_script('kolab_tags.js');
-            $this->rc->output->add_label('cancel', 'save');
-            $this->plugin->add_label('tags', 'add', 'edit', 'delete', 'saving',
-                'nameempty', 'nameexists', 'colorinvalid', 'untag', 'tagname',
-                'tagcolor', 'tagsearchnew', 'newtag');
-
-            $this->rc->output->add_handlers(array(
-                'plugin.taglist' => array($this, 'taglist'),
-            ));
-
-            $ui = $this->rc->output->parse('kolab_tags.ui', false, false);
-            $this->rc->output->add_footer($ui);
-
-            // load miniColors
-            jqueryui::miniColors();
+        if ($this->rc->task != 'mail') {
+            return;
         }
+
+        if ($this->rc->action && !in_array($this->rc->action, array('show', 'preview'))) {
+            return;
+        }
+
+        $this->plugin->add_texts('localization/');
+
+        $this->plugin->include_stylesheet($this->plugin->local_skin_path().'/style.css');
+        $this->plugin->include_script('kolab_tags.js');
+        $this->rc->output->add_label('cancel', 'save');
+        $this->plugin->add_label('tags', 'add', 'edit', 'delete', 'saving',
+            'nameempty', 'nameexists', 'colorinvalid', 'untag', 'tagname',
+            'tagcolor', 'tagsearchnew', 'newtag');
+
+        $this->rc->output->add_handlers(array(
+            'plugin.taglist' => array($this, 'taglist'),
+        ));
+
+        $ui = $this->rc->output->parse('kolab_tags.ui', false, false);
+        $this->rc->output->add_footer($ui);
+
+        // load miniColors
+        jqueryui::miniColors();
     }
 
     /**

From 67f6ef9bbfc5a3c399f6f20ddbd1e81fddd5ec7e Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Wed, 19 Dec 2018 12:25:39 +0100
Subject: [PATCH 02/30] Always fetch xml from IMAP and parse objects on select

Temporarily we still store data and xml as before, but do not use them on select.
---
 plugins/libkolab/lib/kolab_storage_cache.php  | 43 ++-----------------
 plugins/libkolab/lib/kolab_storage_folder.php |  3 +-
 2 files changed, 6 insertions(+), 40 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index 362d53c1..0912ebf1 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -679,7 +679,6 @@ class kolab_storage_cache
         return $result;
     }
 
-
     /**
      * Get number of objects mathing the given query
      *
@@ -897,44 +896,10 @@ class kolab_storage_cache
      */
     protected function _unserialize($sql_arr)
     {
-        // check if data is a base64-encoded string, for backward compat.
-        if (strpos(substr($sql_arr['data'], 0, 64), ':') === false) {
-            $sql_arr['data'] = base64_decode($sql_arr['data']);
-        }
-
-        $object = unserialize($sql_arr['data']);
-
-        // de-serialization failed
-        if ($object === false) {
-            rcube::raise_error(array(
-                'code' => 900, 'type' => 'php',
-                'message' => "Malformed data for {$this->resource_uri}/{$sql_arr['msguid']} object."
-            ), true);
-
-            return null;
-        }
-
-        // decode binary properties
-        foreach ($this->binary_items as $key => $regexp) {
-            if (!empty($object[$key]) && preg_match($regexp, $sql_arr['xml'], $m)) {
-                $object[$key] = base64_decode($m[1]);
-            }
-        }
-
-        $object_type = $sql_arr['type'] ?: $this->folder->type;
-        $format_type = $this->folder->type == 'configuration' ? 'configuration' : $object_type;
-
-        // add meta data
-        $object['_type']      = $object_type;
-        $object['_msguid']    = $sql_arr['msguid'];
-        $object['_mailbox']   = $this->folder->name;
-        $object['_size']      = strlen($sql_arr['xml']);
-        $object['_formatobj'] = kolab_format::factory($format_type, 3.0, $sql_arr['xml']);
-
-        // Fix old broken objects with missing creation date
-        if (empty($object['created']) && method_exists($object['_formatobj'], 'to_array')) {
-            $new_object        = $object['_formatobj']->to_array();
-            $object['created'] = $new_object['created'];
+        // Fetch object xml
+        if ($object = $this->folder->read_object($sql_arr['msguid'])) {
+            // additional meta data
+            $object['_size'] = strlen($xml);
         }
 
         return $object;
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 4018b7df..eb28ed61 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -486,8 +486,9 @@ class kolab_storage_folder extends kolab_storage_folder_api
         $content_type = kolab_format::KTYPE_PREFIX . $object_type;
 
         // check object type header and abort on mismatch
-        if ($type != '*' && $object_type != $type)
+        if ($type != '*' && strpos($object_type, $type) !== 0) {
             return false;
+        }
 
         $attachments = array();
 

From be749b2795bcaadcc38baa7b134591045b116b10 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 16:27:06 +0000
Subject: [PATCH 03/30] Add possibility to bypass indexes cache only

---
 plugins/libkolab/config.inc.php.dist         |  3 ++-
 plugins/libkolab/lib/kolab_storage_cache.php | 11 ++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/plugins/libkolab/config.inc.php.dist b/plugins/libkolab/config.inc.php.dist
index a2c15e8c..a6338e74 100644
--- a/plugins/libkolab/config.inc.php.dist
+++ b/plugins/libkolab/config.inc.php.dist
@@ -35,7 +35,8 @@ $config['kolab_http_request'] = array();
 
 // When kolab_cache is enabled Roundcube's messages cache will be redundant
 // when working on kolab folders. Here we can:
-// 2 - bypass messages/indexes cache completely
+// 3 - bypass only indexes, but use messages cache
+// 2 - bypass both messages and indexes cache
 // 1 - bypass only messages, but use index cache
 $config['kolab_messages_cache_bypass'] = 0;
 
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index 0912ebf1..cc3f2cf0 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -1155,7 +1155,7 @@ class kolab_storage_cache
     }
 
     /**
-     * Set Roundcube storage options and bypass messages cache.
+     * Set Roundcube storage options and bypass messages/indexes cache.
      *
      * We use skip_deleted and threading settings specific to Kolab,
      * we have to change these global settings only temporarily.
@@ -1210,17 +1210,18 @@ class kolab_storage_cache
 
             switch ($cache_bypass) {
                 case 2:
-                    // Disable messages cache completely
+                    // Disable messages and index cache completely
                     $this->imap->set_messages_caching(!$force);
                     break;
 
+                case 3:
                 case 1:
-                    // We'll disable messages cache, but keep index cache.
+                    // We'll disable messages cache, but keep index cache (1) or vice-versa (3)
                     // Default mode is both (MODE_INDEX | MODE_MESSAGE)
-                    $mode = rcube_imap_cache::MODE_INDEX;
+                    $mode = $cache_bypass == 3 ? rcube_imap_cache::MODE_MESSAGE : rcube_imap_cache::MODE_INDEX;
 
                     if (!$force) {
-                        $mode |= rcube_imap_cache::MODE_MESSAGE;
+                        $mode |= $cache_bypass == 3 ? rcube_imap_cache::MODE_INDEX : rcube_imap_cache::MODE_MESSAGE;
                     }
 
                     $this->imap->set_messages_caching(true, $mode);

From 2ac64d32eba0290e8c001d6e35233b33b94b3d54 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 16:51:34 +0000
Subject: [PATCH 04/30] Fix regression in handling contact groups

---
 plugins/libkolab/lib/kolab_storage_folder.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index eb28ed61..45f9bbcd 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -486,7 +486,7 @@ class kolab_storage_folder extends kolab_storage_folder_api
         $content_type = kolab_format::KTYPE_PREFIX . $object_type;
 
         // check object type header and abort on mismatch
-        if ($type != '*' && strpos($object_type, $type) !== 0) {
+        if ($type != '*' && strpos($object_type, $type) !== 0 && !($object_type == 'distribution-list' && $type == 'contact')) {
             return false;
         }
 

From 28932e4c92a9ac7b24d88112b1f0b4946708df96 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 18:36:17 +0000
Subject: [PATCH 05/30] Fast-mode for selecting kolab objects

... for now used only by kolab_notes plugin
---
 plugins/kolab_notes/kolab_notes.php           |  2 +-
 plugins/libkolab/lib/kolab_storage_cache.php  | 33 +++++++++++++++++--
 .../libkolab/lib/kolab_storage_cache_note.php |  2 +-
 plugins/libkolab/lib/kolab_storage_folder.php | 10 +++---
 4 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/plugins/kolab_notes/kolab_notes.php b/plugins/kolab_notes/kolab_notes.php
index d0547ede..16a97276 100644
--- a/plugins/kolab_notes/kolab_notes.php
+++ b/plugins/kolab_notes/kolab_notes.php
@@ -480,7 +480,7 @@ class kolab_notes extends rcube_plugin
 
         $this->_read_lists();
         if ($folder = $this->get_folder($list_id)) {
-            foreach ($folder->select($query) as $record) {
+            foreach ($folder->select($query, empty($query)) as $record) {
                 // post-filter search results
                 if (strlen($search)) {
                     $matches = 0;
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index cc3f2cf0..e1ec56d9 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -48,6 +48,7 @@ class kolab_storage_cache
     protected $max_sync_lock_time = 600;
     protected $binary_items = array();
     protected $extra_cols = array();
+    protected $data_props = array();
     protected $order_by = null;
     protected $limit = null;
     protected $error = 0;
@@ -585,9 +586,12 @@ class kolab_storage_cache
      * @param array Pseudo-SQL query as list of filter parameter triplets
      *   triplet: array('<colname>', '<comparator>', '<value>')
      * @param boolean Set true to only return UIDs instead of complete objects
+     * @param boolean Use fast mode to fetch only minimal set of information
+     *                (no xml fetching and parsing, etc.)
+     *
      * @return array List of Kolab data objects (each represented as hash array) or UIDs
      */
-    public function select($query = array(), $uids = false)
+    public function select($query = array(), $uids = false, $fast = false)
     {
         $result = $uids ? array() : new kolab_storage_dataset($this);
 
@@ -621,6 +625,9 @@ class kolab_storage_cache
             }
 
             while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+                if ($fast) {
+                    $sql_arr['fast-mode'] = true;
+                }
                 if ($uids) {
                     $this->uid2msg[$sql_arr['uid']] = $sql_arr['_msguid'];
                     $result[] = $sql_arr['uid'];
@@ -862,6 +869,11 @@ class kolab_storage_cache
             $sql_data['words'] = ' ' . join(' ', $object['_formatobj']->get_words()) . ' ';
         }
 
+        // TODO: get rid of xml column
+        // TODO: store only small subset of properties in data column, i.e. properties that are
+        //       needed for fast-mode only (use $data_props)
+        // TODO: store data in JSON format and no base64-encoding
+
         // extract object data
         $data = array();
         foreach ($object as $key => $val) {
@@ -896,8 +908,25 @@ class kolab_storage_cache
      */
     protected function _unserialize($sql_arr)
     {
+        if ($sql_arr['fast-mode'] && !empty($sql_arr['data'])
+            && ($object = unserialize(base64_decode($sql_arr['data'])))
+        ) {
+            foreach ($this->data_props as $prop) {
+                if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
+                    $object[$prop] = $sql_arr[$prop];
+
+                    if (($prop == 'created' || $prop == 'changed') && $object[$prop] && is_string($object[$prop])) {
+                        $object[$prop] = new DateTime($object[$prop]);
+                    }
+                }
+            }
+
+            $object['_type']     = $sql_arr['type'] ?: $this->folder->type;
+            $object['_msguid']   = $sql_arr['msguid'];
+            $object['_mailbox']  = $this->folder->name;
+        }
         // Fetch object xml
-        if ($object = $this->folder->read_object($sql_arr['msguid'])) {
+        else if ($object = $this->folder->read_object($sql_arr['msguid'])) {
             // additional meta data
             $object['_size'] = strlen($xml);
         }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_note.php b/plugins/libkolab/lib/kolab_storage_cache_note.php
index 85469275..fc23d5b0 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_note.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_note.php
@@ -23,5 +23,5 @@
 
 class kolab_storage_cache_note extends kolab_storage_cache
 {
-    
+    protected $data_props = array('uid', 'title', 'created', 'changed');
 }
\ No newline at end of file
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 45f9bbcd..02ac6adf 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -281,12 +281,14 @@ class kolab_storage_folder extends kolab_storage_folder_api
     /**
      * Select Kolab objects matching the given query
      *
-     * @param mixed Pseudo-SQL query as list of filter parameter triplets
-     *    or string with object type (e.g. contact, event, todo, journal, note, configuration)
+     * @param mixed   Pseudo-SQL query as list of filter parameter triplets
+     *                or string with object type (e.g. contact, event, todo, journal, note, configuration)
+     * @param boolean Use fast mode to fetch only minimal set of information
+     *                (no xml fetching and parsing, etc.)
      *
      * @return array List of Kolab data objects (each represented as hash array)
      */
-    public function select($query = array())
+    public function select($query = array(), $fast = false)
     {
         if (!$this->valid) {
             return array();
@@ -296,7 +298,7 @@ class kolab_storage_folder extends kolab_storage_folder_api
         $this->cache->synchronize();
 
         // fetch objects from cache
-        return $this->cache->select($this->_prepare_query($query));
+        return $this->cache->select($this->_prepare_query($query), false, $fast);
     }
 
     /**

From 217e861d4dbc28b42da2205a3eab4a17fedf2084 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 19:49:09 +0000
Subject: [PATCH 06/30] Fast-mode for contacts

... there's still a place for improvement, we can make more requests use fast-mode.
---
 .../lib/rcube_kolab_contacts.php                  | 15 +++++++++------
 .../libkolab/lib/kolab_storage_cache_contact.php  |  3 ++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
index b301ffad..081996b6 100644
--- a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
+++ b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
@@ -325,7 +325,9 @@ class rcube_kolab_contacts extends rcube_addressbook
     public function list_records($cols = null, $subset = 0, $nocount = false)
     {
         $this->result = new rcube_result_set(0, ($this->list_page-1) * $this->page_size);
+
         $fetch_all = false;
+        $fast_mode = !empty($cols) && is_array($cols);
 
         // list member of the selected group
         if ($this->gid) {
@@ -355,7 +357,7 @@ class rcube_kolab_contacts extends rcube_addressbook
 
             // get members by UID
             if (!empty($uids)) {
-                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), $fetch_all ? false : count($uids));
+                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), $fetch_all ? false : count($uids), $fast_mode);
                 $this->sortindex = array_merge($this->sortindex, $local_sortindex);
             }
         }
@@ -363,11 +365,11 @@ class rcube_kolab_contacts extends rcube_addressbook
             $ids = $this->filter['ids'];
             if (count($ids)) {
                 $uids = array_map(array($this, 'id2uid'), $this->filter['ids']);
-                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), count($ids));
+                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), count($ids), $fast_mode);
             }
         }
         else {
-            $this->_fetch_contacts($query = 'contact', true);
+            $this->_fetch_contacts($query = 'contact', true, $fast_mode);
         }
 
         if ($fetch_all) {
@@ -1062,7 +1064,7 @@ class rcube_kolab_contacts extends rcube_addressbook
     /**
      * Query storage layer and store records in private member var
      */
-    private function _fetch_contacts($query = array(), $limit = false)
+    private function _fetch_contacts($query = array(), $limit = false, $fast_mode = false)
     {
         if (!isset($this->dataset) || !empty($query)) {
             if ($limit) {
@@ -1071,7 +1073,7 @@ class rcube_kolab_contacts extends rcube_addressbook
             }
 
             $this->sortindex = array();
-            $this->dataset   = $this->storagefolder->select($query);
+            $this->dataset   = $this->storagefolder->select($query, $fast_mode);
 
             foreach ($this->dataset as $idx => $record) {
                 $contact = $this->_to_rcube_contact($record);
@@ -1145,8 +1147,9 @@ class rcube_kolab_contacts extends rcube_addressbook
                     $record['member'][$i]['readonly'] = empty($member['uid']);
                     $this->groupmembers[$mid][] = $record['ID'];
 
-                    if ($with_contacts && empty($member['uid']))
+                    if ($with_contacts && empty($member['uid'])) {
                         $this->contacts[$mid] = $record['member'][$i];
+                    }
                 }
                 $this->distlists[$record['ID']] = $record;
             }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 70aa5f62..531781c8 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -24,7 +24,8 @@
 class kolab_storage_cache_contact extends kolab_storage_cache
 {
     protected $extra_cols_max = 255;
-    protected $extra_cols     = array('type','name','firstname','surname','email');
+    protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
+    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'uid');
     protected $binary_items   = array(
         'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
         'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',

From 83486eca9245b24a2f9c45f69061026bdbd50b31 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Fri, 21 Dec 2018 09:58:40 +0100
Subject: [PATCH 07/30] Fast-mode for files

---
 .../libkolab/lib/kolab_storage_cache_file.php  | 18 ++++++++++++++----
 plugins/libkolab/lib/kolab_storage_folder.php  |  2 ++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache_file.php b/plugins/libkolab/lib/kolab_storage_cache_file.php
index ea1823d6..053438c1 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_file.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_file.php
@@ -24,6 +24,7 @@
 class kolab_storage_cache_file extends kolab_storage_cache
 {
     protected $extra_cols = array('filename');
+    protected $data_props = array('type', 'size', 'filename', 'fileid', 'uid');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
@@ -32,13 +33,22 @@ class kolab_storage_cache_file extends kolab_storage_cache
      */
     protected function _serialize($object)
     {
-        $sql_data = parent::_serialize($object);
-
         if (!empty($object['_attachments'])) {
             reset($object['_attachments']);
-            $sql_data['filename'] = $object['_attachments'][key($object['_attachments'])]['name'];
+
+            $file = $object['_attachments'][key($object['_attachments'])];
+
+            $object['type']   = $file['mimetype'];
+            $object['size']   = $file['size'];
+            $object['fileid'] = $file['id'];
+        }
+
+        $sql_data = parent::_serialize($object);
+
+        if (!empty($file)) {
+            $sql_data['filename'] = $file['name'];
         }
 
         return $sql_data;
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 02ac6adf..1ffbb457 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -421,6 +421,8 @@ class kolab_storage_folder extends kolab_storage_folder_api
             }
             else {
                 // return message part from IMAP directly
+                // TODO: We could improve performance if we cache part's encoding
+                //       without 3rd argument get_message_part() will request BODYSTRUCTURE from IMAP
                 return $this->imap->get_message_part($msguid, $part, null, $print, $fp, $skip_charset_conv);
             }
         }

From a90d4c6015841ada405596194537e84f20668727 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Fri, 21 Dec 2018 12:23:41 +0100
Subject: [PATCH 08/30] Fast-mode for events/tasks/contacts and some code
 improvements

---
 plugins/libkolab/lib/kolab_storage_cache.php  | 23 +++++++++++++----
 .../lib/kolab_storage_cache_contact.php       |  2 +-
 .../lib/kolab_storage_cache_event.php         |  1 +
 .../libkolab/lib/kolab_storage_cache_file.php | 25 ++++++++++++++++++-
 .../lib/kolab_storage_cache_journal.php       |  1 +
 .../libkolab/lib/kolab_storage_cache_note.php |  2 +-
 .../libkolab/lib/kolab_storage_cache_task.php |  3 ++-
 7 files changed, 48 insertions(+), 9 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index e1ec56d9..df851d6d 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -864,7 +864,10 @@ class kolab_storage_cache
         }
 
         if ($object['_formatobj']) {
-            $sql_data['xml']   = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', (string)$object['_formatobj']->write(3.0));
+            $xml  = (string) $object['_formatobj']->write(3.0);
+            $size = strlen($xml);
+
+            $sql_data['xml']   = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', $xml);
             $sql_data['tags']  = ' ' . join(' ', $object['_formatobj']->get_tags()) . ' ';  // pad with spaces for strict/prefix search
             $sql_data['words'] = ' ' . join(' ', $object['_formatobj']->get_words()) . ' ';
         }
@@ -897,6 +900,10 @@ class kolab_storage_cache
             }
         }
 
+        if ($size) {
+            $data['_size'] = $size;
+        }
+
         // use base64 encoding (Bug #1912, #2662)
         $sql_data['data'] = base64_encode(serialize($data));
 
@@ -911,16 +918,22 @@ class kolab_storage_cache
         if ($sql_arr['fast-mode'] && !empty($sql_arr['data'])
             && ($object = unserialize(base64_decode($sql_arr['data'])))
         ) {
+            $object['uid'] = $sql_arr['uid'];
+
             foreach ($this->data_props as $prop) {
                 if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
                     $object[$prop] = $sql_arr[$prop];
-
-                    if (($prop == 'created' || $prop == 'changed') && $object[$prop] && is_string($object[$prop])) {
-                        $object[$prop] = new DateTime($object[$prop]);
-                    }
                 }
             }
 
+            if ($sql_arr['created'] && empty($object['created'])) {
+                $object['created'] = new DateTime($sql_arr['created']);
+            }
+
+            if ($sql_arr['changed'] && empty($object['changed'])) {
+                $object['changed'] = new DateTime($sql_arr['changed']);
+            }
+
             $object['_type']     = $sql_arr['type'] ?: $this->folder->type;
             $object['_msguid']   = $sql_arr['msguid'];
             $object['_mailbox']  = $this->folder->name;
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 531781c8..6f3a0c9a 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -25,7 +25,7 @@ class kolab_storage_cache_contact extends kolab_storage_cache
 {
     protected $extra_cols_max = 255;
     protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
-    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'uid');
+    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization');
     protected $binary_items   = array(
         'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
         'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',
diff --git a/plugins/libkolab/lib/kolab_storage_cache_event.php b/plugins/libkolab/lib/kolab_storage_cache_event.php
index b6d8b15f..13b350b3 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_event.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_event.php
@@ -24,6 +24,7 @@
 class kolab_storage_cache_event extends kolab_storage_cache
 {
     protected $extra_cols = array('dtstart','dtend');
+    protected $data_props = array('categories', 'status', 'attendees'); // start, end
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/libkolab/lib/kolab_storage_cache_file.php b/plugins/libkolab/lib/kolab_storage_cache_file.php
index 053438c1..1f3d4d92 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_file.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_file.php
@@ -24,7 +24,7 @@
 class kolab_storage_cache_file extends kolab_storage_cache
 {
     protected $extra_cols = array('filename');
-    protected $data_props = array('type', 'size', 'filename', 'fileid', 'uid');
+    protected $data_props = array('type', 'size', 'filename', 'fileid');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
@@ -51,4 +51,27 @@ class kolab_storage_cache_file extends kolab_storage_cache
 
         return $sql_data;
     }
+
+    /**
+     * Helper method to turn stored cache data into a valid storage object
+     *
+     * @override
+     */
+    protected function _unserialize($sql_arr)
+    {
+        $object = parent::_unserialize($sql_arr);
+
+        if ($object && $sql_arr['fast-mode']) {
+            if (!empty($object['_attachments'])) {
+                $file = $object['_attachments'][key($object['_attachments'])];
+
+                $object['type']     = $file['mimetype'];
+                $object['size']     = $file['size'];
+                $object['fileid']   = $file['id'];
+                $object['filename'] = $file['name'];
+            }
+        }
+
+        return $object;
+    }
 }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_journal.php b/plugins/libkolab/lib/kolab_storage_cache_journal.php
index 766f1da1..042d4bde 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_journal.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_journal.php
@@ -24,6 +24,7 @@
 class kolab_storage_cache_journal extends kolab_storage_cache
 {
     protected $extra_cols = array('dtstart','dtend');
+    protected $data_props = array('categories');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/libkolab/lib/kolab_storage_cache_note.php b/plugins/libkolab/lib/kolab_storage_cache_note.php
index fc23d5b0..7f5ebc8e 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_note.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_note.php
@@ -23,5 +23,5 @@
 
 class kolab_storage_cache_note extends kolab_storage_cache
 {
-    protected $data_props = array('uid', 'title', 'created', 'changed');
+    protected $data_props = array('title');
 }
\ No newline at end of file
diff --git a/plugins/libkolab/lib/kolab_storage_cache_task.php b/plugins/libkolab/lib/kolab_storage_cache_task.php
index 8b714e63..06b64d39 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_task.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_task.php
@@ -23,7 +23,8 @@
 
 class kolab_storage_cache_task extends kolab_storage_cache
 {
-    protected $extra_cols = array('dtstart','dtend');
+    protected $extra_cols = array('dtstart', 'dtend');
+    protected $data_props = array('categories', 'status'); // start, due
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache

From c9499eeaa61a4e1899caa33c2a533b075413ab50 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Mon, 24 Dec 2018 10:45:19 +0000
Subject: [PATCH 09/30] Fast-mode for configuration objects

---
 .../lib/kolab_storage_cache_configuration.php | 54 +++++++++++++++++--
 .../lib/kolab_storage_cache_contact.php       |  2 +-
 .../libkolab/lib/kolab_storage_cache_note.php |  2 +-
 plugins/libkolab/lib/kolab_storage_config.php |  2 +-
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache_configuration.php b/plugins/libkolab/lib/kolab_storage_cache_configuration.php
index c3c7ac4f..a5ac396d 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_configuration.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_configuration.php
@@ -32,20 +32,37 @@ class kolab_storage_cache_configuration extends kolab_storage_cache
      */
     protected function _serialize($object)
     {
+        $this->set_data_props($object['type']);
+
         $sql_data = parent::_serialize($object);
         $sql_data['type'] = $object['type'];
 
         return $sql_data;
     }
 
+    /**
+     * Helper method to convert database record to the given Kolab object
+     *
+     * @override
+     */
+    protected function _unserialize($sql_arr)
+    {
+        $this->set_data_props($sql_arr['type']);
+
+        return parent::_unserialize($sql_arr);
+    }
+
     /**
      * Select Kolab objects filtered by the given query
      *
-     * @param array Pseudo-SQL query as list of filter parameter triplets
+     * @param array   Pseudo-SQL query as list of filter parameter triplets
      * @param boolean Set true to only return UIDs instead of complete objects
+     * @param boolean Use fast mode to fetch only minimal set of information
+     *                (no xml fetching and parsing, etc.)
+     *
      * @return array List of Kolab data objects (each represented as hash array) or UIDs
      */
-    public function select($query = array(), $uids = false)
+    public function select($query = array(), $uids = false, $fast = false)
     {
         // modify query for IMAP search: query param 'type' is actually a subtype
         if (!$this->ready) {
@@ -57,7 +74,7 @@ class kolab_storage_cache_configuration extends kolab_storage_cache
             }
         }
 
-        return parent::select($query, $uids);
+        return parent::select($query, $uids, $fast);
     }
 
     /**
@@ -85,4 +102,35 @@ class kolab_storage_cache_configuration extends kolab_storage_cache
 
         return parent::_sql_where($query);
     }
+
+    /**
+     * Set $data_props property depending on object type
+     */
+    protected function set_data_props($type)
+    {
+        switch ($type) {
+        case 'dictionary':
+            $this->data_props = array('language', 'e');
+            break;
+
+        case 'file_driver':
+            $this->data_props = array('driver', 'title', 'enabled', 'host', 'port', 'username', 'password');
+            break;
+
+        case 'relation':
+            // Note: Because relations are heavily used, for performance reasons
+            // we store all properties for them
+            $this->data_props = array('name', 'category', 'color', 'parent', 'iconName', 'priority', 'members');
+            break;
+
+        case 'snippet':
+            $this->data_props = array('name');
+            break;
+
+        case 'category':
+        default:
+            // TODO: implement this
+            $this->data_props = array();
+        }
+    }
 }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 6f3a0c9a..a8a475a5 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -70,4 +70,4 @@ class kolab_storage_cache_contact extends kolab_storage_cache
 
         return $sql_data;
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/libkolab/lib/kolab_storage_cache_note.php b/plugins/libkolab/lib/kolab_storage_cache_note.php
index 7f5ebc8e..3211da57 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_note.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_note.php
@@ -24,4 +24,4 @@
 class kolab_storage_cache_note extends kolab_storage_cache
 {
     protected $data_props = array('title');
-}
\ No newline at end of file
+}
diff --git a/plugins/libkolab/lib/kolab_storage_config.php b/plugins/libkolab/lib/kolab_storage_config.php
index fb442ece..b29e7a90 100644
--- a/plugins/libkolab/lib/kolab_storage_config.php
+++ b/plugins/libkolab/lib/kolab_storage_config.php
@@ -131,7 +131,7 @@ class kolab_storage_config
                 $folder->set_order_and_limit(null, $limit);
             }
 
-            foreach ($folder->select($filter) as $object) {
+            foreach ($folder->select($filter, true) as $object) {
                 unset($object['_formatobj']);
                 $list[] = $object;
             }

From d8180bf560d4317b6e11e2a7dd40512d7e9ee9fe Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 27 Dec 2018 09:23:29 +0000
Subject: [PATCH 10/30] Use cache fast-mode for counting tasks

---
 plugins/libkolab/lib/kolab_storage_cache_task.php        | 2 +-
 plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache_task.php b/plugins/libkolab/lib/kolab_storage_cache_task.php
index 06b64d39..9c776d75 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_task.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_task.php
@@ -24,7 +24,7 @@
 class kolab_storage_cache_task extends kolab_storage_cache
 {
     protected $extra_cols = array('dtstart', 'dtend');
-    protected $data_props = array('categories', 'status'); // start, due
+    protected $data_props = array('categories', 'status', 'complete', 'start', 'due');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php b/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php
index 1bfff58b..73142be3 100644
--- a/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php
+++ b/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php
@@ -587,7 +587,7 @@ class tasklist_kolab_driver extends tasklist_driver
                 continue;
             }
 
-            foreach ($folder->select(array(array('tags','!~','x-complete'))) as $record) {
+            foreach ($folder->select(array(array('tags','!~','x-complete')), true) as $record) {
                 $rec = $this->_to_rcube_task($record, $list_id, false);
 
                 if ($this->is_complete($rec))  // don't count complete tasks

From 9904323c4162311f8673770bf000bd5f7ece44a7 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 27 Dec 2018 14:14:44 +0000
Subject: [PATCH 11/30] Use cache fast-mode for contact groups

The same as tags/relations these objects are used quite often, so
we store group members in database and we can access them without
accessing imap.
---
 plugins/kolab_addressbook/lib/rcube_kolab_contacts.php | 2 +-
 plugins/libkolab/lib/kolab_storage_cache_contact.php   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
index 081996b6..cfc9e693 100644
--- a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
+++ b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
@@ -1139,7 +1139,7 @@ class rcube_kolab_contacts extends rcube_addressbook
     {
         if (!isset($this->distlists)) {
             $this->distlists = $this->groupmembers = array();
-            foreach ($this->storagefolder->select('distribution-list') as $record) {
+            foreach ($this->storagefolder->select('distribution-list', true) as $record) {
                 $record['ID'] = $this->uid2id($record['uid']);
                 foreach ((array)$record['member'] as $i => $member) {
                     $mid = $this->uid2id($member['uid'] ? $member['uid'] : 'mailto:' . $member['email']);
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index a8a475a5..5dfb16b7 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -25,7 +25,7 @@ class kolab_storage_cache_contact extends kolab_storage_cache
 {
     protected $extra_cols_max = 255;
     protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
-    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization');
+    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'member');
     protected $binary_items   = array(
         'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
         'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',

From 47f7793ac01603c8f2349c3c520458ed27beec42 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 27 Dec 2018 14:26:35 +0000
Subject: [PATCH 12/30] Slim down kolab cache (drop xml column, change data
 format) (Bifrost#T61991)

---
 plugins/libkolab/SQL/mysql.initial.sql        |  10 +-
 plugins/libkolab/SQL/mysql/2018122700.sql     |  10 +
 plugins/libkolab/SQL/oracle.initial.sql       |  10 +-
 plugins/libkolab/SQL/oracle/2018122700.sql    |  10 +
 plugins/libkolab/SQL/sqlite.initial.sql       |  10 +-
 plugins/libkolab/SQL/sqlite/2018122700.sql    | 137 +++++
 plugins/libkolab/lib/kolab_storage_cache.php  |  77 +--
 .../lib/kolab_storage_cache_contact.php       |   5 -
 .../lib/kolab_storage_cache_mongodb.php       | 561 ------------------
 plugins/libkolab/lib/kolab_storage_folder.php |   1 +
 10 files changed, 189 insertions(+), 642 deletions(-)
 create mode 100644 plugins/libkolab/SQL/mysql/2018122700.sql
 create mode 100644 plugins/libkolab/SQL/oracle/2018122700.sql
 create mode 100644 plugins/libkolab/SQL/sqlite/2018122700.sql
 delete mode 100644 plugins/libkolab/lib/kolab_storage_cache_mongodb.php

diff --git a/plugins/libkolab/SQL/mysql.initial.sql b/plugins/libkolab/SQL/mysql.initial.sql
index 0213c45b..80da02fa 100644
--- a/plugins/libkolab/SQL/mysql.initial.sql
+++ b/plugins/libkolab/SQL/mysql.initial.sql
@@ -33,7 +33,6 @@ CREATE TABLE `kolab_cache_contact` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `type` VARCHAR(32) CHARACTER SET ascii NOT NULL,
@@ -57,7 +56,6 @@ CREATE TABLE `kolab_cache_event` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -77,7 +75,6 @@ CREATE TABLE `kolab_cache_task` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -97,7 +94,6 @@ CREATE TABLE `kolab_cache_journal` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -117,7 +113,6 @@ CREATE TABLE `kolab_cache_note` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   CONSTRAINT `fk_kolab_cache_note_folder` FOREIGN KEY (`folder_id`)
@@ -135,7 +130,6 @@ CREATE TABLE `kolab_cache_file` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `filename` varchar(255) DEFAULT NULL,
@@ -155,7 +149,6 @@ CREATE TABLE `kolab_cache_configuration` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `type` VARCHAR(32) CHARACTER SET ascii NOT NULL,
@@ -175,7 +168,6 @@ CREATE TABLE `kolab_cache_freebusy` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -188,4 +180,4 @@ CREATE TABLE `kolab_cache_freebusy` (
 
 /*!40014  SET FOREIGN_KEY_CHECKS=1 */;
 
-REPLACE INTO `system` (`name`, `value`) VALUES ('libkolab-version', '2018021300');
+REPLACE INTO `system` (`name`, `value`) VALUES ('libkolab-version', '2018122700');
diff --git a/plugins/libkolab/SQL/mysql/2018122700.sql b/plugins/libkolab/SQL/mysql/2018122700.sql
new file mode 100644
index 00000000..08030dd0
--- /dev/null
+++ b/plugins/libkolab/SQL/mysql/2018122700.sql
@@ -0,0 +1,10 @@
+-- remove xml column, and change data format (clear cache needed)
+DELETE FROM `kolab_folders`;
+ALTER TABLE `kolab_cache_contact` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_event` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_task` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_journal` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_note` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_file` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_configuration` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_freebusy` DROP COLUMN `xml`;
diff --git a/plugins/libkolab/SQL/oracle.initial.sql b/plugins/libkolab/SQL/oracle.initial.sql
index 78675783..ebba60dc 100644
--- a/plugins/libkolab/SQL/oracle.initial.sql
+++ b/plugins/libkolab/SQL/oracle.initial.sql
@@ -37,7 +37,6 @@ CREATE TABLE "kolab_cache_contact" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "type" varchar(32) NOT NULL,
@@ -60,7 +59,6 @@ CREATE TABLE "kolab_cache_event" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -79,7 +77,6 @@ CREATE TABLE "kolab_cache_task" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -98,7 +95,6 @@ CREATE TABLE "kolab_cache_journal" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -117,7 +113,6 @@ CREATE TABLE "kolab_cache_note" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     PRIMARY KEY ("folder_id", "msguid")
@@ -134,7 +129,6 @@ CREATE TABLE "kolab_cache_file" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "filename" varchar(255) DEFAULT NULL,
@@ -153,7 +147,6 @@ CREATE TABLE "kolab_cache_configuration" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "type" varchar(32) NOT NULL,
@@ -172,7 +165,6 @@ CREATE TABLE "kolab_cache_freebusy" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -183,4 +175,4 @@ CREATE TABLE "kolab_cache_freebusy" (
 CREATE INDEX "kolab_cache_fb_uid2msguid" ON "kolab_cache_freebusy" ("folder_id", "uid", "msguid");
 
 
-INSERT INTO "system" ("name", "value") VALUES ('libkolab-version', '2018021300');
+INSERT INTO "system" ("name", "value") VALUES ('libkolab-version', '2018122700');
diff --git a/plugins/libkolab/SQL/oracle/2018122700.sql b/plugins/libkolab/SQL/oracle/2018122700.sql
new file mode 100644
index 00000000..45fccb74
--- /dev/null
+++ b/plugins/libkolab/SQL/oracle/2018122700.sql
@@ -0,0 +1,10 @@
+-- remove xml column, and change data format (clear cache needed)
+DELETE FROM "kolab_folders";
+ALTER TABLE "kolab_cache_contact" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_event" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_task" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_journal" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_note" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_file" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_configuration" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_freebusy" DROP COLUMN "xml";
diff --git a/plugins/libkolab/SQL/sqlite.initial.sql b/plugins/libkolab/SQL/sqlite.initial.sql
index 909188ca..815316dc 100644
--- a/plugins/libkolab/SQL/sqlite.initial.sql
+++ b/plugins/libkolab/SQL/sqlite.initial.sql
@@ -25,7 +25,6 @@ CREATE TABLE kolab_cache_contact (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   type VARCHAR(32) NOT NULL,
@@ -46,7 +45,6 @@ CREATE TABLE kolab_cache_event (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -63,7 +61,6 @@ CREATE TABLE kolab_cache_task (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -80,7 +77,6 @@ CREATE TABLE kolab_cache_journal (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -97,7 +93,6 @@ CREATE TABLE kolab_cache_note (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   PRIMARY KEY(folder_id,msguid)
@@ -112,7 +107,6 @@ CREATE TABLE kolab_cache_file (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   filename varchar(255) DEFAULT NULL,
@@ -129,7 +123,6 @@ CREATE TABLE kolab_cache_configuration (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   type VARCHAR(32) NOT NULL,
@@ -146,7 +139,6 @@ CREATE TABLE kolab_cache_freebusy (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -156,4 +148,4 @@ CREATE TABLE kolab_cache_freebusy (
 
 CREATE INDEX ix_freebusy_uid2msguid ON kolab_cache_freebusy(folder_id,uid,msguid);
 
-INSERT INTO system (name, value) VALUES ('libkolab-version', '2018021300');
+INSERT INTO system (name, value) VALUES ('libkolab-version', '2018122700');
diff --git a/plugins/libkolab/SQL/sqlite/2018122700.sql b/plugins/libkolab/SQL/sqlite/2018122700.sql
new file mode 100644
index 00000000..ee24c1a6
--- /dev/null
+++ b/plugins/libkolab/SQL/sqlite/2018122700.sql
@@ -0,0 +1,137 @@
+DROP TABLE kolab_cache_contact;
+CREATE TABLE kolab_cache_contact (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  type VARCHAR(32) NOT NULL,
+  name VARCHAR(255) NOT NULL,
+  firstname VARCHAR(255) NOT NULL,
+  surname VARCHAR(255) NOT NULL,
+  email VARCHAR(255) NOT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_contact_type ON kolab_cache_contact(folder_id,type);
+CREATE INDEX ix_contact_uid2msguid ON kolab_cache_contact(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_event;
+CREATE TABLE kolab_cache_event (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_event_uid2msguid ON kolab_cache_event(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_task;
+CREATE TABLE kolab_cache_task (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_task_uid2msguid ON kolab_cache_task(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_journal;
+CREATE TABLE kolab_cache_journal (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_journal_uid2msguid ON kolab_cache_journal(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_note;
+CREATE TABLE kolab_cache_note (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_note_uid2msguid ON kolab_cache_note(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_file;
+CREATE TABLE kolab_cache_file (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  filename varchar(255) DEFAULT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_folder_filename ON kolab_cache_file(folder_id,filename);
+CREATE INDEX ix_file_uid2msguid ON kolab_cache_file(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_configuration;
+CREATE TABLE kolab_cache_configuration (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  type VARCHAR(32) NOT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_configuration_type ON kolab_cache_configuration(folder_id,type);
+CREATE INDEX ix_configuration_uid2msguid ON kolab_cache_configuration(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_freebusy;
+CREATE TABLE kolab_cache_freebusy (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_freebusy_uid2msguid ON kolab_cache_freebusy(folder_id,uid,msguid);
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index df851d6d..a0d360ed 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -46,7 +46,6 @@ class kolab_storage_cache
     protected $folders_table;
     protected $max_sql_packet;
     protected $max_sync_lock_time = 600;
-    protected $binary_items = array();
     protected $extra_cols = array();
     protected $data_props = array();
     protected $order_by = null;
@@ -202,7 +201,7 @@ class kolab_storage_cache
                   $this->metadata['changed'] < date(self::DB_DATE_FORMAT, time() - $this->cache_refresh) ||
                   $this->metadata['ctag'] != $this->folder->get_ctag() ||
                   intval($this->metadata['objectcount']) !== $this->count()
-                ) {
+            ) {
                 // lock synchronization for this folder or wait if locked
                 $this->_sync_lock();
 
@@ -448,7 +447,7 @@ class kolab_storage_cache
             $sql_data['uid']       = $object['uid'];
 
             $args = array();
-            $cols = array('folder_id', 'msguid', 'uid', 'changed', 'data', 'xml', 'tags', 'words');
+            $cols = array('folder_id', 'msguid', 'uid', 'changed', 'data', 'tags', 'words');
             $cols = array_merge($cols, $this->extra_cols);
 
             foreach ($cols as $idx => $col) {
@@ -857,55 +856,36 @@ class kolab_storage_cache
      */
     protected function _serialize($object)
     {
-        $sql_data = array('changed' => null, 'xml' => '', 'tags' => '', 'words' => '');
+        $data     = array();
+        $sql_data = array('changed' => null, 'tags' => '', 'words' => '');
 
         if ($object['changed']) {
             $sql_data['changed'] = date(self::DB_DATE_FORMAT, is_object($object['changed']) ? $object['changed']->format('U') : $object['changed']);
         }
 
         if ($object['_formatobj']) {
-            $xml  = (string) $object['_formatobj']->write(3.0);
-            $size = strlen($xml);
+            $xml = (string) $object['_formatobj']->write(3.0);
 
-            $sql_data['xml']   = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', $xml);
+            $data['_size']     = strlen($xml);
             $sql_data['tags']  = ' ' . join(' ', $object['_formatobj']->get_tags()) . ' ';  // pad with spaces for strict/prefix search
             $sql_data['words'] = ' ' . join(' ', $object['_formatobj']->get_words()) . ' ';
         }
 
-        // TODO: get rid of xml column
-        // TODO: store only small subset of properties in data column, i.e. properties that are
-        //       needed for fast-mode only (use $data_props)
-        // TODO: store data in JSON format and no base64-encoding
-
-        // extract object data
-        $data = array();
-        foreach ($object as $key => $val) {
-            // skip empty properties
-            if ($val === "" || $val === null) {
-                continue;
-            }
-            // mark binary data to be extracted from xml on unserialize()
-            if (isset($this->binary_items[$key])) {
-                $data[$key] = true;
-            }
-            else if ($key[0] != '_') {
-                $data[$key] = $val;
-            }
-            else if ($key == '_attachments') {
-                foreach ($val as $k => $att) {
-                    unset($att['content'], $att['path']);
-                    if ($att['id'])
-                        $data[$key][$k] = $att;
+        // Store only minimal set of object properties
+        foreach ($this->data_props as $prop) {
+            if (isset($object[$prop])) {
+                $data[$prop] = $object[$prop];
+                if ($data[$prop] instanceof DateTime) {
+                    $data[$prop] = array(
+                        'cl' => 'DateTime',
+                        'dt' => $data[$prop]->format('Y-m-d H:i:s'),
+                        'tz' => $data[$prop]->getTimezone()->getName(),
+                    );
                 }
             }
         }
 
-        if ($size) {
-            $data['_size'] = $size;
-        }
-
-        // use base64 encoding (Bug #1912, #2662)
-        $sql_data['data'] = base64_encode(serialize($data));
+        $sql_data['data'] = json_encode(rcube_charset::clean($data));
 
         return $sql_data;
     }
@@ -915,13 +895,14 @@ class kolab_storage_cache
      */
     protected function _unserialize($sql_arr)
     {
-        if ($sql_arr['fast-mode'] && !empty($sql_arr['data'])
-            && ($object = unserialize(base64_decode($sql_arr['data'])))
-        ) {
+        if ($sql_arr['fast-mode'] && !empty($sql_arr['data']) && ($object = json_decode($sql_arr['data'], true))) {
             $object['uid'] = $sql_arr['uid'];
 
             foreach ($this->data_props as $prop) {
-                if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
+                if (isset($object[$prop]) && is_array($object[$prop]) && $object[$prop]['cl'] == 'DateTime') {
+                    $object[$prop] = new DateTime($object[$prop]['dt'], new DateTimeZone($object[$prop]['tz']));
+                }
+                else if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
                     $object[$prop] = $sql_arr[$prop];
                 }
             }
@@ -939,9 +920,8 @@ class kolab_storage_cache
             $object['_mailbox']  = $this->folder->name;
         }
         // Fetch object xml
-        else if ($object = $this->folder->read_object($sql_arr['msguid'])) {
-            // additional meta data
-            $object['_size'] = strlen($xml);
+        else {
+            $object = $this->folder->read_object($sql_arr['msguid']);
         }
 
         return $object;
@@ -971,7 +951,7 @@ class kolab_storage_cache
                 }
 
                 $params = array($this->folder_id, $msguid, $object['uid'], $sql_data['changed'],
-                    $sql_data['data'], $sql_data['xml'], $sql_data['tags'], $sql_data['words']);
+                    $sql_data['data'], $sql_data['tags'], $sql_data['words']);
 
                 foreach ($this->extra_cols as $col) {
                     $params[] = $sql_data[$col];
@@ -979,8 +959,8 @@ class kolab_storage_cache
 
                 $result = $this->db->query(
                     "INSERT INTO `{$this->cache_table}` "
-                    . " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `xml`, `tags`, `words` $extra_cols)"
-                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ?, ? $extra_args)",
+                    . " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)"
+                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ? $extra_args)",
                     $params
                 );
 
@@ -1001,7 +981,6 @@ class kolab_storage_cache
                 $this->db->now(),
                 $this->db->quote($sql_data['changed']),
                 $this->db->quote($sql_data['data']),
-                $this->db->quote($sql_data['xml']),
                 $this->db->quote($sql_data['tags']),
                 $this->db->quote($sql_data['words']),
             );
@@ -1020,7 +999,7 @@ class kolab_storage_cache
 
             $result = $this->db->query(
                 "INSERT INTO `{$this->cache_table}` ".
-                " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `xml`, `tags`, `words` $extra_cols)".
+                " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)".
                 " VALUES $buffer"
             );
 
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 5dfb16b7..22d0a330 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -26,11 +26,6 @@ class kolab_storage_cache_contact extends kolab_storage_cache
     protected $extra_cols_max = 255;
     protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
     protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'member');
-    protected $binary_items   = array(
-        'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
-        'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',
-        'pkcs7publickey' => '|<key><uri>data:application/pkcs7-mime;base64,([^<]+)</uri></key>|i',
-    );
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/libkolab/lib/kolab_storage_cache_mongodb.php b/plugins/libkolab/lib/kolab_storage_cache_mongodb.php
deleted file mode 100644
index 8ae95e4b..00000000
--- a/plugins/libkolab/lib/kolab_storage_cache_mongodb.php
+++ /dev/null
@@ -1,561 +0,0 @@
-<?php
-
-/**
- * Kolab storage cache class providing a local caching layer for Kolab groupware objects.
- *
- * @version @package_version@
- * @author Thomas Bruederli <bruederli@kolabsys.com>
- *
- * Copyright (C) 2012, Kolab Systems AG <contact@kolabsys.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-class kolab_storage_cache_mongodb
-{
-    private $db;
-    private $imap;
-    private $folder;
-    private $uid2msg;
-    private $objects;
-    private $index = array();
-    private $resource_uri;
-    private $enabled = true;
-    private $synched = false;
-    private $synclock = false;
-    private $ready = false;
-    private $max_sql_packet = 1046576;  // 1 MB - 2000 bytes
-    private $binary_cols = array('photo','pgppublickey','pkcs7publickey');
-
-
-    /**
-     * Default constructor
-     */
-    public function __construct(kolab_storage_folder $storage_folder = null)
-    {
-        $rcmail = rcube::get_instance();
-        $mongo = new Mongo();
-        $this->db = $mongo->kolab_cache;
-        $this->imap = $rcmail->get_storage();
-        $this->enabled = $rcmail->config->get('kolab_cache', false);
-
-        if ($this->enabled) {
-            // remove sync-lock on script termination
-            $rcmail->add_shutdown_function(array($this, '_sync_unlock'));
-        }
-
-        if ($storage_folder)
-            $this->set_folder($storage_folder);
-    }
-
-
-    /**
-     * Connect cache with a storage folder
-     *
-     * @param kolab_storage_folder The storage folder instance to connect with
-     */
-    public function set_folder(kolab_storage_folder $storage_folder)
-    {
-        $this->folder = $storage_folder;
-
-        if (empty($this->folder->name)) {
-            $this->ready = false;
-            return;
-        }
-
-        // compose fully qualified ressource uri for this instance
-        $this->resource_uri = $this->folder->get_resource_uri();
-        $this->ready = $this->enabled;
-    }
-
-
-    /**
-     * Synchronize local cache data with remote
-     */
-    public function synchronize()
-    {
-        // only sync once per request cycle
-        if ($this->synched)
-            return;
-
-        // increase time limit
-        @set_time_limit(500);
-
-        // lock synchronization for this folder or wait if locked
-        $this->_sync_lock();
-
-        // synchronize IMAP mailbox cache
-        $this->imap->folder_sync($this->folder->name);
-
-        // compare IMAP index with object cache index
-        $imap_index = $this->imap->index($this->folder->name);
-        $this->index = $imap_index->get();
-
-        // determine objects to fetch or to invalidate
-        if ($this->ready) {
-            // read cache index
-            $old_index = array();
-            $cursor = $this->db->cache->find(array('resource' => $this->resource_uri), array('msguid' => 1, 'uid' => 1));
-            foreach ($cursor as $doc) {
-                $old_index[] = $doc['msguid'];
-                $this->uid2msg[$doc['uid']] = $doc['msguid'];
-            }
-
-            // fetch new objects from imap
-            foreach (array_diff($this->index, $old_index) as $msguid) {
-                if ($object = $this->folder->read_object($msguid, '*')) {
-                    try {
-                        $this->db->cache->insert($this->_serialize($object, $msguid));
-                    }
-                    catch (Exception $e) {
-                        rcmail::raise_error(array(
-                            'code' => 900, 'type' => 'php',
-                            'message' => "Failed to write to mongodb cache: " . $e->getMessage(),
-                        ), true);
-                    }
-                }
-            }
-
-            // delete invalid entries from local DB
-            $del_index = array_diff($old_index, $this->index);
-            if (!empty($del_index)) {
-                $this->db->cache->remove(array('resource' => $this->resource_uri, 'msguid' => array('$in' => $del_index)));
-            }
-        }
-
-        // remove lock
-        $this->_sync_unlock();
-
-        $this->synched = time();
-    }
-
-
-    /**
-     * Read a single entry from cache or from IMAP directly
-     *
-     * @param string Related IMAP message UID
-     * @param string Object type to read
-     * @param string IMAP folder name the entry relates to
-     * @param array  Hash array with object properties or null if not found
-     */
-    public function get($msguid, $type = null, $foldername = null)
-    {
-        // delegate to another cache instance
-        if ($foldername && $foldername != $this->folder->name) {
-            return kolab_storage::get_folder($foldername)->cache->get($msguid, $object);
-        }
-
-        // load object if not in memory
-        if (!isset($this->objects[$msguid])) {
-            if ($this->ready && ($doc = $this->db->cache->findOne(array('resource' => $this->resource_uri, 'msguid' => $msguid))))
-                $this->objects[$msguid] = $this->_unserialize($doc);
-
-            // fetch from IMAP if not present in cache
-            if (empty($this->objects[$msguid])) {
-                $result = $this->_fetch(array($msguid), $type, $foldername);
-                $this->objects[$msguid] = $result[0];
-            }
-        }
-
-        return $this->objects[$msguid];
-    }
-
-
-    /**
-     * Insert/Update a cache entry
-     *
-     * @param string Related IMAP message UID
-     * @param mixed  Hash array with object properties to save or false to delete the cache entry
-     * @param string IMAP folder name the entry relates to
-     */
-    public function set($msguid, $object, $foldername = null)
-    {
-        // delegate to another cache instance
-        if ($foldername && $foldername != $this->folder->name) {
-            kolab_storage::get_folder($foldername)->cache->set($msguid, $object);
-            return;
-        }
-
-        // write to cache
-        if ($this->ready) {
-            // remove old entry
-            $this->db->cache->remove(array('resource' => $this->resource_uri, 'msguid' => $msguid));
-
-            // write new object data if not false (wich means deleted)
-            if ($object) {
-                try {
-                    $this->db->cache->insert($this->_serialize($object, $msguid));
-                }
-                catch (Exception $e) {
-                    rcmail::raise_error(array(
-                        'code' => 900, 'type' => 'php',
-                        'message' => "Failed to write to mongodb cache: " . $e->getMessage(),
-                    ), true);
-                }
-            }
-        }
-
-        // keep a copy in memory for fast access
-        $this->objects[$msguid] = $object;
-
-        if ($object)
-            $this->uid2msg[$object['uid']] = $msguid;
-    }
-
-    /**
-     * Move an existing cache entry to a new resource
-     *
-     * @param string Entry's IMAP message UID
-     * @param string Entry's Object UID
-     * @param string Target IMAP folder to move it to
-     */
-    public function move($msguid, $objuid, $target_folder)
-    {
-        $target = kolab_storage::get_folder($target_folder);
-
-        // resolve new message UID in target folder
-        if ($new_msguid = $target->cache->uid2msguid($objuid)) {
-/*
-            $this->db->query(
-                "UPDATE kolab_cache SET resource=?, msguid=? ".
-                "WHERE resource=? AND msguid=? AND type<>?",
-                $target->get_resource_uri(),
-                $new_msguid,
-                $this->resource_uri,
-                $msguid,
-                'lock'
-            );
-*/
-        }
-        else {
-            // just clear cache entry
-            $this->set($msguid, false);
-        }
-
-        unset($this->uid2msg[$uid]);
-    }
-
-
-    /**
-     * Remove all objects from local cache
-     */
-    public function purge($type = null)
-    {
-        return $this->db->cache->remove(array(), array('safe' => true));
-    }
-
-
-    /**
-     * Select Kolab objects filtered by the given query
-     *
-     * @param array Pseudo-SQL query as list of filter parameter triplets
-     *   triplet: array('<colname>', '<comparator>', '<value>')
-     * @return array List of Kolab data objects (each represented as hash array)
-     */
-    public function select($query = array())
-    {
-        $result = array();
-
-        // read from local cache DB (assume it to be synchronized)
-        if ($this->ready) {
-            $cursor = $this->db->cache->find(array('resource' => $this->resource_uri) + $this->_mongo_filter($query));
-            foreach ($cursor as $doc) {
-                if ($object = $this->_unserialize($doc))
-                    $result[] = $object;
-            }
-        }
-        else {
-            // extract object type from query parameter
-            $filter = $this->_query2assoc($query);
-
-            // use 'list' for folder's default objects
-            if ($filter['type'] == $this->type) {
-                $index = $this->index;
-            }
-            else {  // search by object type
-                $search = 'UNDELETED HEADER X-Kolab-Type ' . kolab_storage_folder::KTYPE_PREFIX . $filter['type'];
-                $index = $this->imap->search_once($this->folder->name, $search)->get();
-            }
-
-            // fetch all messages in $index from IMAP
-            $result = $this->_fetch($index, $filter['type']);
-
-            // TODO: post-filter result according to query
-        }
-
-        return $result;
-    }
-
-
-    /**
-     * Get number of objects mathing the given query
-     *
-     * @param array  $query Pseudo-SQL query as list of filter parameter triplets
-     * @return integer The number of objects of the given type
-     */
-    public function count($query = array())
-    {
-        $count = 0;
-
-        // cache is in sync, we can count records in local DB
-        if ($this->synched) {
-            $cursor = $this->db->cache->find(array('resource' => $this->resource_uri) + $this->_mongo_filter($query));
-            $count = $cursor->valid() ? $cursor->count() : 0;
-        }
-        else {
-            // search IMAP by object type
-            $filter = $this->_query2assoc($query);
-            $ctype  = kolab_storage_folder::KTYPE_PREFIX . $filter['type'];
-            $index = $this->imap->search_once($this->folder->name, 'UNDELETED HEADER X-Kolab-Type ' . $ctype);
-            $count = $index->count();
-        }
-
-        return $count;
-    }
-
-    /**
-     * Helper method to convert the pseudo-SQL query into a valid mongodb filter
-     */
-    private function _mongo_filter($query)
-    {
-        $filters = array();
-        foreach ($query as $param) {
-            $filter = array();
-            if ($param[1] == '=' && is_array($param[2])) {
-                $filter[$param[0]] = array('$in' => $param[2]);
-                $filters[] = $filter;
-            }
-            else if ($param[1] == '=') {
-                $filters[] = array($param[0] => $param[2]);
-            }
-            else if ($param[1] == 'LIKE' || $param[1] == '~') {
-                $filter[$param[0]] = array('$regex' => preg_quote($param[2]), '$options' => 'i');
-                $filters[] = $filter;
-            }
-            else if ($param[1] == '!~' || $param[1] == '!LIKE') {
-                $filter[$param[0]] = array('$not' => '/' . preg_quote($param[2]) . '/i');
-                $filters[] = $filter;
-            }
-            else {
-                $op = '';
-                switch ($param[1]) {
-                    case '>':  $op = '$gt';  break;
-                    case '>=': $op = '$gte'; break;
-                    case '<':  $op = '$lt';  break;
-                    case '<=': $op = '$lte'; break;
-                    case '!=':
-                    case '<>': $op = '$gte'; break;
-                }
-                if ($op) {
-                    $filter[$param[0]] = array($op => $param[2]);
-                    $filters[] = $filter;
-                }
-            }
-        }
-
-        return array('$and' => $filters);
-    }
-
-    /**
-     * Helper method to convert the given pseudo-query triplets into
-     * an associative filter array with 'equals' values only
-     */
-    private function _query2assoc($query)
-    {
-        // extract object type from query parameter
-        $filter = array();
-        foreach ($query as $param) {
-            if ($param[1] == '=')
-                $filter[$param[0]] = $param[2];
-        }
-        return $filter;
-    }
-
-    /**
-     * Fetch messages from IMAP
-     *
-     * @param array List of message UIDs to fetch
-     * @return array List of parsed Kolab objects
-     */
-    private function _fetch($index, $type = null, $folder = null)
-    {
-        $results = array();
-        foreach ((array)$index as $msguid) {
-            if ($object = $this->folder->read_object($msguid, $type, $folder)) {
-                $results[] = $object;
-                $this->set($msguid, $object);
-            }
-        }
-
-        return $results;
-    }
-
-
-    /**
-     * Helper method to convert the given Kolab object into a dataset to be written to cache
-     */
-    private function _serialize($object, $msguid)
-    {
-        $bincols = array_flip($this->binary_cols);
-        $doc = array(
-            'resource' => $this->resource_uri,
-            'type'     => $object['_type'] ? $object['_type'] : $this->folder->type,
-            'msguid'   => $msguid,
-            'uid'      => $object['uid'],
-            'xml'      => '',
-            'tags'     => array(),
-            'words'    => array(),
-            'objcols'  => array(),
-        );
-
-        // set type specific values
-        if ($this->folder->type == 'event') {
-            // database runs in server's timezone so using date() is what we want
-            $doc['dtstart'] = date('Y-m-d H:i:s', is_object($object['start']) ? $object['start']->format('U') : $object['start']);
-            $doc['dtend']   = date('Y-m-d H:i:s', is_object($object['end'])   ? $object['end']->format('U')   : $object['end']);
-
-            // extend date range for recurring events
-            if ($object['recurrence']) {
-                $doc['dtend'] = date('Y-m-d H:i:s', $object['recurrence']['UNTIL'] ?: strtotime('now + 2 years'));
-            }
-        }
-
-        if ($object['_formatobj']) {
-            $doc['xml'] = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', (string)$object['_formatobj']->write());
-            $doc['tags'] = $object['_formatobj']->get_tags();
-            $doc['words'] = $object['_formatobj']->get_words();
-        }
-
-        // extract object data
-        $data = array();
-        foreach ($object as $key => $val) {
-            if ($val === "" || $val === null) {
-                // skip empty properties
-                continue;
-            }
-            if (isset($bincols[$key])) {
-                $data[$key] = base64_encode($val);
-            }
-            else if (is_object($val)) {
-                if (is_a($val, 'DateTime')) {
-                    $data[$key] = array('_class' => 'DateTime', 'date' => $val->format('Y-m-d H:i:s'), 'timezone' => $val->getTimezone()->getName());
-                    $doc['objcols'][] = $key;
-                }
-            }
-            else if ($key[0] != '_') {
-                $data[$key] = $val;
-            }
-            else if ($key == '_attachments') {
-                foreach ($val as $k => $att) {
-                    unset($att['content'], $att['path']);
-                    if ($att['id'])
-                        $data[$key][$k] = $att;
-                }
-            }
-        }
-
-        $doc['data'] = $data;
-        return $doc;
-    }
-
-    /**
-     * Helper method to turn stored cache data into a valid storage object
-     */
-    private function _unserialize($doc)
-    {
-        $object = $doc['data'];
-
-        // decode binary properties
-        foreach ($this->binary_cols as $key) {
-            if (!empty($object[$key]))
-                $object[$key] = base64_decode($object[$key]);
-        }
-
-        // restore serialized objects
-        foreach ((array)$doc['objcols'] as $key) {
-            switch ($object[$key]['_class']) {
-                case 'DateTime':
-                    $val = new DateTime($object[$key]['date'], new DateTimeZone($object[$key]['timezone']));
-                    $object[$key] = $val;
-                    break;
-            }
-        }
-
-        // add meta data
-        $object['_type'] = $doc['type'];
-        $object['_msguid'] = $doc['msguid'];
-        $object['_mailbox'] = $this->folder->name;
-        $object['_formatobj'] = kolab_format::factory($doc['type'], $doc['xml']);
-
-        return $object;
-    }
-
-    /**
-     * Check lock record for this folder and wait if locked or set lock
-     */
-    private function _sync_lock()
-    {
-        if (!$this->ready)
-            return;
-
-        $this->synclock = true;
-        $lock = $this->db->locks->findOne(array('resource' => $this->resource_uri));
-
-        // create lock record if not exists
-        if (!$lock) {
-            $this->db->locks->insert(array('resource' => $this->resource_uri, 'created' => time()));
-        }
-        // wait if locked (expire locks after 10 minutes)
-        else if ((time() - $lock['created']) < 600) {
-            usleep(500000);
-            return $this->_sync_lock();
-        }
-        // set lock
-        else {
-            $lock['created'] = time();
-            $this->db->locks->update(array('_id' => $lock['_id']), $lock, array('safe' => true));
-        }
-    }
-
-    /**
-     * Remove lock for this folder
-     */
-    public function _sync_unlock()
-    {
-        if (!$this->ready || !$this->synclock)
-            return;
-
-        $this->db->locks->remove(array('resource' => $this->resource_uri));
-    }
-
-    /**
-     * Resolve an object UID into an IMAP message UID
-     *
-     * @param string  Kolab object UID
-     * @param boolean Include deleted objects
-     * @return int The resolved IMAP message UID
-     */
-    public function uid2msguid($uid, $deleted = false)
-    {
-        if (!isset($this->uid2msg[$uid])) {
-            // use IMAP SEARCH to get the right message
-            $index = $this->imap->search_once($this->folder->name, ($deleted ? '' : 'UNDELETED ') . 'HEADER SUBJECT ' . $uid);
-            $results = $index->get();
-            $this->uid2msg[$uid] = $results[0];
-        }
-
-        return $this->uid2msg[$uid];
-    }
-
-}
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 1ffbb457..d0695cb2 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -562,6 +562,7 @@ class kolab_storage_folder extends kolab_storage_folder_api
             $object['_msguid']    = $msguid;
             $object['_mailbox']   = $this->name;
             $object['_formatobj'] = $format;
+            $object['_size']      = strlen($xml);
 
             return $object;
         }

From e848de3129ae59965d18a4194c746a70c8d77c24 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Fri, 28 Dec 2018 14:13:25 +0000
Subject: [PATCH 13/30] Use INSERT ... ON DUPLICATE KEY UPDATE in kolab cache

Bifrost#T61987
---
 plugins/libkolab/lib/kolab_storage_cache.php | 45 +++++++++-----------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index a0d360ed..53aa090c 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -938,36 +938,38 @@ class kolab_storage_cache
         static $buffer = '';
 
         $line = '';
+        $cols = array('folder_id', 'msguid', 'uid', 'created', 'changed', 'data', 'tags', 'words');
+        if ($this->extra_cols) {
+            $cols = array_merge($cols, $this->extra_cols);
+        }
+
         if ($object) {
             $sql_data = $this->_serialize($object);
 
-            // Skip multifolder insert for Oracle, we can't put long data inline
-            if ($this->db->db_provider == 'oracle') {
-                $extra_cols = '';
-                if ($this->extra_cols) {
-                    $extra_cols = array_map(function($n) { return "`{$n}`"; }, $this->extra_cols);
-                    $extra_cols = ', ' . join(', ', $extra_cols);
-                    $extra_args = str_repeat(', ?', count($this->extra_cols));
-                }
-
+            // Skip multi-folder insert for all databases but MySQL
+            // In Oracle we can't put long data inline, others we don't support yet
+            if (strpos($this->db->db_provider, 'mysql') !== 0) {
+                $extra_args = array();
                 $params = array($this->folder_id, $msguid, $object['uid'], $sql_data['changed'],
                     $sql_data['data'], $sql_data['tags'], $sql_data['words']);
 
                 foreach ($this->extra_cols as $col) {
                     $params[] = $sql_data[$col];
+                    $extra_args[] = '?';
                 }
 
+                $cols = implode(', ', array_map(function($n) { return "`{$n}`"; }, $cols));
+                $extra_args = count($extra_args) ? ', ' . implode(', ', $extra_args) : '';
+
                 $result = $this->db->query(
-                    "INSERT INTO `{$this->cache_table}` "
-                    . " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)"
-                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ? $extra_args)",
+                    "INSERT INTO `{$this->cache_table}` ($cols)"
+                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ?$extra_args)",
                     $params
                 );
 
                 if (!$this->db->affected_rows($result)) {
                     rcube::raise_error(array(
-                        'code' => 900, 'type' => 'php',
-                        'message' => "Failed to write to kolab cache"
+                        'code' => 900, 'message' => "Failed to write to kolab cache"
                     ), true);
                 }
 
@@ -991,22 +993,17 @@ class kolab_storage_cache
         }
 
         if ($buffer && (!$msguid || (strlen($buffer) + strlen($line) > $this->max_sql_packet()))) {
-            $extra_cols = '';
-            if ($this->extra_cols) {
-                $extra_cols = array_map(function($n) { return "`{$n}`"; }, $this->extra_cols);
-                $extra_cols = ', ' . join(', ', $extra_cols);
-            }
+            $columns = implode(', ', array_map(function($n) { return "`{$n}`"; }, $cols));
+            $update  = implode(', ', array_map(function($i) { return "`{$i}` = VALUES(`{$i}`)"; }, array_slice($cols, 2)));
 
             $result = $this->db->query(
-                "INSERT INTO `{$this->cache_table}` ".
-                " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)".
-                " VALUES $buffer"
+                "INSERT INTO `{$this->cache_table}` ($columns) VALUES $buffer"
+                . " ON DUPLICATE KEY UPDATE $update"
             );
 
             if (!$this->db->affected_rows($result)) {
                 rcube::raise_error(array(
-                    'code' => 900, 'type' => 'php',
-                    'message' => "Failed to write to kolab cache"
+                    'code' => 900, 'message' => "Failed to write to kolab cache"
                 ), true);
             }
 

From 4d350d599ef069389732af26d5f5fc23d7caac23 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 25 Feb 2016 18:16:54 +0100
Subject: [PATCH 14/30] Performance: Load tags UI (and initialize configuration
 folder/cache) on when it's needed

---
 plugins/kolab_tags/lib/kolab_tags_engine.php | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/plugins/kolab_tags/lib/kolab_tags_engine.php b/plugins/kolab_tags/lib/kolab_tags_engine.php
index 1c45040f..49a37fbf 100644
--- a/plugins/kolab_tags/lib/kolab_tags_engine.php
+++ b/plugins/kolab_tags/lib/kolab_tags_engine.php
@@ -46,7 +46,12 @@ class kolab_tags_engine
      */
     public function ui()
     {
-        if ($this->rc->action && !in_array($this->rc->action, array('show', 'preview', 'dialog-ui'))) {
+        // set templates of Files UI and widgets
+        if ($this->rc->task != 'mail') {
+            return;
+        }
+
+        if ($this->rc->action && !in_array($this->rc->action, array('show', 'preview'))) {
             return;
         }
 
@@ -57,7 +62,7 @@ class kolab_tags_engine
         $this->rc->output->add_label('cancel', 'save');
         $this->plugin->add_label('tags', 'add', 'edit', 'delete', 'saving',
             'nameempty', 'nameexists', 'colorinvalid', 'untag', 'tagname',
-            'tagcolor', 'tagsearchnew', 'newtag', 'notags');
+            'tagcolor', 'tagsearchnew', 'newtag');
 
         $this->rc->output->add_handlers(array(
             'plugin.taglist' => array($this, 'taglist'),

From d4362d8836fea2dbd8550ae0972783a999f5e137 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Wed, 19 Dec 2018 12:25:39 +0100
Subject: [PATCH 15/30] Always fetch xml from IMAP and parse objects on select

Temporarily we still store data and xml as before, but do not use them on select.
---
 plugins/libkolab/lib/kolab_storage_cache.php  | 43 ++-----------------
 plugins/libkolab/lib/kolab_storage_folder.php |  3 +-
 2 files changed, 6 insertions(+), 40 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index 362d53c1..0912ebf1 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -679,7 +679,6 @@ class kolab_storage_cache
         return $result;
     }
 
-
     /**
      * Get number of objects mathing the given query
      *
@@ -897,44 +896,10 @@ class kolab_storage_cache
      */
     protected function _unserialize($sql_arr)
     {
-        // check if data is a base64-encoded string, for backward compat.
-        if (strpos(substr($sql_arr['data'], 0, 64), ':') === false) {
-            $sql_arr['data'] = base64_decode($sql_arr['data']);
-        }
-
-        $object = unserialize($sql_arr['data']);
-
-        // de-serialization failed
-        if ($object === false) {
-            rcube::raise_error(array(
-                'code' => 900, 'type' => 'php',
-                'message' => "Malformed data for {$this->resource_uri}/{$sql_arr['msguid']} object."
-            ), true);
-
-            return null;
-        }
-
-        // decode binary properties
-        foreach ($this->binary_items as $key => $regexp) {
-            if (!empty($object[$key]) && preg_match($regexp, $sql_arr['xml'], $m)) {
-                $object[$key] = base64_decode($m[1]);
-            }
-        }
-
-        $object_type = $sql_arr['type'] ?: $this->folder->type;
-        $format_type = $this->folder->type == 'configuration' ? 'configuration' : $object_type;
-
-        // add meta data
-        $object['_type']      = $object_type;
-        $object['_msguid']    = $sql_arr['msguid'];
-        $object['_mailbox']   = $this->folder->name;
-        $object['_size']      = strlen($sql_arr['xml']);
-        $object['_formatobj'] = kolab_format::factory($format_type, 3.0, $sql_arr['xml']);
-
-        // Fix old broken objects with missing creation date
-        if (empty($object['created']) && method_exists($object['_formatobj'], 'to_array')) {
-            $new_object        = $object['_formatobj']->to_array();
-            $object['created'] = $new_object['created'];
+        // Fetch object xml
+        if ($object = $this->folder->read_object($sql_arr['msguid'])) {
+            // additional meta data
+            $object['_size'] = strlen($xml);
         }
 
         return $object;
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 4018b7df..eb28ed61 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -486,8 +486,9 @@ class kolab_storage_folder extends kolab_storage_folder_api
         $content_type = kolab_format::KTYPE_PREFIX . $object_type;
 
         // check object type header and abort on mismatch
-        if ($type != '*' && $object_type != $type)
+        if ($type != '*' && strpos($object_type, $type) !== 0) {
             return false;
+        }
 
         $attachments = array();
 

From e1a4152820ff87b59e40e44c89edccaff7fd7dde Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 16:27:06 +0000
Subject: [PATCH 16/30] Add possibility to bypass indexes cache only

---
 plugins/libkolab/config.inc.php.dist         |  3 ++-
 plugins/libkolab/lib/kolab_storage_cache.php | 11 ++++++-----
 2 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/plugins/libkolab/config.inc.php.dist b/plugins/libkolab/config.inc.php.dist
index a2c15e8c..a6338e74 100644
--- a/plugins/libkolab/config.inc.php.dist
+++ b/plugins/libkolab/config.inc.php.dist
@@ -35,7 +35,8 @@ $config['kolab_http_request'] = array();
 
 // When kolab_cache is enabled Roundcube's messages cache will be redundant
 // when working on kolab folders. Here we can:
-// 2 - bypass messages/indexes cache completely
+// 3 - bypass only indexes, but use messages cache
+// 2 - bypass both messages and indexes cache
 // 1 - bypass only messages, but use index cache
 $config['kolab_messages_cache_bypass'] = 0;
 
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index 0912ebf1..cc3f2cf0 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -1155,7 +1155,7 @@ class kolab_storage_cache
     }
 
     /**
-     * Set Roundcube storage options and bypass messages cache.
+     * Set Roundcube storage options and bypass messages/indexes cache.
      *
      * We use skip_deleted and threading settings specific to Kolab,
      * we have to change these global settings only temporarily.
@@ -1210,17 +1210,18 @@ class kolab_storage_cache
 
             switch ($cache_bypass) {
                 case 2:
-                    // Disable messages cache completely
+                    // Disable messages and index cache completely
                     $this->imap->set_messages_caching(!$force);
                     break;
 
+                case 3:
                 case 1:
-                    // We'll disable messages cache, but keep index cache.
+                    // We'll disable messages cache, but keep index cache (1) or vice-versa (3)
                     // Default mode is both (MODE_INDEX | MODE_MESSAGE)
-                    $mode = rcube_imap_cache::MODE_INDEX;
+                    $mode = $cache_bypass == 3 ? rcube_imap_cache::MODE_MESSAGE : rcube_imap_cache::MODE_INDEX;
 
                     if (!$force) {
-                        $mode |= rcube_imap_cache::MODE_MESSAGE;
+                        $mode |= $cache_bypass == 3 ? rcube_imap_cache::MODE_INDEX : rcube_imap_cache::MODE_MESSAGE;
                     }
 
                     $this->imap->set_messages_caching(true, $mode);

From f3d4dadffd6213d0e6c94e55c9bb3031f4d71b1d Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 16:51:34 +0000
Subject: [PATCH 17/30] Fix regression in handling contact groups

---
 plugins/libkolab/lib/kolab_storage_folder.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index eb28ed61..45f9bbcd 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -486,7 +486,7 @@ class kolab_storage_folder extends kolab_storage_folder_api
         $content_type = kolab_format::KTYPE_PREFIX . $object_type;
 
         // check object type header and abort on mismatch
-        if ($type != '*' && strpos($object_type, $type) !== 0) {
+        if ($type != '*' && strpos($object_type, $type) !== 0 && !($object_type == 'distribution-list' && $type == 'contact')) {
             return false;
         }
 

From 2c98bf28111aba42492e799334ddd7bc5d012b9d Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 18:36:17 +0000
Subject: [PATCH 18/30] Fast-mode for selecting kolab objects

... for now used only by kolab_notes plugin
---
 plugins/kolab_notes/kolab_notes.php           |  2 +-
 plugins/libkolab/lib/kolab_storage_cache.php  | 33 +++++++++++++++++--
 .../libkolab/lib/kolab_storage_cache_note.php |  2 +-
 plugins/libkolab/lib/kolab_storage_folder.php | 10 +++---
 4 files changed, 39 insertions(+), 8 deletions(-)

diff --git a/plugins/kolab_notes/kolab_notes.php b/plugins/kolab_notes/kolab_notes.php
index 34f0d3c5..9d1c240d 100644
--- a/plugins/kolab_notes/kolab_notes.php
+++ b/plugins/kolab_notes/kolab_notes.php
@@ -480,7 +480,7 @@ class kolab_notes extends rcube_plugin
 
         $this->_read_lists();
         if ($folder = $this->get_folder($list_id)) {
-            foreach ($folder->select($query) as $record) {
+            foreach ($folder->select($query, empty($query)) as $record) {
                 // post-filter search results
                 if (strlen($search)) {
                     $matches = 0;
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index cc3f2cf0..e1ec56d9 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -48,6 +48,7 @@ class kolab_storage_cache
     protected $max_sync_lock_time = 600;
     protected $binary_items = array();
     protected $extra_cols = array();
+    protected $data_props = array();
     protected $order_by = null;
     protected $limit = null;
     protected $error = 0;
@@ -585,9 +586,12 @@ class kolab_storage_cache
      * @param array Pseudo-SQL query as list of filter parameter triplets
      *   triplet: array('<colname>', '<comparator>', '<value>')
      * @param boolean Set true to only return UIDs instead of complete objects
+     * @param boolean Use fast mode to fetch only minimal set of information
+     *                (no xml fetching and parsing, etc.)
+     *
      * @return array List of Kolab data objects (each represented as hash array) or UIDs
      */
-    public function select($query = array(), $uids = false)
+    public function select($query = array(), $uids = false, $fast = false)
     {
         $result = $uids ? array() : new kolab_storage_dataset($this);
 
@@ -621,6 +625,9 @@ class kolab_storage_cache
             }
 
             while ($sql_arr = $this->db->fetch_assoc($sql_result)) {
+                if ($fast) {
+                    $sql_arr['fast-mode'] = true;
+                }
                 if ($uids) {
                     $this->uid2msg[$sql_arr['uid']] = $sql_arr['_msguid'];
                     $result[] = $sql_arr['uid'];
@@ -862,6 +869,11 @@ class kolab_storage_cache
             $sql_data['words'] = ' ' . join(' ', $object['_formatobj']->get_words()) . ' ';
         }
 
+        // TODO: get rid of xml column
+        // TODO: store only small subset of properties in data column, i.e. properties that are
+        //       needed for fast-mode only (use $data_props)
+        // TODO: store data in JSON format and no base64-encoding
+
         // extract object data
         $data = array();
         foreach ($object as $key => $val) {
@@ -896,8 +908,25 @@ class kolab_storage_cache
      */
     protected function _unserialize($sql_arr)
     {
+        if ($sql_arr['fast-mode'] && !empty($sql_arr['data'])
+            && ($object = unserialize(base64_decode($sql_arr['data'])))
+        ) {
+            foreach ($this->data_props as $prop) {
+                if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
+                    $object[$prop] = $sql_arr[$prop];
+
+                    if (($prop == 'created' || $prop == 'changed') && $object[$prop] && is_string($object[$prop])) {
+                        $object[$prop] = new DateTime($object[$prop]);
+                    }
+                }
+            }
+
+            $object['_type']     = $sql_arr['type'] ?: $this->folder->type;
+            $object['_msguid']   = $sql_arr['msguid'];
+            $object['_mailbox']  = $this->folder->name;
+        }
         // Fetch object xml
-        if ($object = $this->folder->read_object($sql_arr['msguid'])) {
+        else if ($object = $this->folder->read_object($sql_arr['msguid'])) {
             // additional meta data
             $object['_size'] = strlen($xml);
         }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_note.php b/plugins/libkolab/lib/kolab_storage_cache_note.php
index 85469275..fc23d5b0 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_note.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_note.php
@@ -23,5 +23,5 @@
 
 class kolab_storage_cache_note extends kolab_storage_cache
 {
-    
+    protected $data_props = array('uid', 'title', 'created', 'changed');
 }
\ No newline at end of file
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 45f9bbcd..02ac6adf 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -281,12 +281,14 @@ class kolab_storage_folder extends kolab_storage_folder_api
     /**
      * Select Kolab objects matching the given query
      *
-     * @param mixed Pseudo-SQL query as list of filter parameter triplets
-     *    or string with object type (e.g. contact, event, todo, journal, note, configuration)
+     * @param mixed   Pseudo-SQL query as list of filter parameter triplets
+     *                or string with object type (e.g. contact, event, todo, journal, note, configuration)
+     * @param boolean Use fast mode to fetch only minimal set of information
+     *                (no xml fetching and parsing, etc.)
      *
      * @return array List of Kolab data objects (each represented as hash array)
      */
-    public function select($query = array())
+    public function select($query = array(), $fast = false)
     {
         if (!$this->valid) {
             return array();
@@ -296,7 +298,7 @@ class kolab_storage_folder extends kolab_storage_folder_api
         $this->cache->synchronize();
 
         // fetch objects from cache
-        return $this->cache->select($this->_prepare_query($query));
+        return $this->cache->select($this->_prepare_query($query), false, $fast);
     }
 
     /**

From 818649624b8355063d8ded1cac73d01978d7bc45 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 13 Dec 2018 19:49:09 +0000
Subject: [PATCH 19/30] Fast-mode for contacts

... there's still a place for improvement, we can make more requests use fast-mode.
---
 .../lib/rcube_kolab_contacts.php                  | 15 +++++++++------
 .../libkolab/lib/kolab_storage_cache_contact.php  |  3 ++-
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
index b301ffad..081996b6 100644
--- a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
+++ b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
@@ -325,7 +325,9 @@ class rcube_kolab_contacts extends rcube_addressbook
     public function list_records($cols = null, $subset = 0, $nocount = false)
     {
         $this->result = new rcube_result_set(0, ($this->list_page-1) * $this->page_size);
+
         $fetch_all = false;
+        $fast_mode = !empty($cols) && is_array($cols);
 
         // list member of the selected group
         if ($this->gid) {
@@ -355,7 +357,7 @@ class rcube_kolab_contacts extends rcube_addressbook
 
             // get members by UID
             if (!empty($uids)) {
-                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), $fetch_all ? false : count($uids));
+                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), $fetch_all ? false : count($uids), $fast_mode);
                 $this->sortindex = array_merge($this->sortindex, $local_sortindex);
             }
         }
@@ -363,11 +365,11 @@ class rcube_kolab_contacts extends rcube_addressbook
             $ids = $this->filter['ids'];
             if (count($ids)) {
                 $uids = array_map(array($this, 'id2uid'), $this->filter['ids']);
-                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), count($ids));
+                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), count($ids), $fast_mode);
             }
         }
         else {
-            $this->_fetch_contacts($query = 'contact', true);
+            $this->_fetch_contacts($query = 'contact', true, $fast_mode);
         }
 
         if ($fetch_all) {
@@ -1062,7 +1064,7 @@ class rcube_kolab_contacts extends rcube_addressbook
     /**
      * Query storage layer and store records in private member var
      */
-    private function _fetch_contacts($query = array(), $limit = false)
+    private function _fetch_contacts($query = array(), $limit = false, $fast_mode = false)
     {
         if (!isset($this->dataset) || !empty($query)) {
             if ($limit) {
@@ -1071,7 +1073,7 @@ class rcube_kolab_contacts extends rcube_addressbook
             }
 
             $this->sortindex = array();
-            $this->dataset   = $this->storagefolder->select($query);
+            $this->dataset   = $this->storagefolder->select($query, $fast_mode);
 
             foreach ($this->dataset as $idx => $record) {
                 $contact = $this->_to_rcube_contact($record);
@@ -1145,8 +1147,9 @@ class rcube_kolab_contacts extends rcube_addressbook
                     $record['member'][$i]['readonly'] = empty($member['uid']);
                     $this->groupmembers[$mid][] = $record['ID'];
 
-                    if ($with_contacts && empty($member['uid']))
+                    if ($with_contacts && empty($member['uid'])) {
                         $this->contacts[$mid] = $record['member'][$i];
+                    }
                 }
                 $this->distlists[$record['ID']] = $record;
             }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 70aa5f62..531781c8 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -24,7 +24,8 @@
 class kolab_storage_cache_contact extends kolab_storage_cache
 {
     protected $extra_cols_max = 255;
-    protected $extra_cols     = array('type','name','firstname','surname','email');
+    protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
+    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'uid');
     protected $binary_items   = array(
         'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
         'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',

From e05d7987f62168e88811d751b9f21c12a86653b4 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Fri, 21 Dec 2018 09:58:40 +0100
Subject: [PATCH 20/30] Fast-mode for files

---
 .../libkolab/lib/kolab_storage_cache_file.php  | 18 ++++++++++++++----
 plugins/libkolab/lib/kolab_storage_folder.php  |  2 ++
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache_file.php b/plugins/libkolab/lib/kolab_storage_cache_file.php
index ea1823d6..053438c1 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_file.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_file.php
@@ -24,6 +24,7 @@
 class kolab_storage_cache_file extends kolab_storage_cache
 {
     protected $extra_cols = array('filename');
+    protected $data_props = array('type', 'size', 'filename', 'fileid', 'uid');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
@@ -32,13 +33,22 @@ class kolab_storage_cache_file extends kolab_storage_cache
      */
     protected function _serialize($object)
     {
-        $sql_data = parent::_serialize($object);
-
         if (!empty($object['_attachments'])) {
             reset($object['_attachments']);
-            $sql_data['filename'] = $object['_attachments'][key($object['_attachments'])]['name'];
+
+            $file = $object['_attachments'][key($object['_attachments'])];
+
+            $object['type']   = $file['mimetype'];
+            $object['size']   = $file['size'];
+            $object['fileid'] = $file['id'];
+        }
+
+        $sql_data = parent::_serialize($object);
+
+        if (!empty($file)) {
+            $sql_data['filename'] = $file['name'];
         }
 
         return $sql_data;
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 02ac6adf..1ffbb457 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -421,6 +421,8 @@ class kolab_storage_folder extends kolab_storage_folder_api
             }
             else {
                 // return message part from IMAP directly
+                // TODO: We could improve performance if we cache part's encoding
+                //       without 3rd argument get_message_part() will request BODYSTRUCTURE from IMAP
                 return $this->imap->get_message_part($msguid, $part, null, $print, $fp, $skip_charset_conv);
             }
         }

From 46af51d3155ba46fd35d17584ca2b9a9d151aa5d Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Fri, 21 Dec 2018 12:23:41 +0100
Subject: [PATCH 21/30] Fast-mode for events/tasks/contacts and some code
 improvements

---
 plugins/libkolab/lib/kolab_storage_cache.php  | 23 +++++++++++++----
 .../lib/kolab_storage_cache_contact.php       |  2 +-
 .../lib/kolab_storage_cache_event.php         |  1 +
 .../libkolab/lib/kolab_storage_cache_file.php | 25 ++++++++++++++++++-
 .../lib/kolab_storage_cache_journal.php       |  1 +
 .../libkolab/lib/kolab_storage_cache_note.php |  2 +-
 .../libkolab/lib/kolab_storage_cache_task.php |  3 ++-
 7 files changed, 48 insertions(+), 9 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index e1ec56d9..df851d6d 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -864,7 +864,10 @@ class kolab_storage_cache
         }
 
         if ($object['_formatobj']) {
-            $sql_data['xml']   = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', (string)$object['_formatobj']->write(3.0));
+            $xml  = (string) $object['_formatobj']->write(3.0);
+            $size = strlen($xml);
+
+            $sql_data['xml']   = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', $xml);
             $sql_data['tags']  = ' ' . join(' ', $object['_formatobj']->get_tags()) . ' ';  // pad with spaces for strict/prefix search
             $sql_data['words'] = ' ' . join(' ', $object['_formatobj']->get_words()) . ' ';
         }
@@ -897,6 +900,10 @@ class kolab_storage_cache
             }
         }
 
+        if ($size) {
+            $data['_size'] = $size;
+        }
+
         // use base64 encoding (Bug #1912, #2662)
         $sql_data['data'] = base64_encode(serialize($data));
 
@@ -911,16 +918,22 @@ class kolab_storage_cache
         if ($sql_arr['fast-mode'] && !empty($sql_arr['data'])
             && ($object = unserialize(base64_decode($sql_arr['data'])))
         ) {
+            $object['uid'] = $sql_arr['uid'];
+
             foreach ($this->data_props as $prop) {
                 if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
                     $object[$prop] = $sql_arr[$prop];
-
-                    if (($prop == 'created' || $prop == 'changed') && $object[$prop] && is_string($object[$prop])) {
-                        $object[$prop] = new DateTime($object[$prop]);
-                    }
                 }
             }
 
+            if ($sql_arr['created'] && empty($object['created'])) {
+                $object['created'] = new DateTime($sql_arr['created']);
+            }
+
+            if ($sql_arr['changed'] && empty($object['changed'])) {
+                $object['changed'] = new DateTime($sql_arr['changed']);
+            }
+
             $object['_type']     = $sql_arr['type'] ?: $this->folder->type;
             $object['_msguid']   = $sql_arr['msguid'];
             $object['_mailbox']  = $this->folder->name;
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 531781c8..6f3a0c9a 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -25,7 +25,7 @@ class kolab_storage_cache_contact extends kolab_storage_cache
 {
     protected $extra_cols_max = 255;
     protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
-    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'uid');
+    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization');
     protected $binary_items   = array(
         'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
         'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',
diff --git a/plugins/libkolab/lib/kolab_storage_cache_event.php b/plugins/libkolab/lib/kolab_storage_cache_event.php
index b6d8b15f..13b350b3 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_event.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_event.php
@@ -24,6 +24,7 @@
 class kolab_storage_cache_event extends kolab_storage_cache
 {
     protected $extra_cols = array('dtstart','dtend');
+    protected $data_props = array('categories', 'status', 'attendees'); // start, end
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/libkolab/lib/kolab_storage_cache_file.php b/plugins/libkolab/lib/kolab_storage_cache_file.php
index 053438c1..1f3d4d92 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_file.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_file.php
@@ -24,7 +24,7 @@
 class kolab_storage_cache_file extends kolab_storage_cache
 {
     protected $extra_cols = array('filename');
-    protected $data_props = array('type', 'size', 'filename', 'fileid', 'uid');
+    protected $data_props = array('type', 'size', 'filename', 'fileid');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
@@ -51,4 +51,27 @@ class kolab_storage_cache_file extends kolab_storage_cache
 
         return $sql_data;
     }
+
+    /**
+     * Helper method to turn stored cache data into a valid storage object
+     *
+     * @override
+     */
+    protected function _unserialize($sql_arr)
+    {
+        $object = parent::_unserialize($sql_arr);
+
+        if ($object && $sql_arr['fast-mode']) {
+            if (!empty($object['_attachments'])) {
+                $file = $object['_attachments'][key($object['_attachments'])];
+
+                $object['type']     = $file['mimetype'];
+                $object['size']     = $file['size'];
+                $object['fileid']   = $file['id'];
+                $object['filename'] = $file['name'];
+            }
+        }
+
+        return $object;
+    }
 }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_journal.php b/plugins/libkolab/lib/kolab_storage_cache_journal.php
index 766f1da1..042d4bde 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_journal.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_journal.php
@@ -24,6 +24,7 @@
 class kolab_storage_cache_journal extends kolab_storage_cache
 {
     protected $extra_cols = array('dtstart','dtend');
+    protected $data_props = array('categories');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/libkolab/lib/kolab_storage_cache_note.php b/plugins/libkolab/lib/kolab_storage_cache_note.php
index fc23d5b0..7f5ebc8e 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_note.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_note.php
@@ -23,5 +23,5 @@
 
 class kolab_storage_cache_note extends kolab_storage_cache
 {
-    protected $data_props = array('uid', 'title', 'created', 'changed');
+    protected $data_props = array('title');
 }
\ No newline at end of file
diff --git a/plugins/libkolab/lib/kolab_storage_cache_task.php b/plugins/libkolab/lib/kolab_storage_cache_task.php
index 8b714e63..06b64d39 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_task.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_task.php
@@ -23,7 +23,8 @@
 
 class kolab_storage_cache_task extends kolab_storage_cache
 {
-    protected $extra_cols = array('dtstart','dtend');
+    protected $extra_cols = array('dtstart', 'dtend');
+    protected $data_props = array('categories', 'status'); // start, due
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache

From 6ca01dc8c9a0e314ffb2812e671cefa5683a0d20 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Mon, 24 Dec 2018 10:45:19 +0000
Subject: [PATCH 22/30] Fast-mode for configuration objects

---
 .../lib/kolab_storage_cache_configuration.php | 54 +++++++++++++++++--
 .../lib/kolab_storage_cache_contact.php       |  2 +-
 .../libkolab/lib/kolab_storage_cache_note.php |  2 +-
 plugins/libkolab/lib/kolab_storage_config.php |  2 +-
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache_configuration.php b/plugins/libkolab/lib/kolab_storage_cache_configuration.php
index c3c7ac4f..a5ac396d 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_configuration.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_configuration.php
@@ -32,20 +32,37 @@ class kolab_storage_cache_configuration extends kolab_storage_cache
      */
     protected function _serialize($object)
     {
+        $this->set_data_props($object['type']);
+
         $sql_data = parent::_serialize($object);
         $sql_data['type'] = $object['type'];
 
         return $sql_data;
     }
 
+    /**
+     * Helper method to convert database record to the given Kolab object
+     *
+     * @override
+     */
+    protected function _unserialize($sql_arr)
+    {
+        $this->set_data_props($sql_arr['type']);
+
+        return parent::_unserialize($sql_arr);
+    }
+
     /**
      * Select Kolab objects filtered by the given query
      *
-     * @param array Pseudo-SQL query as list of filter parameter triplets
+     * @param array   Pseudo-SQL query as list of filter parameter triplets
      * @param boolean Set true to only return UIDs instead of complete objects
+     * @param boolean Use fast mode to fetch only minimal set of information
+     *                (no xml fetching and parsing, etc.)
+     *
      * @return array List of Kolab data objects (each represented as hash array) or UIDs
      */
-    public function select($query = array(), $uids = false)
+    public function select($query = array(), $uids = false, $fast = false)
     {
         // modify query for IMAP search: query param 'type' is actually a subtype
         if (!$this->ready) {
@@ -57,7 +74,7 @@ class kolab_storage_cache_configuration extends kolab_storage_cache
             }
         }
 
-        return parent::select($query, $uids);
+        return parent::select($query, $uids, $fast);
     }
 
     /**
@@ -85,4 +102,35 @@ class kolab_storage_cache_configuration extends kolab_storage_cache
 
         return parent::_sql_where($query);
     }
+
+    /**
+     * Set $data_props property depending on object type
+     */
+    protected function set_data_props($type)
+    {
+        switch ($type) {
+        case 'dictionary':
+            $this->data_props = array('language', 'e');
+            break;
+
+        case 'file_driver':
+            $this->data_props = array('driver', 'title', 'enabled', 'host', 'port', 'username', 'password');
+            break;
+
+        case 'relation':
+            // Note: Because relations are heavily used, for performance reasons
+            // we store all properties for them
+            $this->data_props = array('name', 'category', 'color', 'parent', 'iconName', 'priority', 'members');
+            break;
+
+        case 'snippet':
+            $this->data_props = array('name');
+            break;
+
+        case 'category':
+        default:
+            // TODO: implement this
+            $this->data_props = array();
+        }
+    }
 }
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 6f3a0c9a..a8a475a5 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -70,4 +70,4 @@ class kolab_storage_cache_contact extends kolab_storage_cache
 
         return $sql_data;
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/libkolab/lib/kolab_storage_cache_note.php b/plugins/libkolab/lib/kolab_storage_cache_note.php
index 7f5ebc8e..3211da57 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_note.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_note.php
@@ -24,4 +24,4 @@
 class kolab_storage_cache_note extends kolab_storage_cache
 {
     protected $data_props = array('title');
-}
\ No newline at end of file
+}
diff --git a/plugins/libkolab/lib/kolab_storage_config.php b/plugins/libkolab/lib/kolab_storage_config.php
index fb442ece..b29e7a90 100644
--- a/plugins/libkolab/lib/kolab_storage_config.php
+++ b/plugins/libkolab/lib/kolab_storage_config.php
@@ -131,7 +131,7 @@ class kolab_storage_config
                 $folder->set_order_and_limit(null, $limit);
             }
 
-            foreach ($folder->select($filter) as $object) {
+            foreach ($folder->select($filter, true) as $object) {
                 unset($object['_formatobj']);
                 $list[] = $object;
             }

From bb419c34cbb889d71f899bbc58aeef83daa5f866 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 27 Dec 2018 09:23:29 +0000
Subject: [PATCH 23/30] Use cache fast-mode for counting tasks

---
 plugins/libkolab/lib/kolab_storage_cache_task.php        | 2 +-
 plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache_task.php b/plugins/libkolab/lib/kolab_storage_cache_task.php
index 06b64d39..9c776d75 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_task.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_task.php
@@ -24,7 +24,7 @@
 class kolab_storage_cache_task extends kolab_storage_cache
 {
     protected $extra_cols = array('dtstart', 'dtend');
-    protected $data_props = array('categories', 'status'); // start, due
+    protected $data_props = array('categories', 'status', 'complete', 'start', 'due');
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php b/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php
index 1bfff58b..73142be3 100644
--- a/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php
+++ b/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php
@@ -587,7 +587,7 @@ class tasklist_kolab_driver extends tasklist_driver
                 continue;
             }
 
-            foreach ($folder->select(array(array('tags','!~','x-complete'))) as $record) {
+            foreach ($folder->select(array(array('tags','!~','x-complete')), true) as $record) {
                 $rec = $this->_to_rcube_task($record, $list_id, false);
 
                 if ($this->is_complete($rec))  // don't count complete tasks

From f052c0210346b195b4395a2f822a619e61f2f709 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 27 Dec 2018 14:14:44 +0000
Subject: [PATCH 24/30] Use cache fast-mode for contact groups

The same as tags/relations these objects are used quite often, so
we store group members in database and we can access them without
accessing imap.
---
 plugins/kolab_addressbook/lib/rcube_kolab_contacts.php | 2 +-
 plugins/libkolab/lib/kolab_storage_cache_contact.php   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
index 081996b6..cfc9e693 100644
--- a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
+++ b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php
@@ -1139,7 +1139,7 @@ class rcube_kolab_contacts extends rcube_addressbook
     {
         if (!isset($this->distlists)) {
             $this->distlists = $this->groupmembers = array();
-            foreach ($this->storagefolder->select('distribution-list') as $record) {
+            foreach ($this->storagefolder->select('distribution-list', true) as $record) {
                 $record['ID'] = $this->uid2id($record['uid']);
                 foreach ((array)$record['member'] as $i => $member) {
                     $mid = $this->uid2id($member['uid'] ? $member['uid'] : 'mailto:' . $member['email']);
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index a8a475a5..5dfb16b7 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -25,7 +25,7 @@ class kolab_storage_cache_contact extends kolab_storage_cache
 {
     protected $extra_cols_max = 255;
     protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
-    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization');
+    protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'member');
     protected $binary_items   = array(
         'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
         'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',

From e69f920de392fce980b548c81200492da874dc52 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 27 Dec 2018 14:26:35 +0000
Subject: [PATCH 25/30] Slim down kolab cache (drop xml column, change data
 format) (Bifrost#T61991)

---
 plugins/libkolab/SQL/mysql.initial.sql        |  10 +-
 plugins/libkolab/SQL/mysql/2018122700.sql     |  10 +
 plugins/libkolab/SQL/oracle.initial.sql       |  10 +-
 plugins/libkolab/SQL/oracle/2018122700.sql    |  10 +
 plugins/libkolab/SQL/sqlite.initial.sql       |  10 +-
 plugins/libkolab/SQL/sqlite/2018122700.sql    | 137 +++++
 plugins/libkolab/lib/kolab_storage_cache.php  |  77 +--
 .../lib/kolab_storage_cache_contact.php       |   5 -
 .../lib/kolab_storage_cache_mongodb.php       | 561 ------------------
 plugins/libkolab/lib/kolab_storage_folder.php |   1 +
 10 files changed, 189 insertions(+), 642 deletions(-)
 create mode 100644 plugins/libkolab/SQL/mysql/2018122700.sql
 create mode 100644 plugins/libkolab/SQL/oracle/2018122700.sql
 create mode 100644 plugins/libkolab/SQL/sqlite/2018122700.sql
 delete mode 100644 plugins/libkolab/lib/kolab_storage_cache_mongodb.php

diff --git a/plugins/libkolab/SQL/mysql.initial.sql b/plugins/libkolab/SQL/mysql.initial.sql
index 0213c45b..80da02fa 100644
--- a/plugins/libkolab/SQL/mysql.initial.sql
+++ b/plugins/libkolab/SQL/mysql.initial.sql
@@ -33,7 +33,6 @@ CREATE TABLE `kolab_cache_contact` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `type` VARCHAR(32) CHARACTER SET ascii NOT NULL,
@@ -57,7 +56,6 @@ CREATE TABLE `kolab_cache_event` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -77,7 +75,6 @@ CREATE TABLE `kolab_cache_task` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -97,7 +94,6 @@ CREATE TABLE `kolab_cache_journal` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -117,7 +113,6 @@ CREATE TABLE `kolab_cache_note` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   CONSTRAINT `fk_kolab_cache_note_folder` FOREIGN KEY (`folder_id`)
@@ -135,7 +130,6 @@ CREATE TABLE `kolab_cache_file` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `filename` varchar(255) DEFAULT NULL,
@@ -155,7 +149,6 @@ CREATE TABLE `kolab_cache_configuration` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `type` VARCHAR(32) CHARACTER SET ascii NOT NULL,
@@ -175,7 +168,6 @@ CREATE TABLE `kolab_cache_freebusy` (
   `created` DATETIME DEFAULT NULL,
   `changed` DATETIME DEFAULT NULL,
   `data` LONGTEXT NOT NULL,
-  `xml` LONGBLOB NOT NULL,
   `tags` TEXT NOT NULL,
   `words` TEXT NOT NULL,
   `dtstart` DATETIME,
@@ -188,4 +180,4 @@ CREATE TABLE `kolab_cache_freebusy` (
 
 /*!40014  SET FOREIGN_KEY_CHECKS=1 */;
 
-REPLACE INTO `system` (`name`, `value`) VALUES ('libkolab-version', '2018021300');
+REPLACE INTO `system` (`name`, `value`) VALUES ('libkolab-version', '2018122700');
diff --git a/plugins/libkolab/SQL/mysql/2018122700.sql b/plugins/libkolab/SQL/mysql/2018122700.sql
new file mode 100644
index 00000000..08030dd0
--- /dev/null
+++ b/plugins/libkolab/SQL/mysql/2018122700.sql
@@ -0,0 +1,10 @@
+-- remove xml column, and change data format (clear cache needed)
+DELETE FROM `kolab_folders`;
+ALTER TABLE `kolab_cache_contact` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_event` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_task` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_journal` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_note` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_file` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_configuration` DROP COLUMN `xml`;
+ALTER TABLE `kolab_cache_freebusy` DROP COLUMN `xml`;
diff --git a/plugins/libkolab/SQL/oracle.initial.sql b/plugins/libkolab/SQL/oracle.initial.sql
index 78675783..ebba60dc 100644
--- a/plugins/libkolab/SQL/oracle.initial.sql
+++ b/plugins/libkolab/SQL/oracle.initial.sql
@@ -37,7 +37,6 @@ CREATE TABLE "kolab_cache_contact" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "type" varchar(32) NOT NULL,
@@ -60,7 +59,6 @@ CREATE TABLE "kolab_cache_event" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -79,7 +77,6 @@ CREATE TABLE "kolab_cache_task" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -98,7 +95,6 @@ CREATE TABLE "kolab_cache_journal" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -117,7 +113,6 @@ CREATE TABLE "kolab_cache_note" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     PRIMARY KEY ("folder_id", "msguid")
@@ -134,7 +129,6 @@ CREATE TABLE "kolab_cache_file" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "filename" varchar(255) DEFAULT NULL,
@@ -153,7 +147,6 @@ CREATE TABLE "kolab_cache_configuration" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "type" varchar(32) NOT NULL,
@@ -172,7 +165,6 @@ CREATE TABLE "kolab_cache_freebusy" (
     "created" timestamp DEFAULT NULL,
     "changed" timestamp DEFAULT NULL,
     "data" clob NOT NULL,
-    "xml" clob NOT NULL,
     "tags" clob DEFAULT NULL,
     "words" clob DEFAULT NULL,
     "dtstart" timestamp DEFAULT NULL,
@@ -183,4 +175,4 @@ CREATE TABLE "kolab_cache_freebusy" (
 CREATE INDEX "kolab_cache_fb_uid2msguid" ON "kolab_cache_freebusy" ("folder_id", "uid", "msguid");
 
 
-INSERT INTO "system" ("name", "value") VALUES ('libkolab-version', '2018021300');
+INSERT INTO "system" ("name", "value") VALUES ('libkolab-version', '2018122700');
diff --git a/plugins/libkolab/SQL/oracle/2018122700.sql b/plugins/libkolab/SQL/oracle/2018122700.sql
new file mode 100644
index 00000000..45fccb74
--- /dev/null
+++ b/plugins/libkolab/SQL/oracle/2018122700.sql
@@ -0,0 +1,10 @@
+-- remove xml column, and change data format (clear cache needed)
+DELETE FROM "kolab_folders";
+ALTER TABLE "kolab_cache_contact" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_event" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_task" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_journal" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_note" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_file" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_configuration" DROP COLUMN "xml";
+ALTER TABLE "kolab_cache_freebusy" DROP COLUMN "xml";
diff --git a/plugins/libkolab/SQL/sqlite.initial.sql b/plugins/libkolab/SQL/sqlite.initial.sql
index 909188ca..815316dc 100644
--- a/plugins/libkolab/SQL/sqlite.initial.sql
+++ b/plugins/libkolab/SQL/sqlite.initial.sql
@@ -25,7 +25,6 @@ CREATE TABLE kolab_cache_contact (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   type VARCHAR(32) NOT NULL,
@@ -46,7 +45,6 @@ CREATE TABLE kolab_cache_event (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -63,7 +61,6 @@ CREATE TABLE kolab_cache_task (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -80,7 +77,6 @@ CREATE TABLE kolab_cache_journal (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -97,7 +93,6 @@ CREATE TABLE kolab_cache_note (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   PRIMARY KEY(folder_id,msguid)
@@ -112,7 +107,6 @@ CREATE TABLE kolab_cache_file (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   filename varchar(255) DEFAULT NULL,
@@ -129,7 +123,6 @@ CREATE TABLE kolab_cache_configuration (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   type VARCHAR(32) NOT NULL,
@@ -146,7 +139,6 @@ CREATE TABLE kolab_cache_freebusy (
   created DATETIME DEFAULT NULL,
   changed DATETIME DEFAULT NULL,
   data TEXT NOT NULL,
-  xml TEXT NOT NULL,
   tags TEXT NOT NULL,
   words TEXT NOT NULL,
   dtstart DATETIME,
@@ -156,4 +148,4 @@ CREATE TABLE kolab_cache_freebusy (
 
 CREATE INDEX ix_freebusy_uid2msguid ON kolab_cache_freebusy(folder_id,uid,msguid);
 
-INSERT INTO system (name, value) VALUES ('libkolab-version', '2018021300');
+INSERT INTO system (name, value) VALUES ('libkolab-version', '2018122700');
diff --git a/plugins/libkolab/SQL/sqlite/2018122700.sql b/plugins/libkolab/SQL/sqlite/2018122700.sql
new file mode 100644
index 00000000..ee24c1a6
--- /dev/null
+++ b/plugins/libkolab/SQL/sqlite/2018122700.sql
@@ -0,0 +1,137 @@
+DROP TABLE kolab_cache_contact;
+CREATE TABLE kolab_cache_contact (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  type VARCHAR(32) NOT NULL,
+  name VARCHAR(255) NOT NULL,
+  firstname VARCHAR(255) NOT NULL,
+  surname VARCHAR(255) NOT NULL,
+  email VARCHAR(255) NOT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_contact_type ON kolab_cache_contact(folder_id,type);
+CREATE INDEX ix_contact_uid2msguid ON kolab_cache_contact(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_event;
+CREATE TABLE kolab_cache_event (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_event_uid2msguid ON kolab_cache_event(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_task;
+CREATE TABLE kolab_cache_task (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_task_uid2msguid ON kolab_cache_task(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_journal;
+CREATE TABLE kolab_cache_journal (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_journal_uid2msguid ON kolab_cache_journal(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_note;
+CREATE TABLE kolab_cache_note (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_note_uid2msguid ON kolab_cache_note(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_file;
+CREATE TABLE kolab_cache_file (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  filename varchar(255) DEFAULT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_folder_filename ON kolab_cache_file(folder_id,filename);
+CREATE INDEX ix_file_uid2msguid ON kolab_cache_file(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_configuration;
+CREATE TABLE kolab_cache_configuration (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  type VARCHAR(32) NOT NULL,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_configuration_type ON kolab_cache_configuration(folder_id,type);
+CREATE INDEX ix_configuration_uid2msguid ON kolab_cache_configuration(folder_id,uid,msguid);
+
+DROP TABLE kolab_cache_freebusy;
+CREATE TABLE kolab_cache_freebusy (
+  folder_id INTEGER NOT NULL,
+  msguid INTEGER NOT NULL,
+  uid VARCHAR(512) NOT NULL,
+  created DATETIME DEFAULT NULL,
+  changed DATETIME DEFAULT NULL,
+  data TEXT NOT NULL,
+  tags TEXT NOT NULL,
+  words TEXT NOT NULL,
+  dtstart DATETIME,
+  dtend DATETIME,
+  PRIMARY KEY(folder_id,msguid)
+);
+
+CREATE INDEX ix_freebusy_uid2msguid ON kolab_cache_freebusy(folder_id,uid,msguid);
diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index df851d6d..a0d360ed 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -46,7 +46,6 @@ class kolab_storage_cache
     protected $folders_table;
     protected $max_sql_packet;
     protected $max_sync_lock_time = 600;
-    protected $binary_items = array();
     protected $extra_cols = array();
     protected $data_props = array();
     protected $order_by = null;
@@ -202,7 +201,7 @@ class kolab_storage_cache
                   $this->metadata['changed'] < date(self::DB_DATE_FORMAT, time() - $this->cache_refresh) ||
                   $this->metadata['ctag'] != $this->folder->get_ctag() ||
                   intval($this->metadata['objectcount']) !== $this->count()
-                ) {
+            ) {
                 // lock synchronization for this folder or wait if locked
                 $this->_sync_lock();
 
@@ -448,7 +447,7 @@ class kolab_storage_cache
             $sql_data['uid']       = $object['uid'];
 
             $args = array();
-            $cols = array('folder_id', 'msguid', 'uid', 'changed', 'data', 'xml', 'tags', 'words');
+            $cols = array('folder_id', 'msguid', 'uid', 'changed', 'data', 'tags', 'words');
             $cols = array_merge($cols, $this->extra_cols);
 
             foreach ($cols as $idx => $col) {
@@ -857,55 +856,36 @@ class kolab_storage_cache
      */
     protected function _serialize($object)
     {
-        $sql_data = array('changed' => null, 'xml' => '', 'tags' => '', 'words' => '');
+        $data     = array();
+        $sql_data = array('changed' => null, 'tags' => '', 'words' => '');
 
         if ($object['changed']) {
             $sql_data['changed'] = date(self::DB_DATE_FORMAT, is_object($object['changed']) ? $object['changed']->format('U') : $object['changed']);
         }
 
         if ($object['_formatobj']) {
-            $xml  = (string) $object['_formatobj']->write(3.0);
-            $size = strlen($xml);
+            $xml = (string) $object['_formatobj']->write(3.0);
 
-            $sql_data['xml']   = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', $xml);
+            $data['_size']     = strlen($xml);
             $sql_data['tags']  = ' ' . join(' ', $object['_formatobj']->get_tags()) . ' ';  // pad with spaces for strict/prefix search
             $sql_data['words'] = ' ' . join(' ', $object['_formatobj']->get_words()) . ' ';
         }
 
-        // TODO: get rid of xml column
-        // TODO: store only small subset of properties in data column, i.e. properties that are
-        //       needed for fast-mode only (use $data_props)
-        // TODO: store data in JSON format and no base64-encoding
-
-        // extract object data
-        $data = array();
-        foreach ($object as $key => $val) {
-            // skip empty properties
-            if ($val === "" || $val === null) {
-                continue;
-            }
-            // mark binary data to be extracted from xml on unserialize()
-            if (isset($this->binary_items[$key])) {
-                $data[$key] = true;
-            }
-            else if ($key[0] != '_') {
-                $data[$key] = $val;
-            }
-            else if ($key == '_attachments') {
-                foreach ($val as $k => $att) {
-                    unset($att['content'], $att['path']);
-                    if ($att['id'])
-                        $data[$key][$k] = $att;
+        // Store only minimal set of object properties
+        foreach ($this->data_props as $prop) {
+            if (isset($object[$prop])) {
+                $data[$prop] = $object[$prop];
+                if ($data[$prop] instanceof DateTime) {
+                    $data[$prop] = array(
+                        'cl' => 'DateTime',
+                        'dt' => $data[$prop]->format('Y-m-d H:i:s'),
+                        'tz' => $data[$prop]->getTimezone()->getName(),
+                    );
                 }
             }
         }
 
-        if ($size) {
-            $data['_size'] = $size;
-        }
-
-        // use base64 encoding (Bug #1912, #2662)
-        $sql_data['data'] = base64_encode(serialize($data));
+        $sql_data['data'] = json_encode(rcube_charset::clean($data));
 
         return $sql_data;
     }
@@ -915,13 +895,14 @@ class kolab_storage_cache
      */
     protected function _unserialize($sql_arr)
     {
-        if ($sql_arr['fast-mode'] && !empty($sql_arr['data'])
-            && ($object = unserialize(base64_decode($sql_arr['data'])))
-        ) {
+        if ($sql_arr['fast-mode'] && !empty($sql_arr['data']) && ($object = json_decode($sql_arr['data'], true))) {
             $object['uid'] = $sql_arr['uid'];
 
             foreach ($this->data_props as $prop) {
-                if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
+                if (isset($object[$prop]) && is_array($object[$prop]) && $object[$prop]['cl'] == 'DateTime') {
+                    $object[$prop] = new DateTime($object[$prop]['dt'], new DateTimeZone($object[$prop]['tz']));
+                }
+                else if (!isset($object[$prop]) && isset($sql_arr[$prop])) {
                     $object[$prop] = $sql_arr[$prop];
                 }
             }
@@ -939,9 +920,8 @@ class kolab_storage_cache
             $object['_mailbox']  = $this->folder->name;
         }
         // Fetch object xml
-        else if ($object = $this->folder->read_object($sql_arr['msguid'])) {
-            // additional meta data
-            $object['_size'] = strlen($xml);
+        else {
+            $object = $this->folder->read_object($sql_arr['msguid']);
         }
 
         return $object;
@@ -971,7 +951,7 @@ class kolab_storage_cache
                 }
 
                 $params = array($this->folder_id, $msguid, $object['uid'], $sql_data['changed'],
-                    $sql_data['data'], $sql_data['xml'], $sql_data['tags'], $sql_data['words']);
+                    $sql_data['data'], $sql_data['tags'], $sql_data['words']);
 
                 foreach ($this->extra_cols as $col) {
                     $params[] = $sql_data[$col];
@@ -979,8 +959,8 @@ class kolab_storage_cache
 
                 $result = $this->db->query(
                     "INSERT INTO `{$this->cache_table}` "
-                    . " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `xml`, `tags`, `words` $extra_cols)"
-                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ?, ? $extra_args)",
+                    . " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)"
+                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ? $extra_args)",
                     $params
                 );
 
@@ -1001,7 +981,6 @@ class kolab_storage_cache
                 $this->db->now(),
                 $this->db->quote($sql_data['changed']),
                 $this->db->quote($sql_data['data']),
-                $this->db->quote($sql_data['xml']),
                 $this->db->quote($sql_data['tags']),
                 $this->db->quote($sql_data['words']),
             );
@@ -1020,7 +999,7 @@ class kolab_storage_cache
 
             $result = $this->db->query(
                 "INSERT INTO `{$this->cache_table}` ".
-                " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `xml`, `tags`, `words` $extra_cols)".
+                " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)".
                 " VALUES $buffer"
             );
 
diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php
index 5dfb16b7..22d0a330 100644
--- a/plugins/libkolab/lib/kolab_storage_cache_contact.php
+++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php
@@ -26,11 +26,6 @@ class kolab_storage_cache_contact extends kolab_storage_cache
     protected $extra_cols_max = 255;
     protected $extra_cols     = array('type', 'name', 'firstname', 'surname', 'email');
     protected $data_props     = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'member');
-    protected $binary_items   = array(
-        'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
-        'pgppublickey'   => '|<key><uri>data:application/pgp-keys;base64,([^<]+)</uri></key>|i',
-        'pkcs7publickey' => '|<key><uri>data:application/pkcs7-mime;base64,([^<]+)</uri></key>|i',
-    );
 
     /**
      * Helper method to convert the given Kolab object into a dataset to be written to cache
diff --git a/plugins/libkolab/lib/kolab_storage_cache_mongodb.php b/plugins/libkolab/lib/kolab_storage_cache_mongodb.php
deleted file mode 100644
index 8ae95e4b..00000000
--- a/plugins/libkolab/lib/kolab_storage_cache_mongodb.php
+++ /dev/null
@@ -1,561 +0,0 @@
-<?php
-
-/**
- * Kolab storage cache class providing a local caching layer for Kolab groupware objects.
- *
- * @version @package_version@
- * @author Thomas Bruederli <bruederli@kolabsys.com>
- *
- * Copyright (C) 2012, Kolab Systems AG <contact@kolabsys.com>
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as
- * published by the Free Software Foundation, either version 3 of the
- * License, or (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-class kolab_storage_cache_mongodb
-{
-    private $db;
-    private $imap;
-    private $folder;
-    private $uid2msg;
-    private $objects;
-    private $index = array();
-    private $resource_uri;
-    private $enabled = true;
-    private $synched = false;
-    private $synclock = false;
-    private $ready = false;
-    private $max_sql_packet = 1046576;  // 1 MB - 2000 bytes
-    private $binary_cols = array('photo','pgppublickey','pkcs7publickey');
-
-
-    /**
-     * Default constructor
-     */
-    public function __construct(kolab_storage_folder $storage_folder = null)
-    {
-        $rcmail = rcube::get_instance();
-        $mongo = new Mongo();
-        $this->db = $mongo->kolab_cache;
-        $this->imap = $rcmail->get_storage();
-        $this->enabled = $rcmail->config->get('kolab_cache', false);
-
-        if ($this->enabled) {
-            // remove sync-lock on script termination
-            $rcmail->add_shutdown_function(array($this, '_sync_unlock'));
-        }
-
-        if ($storage_folder)
-            $this->set_folder($storage_folder);
-    }
-
-
-    /**
-     * Connect cache with a storage folder
-     *
-     * @param kolab_storage_folder The storage folder instance to connect with
-     */
-    public function set_folder(kolab_storage_folder $storage_folder)
-    {
-        $this->folder = $storage_folder;
-
-        if (empty($this->folder->name)) {
-            $this->ready = false;
-            return;
-        }
-
-        // compose fully qualified ressource uri for this instance
-        $this->resource_uri = $this->folder->get_resource_uri();
-        $this->ready = $this->enabled;
-    }
-
-
-    /**
-     * Synchronize local cache data with remote
-     */
-    public function synchronize()
-    {
-        // only sync once per request cycle
-        if ($this->synched)
-            return;
-
-        // increase time limit
-        @set_time_limit(500);
-
-        // lock synchronization for this folder or wait if locked
-        $this->_sync_lock();
-
-        // synchronize IMAP mailbox cache
-        $this->imap->folder_sync($this->folder->name);
-
-        // compare IMAP index with object cache index
-        $imap_index = $this->imap->index($this->folder->name);
-        $this->index = $imap_index->get();
-
-        // determine objects to fetch or to invalidate
-        if ($this->ready) {
-            // read cache index
-            $old_index = array();
-            $cursor = $this->db->cache->find(array('resource' => $this->resource_uri), array('msguid' => 1, 'uid' => 1));
-            foreach ($cursor as $doc) {
-                $old_index[] = $doc['msguid'];
-                $this->uid2msg[$doc['uid']] = $doc['msguid'];
-            }
-
-            // fetch new objects from imap
-            foreach (array_diff($this->index, $old_index) as $msguid) {
-                if ($object = $this->folder->read_object($msguid, '*')) {
-                    try {
-                        $this->db->cache->insert($this->_serialize($object, $msguid));
-                    }
-                    catch (Exception $e) {
-                        rcmail::raise_error(array(
-                            'code' => 900, 'type' => 'php',
-                            'message' => "Failed to write to mongodb cache: " . $e->getMessage(),
-                        ), true);
-                    }
-                }
-            }
-
-            // delete invalid entries from local DB
-            $del_index = array_diff($old_index, $this->index);
-            if (!empty($del_index)) {
-                $this->db->cache->remove(array('resource' => $this->resource_uri, 'msguid' => array('$in' => $del_index)));
-            }
-        }
-
-        // remove lock
-        $this->_sync_unlock();
-
-        $this->synched = time();
-    }
-
-
-    /**
-     * Read a single entry from cache or from IMAP directly
-     *
-     * @param string Related IMAP message UID
-     * @param string Object type to read
-     * @param string IMAP folder name the entry relates to
-     * @param array  Hash array with object properties or null if not found
-     */
-    public function get($msguid, $type = null, $foldername = null)
-    {
-        // delegate to another cache instance
-        if ($foldername && $foldername != $this->folder->name) {
-            return kolab_storage::get_folder($foldername)->cache->get($msguid, $object);
-        }
-
-        // load object if not in memory
-        if (!isset($this->objects[$msguid])) {
-            if ($this->ready && ($doc = $this->db->cache->findOne(array('resource' => $this->resource_uri, 'msguid' => $msguid))))
-                $this->objects[$msguid] = $this->_unserialize($doc);
-
-            // fetch from IMAP if not present in cache
-            if (empty($this->objects[$msguid])) {
-                $result = $this->_fetch(array($msguid), $type, $foldername);
-                $this->objects[$msguid] = $result[0];
-            }
-        }
-
-        return $this->objects[$msguid];
-    }
-
-
-    /**
-     * Insert/Update a cache entry
-     *
-     * @param string Related IMAP message UID
-     * @param mixed  Hash array with object properties to save or false to delete the cache entry
-     * @param string IMAP folder name the entry relates to
-     */
-    public function set($msguid, $object, $foldername = null)
-    {
-        // delegate to another cache instance
-        if ($foldername && $foldername != $this->folder->name) {
-            kolab_storage::get_folder($foldername)->cache->set($msguid, $object);
-            return;
-        }
-
-        // write to cache
-        if ($this->ready) {
-            // remove old entry
-            $this->db->cache->remove(array('resource' => $this->resource_uri, 'msguid' => $msguid));
-
-            // write new object data if not false (wich means deleted)
-            if ($object) {
-                try {
-                    $this->db->cache->insert($this->_serialize($object, $msguid));
-                }
-                catch (Exception $e) {
-                    rcmail::raise_error(array(
-                        'code' => 900, 'type' => 'php',
-                        'message' => "Failed to write to mongodb cache: " . $e->getMessage(),
-                    ), true);
-                }
-            }
-        }
-
-        // keep a copy in memory for fast access
-        $this->objects[$msguid] = $object;
-
-        if ($object)
-            $this->uid2msg[$object['uid']] = $msguid;
-    }
-
-    /**
-     * Move an existing cache entry to a new resource
-     *
-     * @param string Entry's IMAP message UID
-     * @param string Entry's Object UID
-     * @param string Target IMAP folder to move it to
-     */
-    public function move($msguid, $objuid, $target_folder)
-    {
-        $target = kolab_storage::get_folder($target_folder);
-
-        // resolve new message UID in target folder
-        if ($new_msguid = $target->cache->uid2msguid($objuid)) {
-/*
-            $this->db->query(
-                "UPDATE kolab_cache SET resource=?, msguid=? ".
-                "WHERE resource=? AND msguid=? AND type<>?",
-                $target->get_resource_uri(),
-                $new_msguid,
-                $this->resource_uri,
-                $msguid,
-                'lock'
-            );
-*/
-        }
-        else {
-            // just clear cache entry
-            $this->set($msguid, false);
-        }
-
-        unset($this->uid2msg[$uid]);
-    }
-
-
-    /**
-     * Remove all objects from local cache
-     */
-    public function purge($type = null)
-    {
-        return $this->db->cache->remove(array(), array('safe' => true));
-    }
-
-
-    /**
-     * Select Kolab objects filtered by the given query
-     *
-     * @param array Pseudo-SQL query as list of filter parameter triplets
-     *   triplet: array('<colname>', '<comparator>', '<value>')
-     * @return array List of Kolab data objects (each represented as hash array)
-     */
-    public function select($query = array())
-    {
-        $result = array();
-
-        // read from local cache DB (assume it to be synchronized)
-        if ($this->ready) {
-            $cursor = $this->db->cache->find(array('resource' => $this->resource_uri) + $this->_mongo_filter($query));
-            foreach ($cursor as $doc) {
-                if ($object = $this->_unserialize($doc))
-                    $result[] = $object;
-            }
-        }
-        else {
-            // extract object type from query parameter
-            $filter = $this->_query2assoc($query);
-
-            // use 'list' for folder's default objects
-            if ($filter['type'] == $this->type) {
-                $index = $this->index;
-            }
-            else {  // search by object type
-                $search = 'UNDELETED HEADER X-Kolab-Type ' . kolab_storage_folder::KTYPE_PREFIX . $filter['type'];
-                $index = $this->imap->search_once($this->folder->name, $search)->get();
-            }
-
-            // fetch all messages in $index from IMAP
-            $result = $this->_fetch($index, $filter['type']);
-
-            // TODO: post-filter result according to query
-        }
-
-        return $result;
-    }
-
-
-    /**
-     * Get number of objects mathing the given query
-     *
-     * @param array  $query Pseudo-SQL query as list of filter parameter triplets
-     * @return integer The number of objects of the given type
-     */
-    public function count($query = array())
-    {
-        $count = 0;
-
-        // cache is in sync, we can count records in local DB
-        if ($this->synched) {
-            $cursor = $this->db->cache->find(array('resource' => $this->resource_uri) + $this->_mongo_filter($query));
-            $count = $cursor->valid() ? $cursor->count() : 0;
-        }
-        else {
-            // search IMAP by object type
-            $filter = $this->_query2assoc($query);
-            $ctype  = kolab_storage_folder::KTYPE_PREFIX . $filter['type'];
-            $index = $this->imap->search_once($this->folder->name, 'UNDELETED HEADER X-Kolab-Type ' . $ctype);
-            $count = $index->count();
-        }
-
-        return $count;
-    }
-
-    /**
-     * Helper method to convert the pseudo-SQL query into a valid mongodb filter
-     */
-    private function _mongo_filter($query)
-    {
-        $filters = array();
-        foreach ($query as $param) {
-            $filter = array();
-            if ($param[1] == '=' && is_array($param[2])) {
-                $filter[$param[0]] = array('$in' => $param[2]);
-                $filters[] = $filter;
-            }
-            else if ($param[1] == '=') {
-                $filters[] = array($param[0] => $param[2]);
-            }
-            else if ($param[1] == 'LIKE' || $param[1] == '~') {
-                $filter[$param[0]] = array('$regex' => preg_quote($param[2]), '$options' => 'i');
-                $filters[] = $filter;
-            }
-            else if ($param[1] == '!~' || $param[1] == '!LIKE') {
-                $filter[$param[0]] = array('$not' => '/' . preg_quote($param[2]) . '/i');
-                $filters[] = $filter;
-            }
-            else {
-                $op = '';
-                switch ($param[1]) {
-                    case '>':  $op = '$gt';  break;
-                    case '>=': $op = '$gte'; break;
-                    case '<':  $op = '$lt';  break;
-                    case '<=': $op = '$lte'; break;
-                    case '!=':
-                    case '<>': $op = '$gte'; break;
-                }
-                if ($op) {
-                    $filter[$param[0]] = array($op => $param[2]);
-                    $filters[] = $filter;
-                }
-            }
-        }
-
-        return array('$and' => $filters);
-    }
-
-    /**
-     * Helper method to convert the given pseudo-query triplets into
-     * an associative filter array with 'equals' values only
-     */
-    private function _query2assoc($query)
-    {
-        // extract object type from query parameter
-        $filter = array();
-        foreach ($query as $param) {
-            if ($param[1] == '=')
-                $filter[$param[0]] = $param[2];
-        }
-        return $filter;
-    }
-
-    /**
-     * Fetch messages from IMAP
-     *
-     * @param array List of message UIDs to fetch
-     * @return array List of parsed Kolab objects
-     */
-    private function _fetch($index, $type = null, $folder = null)
-    {
-        $results = array();
-        foreach ((array)$index as $msguid) {
-            if ($object = $this->folder->read_object($msguid, $type, $folder)) {
-                $results[] = $object;
-                $this->set($msguid, $object);
-            }
-        }
-
-        return $results;
-    }
-
-
-    /**
-     * Helper method to convert the given Kolab object into a dataset to be written to cache
-     */
-    private function _serialize($object, $msguid)
-    {
-        $bincols = array_flip($this->binary_cols);
-        $doc = array(
-            'resource' => $this->resource_uri,
-            'type'     => $object['_type'] ? $object['_type'] : $this->folder->type,
-            'msguid'   => $msguid,
-            'uid'      => $object['uid'],
-            'xml'      => '',
-            'tags'     => array(),
-            'words'    => array(),
-            'objcols'  => array(),
-        );
-
-        // set type specific values
-        if ($this->folder->type == 'event') {
-            // database runs in server's timezone so using date() is what we want
-            $doc['dtstart'] = date('Y-m-d H:i:s', is_object($object['start']) ? $object['start']->format('U') : $object['start']);
-            $doc['dtend']   = date('Y-m-d H:i:s', is_object($object['end'])   ? $object['end']->format('U')   : $object['end']);
-
-            // extend date range for recurring events
-            if ($object['recurrence']) {
-                $doc['dtend'] = date('Y-m-d H:i:s', $object['recurrence']['UNTIL'] ?: strtotime('now + 2 years'));
-            }
-        }
-
-        if ($object['_formatobj']) {
-            $doc['xml'] = preg_replace('!(</?[a-z0-9:-]+>)[\n\r\t\s]+!ms', '$1', (string)$object['_formatobj']->write());
-            $doc['tags'] = $object['_formatobj']->get_tags();
-            $doc['words'] = $object['_formatobj']->get_words();
-        }
-
-        // extract object data
-        $data = array();
-        foreach ($object as $key => $val) {
-            if ($val === "" || $val === null) {
-                // skip empty properties
-                continue;
-            }
-            if (isset($bincols[$key])) {
-                $data[$key] = base64_encode($val);
-            }
-            else if (is_object($val)) {
-                if (is_a($val, 'DateTime')) {
-                    $data[$key] = array('_class' => 'DateTime', 'date' => $val->format('Y-m-d H:i:s'), 'timezone' => $val->getTimezone()->getName());
-                    $doc['objcols'][] = $key;
-                }
-            }
-            else if ($key[0] != '_') {
-                $data[$key] = $val;
-            }
-            else if ($key == '_attachments') {
-                foreach ($val as $k => $att) {
-                    unset($att['content'], $att['path']);
-                    if ($att['id'])
-                        $data[$key][$k] = $att;
-                }
-            }
-        }
-
-        $doc['data'] = $data;
-        return $doc;
-    }
-
-    /**
-     * Helper method to turn stored cache data into a valid storage object
-     */
-    private function _unserialize($doc)
-    {
-        $object = $doc['data'];
-
-        // decode binary properties
-        foreach ($this->binary_cols as $key) {
-            if (!empty($object[$key]))
-                $object[$key] = base64_decode($object[$key]);
-        }
-
-        // restore serialized objects
-        foreach ((array)$doc['objcols'] as $key) {
-            switch ($object[$key]['_class']) {
-                case 'DateTime':
-                    $val = new DateTime($object[$key]['date'], new DateTimeZone($object[$key]['timezone']));
-                    $object[$key] = $val;
-                    break;
-            }
-        }
-
-        // add meta data
-        $object['_type'] = $doc['type'];
-        $object['_msguid'] = $doc['msguid'];
-        $object['_mailbox'] = $this->folder->name;
-        $object['_formatobj'] = kolab_format::factory($doc['type'], $doc['xml']);
-
-        return $object;
-    }
-
-    /**
-     * Check lock record for this folder and wait if locked or set lock
-     */
-    private function _sync_lock()
-    {
-        if (!$this->ready)
-            return;
-
-        $this->synclock = true;
-        $lock = $this->db->locks->findOne(array('resource' => $this->resource_uri));
-
-        // create lock record if not exists
-        if (!$lock) {
-            $this->db->locks->insert(array('resource' => $this->resource_uri, 'created' => time()));
-        }
-        // wait if locked (expire locks after 10 minutes)
-        else if ((time() - $lock['created']) < 600) {
-            usleep(500000);
-            return $this->_sync_lock();
-        }
-        // set lock
-        else {
-            $lock['created'] = time();
-            $this->db->locks->update(array('_id' => $lock['_id']), $lock, array('safe' => true));
-        }
-    }
-
-    /**
-     * Remove lock for this folder
-     */
-    public function _sync_unlock()
-    {
-        if (!$this->ready || !$this->synclock)
-            return;
-
-        $this->db->locks->remove(array('resource' => $this->resource_uri));
-    }
-
-    /**
-     * Resolve an object UID into an IMAP message UID
-     *
-     * @param string  Kolab object UID
-     * @param boolean Include deleted objects
-     * @return int The resolved IMAP message UID
-     */
-    public function uid2msguid($uid, $deleted = false)
-    {
-        if (!isset($this->uid2msg[$uid])) {
-            // use IMAP SEARCH to get the right message
-            $index = $this->imap->search_once($this->folder->name, ($deleted ? '' : 'UNDELETED ') . 'HEADER SUBJECT ' . $uid);
-            $results = $index->get();
-            $this->uid2msg[$uid] = $results[0];
-        }
-
-        return $this->uid2msg[$uid];
-    }
-
-}
diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php
index 1ffbb457..d0695cb2 100644
--- a/plugins/libkolab/lib/kolab_storage_folder.php
+++ b/plugins/libkolab/lib/kolab_storage_folder.php
@@ -562,6 +562,7 @@ class kolab_storage_folder extends kolab_storage_folder_api
             $object['_msguid']    = $msguid;
             $object['_mailbox']   = $this->name;
             $object['_formatobj'] = $format;
+            $object['_size']      = strlen($xml);
 
             return $object;
         }

From 4b955adbbdceb4c77a1f0aac7b44a0c378212f5a Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Fri, 28 Dec 2018 14:13:25 +0000
Subject: [PATCH 26/30] Use INSERT ... ON DUPLICATE KEY UPDATE in kolab cache

Bifrost#T61987
---
 plugins/libkolab/lib/kolab_storage_cache.php | 45 +++++++++-----------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index a0d360ed..53aa090c 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -938,36 +938,38 @@ class kolab_storage_cache
         static $buffer = '';
 
         $line = '';
+        $cols = array('folder_id', 'msguid', 'uid', 'created', 'changed', 'data', 'tags', 'words');
+        if ($this->extra_cols) {
+            $cols = array_merge($cols, $this->extra_cols);
+        }
+
         if ($object) {
             $sql_data = $this->_serialize($object);
 
-            // Skip multifolder insert for Oracle, we can't put long data inline
-            if ($this->db->db_provider == 'oracle') {
-                $extra_cols = '';
-                if ($this->extra_cols) {
-                    $extra_cols = array_map(function($n) { return "`{$n}`"; }, $this->extra_cols);
-                    $extra_cols = ', ' . join(', ', $extra_cols);
-                    $extra_args = str_repeat(', ?', count($this->extra_cols));
-                }
-
+            // Skip multi-folder insert for all databases but MySQL
+            // In Oracle we can't put long data inline, others we don't support yet
+            if (strpos($this->db->db_provider, 'mysql') !== 0) {
+                $extra_args = array();
                 $params = array($this->folder_id, $msguid, $object['uid'], $sql_data['changed'],
                     $sql_data['data'], $sql_data['tags'], $sql_data['words']);
 
                 foreach ($this->extra_cols as $col) {
                     $params[] = $sql_data[$col];
+                    $extra_args[] = '?';
                 }
 
+                $cols = implode(', ', array_map(function($n) { return "`{$n}`"; }, $cols));
+                $extra_args = count($extra_args) ? ', ' . implode(', ', $extra_args) : '';
+
                 $result = $this->db->query(
-                    "INSERT INTO `{$this->cache_table}` "
-                    . " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)"
-                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ? $extra_args)",
+                    "INSERT INTO `{$this->cache_table}` ($cols)"
+                    . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ?$extra_args)",
                     $params
                 );
 
                 if (!$this->db->affected_rows($result)) {
                     rcube::raise_error(array(
-                        'code' => 900, 'type' => 'php',
-                        'message' => "Failed to write to kolab cache"
+                        'code' => 900, 'message' => "Failed to write to kolab cache"
                     ), true);
                 }
 
@@ -991,22 +993,17 @@ class kolab_storage_cache
         }
 
         if ($buffer && (!$msguid || (strlen($buffer) + strlen($line) > $this->max_sql_packet()))) {
-            $extra_cols = '';
-            if ($this->extra_cols) {
-                $extra_cols = array_map(function($n) { return "`{$n}`"; }, $this->extra_cols);
-                $extra_cols = ', ' . join(', ', $extra_cols);
-            }
+            $columns = implode(', ', array_map(function($n) { return "`{$n}`"; }, $cols));
+            $update  = implode(', ', array_map(function($i) { return "`{$i}` = VALUES(`{$i}`)"; }, array_slice($cols, 2)));
 
             $result = $this->db->query(
-                "INSERT INTO `{$this->cache_table}` ".
-                " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `tags`, `words`$extra_cols)".
-                " VALUES $buffer"
+                "INSERT INTO `{$this->cache_table}` ($columns) VALUES $buffer"
+                . " ON DUPLICATE KEY UPDATE $update"
             );
 
             if (!$this->db->affected_rows($result)) {
                 rcube::raise_error(array(
-                    'code' => 900, 'type' => 'php',
-                    'message' => "Failed to write to kolab cache"
+                    'code' => 900, 'message' => "Failed to write to kolab cache"
                 ), true);
             }
 

From a71b1342ee2a2e1faee2d1ce77f2f47efeeedb54 Mon Sep 17 00:00:00 2001
From: "Jeroen van Meeuwen (Kolab Systems)" <vanmeeuwen@kolabsys.com>
Date: Fri, 19 Apr 2019 11:59:10 +0200
Subject: [PATCH 27/30] Drop the removal of records from kolab_folders

---
 plugins/libkolab/SQL/mysql/2018122700.sql  | 3 +--
 plugins/libkolab/SQL/oracle/2018122700.sql | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/plugins/libkolab/SQL/mysql/2018122700.sql b/plugins/libkolab/SQL/mysql/2018122700.sql
index 08030dd0..519a1e8a 100644
--- a/plugins/libkolab/SQL/mysql/2018122700.sql
+++ b/plugins/libkolab/SQL/mysql/2018122700.sql
@@ -1,5 +1,4 @@
--- remove xml column, and change data format (clear cache needed)
-DELETE FROM `kolab_folders`;
+-- remove xml column
 ALTER TABLE `kolab_cache_contact` DROP COLUMN `xml`;
 ALTER TABLE `kolab_cache_event` DROP COLUMN `xml`;
 ALTER TABLE `kolab_cache_task` DROP COLUMN `xml`;
diff --git a/plugins/libkolab/SQL/oracle/2018122700.sql b/plugins/libkolab/SQL/oracle/2018122700.sql
index 45fccb74..4f4892b4 100644
--- a/plugins/libkolab/SQL/oracle/2018122700.sql
+++ b/plugins/libkolab/SQL/oracle/2018122700.sql
@@ -1,5 +1,4 @@
--- remove xml column, and change data format (clear cache needed)
-DELETE FROM "kolab_folders";
+-- remove xml column
 ALTER TABLE "kolab_cache_contact" DROP COLUMN "xml";
 ALTER TABLE "kolab_cache_event" DROP COLUMN "xml";
 ALTER TABLE "kolab_cache_task" DROP COLUMN "xml";

From ace036d6c8b0a21cecfac7526e2b1b3d97d726a6 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Wed, 15 May 2019 16:22:17 +0200
Subject: [PATCH 28/30] Fix regression where tags were not available in Add a
 Note dialog (Bifrost#T209710)

---
 plugins/kolab_tags/lib/kolab_tags_engine.php | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/plugins/kolab_tags/lib/kolab_tags_engine.php b/plugins/kolab_tags/lib/kolab_tags_engine.php
index 49a37fbf..1c45040f 100644
--- a/plugins/kolab_tags/lib/kolab_tags_engine.php
+++ b/plugins/kolab_tags/lib/kolab_tags_engine.php
@@ -46,12 +46,7 @@ class kolab_tags_engine
      */
     public function ui()
     {
-        // set templates of Files UI and widgets
-        if ($this->rc->task != 'mail') {
-            return;
-        }
-
-        if ($this->rc->action && !in_array($this->rc->action, array('show', 'preview'))) {
+        if ($this->rc->action && !in_array($this->rc->action, array('show', 'preview', 'dialog-ui'))) {
             return;
         }
 
@@ -62,7 +57,7 @@ class kolab_tags_engine
         $this->rc->output->add_label('cancel', 'save');
         $this->plugin->add_label('tags', 'add', 'edit', 'delete', 'saving',
             'nameempty', 'nameexists', 'colorinvalid', 'untag', 'tagname',
-            'tagcolor', 'tagsearchnew', 'newtag');
+            'tagcolor', 'tagsearchnew', 'newtag', 'notags');
 
         $this->rc->output->add_handlers(array(
             'plugin.taglist' => array($this, 'taglist'),

From 4136938a7ac843afa883e993d14bfb64cf18b5f0 Mon Sep 17 00:00:00 2001
From: Aleksander Machniak <machniak@kolabsys.com>
Date: Thu, 16 May 2019 08:33:27 +0000
Subject: [PATCH 29/30] Don't store invalid objects in cache

For example, before the change it was possible to end up with
a task object in a calendar folder cache.
---
 plugins/libkolab/lib/kolab_storage_cache.php | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php
index 53aa090c..8f682e54 100644
--- a/plugins/libkolab/lib/kolab_storage_cache.php
+++ b/plugins/libkolab/lib/kolab_storage_cache.php
@@ -242,7 +242,9 @@ class kolab_storage_cache
                     // fetch new objects from imap
                     $i = 0;
                     foreach (array_diff($imap_index, $old_index) as $msguid) {
-                        if ($object = $this->folder->read_object($msguid, '*')) {
+                        // Note: We'll store only objects matching the folder type
+                        // anything else will be silently ignored
+                        if ($object = $this->folder->read_object($msguid)) {
                             // Deduplication: remove older objects with the same UID
                             // Here we do not resolve conflicts, we just make sure
                             // the most recent version of the object will be used
@@ -921,7 +923,12 @@ class kolab_storage_cache
         }
         // Fetch object xml
         else {
-            $object = $this->folder->read_object($sql_arr['msguid']);
+            // FIXME: Because old cache solution allowed storing objects that
+            // do not match folder type we may end up with invalid objects.
+            // 2nd argument of read_object() here makes sure they are still
+            // usable. However, not allowing them here might be also an intended
+            // solution in future.
+            $object = $this->folder->read_object($sql_arr['msguid'], '*');
         }
 
         return $object;

From f828f9925e747da218bbc92168a9186b139dd0cd Mon Sep 17 00:00:00 2001
From: "Jeroen van Meeuwen (Kolab Systems)" <vanmeeuwen@kolabsys.com>
Date: Mon, 8 Jul 2019 11:51:20 +0200
Subject: [PATCH 30/30] Revert "Drop the removal of records from kolab_folders"

This reverts commit a71b1342ee2a2e1faee2d1ce77f2f47efeeedb54.

The payload in the 'data' column is restructured.
---
 plugins/libkolab/SQL/mysql/2018122700.sql  | 3 ++-
 plugins/libkolab/SQL/oracle/2018122700.sql | 3 ++-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/plugins/libkolab/SQL/mysql/2018122700.sql b/plugins/libkolab/SQL/mysql/2018122700.sql
index 519a1e8a..08030dd0 100644
--- a/plugins/libkolab/SQL/mysql/2018122700.sql
+++ b/plugins/libkolab/SQL/mysql/2018122700.sql
@@ -1,4 +1,5 @@
--- remove xml column
+-- remove xml column, and change data format (clear cache needed)
+DELETE FROM `kolab_folders`;
 ALTER TABLE `kolab_cache_contact` DROP COLUMN `xml`;
 ALTER TABLE `kolab_cache_event` DROP COLUMN `xml`;
 ALTER TABLE `kolab_cache_task` DROP COLUMN `xml`;
diff --git a/plugins/libkolab/SQL/oracle/2018122700.sql b/plugins/libkolab/SQL/oracle/2018122700.sql
index 4f4892b4..45fccb74 100644
--- a/plugins/libkolab/SQL/oracle/2018122700.sql
+++ b/plugins/libkolab/SQL/oracle/2018122700.sql
@@ -1,4 +1,5 @@
--- remove xml column
+-- remove xml column, and change data format (clear cache needed)
+DELETE FROM "kolab_folders";
 ALTER TABLE "kolab_cache_contact" DROP COLUMN "xml";
 ALTER TABLE "kolab_cache_event" DROP COLUMN "xml";
 ALTER TABLE "kolab_cache_task" DROP COLUMN "xml";