diff --git a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php index b301ffad..cfc9e693 100644 --- a/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php +++ b/plugins/kolab_addressbook/lib/rcube_kolab_contacts.php @@ -325,7 +325,9 @@ class rcube_kolab_contacts extends rcube_addressbook public function list_records($cols = null, $subset = 0, $nocount = false) { $this->result = new rcube_result_set(0, ($this->list_page-1) * $this->page_size); + $fetch_all = false; + $fast_mode = !empty($cols) && is_array($cols); // list member of the selected group if ($this->gid) { @@ -355,7 +357,7 @@ class rcube_kolab_contacts extends rcube_addressbook // get members by UID if (!empty($uids)) { - $this->_fetch_contacts($query = array(array('uid', '=', $uids)), $fetch_all ? false : count($uids)); + $this->_fetch_contacts($query = array(array('uid', '=', $uids)), $fetch_all ? false : count($uids), $fast_mode); $this->sortindex = array_merge($this->sortindex, $local_sortindex); } } @@ -363,11 +365,11 @@ class rcube_kolab_contacts extends rcube_addressbook $ids = $this->filter['ids']; if (count($ids)) { $uids = array_map(array($this, 'id2uid'), $this->filter['ids']); - $this->_fetch_contacts($query = array(array('uid', '=', $uids)), count($ids)); + $this->_fetch_contacts($query = array(array('uid', '=', $uids)), count($ids), $fast_mode); } } else { - $this->_fetch_contacts($query = 'contact', true); + $this->_fetch_contacts($query = 'contact', true, $fast_mode); } if ($fetch_all) { @@ -1062,7 +1064,7 @@ class rcube_kolab_contacts extends rcube_addressbook /** * Query storage layer and store records in private member var */ - private function _fetch_contacts($query = array(), $limit = false) + private function _fetch_contacts($query = array(), $limit = false, $fast_mode = false) { if (!isset($this->dataset) || !empty($query)) { if ($limit) { @@ -1071,7 +1073,7 @@ class rcube_kolab_contacts extends rcube_addressbook } $this->sortindex = array(); - $this->dataset = $this->storagefolder->select($query); + $this->dataset = $this->storagefolder->select($query, $fast_mode); foreach ($this->dataset as $idx => $record) { $contact = $this->_to_rcube_contact($record); @@ -1137,7 +1139,7 @@ class rcube_kolab_contacts extends rcube_addressbook { if (!isset($this->distlists)) { $this->distlists = $this->groupmembers = array(); - foreach ($this->storagefolder->select('distribution-list') as $record) { + foreach ($this->storagefolder->select('distribution-list', true) as $record) { $record['ID'] = $this->uid2id($record['uid']); foreach ((array)$record['member'] as $i => $member) { $mid = $this->uid2id($member['uid'] ? $member['uid'] : 'mailto:' . $member['email']); @@ -1145,8 +1147,9 @@ class rcube_kolab_contacts extends rcube_addressbook $record['member'][$i]['readonly'] = empty($member['uid']); $this->groupmembers[$mid][] = $record['ID']; - if ($with_contacts && empty($member['uid'])) + if ($with_contacts && empty($member['uid'])) { $this->contacts[$mid] = $record['member'][$i]; + } } $this->distlists[$record['ID']] = $record; } diff --git a/plugins/kolab_notes/kolab_notes.php b/plugins/kolab_notes/kolab_notes.php index 34f0d3c5..9d1c240d 100644 --- a/plugins/kolab_notes/kolab_notes.php +++ b/plugins/kolab_notes/kolab_notes.php @@ -480,7 +480,7 @@ class kolab_notes extends rcube_plugin $this->_read_lists(); if ($folder = $this->get_folder($list_id)) { - foreach ($folder->select($query) as $record) { + foreach ($folder->select($query, empty($query)) as $record) { // post-filter search results if (strlen($search)) { $matches = 0; diff --git a/plugins/libkolab/SQL/mysql.initial.sql b/plugins/libkolab/SQL/mysql.initial.sql index 0213c45b..80da02fa 100644 --- a/plugins/libkolab/SQL/mysql.initial.sql +++ b/plugins/libkolab/SQL/mysql.initial.sql @@ -33,7 +33,6 @@ CREATE TABLE `kolab_cache_contact` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, `type` VARCHAR(32) CHARACTER SET ascii NOT NULL, @@ -57,7 +56,6 @@ CREATE TABLE `kolab_cache_event` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, `dtstart` DATETIME, @@ -77,7 +75,6 @@ CREATE TABLE `kolab_cache_task` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, `dtstart` DATETIME, @@ -97,7 +94,6 @@ CREATE TABLE `kolab_cache_journal` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, `dtstart` DATETIME, @@ -117,7 +113,6 @@ CREATE TABLE `kolab_cache_note` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, CONSTRAINT `fk_kolab_cache_note_folder` FOREIGN KEY (`folder_id`) @@ -135,7 +130,6 @@ CREATE TABLE `kolab_cache_file` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, `filename` varchar(255) DEFAULT NULL, @@ -155,7 +149,6 @@ CREATE TABLE `kolab_cache_configuration` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, `type` VARCHAR(32) CHARACTER SET ascii NOT NULL, @@ -175,7 +168,6 @@ CREATE TABLE `kolab_cache_freebusy` ( `created` DATETIME DEFAULT NULL, `changed` DATETIME DEFAULT NULL, `data` LONGTEXT NOT NULL, - `xml` LONGBLOB NOT NULL, `tags` TEXT NOT NULL, `words` TEXT NOT NULL, `dtstart` DATETIME, @@ -188,4 +180,4 @@ CREATE TABLE `kolab_cache_freebusy` ( /*!40014 SET FOREIGN_KEY_CHECKS=1 */; -REPLACE INTO `system` (`name`, `value`) VALUES ('libkolab-version', '2018021300'); +REPLACE INTO `system` (`name`, `value`) VALUES ('libkolab-version', '2018122700'); diff --git a/plugins/libkolab/SQL/mysql/2018122700.sql b/plugins/libkolab/SQL/mysql/2018122700.sql new file mode 100644 index 00000000..08030dd0 --- /dev/null +++ b/plugins/libkolab/SQL/mysql/2018122700.sql @@ -0,0 +1,10 @@ +-- remove xml column, and change data format (clear cache needed) +DELETE FROM `kolab_folders`; +ALTER TABLE `kolab_cache_contact` DROP COLUMN `xml`; +ALTER TABLE `kolab_cache_event` DROP COLUMN `xml`; +ALTER TABLE `kolab_cache_task` DROP COLUMN `xml`; +ALTER TABLE `kolab_cache_journal` DROP COLUMN `xml`; +ALTER TABLE `kolab_cache_note` DROP COLUMN `xml`; +ALTER TABLE `kolab_cache_file` DROP COLUMN `xml`; +ALTER TABLE `kolab_cache_configuration` DROP COLUMN `xml`; +ALTER TABLE `kolab_cache_freebusy` DROP COLUMN `xml`; diff --git a/plugins/libkolab/SQL/oracle.initial.sql b/plugins/libkolab/SQL/oracle.initial.sql index 78675783..ebba60dc 100644 --- a/plugins/libkolab/SQL/oracle.initial.sql +++ b/plugins/libkolab/SQL/oracle.initial.sql @@ -37,7 +37,6 @@ CREATE TABLE "kolab_cache_contact" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, "type" varchar(32) NOT NULL, @@ -60,7 +59,6 @@ CREATE TABLE "kolab_cache_event" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, "dtstart" timestamp DEFAULT NULL, @@ -79,7 +77,6 @@ CREATE TABLE "kolab_cache_task" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, "dtstart" timestamp DEFAULT NULL, @@ -98,7 +95,6 @@ CREATE TABLE "kolab_cache_journal" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, "dtstart" timestamp DEFAULT NULL, @@ -117,7 +113,6 @@ CREATE TABLE "kolab_cache_note" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, PRIMARY KEY ("folder_id", "msguid") @@ -134,7 +129,6 @@ CREATE TABLE "kolab_cache_file" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, "filename" varchar(255) DEFAULT NULL, @@ -153,7 +147,6 @@ CREATE TABLE "kolab_cache_configuration" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, "type" varchar(32) NOT NULL, @@ -172,7 +165,6 @@ CREATE TABLE "kolab_cache_freebusy" ( "created" timestamp DEFAULT NULL, "changed" timestamp DEFAULT NULL, "data" clob NOT NULL, - "xml" clob NOT NULL, "tags" clob DEFAULT NULL, "words" clob DEFAULT NULL, "dtstart" timestamp DEFAULT NULL, @@ -183,4 +175,4 @@ CREATE TABLE "kolab_cache_freebusy" ( CREATE INDEX "kolab_cache_fb_uid2msguid" ON "kolab_cache_freebusy" ("folder_id", "uid", "msguid"); -INSERT INTO "system" ("name", "value") VALUES ('libkolab-version', '2018021300'); +INSERT INTO "system" ("name", "value") VALUES ('libkolab-version', '2018122700'); diff --git a/plugins/libkolab/SQL/oracle/2018122700.sql b/plugins/libkolab/SQL/oracle/2018122700.sql new file mode 100644 index 00000000..45fccb74 --- /dev/null +++ b/plugins/libkolab/SQL/oracle/2018122700.sql @@ -0,0 +1,10 @@ +-- remove xml column, and change data format (clear cache needed) +DELETE FROM "kolab_folders"; +ALTER TABLE "kolab_cache_contact" DROP COLUMN "xml"; +ALTER TABLE "kolab_cache_event" DROP COLUMN "xml"; +ALTER TABLE "kolab_cache_task" DROP COLUMN "xml"; +ALTER TABLE "kolab_cache_journal" DROP COLUMN "xml"; +ALTER TABLE "kolab_cache_note" DROP COLUMN "xml"; +ALTER TABLE "kolab_cache_file" DROP COLUMN "xml"; +ALTER TABLE "kolab_cache_configuration" DROP COLUMN "xml"; +ALTER TABLE "kolab_cache_freebusy" DROP COLUMN "xml"; diff --git a/plugins/libkolab/SQL/sqlite.initial.sql b/plugins/libkolab/SQL/sqlite.initial.sql index 909188ca..815316dc 100644 --- a/plugins/libkolab/SQL/sqlite.initial.sql +++ b/plugins/libkolab/SQL/sqlite.initial.sql @@ -25,7 +25,6 @@ CREATE TABLE kolab_cache_contact ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, type VARCHAR(32) NOT NULL, @@ -46,7 +45,6 @@ CREATE TABLE kolab_cache_event ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, dtstart DATETIME, @@ -63,7 +61,6 @@ CREATE TABLE kolab_cache_task ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, dtstart DATETIME, @@ -80,7 +77,6 @@ CREATE TABLE kolab_cache_journal ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, dtstart DATETIME, @@ -97,7 +93,6 @@ CREATE TABLE kolab_cache_note ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, PRIMARY KEY(folder_id,msguid) @@ -112,7 +107,6 @@ CREATE TABLE kolab_cache_file ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, filename varchar(255) DEFAULT NULL, @@ -129,7 +123,6 @@ CREATE TABLE kolab_cache_configuration ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, type VARCHAR(32) NOT NULL, @@ -146,7 +139,6 @@ CREATE TABLE kolab_cache_freebusy ( created DATETIME DEFAULT NULL, changed DATETIME DEFAULT NULL, data TEXT NOT NULL, - xml TEXT NOT NULL, tags TEXT NOT NULL, words TEXT NOT NULL, dtstart DATETIME, @@ -156,4 +148,4 @@ CREATE TABLE kolab_cache_freebusy ( CREATE INDEX ix_freebusy_uid2msguid ON kolab_cache_freebusy(folder_id,uid,msguid); -INSERT INTO system (name, value) VALUES ('libkolab-version', '2018021300'); +INSERT INTO system (name, value) VALUES ('libkolab-version', '2018122700'); diff --git a/plugins/libkolab/SQL/sqlite/2018122700.sql b/plugins/libkolab/SQL/sqlite/2018122700.sql new file mode 100644 index 00000000..ee24c1a6 --- /dev/null +++ b/plugins/libkolab/SQL/sqlite/2018122700.sql @@ -0,0 +1,137 @@ +DROP TABLE kolab_cache_contact; +CREATE TABLE kolab_cache_contact ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + type VARCHAR(32) NOT NULL, + name VARCHAR(255) NOT NULL, + firstname VARCHAR(255) NOT NULL, + surname VARCHAR(255) NOT NULL, + email VARCHAR(255) NOT NULL, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_contact_type ON kolab_cache_contact(folder_id,type); +CREATE INDEX ix_contact_uid2msguid ON kolab_cache_contact(folder_id,uid,msguid); + +DROP TABLE kolab_cache_event; +CREATE TABLE kolab_cache_event ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + dtstart DATETIME, + dtend DATETIME, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_event_uid2msguid ON kolab_cache_event(folder_id,uid,msguid); + +DROP TABLE kolab_cache_task; +CREATE TABLE kolab_cache_task ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + dtstart DATETIME, + dtend DATETIME, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_task_uid2msguid ON kolab_cache_task(folder_id,uid,msguid); + +DROP TABLE kolab_cache_journal; +CREATE TABLE kolab_cache_journal ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + dtstart DATETIME, + dtend DATETIME, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_journal_uid2msguid ON kolab_cache_journal(folder_id,uid,msguid); + +DROP TABLE kolab_cache_note; +CREATE TABLE kolab_cache_note ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_note_uid2msguid ON kolab_cache_note(folder_id,uid,msguid); + +DROP TABLE kolab_cache_file; +CREATE TABLE kolab_cache_file ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + filename varchar(255) DEFAULT NULL, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_folder_filename ON kolab_cache_file(folder_id,filename); +CREATE INDEX ix_file_uid2msguid ON kolab_cache_file(folder_id,uid,msguid); + +DROP TABLE kolab_cache_configuration; +CREATE TABLE kolab_cache_configuration ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + type VARCHAR(32) NOT NULL, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_configuration_type ON kolab_cache_configuration(folder_id,type); +CREATE INDEX ix_configuration_uid2msguid ON kolab_cache_configuration(folder_id,uid,msguid); + +DROP TABLE kolab_cache_freebusy; +CREATE TABLE kolab_cache_freebusy ( + folder_id INTEGER NOT NULL, + msguid INTEGER NOT NULL, + uid VARCHAR(512) NOT NULL, + created DATETIME DEFAULT NULL, + changed DATETIME DEFAULT NULL, + data TEXT NOT NULL, + tags TEXT NOT NULL, + words TEXT NOT NULL, + dtstart DATETIME, + dtend DATETIME, + PRIMARY KEY(folder_id,msguid) +); + +CREATE INDEX ix_freebusy_uid2msguid ON kolab_cache_freebusy(folder_id,uid,msguid); diff --git a/plugins/libkolab/config.inc.php.dist b/plugins/libkolab/config.inc.php.dist index a4a7a7a6..eddafbd6 100644 --- a/plugins/libkolab/config.inc.php.dist +++ b/plugins/libkolab/config.inc.php.dist @@ -35,7 +35,8 @@ $config['kolab_http_request'] = array(); // When kolab_cache is enabled Roundcube's messages cache will be redundant // when working on kolab folders. Here we can: -// 2 - bypass messages/indexes cache completely +// 3 - bypass only indexes, but use messages cache +// 2 - bypass both messages and indexes cache // 1 - bypass only messages, but use index cache $config['kolab_messages_cache_bypass'] = 0; diff --git a/plugins/libkolab/lib/kolab_storage_cache.php b/plugins/libkolab/lib/kolab_storage_cache.php index 362d53c1..8f682e54 100644 --- a/plugins/libkolab/lib/kolab_storage_cache.php +++ b/plugins/libkolab/lib/kolab_storage_cache.php @@ -46,8 +46,8 @@ class kolab_storage_cache protected $folders_table; protected $max_sql_packet; protected $max_sync_lock_time = 600; - protected $binary_items = array(); protected $extra_cols = array(); + protected $data_props = array(); protected $order_by = null; protected $limit = null; protected $error = 0; @@ -201,7 +201,7 @@ class kolab_storage_cache $this->metadata['changed'] < date(self::DB_DATE_FORMAT, time() - $this->cache_refresh) || $this->metadata['ctag'] != $this->folder->get_ctag() || intval($this->metadata['objectcount']) !== $this->count() - ) { + ) { // lock synchronization for this folder or wait if locked $this->_sync_lock(); @@ -242,7 +242,9 @@ class kolab_storage_cache // fetch new objects from imap $i = 0; foreach (array_diff($imap_index, $old_index) as $msguid) { - if ($object = $this->folder->read_object($msguid, '*')) { + // Note: We'll store only objects matching the folder type + // anything else will be silently ignored + if ($object = $this->folder->read_object($msguid)) { // Deduplication: remove older objects with the same UID // Here we do not resolve conflicts, we just make sure // the most recent version of the object will be used @@ -447,7 +449,7 @@ class kolab_storage_cache $sql_data['uid'] = $object['uid']; $args = array(); - $cols = array('folder_id', 'msguid', 'uid', 'changed', 'data', 'xml', 'tags', 'words'); + $cols = array('folder_id', 'msguid', 'uid', 'changed', 'data', 'tags', 'words'); $cols = array_merge($cols, $this->extra_cols); foreach ($cols as $idx => $col) { @@ -585,9 +587,12 @@ class kolab_storage_cache * @param array Pseudo-SQL query as list of filter parameter triplets * triplet: array('', '', '') * @param boolean Set true to only return UIDs instead of complete objects + * @param boolean Use fast mode to fetch only minimal set of information + * (no xml fetching and parsing, etc.) + * * @return array List of Kolab data objects (each represented as hash array) or UIDs */ - public function select($query = array(), $uids = false) + public function select($query = array(), $uids = false, $fast = false) { $result = $uids ? array() : new kolab_storage_dataset($this); @@ -621,6 +626,9 @@ class kolab_storage_cache } while ($sql_arr = $this->db->fetch_assoc($sql_result)) { + if ($fast) { + $sql_arr['fast-mode'] = true; + } if ($uids) { $this->uid2msg[$sql_arr['uid']] = $sql_arr['_msguid']; $result[] = $sql_arr['uid']; @@ -679,7 +687,6 @@ class kolab_storage_cache return $result; } - /** * Get number of objects mathing the given query * @@ -851,43 +858,36 @@ class kolab_storage_cache */ protected function _serialize($object) { - $sql_data = array('changed' => null, 'xml' => '', 'tags' => '', 'words' => ''); + $data = array(); + $sql_data = array('changed' => null, 'tags' => '', 'words' => ''); if ($object['changed']) { $sql_data['changed'] = date(self::DB_DATE_FORMAT, is_object($object['changed']) ? $object['changed']->format('U') : $object['changed']); } if ($object['_formatobj']) { - $sql_data['xml'] = preg_replace('!()[\n\r\t\s]+!ms', '$1', (string)$object['_formatobj']->write(3.0)); + $xml = (string) $object['_formatobj']->write(3.0); + + $data['_size'] = strlen($xml); $sql_data['tags'] = ' ' . join(' ', $object['_formatobj']->get_tags()) . ' '; // pad with spaces for strict/prefix search $sql_data['words'] = ' ' . join(' ', $object['_formatobj']->get_words()) . ' '; } - // extract object data - $data = array(); - foreach ($object as $key => $val) { - // skip empty properties - if ($val === "" || $val === null) { - continue; - } - // mark binary data to be extracted from xml on unserialize() - if (isset($this->binary_items[$key])) { - $data[$key] = true; - } - else if ($key[0] != '_') { - $data[$key] = $val; - } - else if ($key == '_attachments') { - foreach ($val as $k => $att) { - unset($att['content'], $att['path']); - if ($att['id']) - $data[$key][$k] = $att; + // Store only minimal set of object properties + foreach ($this->data_props as $prop) { + if (isset($object[$prop])) { + $data[$prop] = $object[$prop]; + if ($data[$prop] instanceof DateTime) { + $data[$prop] = array( + 'cl' => 'DateTime', + 'dt' => $data[$prop]->format('Y-m-d H:i:s'), + 'tz' => $data[$prop]->getTimezone()->getName(), + ); } } } - // use base64 encoding (Bug #1912, #2662) - $sql_data['data'] = base64_encode(serialize($data)); + $sql_data['data'] = json_encode(rcube_charset::clean($data)); return $sql_data; } @@ -897,44 +897,38 @@ class kolab_storage_cache */ protected function _unserialize($sql_arr) { - // check if data is a base64-encoded string, for backward compat. - if (strpos(substr($sql_arr['data'], 0, 64), ':') === false) { - $sql_arr['data'] = base64_decode($sql_arr['data']); - } + if ($sql_arr['fast-mode'] && !empty($sql_arr['data']) && ($object = json_decode($sql_arr['data'], true))) { + $object['uid'] = $sql_arr['uid']; - $object = unserialize($sql_arr['data']); - - // de-serialization failed - if ($object === false) { - rcube::raise_error(array( - 'code' => 900, 'type' => 'php', - 'message' => "Malformed data for {$this->resource_uri}/{$sql_arr['msguid']} object." - ), true); - - return null; - } - - // decode binary properties - foreach ($this->binary_items as $key => $regexp) { - if (!empty($object[$key]) && preg_match($regexp, $sql_arr['xml'], $m)) { - $object[$key] = base64_decode($m[1]); + foreach ($this->data_props as $prop) { + if (isset($object[$prop]) && is_array($object[$prop]) && $object[$prop]['cl'] == 'DateTime') { + $object[$prop] = new DateTime($object[$prop]['dt'], new DateTimeZone($object[$prop]['tz'])); + } + else if (!isset($object[$prop]) && isset($sql_arr[$prop])) { + $object[$prop] = $sql_arr[$prop]; + } } + + if ($sql_arr['created'] && empty($object['created'])) { + $object['created'] = new DateTime($sql_arr['created']); + } + + if ($sql_arr['changed'] && empty($object['changed'])) { + $object['changed'] = new DateTime($sql_arr['changed']); + } + + $object['_type'] = $sql_arr['type'] ?: $this->folder->type; + $object['_msguid'] = $sql_arr['msguid']; + $object['_mailbox'] = $this->folder->name; } - - $object_type = $sql_arr['type'] ?: $this->folder->type; - $format_type = $this->folder->type == 'configuration' ? 'configuration' : $object_type; - - // add meta data - $object['_type'] = $object_type; - $object['_msguid'] = $sql_arr['msguid']; - $object['_mailbox'] = $this->folder->name; - $object['_size'] = strlen($sql_arr['xml']); - $object['_formatobj'] = kolab_format::factory($format_type, 3.0, $sql_arr['xml']); - - // Fix old broken objects with missing creation date - if (empty($object['created']) && method_exists($object['_formatobj'], 'to_array')) { - $new_object = $object['_formatobj']->to_array(); - $object['created'] = $new_object['created']; + // Fetch object xml + else { + // FIXME: Because old cache solution allowed storing objects that + // do not match folder type we may end up with invalid objects. + // 2nd argument of read_object() here makes sure they are still + // usable. However, not allowing them here might be also an intended + // solution in future. + $object = $this->folder->read_object($sql_arr['msguid'], '*'); } return $object; @@ -951,36 +945,38 @@ class kolab_storage_cache static $buffer = ''; $line = ''; + $cols = array('folder_id', 'msguid', 'uid', 'created', 'changed', 'data', 'tags', 'words'); + if ($this->extra_cols) { + $cols = array_merge($cols, $this->extra_cols); + } + if ($object) { $sql_data = $this->_serialize($object); - // Skip multifolder insert for Oracle, we can't put long data inline - if ($this->db->db_provider == 'oracle') { - $extra_cols = ''; - if ($this->extra_cols) { - $extra_cols = array_map(function($n) { return "`{$n}`"; }, $this->extra_cols); - $extra_cols = ', ' . join(', ', $extra_cols); - $extra_args = str_repeat(', ?', count($this->extra_cols)); - } - + // Skip multi-folder insert for all databases but MySQL + // In Oracle we can't put long data inline, others we don't support yet + if (strpos($this->db->db_provider, 'mysql') !== 0) { + $extra_args = array(); $params = array($this->folder_id, $msguid, $object['uid'], $sql_data['changed'], - $sql_data['data'], $sql_data['xml'], $sql_data['tags'], $sql_data['words']); + $sql_data['data'], $sql_data['tags'], $sql_data['words']); foreach ($this->extra_cols as $col) { $params[] = $sql_data[$col]; + $extra_args[] = '?'; } + $cols = implode(', ', array_map(function($n) { return "`{$n}`"; }, $cols)); + $extra_args = count($extra_args) ? ', ' . implode(', ', $extra_args) : ''; + $result = $this->db->query( - "INSERT INTO `{$this->cache_table}` " - . " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `xml`, `tags`, `words` $extra_cols)" - . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ?, ? $extra_args)", + "INSERT INTO `{$this->cache_table}` ($cols)" + . " VALUES (?, ?, ?, " . $this->db->now() . ", ?, ?, ?, ?$extra_args)", $params ); if (!$this->db->affected_rows($result)) { rcube::raise_error(array( - 'code' => 900, 'type' => 'php', - 'message' => "Failed to write to kolab cache" + 'code' => 900, 'message' => "Failed to write to kolab cache" ), true); } @@ -994,7 +990,6 @@ class kolab_storage_cache $this->db->now(), $this->db->quote($sql_data['changed']), $this->db->quote($sql_data['data']), - $this->db->quote($sql_data['xml']), $this->db->quote($sql_data['tags']), $this->db->quote($sql_data['words']), ); @@ -1005,22 +1000,17 @@ class kolab_storage_cache } if ($buffer && (!$msguid || (strlen($buffer) + strlen($line) > $this->max_sql_packet()))) { - $extra_cols = ''; - if ($this->extra_cols) { - $extra_cols = array_map(function($n) { return "`{$n}`"; }, $this->extra_cols); - $extra_cols = ', ' . join(', ', $extra_cols); - } + $columns = implode(', ', array_map(function($n) { return "`{$n}`"; }, $cols)); + $update = implode(', ', array_map(function($i) { return "`{$i}` = VALUES(`{$i}`)"; }, array_slice($cols, 2))); $result = $this->db->query( - "INSERT INTO `{$this->cache_table}` ". - " (`folder_id`, `msguid`, `uid`, `created`, `changed`, `data`, `xml`, `tags`, `words` $extra_cols)". - " VALUES $buffer" + "INSERT INTO `{$this->cache_table}` ($columns) VALUES $buffer" + . " ON DUPLICATE KEY UPDATE $update" ); if (!$this->db->affected_rows($result)) { rcube::raise_error(array( - 'code' => 900, 'type' => 'php', - 'message' => "Failed to write to kolab cache" + 'code' => 900, 'message' => "Failed to write to kolab cache" ), true); } @@ -1190,7 +1180,7 @@ class kolab_storage_cache } /** - * Set Roundcube storage options and bypass messages cache. + * Set Roundcube storage options and bypass messages/indexes cache. * * We use skip_deleted and threading settings specific to Kolab, * we have to change these global settings only temporarily. @@ -1245,17 +1235,18 @@ class kolab_storage_cache switch ($cache_bypass) { case 2: - // Disable messages cache completely + // Disable messages and index cache completely $this->imap->set_messages_caching(!$force); break; + case 3: case 1: - // We'll disable messages cache, but keep index cache. + // We'll disable messages cache, but keep index cache (1) or vice-versa (3) // Default mode is both (MODE_INDEX | MODE_MESSAGE) - $mode = rcube_imap_cache::MODE_INDEX; + $mode = $cache_bypass == 3 ? rcube_imap_cache::MODE_MESSAGE : rcube_imap_cache::MODE_INDEX; if (!$force) { - $mode |= rcube_imap_cache::MODE_MESSAGE; + $mode |= $cache_bypass == 3 ? rcube_imap_cache::MODE_INDEX : rcube_imap_cache::MODE_MESSAGE; } $this->imap->set_messages_caching(true, $mode); diff --git a/plugins/libkolab/lib/kolab_storage_cache_configuration.php b/plugins/libkolab/lib/kolab_storage_cache_configuration.php index c3c7ac4f..a5ac396d 100644 --- a/plugins/libkolab/lib/kolab_storage_cache_configuration.php +++ b/plugins/libkolab/lib/kolab_storage_cache_configuration.php @@ -32,20 +32,37 @@ class kolab_storage_cache_configuration extends kolab_storage_cache */ protected function _serialize($object) { + $this->set_data_props($object['type']); + $sql_data = parent::_serialize($object); $sql_data['type'] = $object['type']; return $sql_data; } + /** + * Helper method to convert database record to the given Kolab object + * + * @override + */ + protected function _unserialize($sql_arr) + { + $this->set_data_props($sql_arr['type']); + + return parent::_unserialize($sql_arr); + } + /** * Select Kolab objects filtered by the given query * - * @param array Pseudo-SQL query as list of filter parameter triplets + * @param array Pseudo-SQL query as list of filter parameter triplets * @param boolean Set true to only return UIDs instead of complete objects + * @param boolean Use fast mode to fetch only minimal set of information + * (no xml fetching and parsing, etc.) + * * @return array List of Kolab data objects (each represented as hash array) or UIDs */ - public function select($query = array(), $uids = false) + public function select($query = array(), $uids = false, $fast = false) { // modify query for IMAP search: query param 'type' is actually a subtype if (!$this->ready) { @@ -57,7 +74,7 @@ class kolab_storage_cache_configuration extends kolab_storage_cache } } - return parent::select($query, $uids); + return parent::select($query, $uids, $fast); } /** @@ -85,4 +102,35 @@ class kolab_storage_cache_configuration extends kolab_storage_cache return parent::_sql_where($query); } + + /** + * Set $data_props property depending on object type + */ + protected function set_data_props($type) + { + switch ($type) { + case 'dictionary': + $this->data_props = array('language', 'e'); + break; + + case 'file_driver': + $this->data_props = array('driver', 'title', 'enabled', 'host', 'port', 'username', 'password'); + break; + + case 'relation': + // Note: Because relations are heavily used, for performance reasons + // we store all properties for them + $this->data_props = array('name', 'category', 'color', 'parent', 'iconName', 'priority', 'members'); + break; + + case 'snippet': + $this->data_props = array('name'); + break; + + case 'category': + default: + // TODO: implement this + $this->data_props = array(); + } + } } diff --git a/plugins/libkolab/lib/kolab_storage_cache_contact.php b/plugins/libkolab/lib/kolab_storage_cache_contact.php index 70aa5f62..22d0a330 100644 --- a/plugins/libkolab/lib/kolab_storage_cache_contact.php +++ b/plugins/libkolab/lib/kolab_storage_cache_contact.php @@ -24,12 +24,8 @@ class kolab_storage_cache_contact extends kolab_storage_cache { protected $extra_cols_max = 255; - protected $extra_cols = array('type','name','firstname','surname','email'); - protected $binary_items = array( - 'photo' => '|[^;]+;base64,([^<]+)|i', - 'pgppublickey' => '|data:application/pgp-keys;base64,([^<]+)|i', - 'pkcs7publickey' => '|data:application/pkcs7-mime;base64,([^<]+)|i', - ); + protected $extra_cols = array('type', 'name', 'firstname', 'surname', 'email'); + protected $data_props = array('type', 'name', 'firstname', 'middlename', 'prefix', 'suffix', 'surname', 'email', 'organization', 'member'); /** * Helper method to convert the given Kolab object into a dataset to be written to cache @@ -69,4 +65,4 @@ class kolab_storage_cache_contact extends kolab_storage_cache return $sql_data; } -} \ No newline at end of file +} diff --git a/plugins/libkolab/lib/kolab_storage_cache_event.php b/plugins/libkolab/lib/kolab_storage_cache_event.php index b6d8b15f..13b350b3 100644 --- a/plugins/libkolab/lib/kolab_storage_cache_event.php +++ b/plugins/libkolab/lib/kolab_storage_cache_event.php @@ -24,6 +24,7 @@ class kolab_storage_cache_event extends kolab_storage_cache { protected $extra_cols = array('dtstart','dtend'); + protected $data_props = array('categories', 'status', 'attendees'); // start, end /** * Helper method to convert the given Kolab object into a dataset to be written to cache diff --git a/plugins/libkolab/lib/kolab_storage_cache_file.php b/plugins/libkolab/lib/kolab_storage_cache_file.php index ea1823d6..1f3d4d92 100644 --- a/plugins/libkolab/lib/kolab_storage_cache_file.php +++ b/plugins/libkolab/lib/kolab_storage_cache_file.php @@ -24,6 +24,7 @@ class kolab_storage_cache_file extends kolab_storage_cache { protected $extra_cols = array('filename'); + protected $data_props = array('type', 'size', 'filename', 'fileid'); /** * Helper method to convert the given Kolab object into a dataset to be written to cache @@ -32,13 +33,45 @@ class kolab_storage_cache_file extends kolab_storage_cache */ protected function _serialize($object) { - $sql_data = parent::_serialize($object); - if (!empty($object['_attachments'])) { reset($object['_attachments']); - $sql_data['filename'] = $object['_attachments'][key($object['_attachments'])]['name']; + + $file = $object['_attachments'][key($object['_attachments'])]; + + $object['type'] = $file['mimetype']; + $object['size'] = $file['size']; + $object['fileid'] = $file['id']; + } + + $sql_data = parent::_serialize($object); + + if (!empty($file)) { + $sql_data['filename'] = $file['name']; } return $sql_data; } -} \ No newline at end of file + + /** + * Helper method to turn stored cache data into a valid storage object + * + * @override + */ + protected function _unserialize($sql_arr) + { + $object = parent::_unserialize($sql_arr); + + if ($object && $sql_arr['fast-mode']) { + if (!empty($object['_attachments'])) { + $file = $object['_attachments'][key($object['_attachments'])]; + + $object['type'] = $file['mimetype']; + $object['size'] = $file['size']; + $object['fileid'] = $file['id']; + $object['filename'] = $file['name']; + } + } + + return $object; + } +} diff --git a/plugins/libkolab/lib/kolab_storage_cache_journal.php b/plugins/libkolab/lib/kolab_storage_cache_journal.php index 766f1da1..042d4bde 100644 --- a/plugins/libkolab/lib/kolab_storage_cache_journal.php +++ b/plugins/libkolab/lib/kolab_storage_cache_journal.php @@ -24,6 +24,7 @@ class kolab_storage_cache_journal extends kolab_storage_cache { protected $extra_cols = array('dtstart','dtend'); + protected $data_props = array('categories'); /** * Helper method to convert the given Kolab object into a dataset to be written to cache diff --git a/plugins/libkolab/lib/kolab_storage_cache_mongodb.php b/plugins/libkolab/lib/kolab_storage_cache_mongodb.php deleted file mode 100644 index 8ae95e4b..00000000 --- a/plugins/libkolab/lib/kolab_storage_cache_mongodb.php +++ /dev/null @@ -1,561 +0,0 @@ - - * - * Copyright (C) 2012, Kolab Systems AG - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -class kolab_storage_cache_mongodb -{ - private $db; - private $imap; - private $folder; - private $uid2msg; - private $objects; - private $index = array(); - private $resource_uri; - private $enabled = true; - private $synched = false; - private $synclock = false; - private $ready = false; - private $max_sql_packet = 1046576; // 1 MB - 2000 bytes - private $binary_cols = array('photo','pgppublickey','pkcs7publickey'); - - - /** - * Default constructor - */ - public function __construct(kolab_storage_folder $storage_folder = null) - { - $rcmail = rcube::get_instance(); - $mongo = new Mongo(); - $this->db = $mongo->kolab_cache; - $this->imap = $rcmail->get_storage(); - $this->enabled = $rcmail->config->get('kolab_cache', false); - - if ($this->enabled) { - // remove sync-lock on script termination - $rcmail->add_shutdown_function(array($this, '_sync_unlock')); - } - - if ($storage_folder) - $this->set_folder($storage_folder); - } - - - /** - * Connect cache with a storage folder - * - * @param kolab_storage_folder The storage folder instance to connect with - */ - public function set_folder(kolab_storage_folder $storage_folder) - { - $this->folder = $storage_folder; - - if (empty($this->folder->name)) { - $this->ready = false; - return; - } - - // compose fully qualified ressource uri for this instance - $this->resource_uri = $this->folder->get_resource_uri(); - $this->ready = $this->enabled; - } - - - /** - * Synchronize local cache data with remote - */ - public function synchronize() - { - // only sync once per request cycle - if ($this->synched) - return; - - // increase time limit - @set_time_limit(500); - - // lock synchronization for this folder or wait if locked - $this->_sync_lock(); - - // synchronize IMAP mailbox cache - $this->imap->folder_sync($this->folder->name); - - // compare IMAP index with object cache index - $imap_index = $this->imap->index($this->folder->name); - $this->index = $imap_index->get(); - - // determine objects to fetch or to invalidate - if ($this->ready) { - // read cache index - $old_index = array(); - $cursor = $this->db->cache->find(array('resource' => $this->resource_uri), array('msguid' => 1, 'uid' => 1)); - foreach ($cursor as $doc) { - $old_index[] = $doc['msguid']; - $this->uid2msg[$doc['uid']] = $doc['msguid']; - } - - // fetch new objects from imap - foreach (array_diff($this->index, $old_index) as $msguid) { - if ($object = $this->folder->read_object($msguid, '*')) { - try { - $this->db->cache->insert($this->_serialize($object, $msguid)); - } - catch (Exception $e) { - rcmail::raise_error(array( - 'code' => 900, 'type' => 'php', - 'message' => "Failed to write to mongodb cache: " . $e->getMessage(), - ), true); - } - } - } - - // delete invalid entries from local DB - $del_index = array_diff($old_index, $this->index); - if (!empty($del_index)) { - $this->db->cache->remove(array('resource' => $this->resource_uri, 'msguid' => array('$in' => $del_index))); - } - } - - // remove lock - $this->_sync_unlock(); - - $this->synched = time(); - } - - - /** - * Read a single entry from cache or from IMAP directly - * - * @param string Related IMAP message UID - * @param string Object type to read - * @param string IMAP folder name the entry relates to - * @param array Hash array with object properties or null if not found - */ - public function get($msguid, $type = null, $foldername = null) - { - // delegate to another cache instance - if ($foldername && $foldername != $this->folder->name) { - return kolab_storage::get_folder($foldername)->cache->get($msguid, $object); - } - - // load object if not in memory - if (!isset($this->objects[$msguid])) { - if ($this->ready && ($doc = $this->db->cache->findOne(array('resource' => $this->resource_uri, 'msguid' => $msguid)))) - $this->objects[$msguid] = $this->_unserialize($doc); - - // fetch from IMAP if not present in cache - if (empty($this->objects[$msguid])) { - $result = $this->_fetch(array($msguid), $type, $foldername); - $this->objects[$msguid] = $result[0]; - } - } - - return $this->objects[$msguid]; - } - - - /** - * Insert/Update a cache entry - * - * @param string Related IMAP message UID - * @param mixed Hash array with object properties to save or false to delete the cache entry - * @param string IMAP folder name the entry relates to - */ - public function set($msguid, $object, $foldername = null) - { - // delegate to another cache instance - if ($foldername && $foldername != $this->folder->name) { - kolab_storage::get_folder($foldername)->cache->set($msguid, $object); - return; - } - - // write to cache - if ($this->ready) { - // remove old entry - $this->db->cache->remove(array('resource' => $this->resource_uri, 'msguid' => $msguid)); - - // write new object data if not false (wich means deleted) - if ($object) { - try { - $this->db->cache->insert($this->_serialize($object, $msguid)); - } - catch (Exception $e) { - rcmail::raise_error(array( - 'code' => 900, 'type' => 'php', - 'message' => "Failed to write to mongodb cache: " . $e->getMessage(), - ), true); - } - } - } - - // keep a copy in memory for fast access - $this->objects[$msguid] = $object; - - if ($object) - $this->uid2msg[$object['uid']] = $msguid; - } - - /** - * Move an existing cache entry to a new resource - * - * @param string Entry's IMAP message UID - * @param string Entry's Object UID - * @param string Target IMAP folder to move it to - */ - public function move($msguid, $objuid, $target_folder) - { - $target = kolab_storage::get_folder($target_folder); - - // resolve new message UID in target folder - if ($new_msguid = $target->cache->uid2msguid($objuid)) { -/* - $this->db->query( - "UPDATE kolab_cache SET resource=?, msguid=? ". - "WHERE resource=? AND msguid=? AND type<>?", - $target->get_resource_uri(), - $new_msguid, - $this->resource_uri, - $msguid, - 'lock' - ); -*/ - } - else { - // just clear cache entry - $this->set($msguid, false); - } - - unset($this->uid2msg[$uid]); - } - - - /** - * Remove all objects from local cache - */ - public function purge($type = null) - { - return $this->db->cache->remove(array(), array('safe' => true)); - } - - - /** - * Select Kolab objects filtered by the given query - * - * @param array Pseudo-SQL query as list of filter parameter triplets - * triplet: array('', '', '') - * @return array List of Kolab data objects (each represented as hash array) - */ - public function select($query = array()) - { - $result = array(); - - // read from local cache DB (assume it to be synchronized) - if ($this->ready) { - $cursor = $this->db->cache->find(array('resource' => $this->resource_uri) + $this->_mongo_filter($query)); - foreach ($cursor as $doc) { - if ($object = $this->_unserialize($doc)) - $result[] = $object; - } - } - else { - // extract object type from query parameter - $filter = $this->_query2assoc($query); - - // use 'list' for folder's default objects - if ($filter['type'] == $this->type) { - $index = $this->index; - } - else { // search by object type - $search = 'UNDELETED HEADER X-Kolab-Type ' . kolab_storage_folder::KTYPE_PREFIX . $filter['type']; - $index = $this->imap->search_once($this->folder->name, $search)->get(); - } - - // fetch all messages in $index from IMAP - $result = $this->_fetch($index, $filter['type']); - - // TODO: post-filter result according to query - } - - return $result; - } - - - /** - * Get number of objects mathing the given query - * - * @param array $query Pseudo-SQL query as list of filter parameter triplets - * @return integer The number of objects of the given type - */ - public function count($query = array()) - { - $count = 0; - - // cache is in sync, we can count records in local DB - if ($this->synched) { - $cursor = $this->db->cache->find(array('resource' => $this->resource_uri) + $this->_mongo_filter($query)); - $count = $cursor->valid() ? $cursor->count() : 0; - } - else { - // search IMAP by object type - $filter = $this->_query2assoc($query); - $ctype = kolab_storage_folder::KTYPE_PREFIX . $filter['type']; - $index = $this->imap->search_once($this->folder->name, 'UNDELETED HEADER X-Kolab-Type ' . $ctype); - $count = $index->count(); - } - - return $count; - } - - /** - * Helper method to convert the pseudo-SQL query into a valid mongodb filter - */ - private function _mongo_filter($query) - { - $filters = array(); - foreach ($query as $param) { - $filter = array(); - if ($param[1] == '=' && is_array($param[2])) { - $filter[$param[0]] = array('$in' => $param[2]); - $filters[] = $filter; - } - else if ($param[1] == '=') { - $filters[] = array($param[0] => $param[2]); - } - else if ($param[1] == 'LIKE' || $param[1] == '~') { - $filter[$param[0]] = array('$regex' => preg_quote($param[2]), '$options' => 'i'); - $filters[] = $filter; - } - else if ($param[1] == '!~' || $param[1] == '!LIKE') { - $filter[$param[0]] = array('$not' => '/' . preg_quote($param[2]) . '/i'); - $filters[] = $filter; - } - else { - $op = ''; - switch ($param[1]) { - case '>': $op = '$gt'; break; - case '>=': $op = '$gte'; break; - case '<': $op = '$lt'; break; - case '<=': $op = '$lte'; break; - case '!=': - case '<>': $op = '$gte'; break; - } - if ($op) { - $filter[$param[0]] = array($op => $param[2]); - $filters[] = $filter; - } - } - } - - return array('$and' => $filters); - } - - /** - * Helper method to convert the given pseudo-query triplets into - * an associative filter array with 'equals' values only - */ - private function _query2assoc($query) - { - // extract object type from query parameter - $filter = array(); - foreach ($query as $param) { - if ($param[1] == '=') - $filter[$param[0]] = $param[2]; - } - return $filter; - } - - /** - * Fetch messages from IMAP - * - * @param array List of message UIDs to fetch - * @return array List of parsed Kolab objects - */ - private function _fetch($index, $type = null, $folder = null) - { - $results = array(); - foreach ((array)$index as $msguid) { - if ($object = $this->folder->read_object($msguid, $type, $folder)) { - $results[] = $object; - $this->set($msguid, $object); - } - } - - return $results; - } - - - /** - * Helper method to convert the given Kolab object into a dataset to be written to cache - */ - private function _serialize($object, $msguid) - { - $bincols = array_flip($this->binary_cols); - $doc = array( - 'resource' => $this->resource_uri, - 'type' => $object['_type'] ? $object['_type'] : $this->folder->type, - 'msguid' => $msguid, - 'uid' => $object['uid'], - 'xml' => '', - 'tags' => array(), - 'words' => array(), - 'objcols' => array(), - ); - - // set type specific values - if ($this->folder->type == 'event') { - // database runs in server's timezone so using date() is what we want - $doc['dtstart'] = date('Y-m-d H:i:s', is_object($object['start']) ? $object['start']->format('U') : $object['start']); - $doc['dtend'] = date('Y-m-d H:i:s', is_object($object['end']) ? $object['end']->format('U') : $object['end']); - - // extend date range for recurring events - if ($object['recurrence']) { - $doc['dtend'] = date('Y-m-d H:i:s', $object['recurrence']['UNTIL'] ?: strtotime('now + 2 years')); - } - } - - if ($object['_formatobj']) { - $doc['xml'] = preg_replace('!()[\n\r\t\s]+!ms', '$1', (string)$object['_formatobj']->write()); - $doc['tags'] = $object['_formatobj']->get_tags(); - $doc['words'] = $object['_formatobj']->get_words(); - } - - // extract object data - $data = array(); - foreach ($object as $key => $val) { - if ($val === "" || $val === null) { - // skip empty properties - continue; - } - if (isset($bincols[$key])) { - $data[$key] = base64_encode($val); - } - else if (is_object($val)) { - if (is_a($val, 'DateTime')) { - $data[$key] = array('_class' => 'DateTime', 'date' => $val->format('Y-m-d H:i:s'), 'timezone' => $val->getTimezone()->getName()); - $doc['objcols'][] = $key; - } - } - else if ($key[0] != '_') { - $data[$key] = $val; - } - else if ($key == '_attachments') { - foreach ($val as $k => $att) { - unset($att['content'], $att['path']); - if ($att['id']) - $data[$key][$k] = $att; - } - } - } - - $doc['data'] = $data; - return $doc; - } - - /** - * Helper method to turn stored cache data into a valid storage object - */ - private function _unserialize($doc) - { - $object = $doc['data']; - - // decode binary properties - foreach ($this->binary_cols as $key) { - if (!empty($object[$key])) - $object[$key] = base64_decode($object[$key]); - } - - // restore serialized objects - foreach ((array)$doc['objcols'] as $key) { - switch ($object[$key]['_class']) { - case 'DateTime': - $val = new DateTime($object[$key]['date'], new DateTimeZone($object[$key]['timezone'])); - $object[$key] = $val; - break; - } - } - - // add meta data - $object['_type'] = $doc['type']; - $object['_msguid'] = $doc['msguid']; - $object['_mailbox'] = $this->folder->name; - $object['_formatobj'] = kolab_format::factory($doc['type'], $doc['xml']); - - return $object; - } - - /** - * Check lock record for this folder and wait if locked or set lock - */ - private function _sync_lock() - { - if (!$this->ready) - return; - - $this->synclock = true; - $lock = $this->db->locks->findOne(array('resource' => $this->resource_uri)); - - // create lock record if not exists - if (!$lock) { - $this->db->locks->insert(array('resource' => $this->resource_uri, 'created' => time())); - } - // wait if locked (expire locks after 10 minutes) - else if ((time() - $lock['created']) < 600) { - usleep(500000); - return $this->_sync_lock(); - } - // set lock - else { - $lock['created'] = time(); - $this->db->locks->update(array('_id' => $lock['_id']), $lock, array('safe' => true)); - } - } - - /** - * Remove lock for this folder - */ - public function _sync_unlock() - { - if (!$this->ready || !$this->synclock) - return; - - $this->db->locks->remove(array('resource' => $this->resource_uri)); - } - - /** - * Resolve an object UID into an IMAP message UID - * - * @param string Kolab object UID - * @param boolean Include deleted objects - * @return int The resolved IMAP message UID - */ - public function uid2msguid($uid, $deleted = false) - { - if (!isset($this->uid2msg[$uid])) { - // use IMAP SEARCH to get the right message - $index = $this->imap->search_once($this->folder->name, ($deleted ? '' : 'UNDELETED ') . 'HEADER SUBJECT ' . $uid); - $results = $index->get(); - $this->uid2msg[$uid] = $results[0]; - } - - return $this->uid2msg[$uid]; - } - -} diff --git a/plugins/libkolab/lib/kolab_storage_cache_note.php b/plugins/libkolab/lib/kolab_storage_cache_note.php index 85469275..3211da57 100644 --- a/plugins/libkolab/lib/kolab_storage_cache_note.php +++ b/plugins/libkolab/lib/kolab_storage_cache_note.php @@ -23,5 +23,5 @@ class kolab_storage_cache_note extends kolab_storage_cache { - -} \ No newline at end of file + protected $data_props = array('title'); +} diff --git a/plugins/libkolab/lib/kolab_storage_cache_task.php b/plugins/libkolab/lib/kolab_storage_cache_task.php index 8b714e63..9c776d75 100644 --- a/plugins/libkolab/lib/kolab_storage_cache_task.php +++ b/plugins/libkolab/lib/kolab_storage_cache_task.php @@ -23,7 +23,8 @@ class kolab_storage_cache_task extends kolab_storage_cache { - protected $extra_cols = array('dtstart','dtend'); + protected $extra_cols = array('dtstart', 'dtend'); + protected $data_props = array('categories', 'status', 'complete', 'start', 'due'); /** * Helper method to convert the given Kolab object into a dataset to be written to cache diff --git a/plugins/libkolab/lib/kolab_storage_config.php b/plugins/libkolab/lib/kolab_storage_config.php index fb442ece..b29e7a90 100644 --- a/plugins/libkolab/lib/kolab_storage_config.php +++ b/plugins/libkolab/lib/kolab_storage_config.php @@ -131,7 +131,7 @@ class kolab_storage_config $folder->set_order_and_limit(null, $limit); } - foreach ($folder->select($filter) as $object) { + foreach ($folder->select($filter, true) as $object) { unset($object['_formatobj']); $list[] = $object; } diff --git a/plugins/libkolab/lib/kolab_storage_folder.php b/plugins/libkolab/lib/kolab_storage_folder.php index 4018b7df..d0695cb2 100644 --- a/plugins/libkolab/lib/kolab_storage_folder.php +++ b/plugins/libkolab/lib/kolab_storage_folder.php @@ -281,12 +281,14 @@ class kolab_storage_folder extends kolab_storage_folder_api /** * Select Kolab objects matching the given query * - * @param mixed Pseudo-SQL query as list of filter parameter triplets - * or string with object type (e.g. contact, event, todo, journal, note, configuration) + * @param mixed Pseudo-SQL query as list of filter parameter triplets + * or string with object type (e.g. contact, event, todo, journal, note, configuration) + * @param boolean Use fast mode to fetch only minimal set of information + * (no xml fetching and parsing, etc.) * * @return array List of Kolab data objects (each represented as hash array) */ - public function select($query = array()) + public function select($query = array(), $fast = false) { if (!$this->valid) { return array(); @@ -296,7 +298,7 @@ class kolab_storage_folder extends kolab_storage_folder_api $this->cache->synchronize(); // fetch objects from cache - return $this->cache->select($this->_prepare_query($query)); + return $this->cache->select($this->_prepare_query($query), false, $fast); } /** @@ -419,6 +421,8 @@ class kolab_storage_folder extends kolab_storage_folder_api } else { // return message part from IMAP directly + // TODO: We could improve performance if we cache part's encoding + // without 3rd argument get_message_part() will request BODYSTRUCTURE from IMAP return $this->imap->get_message_part($msguid, $part, null, $print, $fp, $skip_charset_conv); } } @@ -486,8 +490,9 @@ class kolab_storage_folder extends kolab_storage_folder_api $content_type = kolab_format::KTYPE_PREFIX . $object_type; // check object type header and abort on mismatch - if ($type != '*' && $object_type != $type) + if ($type != '*' && strpos($object_type, $type) !== 0 && !($object_type == 'distribution-list' && $type == 'contact')) { return false; + } $attachments = array(); @@ -557,6 +562,7 @@ class kolab_storage_folder extends kolab_storage_folder_api $object['_msguid'] = $msguid; $object['_mailbox'] = $this->name; $object['_formatobj'] = $format; + $object['_size'] = strlen($xml); return $object; } diff --git a/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php b/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php index 1bfff58b..73142be3 100644 --- a/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php +++ b/plugins/tasklist/drivers/kolab/tasklist_kolab_driver.php @@ -587,7 +587,7 @@ class tasklist_kolab_driver extends tasklist_driver continue; } - foreach ($folder->select(array(array('tags','!~','x-complete'))) as $record) { + foreach ($folder->select(array(array('tags','!~','x-complete')), true) as $record) { $rec = $this->_to_rcube_task($record, $list_id, false); if ($this->is_complete($rec)) // don't count complete tasks