Optimize access to kolab contacts using a sorted and limited query (#2828)

- Add columns for sorting in kolab_cache_contact
- Extend bin/modcache.sh script to update existing cache records
- Add setters for ORDER BY and LIMIT clauses
- Adapt the kolab_addressbook plugin to fetch contacts page-wise

ATTENTION: This changeset contains database schema changes!
Run `bin/updatedb.sh --dir plugins/libkolab/SQL --package libkolab`

Afterwards, the cached data needs to be updated. To do so, either run
  `plugins/libkolab/bin/modcache.sh update --type=contact`
or execute the following query
  DELETE FROM `kolab_folders` WHERE `type`='contact';

plugins/kolab_addressbook/lib/rcube_kolab_contacts.php

@@ -275,6 +275,7 @@ class rcube_kolab_contacts extends rcube_addressbook
     public function list_records($cols = null, $subset = 0, $nocount = false)
     {
         $this->result = new rcube_result_set(0, ($this->list_page-1) * $this->page_size);
+        $fetch_all = false;
 
         // list member of the selected group
         if ($this->gid) {
@@ -298,12 +299,13 @@ class rcube_kolab_contacts extends rcube_addressbook
                 else if (!empty($member['email'])) {
                     $this->contacts[$member['ID']] = $member;
                     $local_sortindex[$member['ID']] = $this->_sort_string($member);
+                    $fetch_all = true;
                 }
             }
 
             // get members by UID
             if (!empty($uids)) {
-                $this->_fetch_contacts(array(array('uid', '=', $uids)));
+                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), !$fetch_all);
                 $this->sortindex = array_merge($this->sortindex, $local_sortindex);
             }
         }
@@ -311,13 +313,14 @@ class rcube_kolab_contacts extends rcube_addressbook
             $ids = $this->filter['ids'];
             if (count($ids)) {
                 $uids = array_map(array($this, 'id2uid'), $this->filter['ids']);
-                $this->_fetch_contacts(array(array('uid', '=', $uids)));
+                $this->_fetch_contacts($query = array(array('uid', '=', $uids)), true);
             }
         }
         else {
-            $this->_fetch_contacts();
+            $this->_fetch_contacts($query = array(), true);
         }
 
+        if ($fetch_all) {
             // sort results (index only)
             asort($this->sortindex, SORT_LOCALE_STRING);
             $ids = array_keys($this->sortindex);
@@ -333,6 +336,13 @@ class rcube_kolab_contacts extends rcube_addressbook
                     $this->result->add($this->contacts[$idx] ?: $this->_to_rcube_contact($this->dataset[$idx]));
                 }
             }
+        }
+        else {
+            $this->result->count = $this->storagefolder->count($query);
+            foreach ($this->dataset as $idx => $record) {
+                $this->result->add($this->_to_rcube_contact($record));
+            }
+        }
 
         return $this->result;
     }
@@ -971,9 +981,12 @@ class rcube_kolab_contacts extends rcube_addressbook
     /**
      * Query storage layer and store records in private member var
      */
-    private function _fetch_contacts($query = array())
+    private function _fetch_contacts($query = array(), $limit = false)
     {
         if (!isset($this->dataset) || !empty($query)) {
+            if ($limit) {
+                $this->storagefolder->set_order_and_limit($this->_sort_columns(), $this->page_size, ($this->list_page-1) * $this->page_size);
+            }
             $this->sortindex = array();
             $this->dataset = $this->storagefolder->select($query);
             foreach ($this->dataset as $idx => $record) {
@@ -1010,6 +1023,29 @@ class rcube_kolab_contacts extends rcube_addressbook
         return mb_strtolower($str);
     }
 
+    /**
+     * Return the cache table columns to order by
+     */
+    private function _sort_columns()
+    {
+        $sortcols = array();
+
+        switch ($this->sort_col) {
+        case 'name':
+            $sortcols[] = 'name';
+        case 'firstname':
+            $sortcols[] = 'firstname';
+            break;
+
+        case 'surname':
+            $sortcols[] = 'surname';
+            break;
+        }
+
+        $sortcols[] = 'email';
+        return $sortcols;
+    }
+
     /**
      * Read distribution-lists AKA groups from server
      */

plugins/libkolab/SQL/mysql/2014021000.sql

@@ -0,0 +1,9 @@
+ALTER TABLE `kolab_cache_contact` ADD `name` VARCHAR(255) NOT NULL,
+  ADD `firstname` VARCHAR(255) NOT NULL,
+  ADD `surname` VARCHAR(255) NOT NULL,
+  ADD `email` VARCHAR(255) NOT NULL;
+
+-- updating or clearing all contacts caches is required.
+-- either run `bin/modcache.sh update --type=contact` or execute the following query:
+--   DELETE FROM `kolab_folders` WHERE `type`='contact';
+

plugins/libkolab/bin/modcache.sh

@@ -7,7 +7,7 @@
  * @version 3.1
  * @author Thomas Bruederli <bruederli@kolabsys.com>
  *
- * Copyright (C) 2012, Kolab Systems AG <contact@kolabsys.com>
+ * Copyright (C) 2012-2014, Kolab Systems AG <contact@kolabsys.com>
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as
@@ -65,6 +65,7 @@ $db->db_connect('w');
 if (!$db->is_connected() || $db->is_error())
     die("No DB connection\n");
 
+ini_set('display_errors', 1);
 
 /*
  * Script controller
@@ -142,6 +143,32 @@ case 'prewarm':
         die("Authentication failed for " . $opts['user']);
     break;
 
+/**
+ * Update the cache meta columns from the serialized/xml data
+ * (might be run after a schema update)
+ */
+case 'update':
+    // make sure libkolab classes are loaded
+    $rcmail->plugins->load_plugin('libkolab');
+
+    $folder_types = $opts['type'] ? explode(',', $opts['type']) : array('contact','configuration','event','file','task');
+    foreach ($folder_types as $type) {
+        $class = 'kolab_storage_cache_' . $type;
+        $sql_result = $db->query("SELECT folder_id FROM kolab_folders WHERE type=? AND synclock = 0", $type);
+        while ($sql_result && ($sql_arr = $db->fetch_assoc($sql_result))) {
+            $folder = new $class;
+            $folder->select_by_id($sql_arr['folder_id']);
+            echo "Updating " . $sql_arr['folder_id'] . " ($type) ";
+            foreach ($folder->select() as $object) {
+                $object['_formatobj']->to_array();  // load data
+                $folder->save($object['_msguid'], $object, $object['_msguid']);
+                echo ".";
+            }
+            echo "done.\n";
+        }
+    }
+    break;
+
 
 /*
  * Unknown action => show usage

plugins/libkolab/lib/kolab_storage_cache.php

@@ -44,6 +44,8 @@ class kolab_storage_cache
     protected $max_sync_lock_time = 600;
     protected $binary_items = array();
     protected $extra_cols = array();
+    protected $order_by = null;
+    protected $limit = null;
 
 
     /**
@@ -88,6 +90,24 @@ class kolab_storage_cache
             $this->set_folder($storage_folder);
     }
 
+    /**
+     * Direct access to cache by folder_id
+     * (only for internal use)
+     */
+    public function select_by_id($folder_id)
+    {
+        $folders_table = $this->db->table_name('kolab_folders');
+        $sql_arr = $this->db->fetch_assoc($this->db->query("SELECT * FROM $folders_table WHERE folder_id=?", $folder_id));
+        if ($sql_arr) {
+            $this->metadata = $sql_arr;
+            $this->folder_id = $sql_arr['folder_id'];
+            $this->folder = new StdClass;
+            $this->folder->type = $sql_arr['type'];
+            $this->resource_uri = $sql_arr['resource'];
+            $this->cache_table = $this->db->table_name('kolab_cache_' . $sql_arr['type']);
+            $this->ready = true;
+        }
+    }
 
     /**
      * Connect cache with a storage folder
@@ -445,12 +465,15 @@ class kolab_storage_cache
             $this->_read_folder_data();
 
             // fetch full object data on one query if a small result set is expected
-            $fetchall = !$uids && $this->count($query) < 500;
+            $fetchall = !$uids && ($this->limit ? $this->limit[0] : $this->count($query)) < 500;
-            $sql_result = $this->db->query(
+            $sql_query = "SELECT " . ($fetchall ? '*' : 'msguid AS _msguid, uid') . " FROM $this->cache_table ".
-                "SELECT " . ($fetchall ? '*' : 'msguid AS _msguid, uid') . " FROM $this->cache_table ".
+                         "WHERE folder_id=? " . $this->_sql_where($query);
-                "WHERE folder_id=? " . $this->_sql_where($query),
+            if (!empty($this->order_by)) {
-                $this->folder_id
+                $sql_query .= ' ORDER BY ' . $this->order_by;
-            );
+            }
+            $sql_result = $this->limit ?
+                $this->db->limitquery($sql_query, $this->limit[1], $this->limit[0], $this->folder_id) :
+                $this->db->query($sql_query, $this->folder_id);
 
             if ($this->db->is_error($sql_result)) {
                 if ($uids) {
@@ -562,6 +585,26 @@ class kolab_storage_cache
         return $count;
     }
 
+    /**
+     * Define ORDER BY clause for cache queries
+     */
+    public function set_order_by($sortcols)
+    {
+        if (!empty($sortcols)) {
+            $this->order_by = join(', ', (array)$sortcols);
+        }
+        else {
+            $this->order_by = null;
+        }
+    }
+
+    /**
+     * Define LIMIT clause for cache queries
+     */
+    public function set_limit($length, $offset = 0)
+    {
+        $this->limit = array($length, $offset);
+    }
 
     /**
      * Helper method to compose a valid SQL query from pseudo filter triplets

plugins/libkolab/lib/kolab_storage_cache_contact.php

@@ -23,7 +23,7 @@
 
 class kolab_storage_cache_contact extends kolab_storage_cache
 {
-    protected $extra_cols = array('type');
+    protected $extra_cols = array('type','name','firstname','surname','email');
     protected $binary_items = array(
         'photo'          => '|<photo><uri>[^;]+;base64,([^<]+)</uri></photo>|i',
         'pgppublickey'   => '|<key><uri>date:application/pgp-keys;base64,([^<]+)</uri></key>|i',
@@ -40,6 +40,16 @@ class kolab_storage_cache_contact extends kolab_storage_cache
         $sql_data = parent::_serialize($object);
         $sql_data['type'] = $object['_type'];
 
+        // columns for sorting
+        $sql_data['name']      = $object['name'] . $object['prefix'];
+        $sql_data['firstname'] = $object['firstname'] . $object['middlename'] . $object['surname'];
+        $sql_data['surname']   = $object['surname']   . $object['firstname']  . $object['middlename'];
+        $sql_data['email']     = is_array($object['email']) ? $object['email'][0] : $object['email'];
+
+        if (is_array($sql_data['email'])) {
+            $sql_data['email'] = $sql_data['email']['address'];
+        }
+
         return $sql_data;
     }
 }

plugins/libkolab/lib/kolab_storage_folder.php

@@ -92,7 +92,6 @@ class kolab_storage_folder
         $this->cache->set_folder($this);
     }
 
-
     /**
      *
      */
@@ -424,6 +423,21 @@ class kolab_storage_folder
         return $this->cache->select($this->_prepare_query($query), true);
     }
 
+    /**
+     * Setter for ORDER BY and LIMIT parameters for cache queries
+     *
+     * @param array   List of columns to order by
+     * @param integer Limit result set to this length
+     * @param integer Offset row
+     */
+    public function set_order_and_limit($sortcols, $length = null, $offset = 0)
+    {
+        $this->cache->set_order_by($sortcols);
+
+        if ($length !== null) {
+            $this->cache->set_limit($length, $offset);
+        }
+    }
 
     /**
      * Helper method to sanitize query arguments