Return a kolab_storage_dataset itertor object from kolab_storage_cache::select()

to manage memory usage for large result sets (#2828).

Attention!
Do not cast the return value of kolab_storage_folder::select() calls into an array anymore.

plugins/libkolab/lib/kolab_storage_cache.php

@@ -239,14 +239,14 @@ class kolab_storage_cache
                 );
 
                 if ($sql_arr = $this->db->fetch_assoc($sql_result)) {
-                    $this->objects[$msguid] = $this->_unserialize($sql_arr);
+                    $this->objects = array($msguid => $this->_unserialize($sql_arr));  // store only this object in memory (#2827)
                 }
             }
 
             // fetch from IMAP if not present in cache
             if (empty($this->objects[$msguid])) {
                 $result = $this->_fetch(array($msguid), $type, $foldername);
-                $this->objects[$msguid] = $result[0];
+                $this->objects = array($msguid => $result[0]);  // store only this object in memory (#2827)
             }
         }
 
@@ -438,14 +438,16 @@ class kolab_storage_cache
      */
     public function select($query = array(), $uids = false)
     {
-        $result = array();
+        $result = $uids ? array() : new kolab_storage_dataset($this);
 
         // read from local cache DB (assume it to be synchronized)
         if ($this->ready) {
             $this->_read_folder_data();
 
+            // fetch full object data on one query if a small result set is expected
+            $fetchall = !$uids && $this->count($query) < 500;
             $sql_result = $this->db->query(
-                "SELECT " . ($uids ? 'msguid, uid' : '*') . " FROM $this->cache_table ".
+                "SELECT " . (!$fetchall ? 'msguid, msguid AS _msguid, uid' : '*') . " FROM $this->cache_table ".
                 "WHERE folder_id=? " . $this->_sql_where($query),
                 $this->folder_id
             );
@@ -459,9 +461,13 @@ class kolab_storage_cache
                     $this->uid2msg[$sql_arr['uid']] = $sql_arr['msguid'];
                     $result[] = $sql_arr['uid'];
                 }
-                else if ($object = $this->_unserialize($sql_arr)) {
+                else if ($fetchall && ($object = $this->_unserialize($sql_arr))) {
                     $result[] = $object;
                 }
+                else {
+                    // only add msguid to dataset index
+                    $result[] = $sql_arr;
+                }
             }
         }
         // use IMAP
@@ -491,7 +497,7 @@ class kolab_storage_cache
         if (!$uids && count($result) == 1) {
             if ($msguid = $result[0]['_msguid']) {
                 $this->uid2msg[$result[0]['uid']] = $msguid;
-                $this->objects[$msguid] = $result[0];
+                $this->objects = array($msguid => $result[0]);
             }
         }
 
@@ -618,7 +624,7 @@ class kolab_storage_cache
      */
     protected function _fetch($index, $type = null, $folder = null)
     {
-        $results = array();
+        $results = new kolab_storage_dataset($this);
         foreach ((array)$index as $msguid) {
             if ($object = $this->folder->read_object($msguid, $type, $folder)) {
                 $results[] = $object;

plugins/libkolab/lib/kolab_storage_dataset.php

@@ -0,0 +1,129 @@
+<?php
+
+/**
+ * Dataset class providing the results of a select operation on a kolab_storage_folder.
+ *
+ * Can be used as a normal array as well as an iterator in foreach() loops.
+ *
+ * @version @package_version@
+ * @author Thomas Bruederli <bruederli@kolabsys.com>
+ *
+ * Copyright (C) 2014, Kolab Systems AG <contact@kolabsys.com>
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+class kolab_storage_dataset implements Iterator, ArrayAccess
+{
+    private $cache;  // kolab_storage_cache instance to use for fetching data
+    private $memlimit = 0;
+    private $buffer = false;
+    private $index = array();
+    private $data = array();
+    private $iteratorkey = -1;
+
+    /**
+     * Default constructor
+     *
+     * @param object kolab_storage_cache instance to be used for fetching objects upon access
+     */
+    public function __construct($cache)
+    {
+        $this->cache = $cache;
+
+        // enable in-memory buffering up until 1/5 of the available memory
+        if (function_exists('memory_get_usage')) {
+            $this->memlimit = parse_bytes(ini_get('memory_limit')) / 5;
+            $this->buffer = true;
+        }
+    }
+
+
+    /*** Implement PHP ArrayAccess interface ***/
+
+    public function offsetSet($offset, $value)
+    {
+        $uid = $value['_msguid'];
+
+        if (is_null($offset)) {
+            $offset = count($this->index);
+            $this->index[] = $uid;
+        }
+        else {
+            $this->index[$offset] = $uid;
+        }
+
+        // keep full payload data in memory if possible
+        if ($this->memlimit && $this->buffer && isset($value['_mailbox'])) {
+            $this->data[$offset] = $value;
+
+            // check memory usage and stop buffering
+            if ($offset % 10 == 0) {
+                $this->buffer = memory_get_usage() < $this->memlimit;
+            }
+        }
+    }
+
+    public function offsetExists($offset)
+    {
+        return isset($this->index[$offset]);
+    }
+
+    public function offsetUnset($offset)
+    {
+        unset($this->index[$offset]);
+    }
+
+    public function offsetGet($offset)
+    {
+        if (isset($this->data[$offset])) {
+            return $this->data[$offset];
+        }
+        else if ($msguid = $this->index[$offset]) {
+            return $this->cache->get($msguid);
+        }
+
+        return null;
+    }
+
+
+    /*** Implement PHP Iterator interface ***/
+
+    public function current()
+    {
+        return $this->offsetGet($this->iteratorkey);
+    }
+
+    public function key()
+    {
+        return $this->iteratorkey;
+    }
+
+    public function next()
+    {
+        $this->iteratorkey++;
+        return $this->valid();
+    }
+
+    public function rewind()
+    {
+        $this->iteratorkey = 0;
+    }
+
+    public function valid()
+    {
+        return !empty($this->index[$this->iteratorkey]);
+    }
+
+}