Performance: Limit number of SQL queries when "attaching" relations/tags to list of tasks/notes (#3969)
This commit is contained in:
Aleksander Machniak
parent
446cd0dfe1
commit
11ce37ef17
4 changed files with 93 additions and 21 deletions
|
|
@ -441,6 +441,7 @@ class kolab_notes extends rcube_plugin
|
|||
{
|
||||
$config = kolab_storage_config::get_instance();
|
||||
$tags = $config->apply_tags($records);
|
||||
$config->apply_links($records, 'note');
|
||||
|
||||
foreach ($records as $i => $rec) {
|
||||
unset($records[$i]['description']);
|
||||
|
|
@ -586,9 +587,11 @@ class kolab_notes extends rcube_plugin
|
|||
}
|
||||
|
||||
// resolve message links
|
||||
$note['links'] = array_map(function($link) {
|
||||
return kolab_storage_config::get_message_reference($link, 'note') ?: array('uri' => $link);
|
||||
}, $this->get_links($note['uid']));
|
||||
if (!array_key_exists('links', $note)) {
|
||||
$note['links'] = array_map(function($link) {
|
||||
return kolab_storage_config::get_message_reference($link, 'note') ?: array('uri' => $link);
|
||||
}, $this->get_links($note['uid']));
|
||||
}
|
||||
|
||||
return $note;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@
|
|||
class kolab_storage_cache
|
||||
{
|
||||
const DB_DATE_FORMAT = 'Y-m-d H:i:s';
|
||||
const MAX_RECORDS = 500;
|
||||
|
||||
public $sync_complete = false;
|
||||
|
||||
|
|
@ -528,7 +529,7 @@ class kolab_storage_cache
|
|||
$this->_read_folder_data();
|
||||
|
||||
// fetch full object data on one query if a small result set is expected
|
||||
$fetchall = !$uids && ($this->limit ? $this->limit[0] : ($count = $this->count($query))) < 500;
|
||||
$fetchall = !$uids && ($this->limit ? $this->limit[0] : ($count = $this->count($query))) < self::MAX_RECORDS;
|
||||
|
||||
// skip SELECT if we know it will return nothing
|
||||
if ($count === 0) {
|
||||
|
|
|
|||
|
|
@ -25,8 +25,8 @@
|
|||
|
||||
class kolab_storage_config
|
||||
{
|
||||
const FOLDER_TYPE = 'configuration';
|
||||
|
||||
const FOLDER_TYPE = 'configuration';
|
||||
const MAX_RELATIONS = 499; // should be less than kolab_storage_cache::MAX_RECORDS
|
||||
|
||||
/**
|
||||
* Singleton instace of kolab_storage_config
|
||||
|
|
@ -592,6 +592,72 @@ class kolab_storage_config
|
|||
return $tags;
|
||||
}
|
||||
|
||||
/**
|
||||
* Assign links (relations) to kolab objects
|
||||
*
|
||||
* @param array $records List of kolab objects
|
||||
* @param string $type Object type
|
||||
*/
|
||||
public function apply_links(&$records, $type = null)
|
||||
{
|
||||
$links = array();
|
||||
$uids = array();
|
||||
$ids = array();
|
||||
$limit = 25;
|
||||
|
||||
// get list of object UIDs and UIRs map
|
||||
foreach ($records as $i => $rec) {
|
||||
$uids[] = $rec['uid'];
|
||||
$ids[self::build_member_url($rec['uid'])] = $i;
|
||||
$records[$i]['links'] = array();
|
||||
}
|
||||
|
||||
// The whole story here is to not do SELECT for every object.
|
||||
// We'll build one SELECT for many (limit above) objects at once
|
||||
|
||||
while (!empty($uids)) {
|
||||
$chunk = array_splice($uids, 0, $limit);
|
||||
$chunk = array_map(function($v) { return array('member', '=', $v); }, $chunk);
|
||||
|
||||
$filter = array(
|
||||
array('type', '=', 'relation'),
|
||||
array('category', '=', 'generic'),
|
||||
array($chunk, 'OR'),
|
||||
);
|
||||
|
||||
$relations = $this->get_objects($filter, true, self::MAX_RELATIONS);
|
||||
|
||||
foreach ($relations as $relation) {
|
||||
$links[$relation['uid']] = $relation;
|
||||
}
|
||||
}
|
||||
|
||||
if (empty($links)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// assign links of related messages
|
||||
foreach ($links as $relation) {
|
||||
// make relation members up-to-date
|
||||
kolab_storage_config::resolve_members($relation);
|
||||
|
||||
// replace link URIs with message reference URLs
|
||||
$members = array();
|
||||
foreach ((array) $relation['members'] as $member) {
|
||||
if (strpos($member, 'imap://') === 0) {
|
||||
$members[$member] = kolab_storage_config::get_message_reference($link, $type) ?: array('uri' => $link);
|
||||
}
|
||||
}
|
||||
|
||||
// assign links to objects
|
||||
foreach ((array) $relation['members'] as $member) {
|
||||
if (($id = $ids[$member]) !== null) {
|
||||
$records[$id]['links'] = array_unique(array_merge($records[$id]['links'], $members));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Update object tags
|
||||
*
|
||||
|
|
@ -650,12 +716,10 @@ class kolab_storage_config
|
|||
* Get tags (all or referring to specified object)
|
||||
*
|
||||
* @param string $member Optional object UID or mail message-id
|
||||
* @param int $limit Max. number of records (per-folder)
|
||||
* Used when searching by member
|
||||
*
|
||||
* @return array List of Relation objects
|
||||
*/
|
||||
public function get_tags($member = '*', $limit = 0)
|
||||
public function get_tags($member = '*')
|
||||
{
|
||||
if (!isset($this->tags)) {
|
||||
$default = true;
|
||||
|
|
@ -667,10 +731,10 @@ class kolab_storage_config
|
|||
// use faster method
|
||||
if ($member && $member != '*') {
|
||||
$filter[] = array('member', '=', $member);
|
||||
$tags = $this->get_objects($filter, $default, $limit);
|
||||
$tags = $this->get_objects($filter, $default, self::MAX_RELATIONS);
|
||||
}
|
||||
else {
|
||||
$this->tags = $tags = $this->get_objects($filter, $default);
|
||||
$this->tags = $tags = $this->get_objects($filter, $default, self::MAX_RELATIONS);
|
||||
}
|
||||
}
|
||||
else {
|
||||
|
|
@ -802,7 +866,7 @@ class kolab_storage_config
|
|||
array('member', '=', $uid),
|
||||
);
|
||||
|
||||
return $this->get_objects($filter, $default, 100);
|
||||
return $this->get_objects($filter, $default, self::MAX_RELATIONS);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -563,6 +563,7 @@ class tasklist_kolab_driver extends tasklist_driver
|
|||
$lists = explode(',', $lists);
|
||||
}
|
||||
|
||||
$config = kolab_storage_config::get_instance();
|
||||
$results = array();
|
||||
|
||||
// query Kolab storage
|
||||
|
|
@ -584,23 +585,25 @@ class tasklist_kolab_driver extends tasklist_driver
|
|||
$query[] = array('changed', '>=', $filter['since']);
|
||||
}
|
||||
|
||||
// load all tags into memory first
|
||||
kolab_storage_config::get_instance()->get_tags();
|
||||
|
||||
foreach ($lists as $list_id) {
|
||||
if (!$folder = $this->get_folder($list_id)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach ($folder->select($query) as $record) {
|
||||
$this->load_tags($record);
|
||||
$task = $this->_to_rcube_task($record, $list_id);
|
||||
|
||||
// TODO: post-filter tasks returned from storage
|
||||
|
||||
$results[] = $task;
|
||||
$record['list_id'] = $list_id;
|
||||
$results[] = $record;
|
||||
}
|
||||
}
|
||||
|
||||
$config->apply_tags($results);
|
||||
$config->apply_links($results);
|
||||
|
||||
foreach (array_keys($results) as $idx) {
|
||||
$results[$idx] = $this->_to_rcube_task($results[$idx], $results[$idx]['list_id']);
|
||||
}
|
||||
|
||||
// avoid session race conditions that will loose temporary subscriptions
|
||||
$this->plugin->rc->session->nowrite = true;
|
||||
|
||||
|
|
@ -1188,10 +1191,11 @@ class tasklist_kolab_driver extends tasklist_driver
|
|||
'sequence' => $record['sequence'],
|
||||
'tags' => $record['tags'],
|
||||
'list' => $list_id,
|
||||
'links' => $record['links'],
|
||||
);
|
||||
|
||||
// we can sometimes skip this expensive operation
|
||||
if ($all) {
|
||||
if ($all && !array_key_exists('links', $task)) {
|
||||
$task['links'] = $this->get_links($task['uid']);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue