From 48027bc26ec3accae9776e437ae3f345b6cd3ae7 Mon Sep 17 00:00:00 2001 From: Aleksander Machniak Date: Mon, 28 Nov 2022 12:28:04 +0100 Subject: [PATCH] Optionally store original DAV object content in the cache ... up to the specified size limit --- plugins/libkolab/lib/kolab_dav_client.php | 9 +- .../libkolab/lib/kolab_storage_dataset.php | 5 ++ .../libkolab/lib/kolab_storage_dav_cache.php | 84 +++++++++++++++---- .../libkolab/lib/kolab_storage_dav_folder.php | 6 +- 4 files changed, 84 insertions(+), 20 deletions(-) diff --git a/plugins/libkolab/lib/kolab_dav_client.php b/plugins/libkolab/lib/kolab_dav_client.php index 1659372b..c5d3fb29 100644 --- a/plugins/libkolab/lib/kolab_dav_client.php +++ b/plugins/libkolab/lib/kolab_dav_client.php @@ -156,7 +156,7 @@ class kolab_dav_client $elements = $response->getElementsByTagName('response'); foreach ($elements as $element) { - foreach ($element->getElementsByTagName('prop') as $prop) { + foreach ($element->getElementsByTagName('current-user-principal') as $prop) { $principal_href = $prop->nodeValue; break; } @@ -194,7 +194,7 @@ class kolab_dav_client $elements = $response->getElementsByTagName('response'); foreach ($elements as $element) { - foreach ($element->getElementsByTagName('prop') as $prop) { + foreach ($element->getElementsByTagName($homes[$component]) as $prop) { $root_href = $prop->nodeValue; break; } @@ -297,9 +297,10 @@ class kolab_dav_client $response = $this->request($location, 'PUT', $content, $headers); if ($response !== false) { - $etag = $this->responseHeaders['etag']; + // Note: ETag is not always returned, e.g. https://github.com/cyrusimap/cyrus-imapd/issues/2456 + $etag = isset($this->responseHeaders['etag']) ? $this->responseHeaders['etag'] : null; - if (preg_match('|^".*"$|', $etag)) { + if (is_string($etag) && preg_match('|^".*"$|', $etag)) { $etag = substr($etag, 1, -1); } diff --git a/plugins/libkolab/lib/kolab_storage_dataset.php b/plugins/libkolab/lib/kolab_storage_dataset.php index 819b8d8b..97717e2a 100644 --- a/plugins/libkolab/lib/kolab_storage_dataset.php +++ b/plugins/libkolab/lib/kolab_storage_dataset.php @@ -129,6 +129,11 @@ class kolab_storage_dataset implements Iterator, ArrayAccess, Countable $uids = []; while (isset($this->index[$idx]) && count($uids) < self::CHUNK_SIZE) { + if (isset($this->data[$idx]) && !is_string($this->data[$idx])) { + // skip objects that had the raw content in the cache (are not empty) + continue; + } + $uids[$idx] = $this->index[$idx]; $idx++; } diff --git a/plugins/libkolab/lib/kolab_storage_dav_cache.php b/plugins/libkolab/lib/kolab_storage_dav_cache.php index 28c6e4c8..e22518f9 100644 --- a/plugins/libkolab/lib/kolab_storage_dav_cache.php +++ b/plugins/libkolab/lib/kolab_storage_dav_cache.php @@ -170,9 +170,11 @@ class kolab_storage_dav_cache extends kolab_storage_cache return false; } - foreach ($objects as $object) { - if ($object = $this->folder->from_dav($object)) { + foreach ($objects as $dav_object) { + if ($object = $this->folder->from_dav($dav_object)) { + $object['_raw'] = $dav_object['data']; $this->_extended_insert(false, $object); + unset($object['_raw']); } } @@ -198,9 +200,11 @@ class kolab_storage_dav_cache extends kolab_storage_cache return false; } - foreach ($objects as $object) { - if ($object = $this->folder->from_dav($object)) { + foreach ($objects as $dav_object) { + if ($object = $this->folder->from_dav($dav_object)) { + $object['_raw'] = $dav_object['data']; $this->save($object, $object['uid']); + unset($object['_raw']); } } @@ -438,17 +442,13 @@ class kolab_storage_dav_cache extends kolab_storage_cache } while ($sql_arr = $this->db->fetch_assoc($sql_result)) { - if ($fast) { - $sql_arr['fast-mode'] = true; - } - if ($uids) { $result[] = $sql_arr['uid']; } else if (!$fetchall) { $result[] = $sql_arr; } - else if (($object = $this->_unserialize($sql_arr, true))) { + else if (($object = $this->_unserialize($sql_arr, true, $fast))) { $result[] = $object; } else { @@ -604,12 +604,62 @@ class kolab_storage_dav_cache extends kolab_storage_cache $buffer .= ($buffer ? ',' : '') . $line; } + /** + * Helper method to convert the given Kolab object into a dataset to be written to cache + */ + protected function _serialize($object) + { + static $threshold; + + if ($threshold === null) { + $rcube = rcube::get_instance(); + $threshold = parse_bytes(rcube::get_instance()->config->get('dav_cache_threshold', 0)); + } + + $data = []; + $sql_data = ['changed' => null, 'tags' => '', 'words' => '']; + + if (!empty($object['changed'])) { + $sql_data['changed'] = date(self::DB_DATE_FORMAT, is_object($object['changed']) ? $object['changed']->format('U') : $object['changed']); + } + + // Store only minimal set of object properties + foreach ($this->data_props as $prop) { + if (isset($object[$prop])) { + $data[$prop] = $object[$prop]; + if ($data[$prop] instanceof DateTime) { + $data[$prop] = array( + 'cl' => 'DateTime', + 'dt' => $data[$prop]->format('Y-m-d H:i:s'), + 'tz' => $data[$prop]->getTimezone()->getName(), + ); + } + } + } + + if (!empty($object['_raw']) && $threshold > 0 && strlen($object['_raw']) <= $threshold) { + $data['_raw'] = $object['_raw']; + } + + $sql_data['data'] = json_encode(rcube_charset::clean($data)); + + return $sql_data; + } + /** * Helper method to turn stored cache data into a valid storage object */ - protected function _unserialize($sql_arr, $noread = false) + protected function _unserialize($sql_arr, $noread = false, $fast_mode = false) { - if ($sql_arr['fast-mode'] && !empty($sql_arr['data']) && ($object = json_decode($sql_arr['data'], true))) { + if (!empty($sql_arr['data'])) { + if ($object = json_decode($sql_arr['data'], true)) { + $object['_type'] = $sql_arr['type'] ?: $this->folder->type; + $object['uid'] = $sql_arr['uid']; + $object['etag'] = $sql_arr['etag']; + } + } + + if (!empty($fast_mode) && !empty($object)) { foreach ($this->data_props as $prop) { if (isset($object[$prop]) && is_array($object[$prop]) && $object[$prop]['cl'] == 'DateTime') { $object[$prop] = new DateTime($object[$prop]['dt'], new DateTimeZone($object[$prop]['tz'])); @@ -627,11 +677,17 @@ class kolab_storage_dav_cache extends kolab_storage_cache $object['changed'] = new DateTime($sql_arr['changed']); } - $object['_type'] = $sql_arr['type'] ?: $this->folder->type; - $object['uid'] = $sql_arr['uid']; - $object['etag'] = $sql_arr['etag']; + unset($object['_raw']); } else if ($noread) { + // We have the raw content already, parse it + if (!empty($object['_raw'])) { + $object['data'] = $object['_raw']; + if ($object = $this->folder->from_dav($object)) { + return $object; + } + } + return null; } else { diff --git a/plugins/libkolab/lib/kolab_storage_dav_folder.php b/plugins/libkolab/lib/kolab_storage_dav_folder.php index 118c7b91..e73a6882 100644 --- a/plugins/libkolab/lib/kolab_storage_dav_folder.php +++ b/plugins/libkolab/lib/kolab_storage_dav_folder.php @@ -422,8 +422,10 @@ class kolab_storage_dav_folder extends kolab_storage_folder if ($result !== false) { // insert/update object in the cache $object['etag'] = $result; + $object['_raw'] = $content; $this->cache->save($object, $uid); $result = true; + unset($object['_raw']); } } @@ -584,8 +586,8 @@ class kolab_storage_dav_folder extends kolab_storage_folder } $result['etag'] = $object['etag']; - $result['href'] = $object['href']; - $result['uid'] = $object['uid'] ?: $result['uid']; + $result['href'] = !empty($object['href']) ? $object['href'] : null; + $result['uid'] = !empty($object['uid']) ? $object['uid'] : $result['uid']; return $result; }