You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@libcloud.apache.org by to...@apache.org on 2023/01/28 14:19:04 UTC
[libcloud] branch trunk updated (4c2ee783e -> d8f0d3a9f)
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a change to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
from 4c2ee783e Merge branch 'deps-updates-2' into trunk
new a57d9d2bf Clarify changelog item.
new d8f0d3a9f Add additional corectness tests for read_in_chunks function.
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
CHANGES.rst | 3 ++
libcloud/test/test_utils.py | 69 +++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 72 insertions(+)
[libcloud] 02/02: Add additional corectness tests for read_in_chunks function.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit d8f0d3a9fbd8c1edd2d116ccfcaff1471fdb324e
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Sat Jan 28 15:15:55 2023 +0100
Add additional corectness tests for read_in_chunks function.
---
libcloud/test/test_utils.py | 69 +++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 69 insertions(+)
diff --git a/libcloud/test/test_utils.py b/libcloud/test/test_utils.py
index 395bfea47..69d37059b 100644
--- a/libcloud/test/test_utils.py
+++ b/libcloud/test/test_utils.py
@@ -14,7 +14,9 @@
# limitations under the License.
import sys
+import random
import socket
+import string
import os.path
import platform
import unittest
@@ -267,6 +269,73 @@ class TestUtils(unittest.TestCase):
self.assertEqual(index, 548)
+ def test_read_in_chunks_corectness(self):
+ data = "".join(
+ random.choice(string.ascii_lowercase) for i in range(5 * 1024 * 1024)
+ ).encode("utf-8")
+
+ # fill_size = False
+ chunk_size = None
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=False
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=False
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 2 * 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=False
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 5 * 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=False
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 10 * 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=False
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ # fill_size = True
+ chunk_size = None
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=True
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=True
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 2 * 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=True
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 5 * 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=True
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
+ chunk_size = 10 * 1024 * 1024
+ iterator = libcloud.utils.files.read_in_chunks(
+ iter([data]), chunk_size=chunk_size, fill_size=True
+ )
+ self.assertEqual(data, libcloud.utils.files.exhaust_iterator(iterator))
+
def test_exhaust_iterator(self):
def iterator_func():
for x in range(0, 1000):
[libcloud] 01/02: Clarify changelog item.
Posted by to...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tomaz pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/libcloud.git
commit a57d9d2bf6c7440c914ee806d973be93bf0564bb
Author: Tomaz Muraus <to...@tomaz.me>
AuthorDate: Sat Jan 28 15:15:35 2023 +0100
Clarify changelog item.
---
CHANGES.rst | 3 +++
1 file changed, 3 insertions(+)
diff --git a/CHANGES.rst b/CHANGES.rst
index 970d5f635..93418617f 100644
--- a/CHANGES.rst
+++ b/CHANGES.rst
@@ -11,6 +11,9 @@ Storage
This should result in large performance speedups and lower memory usage when
uploading or downloading a large file with a mismatching chunk size.
+
+ Keep in mind that this only affects code paths where the function is called
+ with ``fill_size=True`` argument (such as in the S3 driver, etc).
(GITHUB-1847)
[Tobias Biester - @Tobi995]