You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/12/23 04:21:19 UTC

[1/4] incubator-beam git commit: Decreasing the number of copies of things in scope for reduced peak memory utilization in Avro and Pickler.

Repository: incubator-beam
Updated Branches:
  refs/heads/python-sdk a46091a79 -> 82d7f0f77


Decreasing the number of copies of things in scope for reduced peak
memory utilization in Avro and Pickler.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bbcdb12f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bbcdb12f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bbcdb12f

Branch: refs/heads/python-sdk
Commit: bbcdb12fbfeac54158ff065ffb0b95fee044d935
Parents: a46091a
Author: Gus Katsiapis <ka...@katsiapis-linux.mtv.corp.google.com>
Authored: Tue Dec 20 19:26:11 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:20:59 2016 -0800

----------------------------------------------------------------------
 sdks/python/apache_beam/internal/pickler.py | 15 +++++++++++++--
 sdks/python/apache_beam/io/avroio.py        | 20 +++++++++++---------
 2 files changed, 24 insertions(+), 11 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bbcdb12f/sdks/python/apache_beam/internal/pickler.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index d39a497..e995caa 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -189,11 +189,22 @@ def dumps(o):
     s = dill.dumps(o)
   finally:
     dill.dill._trace(False)  # pylint: disable=protected-access
-  return base64.b64encode(zlib.compress(s))
+
+  # Compress as compactly as possible to decrease peak memory usage (of multiple
+  # in-memory copies). Also use a separate statement in order to decrease peak
+  # memory usage (by allowing unneeded data to go out of scope).
+  s = zlib.compress(s, 9)
+
+  return base64.b64encode(s)
 
 
 def loads(encoded):
-  s = zlib.decompress(base64.b64decode(encoded))
+  s = base64.b64decode(encoded)
+
+  # Separate statement in order to decrease peak memory usage (by allowing
+  # unneeded data to go out of scope).
+  s = zlib.decompress(s)
+
   try:
     return dill.loads(s)
   except Exception:          # pylint: disable=broad-except

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bbcdb12f/sdks/python/apache_beam/io/avroio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index 3663bdb..b08bc83 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -173,20 +173,23 @@ class _AvroBlock(object):
   """Represents a block of an Avro file."""
 
   def __init__(self, block_bytes, num_records, codec, schema_string):
-    self._block_bytes = block_bytes
+    # Decompress data early on (if needed) and thus decrease the number of
+    # parallel copies of the data in memory at any given in time during
+    # block iteration.
+    self._block_bytes = self._decompress_bytes(block_bytes, codec)
     self._num_records = num_records
-    self._codec = codec
     self._schema = schema.parse(schema_string)
 
-  def _decompress_bytes(self, data):
-    if self._codec == 'null':
+  @staticmethod
+  def _decompress_bytes(data, codec):
+    if codec == 'null':
       return data
-    elif self._codec == 'deflate':
+    elif codec == 'deflate':
       # zlib.MAX_WBITS is the window size. '-' sign indicates that this is
       # raw data (without headers). See zlib and Avro documentations for more
       # details.
       return zlib.decompress(data, -zlib.MAX_WBITS)
-    elif self._codec == 'snappy':
+    elif codec == 'snappy':
       # Snappy is an optional avro codec.
       # See Snappy and Avro documentation for more details.
       try:
@@ -199,14 +202,13 @@ class _AvroBlock(object):
       avroio.BinaryDecoder(StringIO.StringIO(data[-4:])).check_crc32(result)
       return result
     else:
-      raise ValueError('Unknown codec: %r', self._codec)
+      raise ValueError('Unknown codec: %r', codec)
 
   def num_records(self):
     return self._num_records
 
   def records(self):
-    decompressed_bytes = self._decompress_bytes(self._block_bytes)
-    decoder = avroio.BinaryDecoder(StringIO.StringIO(decompressed_bytes))
+    decoder = avroio.BinaryDecoder(StringIO.StringIO(self._block_bytes))
     reader = avroio.DatumReader(
         writers_schema=self._schema, readers_schema=self._schema)
 


[4/4] incubator-beam git commit: Closes #1672

Posted by ro...@apache.org.
Closes #1672


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/82d7f0f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/82d7f0f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/82d7f0f7

Branch: refs/heads/python-sdk
Commit: 82d7f0f775e591acc405f8660c7b48e6638ce049
Parents: a46091a 26bb459
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Thu Dec 22 20:21:01 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:21:01 2016 -0800

----------------------------------------------------------------------
 sdks/python/apache_beam/internal/pickler.py | 15 +++++++++++++--
 sdks/python/apache_beam/io/avroio.py        | 21 ++++++++++++---------
 2 files changed, 25 insertions(+), 11 deletions(-)
----------------------------------------------------------------------



[2/4] incubator-beam git commit: Instance variable rename.

Posted by ro...@apache.org.
Instance variable rename.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e0de5ab1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e0de5ab1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e0de5ab1

Branch: refs/heads/python-sdk
Commit: e0de5ab1dc706a327f914cf8ad1e1666e6289915
Parents: bbcdb12
Author: Gus Katsiapis <ka...@katsiapis-linux.mtv.corp.google.com>
Authored: Wed Dec 21 15:55:18 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:21:00 2016 -0800

----------------------------------------------------------------------
 sdks/python/apache_beam/io/avroio.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e0de5ab1/sdks/python/apache_beam/io/avroio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index b08bc83..75a9f2a 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -176,7 +176,7 @@ class _AvroBlock(object):
     # Decompress data early on (if needed) and thus decrease the number of
     # parallel copies of the data in memory at any given in time during
     # block iteration.
-    self._block_bytes = self._decompress_bytes(block_bytes, codec)
+    self._decompressed_block_bytes = self._decompress_bytes(block_bytes, codec)
     self._num_records = num_records
     self._schema = schema.parse(schema_string)
 
@@ -208,7 +208,8 @@ class _AvroBlock(object):
     return self._num_records
 
   def records(self):
-    decoder = avroio.BinaryDecoder(StringIO.StringIO(self._block_bytes))
+    decoder = avroio.BinaryDecoder(
+        StringIO.StringIO(self._decompressed_block_bytes))
     reader = avroio.DatumReader(
         writers_schema=self._schema, readers_schema=self._schema)
 


[3/4] incubator-beam git commit: Using the "del" keyword to more explicitly document the memory optimization.

Posted by ro...@apache.org.
Using the "del" keyword to more explicitly document the memory
optimization.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/26bb459c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/26bb459c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/26bb459c

Branch: refs/heads/python-sdk
Commit: 26bb459c9f5896c0ee1e4e9f00f00540955b1b3f
Parents: e0de5ab
Author: Gus Katsiapis <ka...@katsiapis-linux.mtv.corp.google.com>
Authored: Thu Dec 22 16:02:26 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:21:00 2016 -0800

----------------------------------------------------------------------
 sdks/python/apache_beam/internal/pickler.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/26bb459c/sdks/python/apache_beam/internal/pickler.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index e995caa..67f9fc3 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -191,19 +191,19 @@ def dumps(o):
     dill.dill._trace(False)  # pylint: disable=protected-access
 
   # Compress as compactly as possible to decrease peak memory usage (of multiple
-  # in-memory copies). Also use a separate statement in order to decrease peak
-  # memory usage (by allowing unneeded data to go out of scope).
-  s = zlib.compress(s, 9)
+  # in-memory copies) and free up some possibly large and no-longer-needed
+  # memory.
+  c = zlib.compress(s, 9)
+  del s
 
-  return base64.b64encode(s)
+  return base64.b64encode(c)
 
 
 def loads(encoded):
-  s = base64.b64decode(encoded)
+  c = base64.b64decode(encoded)
 
-  # Separate statement in order to decrease peak memory usage (by allowing
-  # unneeded data to go out of scope).
-  s = zlib.decompress(s)
+  s = zlib.decompress(c)
+  del c  # Free up some possibly large and no-longer-needed memory.
 
   try:
     return dill.loads(s)