You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/12/23 04:21:19 UTC
[1/4] incubator-beam git commit: Decreasing the number of copies of
things in scope for reduced peak memory utilization in Avro and Pickler.
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk a46091a79 -> 82d7f0f77
Decreasing the number of copies of things in scope for reduced peak
memory utilization in Avro and Pickler.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bbcdb12f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bbcdb12f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bbcdb12f
Branch: refs/heads/python-sdk
Commit: bbcdb12fbfeac54158ff065ffb0b95fee044d935
Parents: a46091a
Author: Gus Katsiapis <ka...@katsiapis-linux.mtv.corp.google.com>
Authored: Tue Dec 20 19:26:11 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:20:59 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/internal/pickler.py | 15 +++++++++++++--
sdks/python/apache_beam/io/avroio.py | 20 +++++++++++---------
2 files changed, 24 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bbcdb12f/sdks/python/apache_beam/internal/pickler.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index d39a497..e995caa 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -189,11 +189,22 @@ def dumps(o):
s = dill.dumps(o)
finally:
dill.dill._trace(False) # pylint: disable=protected-access
- return base64.b64encode(zlib.compress(s))
+
+ # Compress as compactly as possible to decrease peak memory usage (of multiple
+ # in-memory copies). Also use a separate statement in order to decrease peak
+ # memory usage (by allowing unneeded data to go out of scope).
+ s = zlib.compress(s, 9)
+
+ return base64.b64encode(s)
def loads(encoded):
- s = zlib.decompress(base64.b64decode(encoded))
+ s = base64.b64decode(encoded)
+
+ # Separate statement in order to decrease peak memory usage (by allowing
+ # unneeded data to go out of scope).
+ s = zlib.decompress(s)
+
try:
return dill.loads(s)
except Exception: # pylint: disable=broad-except
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/bbcdb12f/sdks/python/apache_beam/io/avroio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index 3663bdb..b08bc83 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -173,20 +173,23 @@ class _AvroBlock(object):
"""Represents a block of an Avro file."""
def __init__(self, block_bytes, num_records, codec, schema_string):
- self._block_bytes = block_bytes
+ # Decompress data early on (if needed) and thus decrease the number of
+ # parallel copies of the data in memory at any given in time during
+ # block iteration.
+ self._block_bytes = self._decompress_bytes(block_bytes, codec)
self._num_records = num_records
- self._codec = codec
self._schema = schema.parse(schema_string)
- def _decompress_bytes(self, data):
- if self._codec == 'null':
+ @staticmethod
+ def _decompress_bytes(data, codec):
+ if codec == 'null':
return data
- elif self._codec == 'deflate':
+ elif codec == 'deflate':
# zlib.MAX_WBITS is the window size. '-' sign indicates that this is
# raw data (without headers). See zlib and Avro documentations for more
# details.
return zlib.decompress(data, -zlib.MAX_WBITS)
- elif self._codec == 'snappy':
+ elif codec == 'snappy':
# Snappy is an optional avro codec.
# See Snappy and Avro documentation for more details.
try:
@@ -199,14 +202,13 @@ class _AvroBlock(object):
avroio.BinaryDecoder(StringIO.StringIO(data[-4:])).check_crc32(result)
return result
else:
- raise ValueError('Unknown codec: %r', self._codec)
+ raise ValueError('Unknown codec: %r', codec)
def num_records(self):
return self._num_records
def records(self):
- decompressed_bytes = self._decompress_bytes(self._block_bytes)
- decoder = avroio.BinaryDecoder(StringIO.StringIO(decompressed_bytes))
+ decoder = avroio.BinaryDecoder(StringIO.StringIO(self._block_bytes))
reader = avroio.DatumReader(
writers_schema=self._schema, readers_schema=self._schema)
[4/4] incubator-beam git commit: Closes #1672
Posted by ro...@apache.org.
Closes #1672
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/82d7f0f7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/82d7f0f7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/82d7f0f7
Branch: refs/heads/python-sdk
Commit: 82d7f0f775e591acc405f8660c7b48e6638ce049
Parents: a46091a 26bb459
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Thu Dec 22 20:21:01 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:21:01 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/internal/pickler.py | 15 +++++++++++++--
sdks/python/apache_beam/io/avroio.py | 21 ++++++++++++---------
2 files changed, 25 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
[2/4] incubator-beam git commit: Instance variable rename.
Posted by ro...@apache.org.
Instance variable rename.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/e0de5ab1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/e0de5ab1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/e0de5ab1
Branch: refs/heads/python-sdk
Commit: e0de5ab1dc706a327f914cf8ad1e1666e6289915
Parents: bbcdb12
Author: Gus Katsiapis <ka...@katsiapis-linux.mtv.corp.google.com>
Authored: Wed Dec 21 15:55:18 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:21:00 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/io/avroio.py | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/e0de5ab1/sdks/python/apache_beam/io/avroio.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/avroio.py b/sdks/python/apache_beam/io/avroio.py
index b08bc83..75a9f2a 100644
--- a/sdks/python/apache_beam/io/avroio.py
+++ b/sdks/python/apache_beam/io/avroio.py
@@ -176,7 +176,7 @@ class _AvroBlock(object):
# Decompress data early on (if needed) and thus decrease the number of
# parallel copies of the data in memory at any given in time during
# block iteration.
- self._block_bytes = self._decompress_bytes(block_bytes, codec)
+ self._decompressed_block_bytes = self._decompress_bytes(block_bytes, codec)
self._num_records = num_records
self._schema = schema.parse(schema_string)
@@ -208,7 +208,8 @@ class _AvroBlock(object):
return self._num_records
def records(self):
- decoder = avroio.BinaryDecoder(StringIO.StringIO(self._block_bytes))
+ decoder = avroio.BinaryDecoder(
+ StringIO.StringIO(self._decompressed_block_bytes))
reader = avroio.DatumReader(
writers_schema=self._schema, readers_schema=self._schema)
[3/4] incubator-beam git commit: Using the "del" keyword to more
explicitly document the memory optimization.
Posted by ro...@apache.org.
Using the "del" keyword to more explicitly document the memory
optimization.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/26bb459c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/26bb459c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/26bb459c
Branch: refs/heads/python-sdk
Commit: 26bb459c9f5896c0ee1e4e9f00f00540955b1b3f
Parents: e0de5ab
Author: Gus Katsiapis <ka...@katsiapis-linux.mtv.corp.google.com>
Authored: Thu Dec 22 16:02:26 2016 -0800
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Thu Dec 22 20:21:00 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/internal/pickler.py | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/26bb459c/sdks/python/apache_beam/internal/pickler.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/pickler.py b/sdks/python/apache_beam/internal/pickler.py
index e995caa..67f9fc3 100644
--- a/sdks/python/apache_beam/internal/pickler.py
+++ b/sdks/python/apache_beam/internal/pickler.py
@@ -191,19 +191,19 @@ def dumps(o):
dill.dill._trace(False) # pylint: disable=protected-access
# Compress as compactly as possible to decrease peak memory usage (of multiple
- # in-memory copies). Also use a separate statement in order to decrease peak
- # memory usage (by allowing unneeded data to go out of scope).
- s = zlib.compress(s, 9)
+ # in-memory copies) and free up some possibly large and no-longer-needed
+ # memory.
+ c = zlib.compress(s, 9)
+ del s
- return base64.b64encode(s)
+ return base64.b64encode(c)
def loads(encoded):
- s = base64.b64decode(encoded)
+ c = base64.b64decode(encoded)
- # Separate statement in order to decrease peak memory usage (by allowing
- # unneeded data to go out of scope).
- s = zlib.decompress(s)
+ s = zlib.decompress(c)
+ del c # Free up some possibly large and no-longer-needed memory.
try:
return dill.loads(s)