You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/10/31 16:06:34 UTC

[2/2] incubator-beam git commit: Adds IterableCoder, fast coding for sets, booleans.

Adds IterableCoder, fast coding for sets, booleans.


Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/9dd16455
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/9dd16455
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/9dd16455

Branch: refs/heads/python-sdk
Commit: 9dd16455438a8321fdb5a35dc91a097c66e35fcf
Parents: ef01140
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Fri Oct 28 13:44:06 2016 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Mon Oct 31 09:05:55 2016 -0700

----------------------------------------------------------------------
 sdks/python/apache_beam/coders/coder_impl.pxd   |  8 ++++--
 sdks/python/apache_beam/coders/coder_impl.py    | 27 ++++++++++++++++----
 sdks/python/apache_beam/coders/coders.py        | 23 +++++++++++++++++
 .../apache_beam/coders/coders_test_common.py    | 10 ++++++++
 4 files changed, 61 insertions(+), 7 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9dd16455/sdks/python/apache_beam/coders/coder_impl.pxd
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coder_impl.pxd b/sdks/python/apache_beam/coders/coder_impl.pxd
index 2ca6743..194cbb8 100644
--- a/sdks/python/apache_beam/coders/coder_impl.pxd
+++ b/sdks/python/apache_beam/coders/coder_impl.pxd
@@ -66,8 +66,8 @@ cdef class DeterministicPickleCoderImpl(CoderImpl):
 
 
 cdef object NoneType
-cdef char UNKNOWN_TYPE, NONE_TYPE, INT_TYPE, FLOAT_TYPE
-cdef char STR_TYPE, UNICODE_TYPE, LIST_TYPE, TUPLE_TYPE, DICT_TYPE
+cdef char UNKNOWN_TYPE, NONE_TYPE, INT_TYPE, FLOAT_TYPE, BOOL_TYPE
+cdef char STR_TYPE, UNICODE_TYPE, LIST_TYPE, TUPLE_TYPE, DICT_TYPE, SET_TYPE
 
 cdef class FastPrimitivesCoderImpl(StreamCoderImpl):
   cdef CoderImpl fallback_coder_impl
@@ -125,6 +125,10 @@ cdef class TupleSequenceCoderImpl(SequenceCoderImpl):
   pass
 
 
+cdef class IterableCoderImpl(SequenceCoderImpl):
+  pass
+
+
 cdef class WindowedValueCoderImpl(StreamCoderImpl):
   """A coder for windowed values."""
   cdef CoderImpl _value_coder

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9dd16455/sdks/python/apache_beam/coders/coder_impl.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coder_impl.py b/sdks/python/apache_beam/coders/coder_impl.py
index 0024fd8..606d51c 100644
--- a/sdks/python/apache_beam/coders/coder_impl.py
+++ b/sdks/python/apache_beam/coders/coder_impl.py
@@ -202,9 +202,11 @@ INT_TYPE = 1
 FLOAT_TYPE = 2
 STR_TYPE = 3
 UNICODE_TYPE = 4
+BOOL_TYPE = 9
 LIST_TYPE = 5
 TUPLE_TYPE = 6
 DICT_TYPE = 7
+SET_TYPE = 8
 
 
 class FastPrimitivesCoderImpl(StreamCoderImpl):
@@ -229,18 +231,22 @@ class FastPrimitivesCoderImpl(StreamCoderImpl):
       unicode_value = value  # for typing
       stream.write_byte(UNICODE_TYPE)
       stream.write(unicode_value.encode('utf-8'), nested)
-    elif t is list or t is tuple:
-      stream.write_byte(LIST_TYPE if t is list else TUPLE_TYPE)
+    elif t is list or t is tuple or t is set:
+      stream.write_byte(
+          LIST_TYPE if t is list else TUPLE_TYPE if t is tuple else SET_TYPE)
       stream.write_var_int64(len(value))
       for e in value:
         self.encode_to_stream(e, stream, True)
     elif t is dict:
       dict_value = value  # for typing
       stream.write_byte(DICT_TYPE)
-      stream.write_var_int64(len(value))
+      stream.write_var_int64(len(dict_value))
       for k, v in dict_value.iteritems():
         self.encode_to_stream(k, stream, True)
         self.encode_to_stream(v, stream, True)
+    elif t is bool:
+      stream.write_byte(BOOL_TYPE)
+      stream.write_byte(value)
     else:
       stream.write_byte(UNKNOWN_TYPE)
       self.fallback_coder_impl.encode_to_stream(value, stream, nested)
@@ -257,13 +263,15 @@ class FastPrimitivesCoderImpl(StreamCoderImpl):
       return stream.read_all(nested)
     elif t == UNICODE_TYPE:
       return stream.read_all(nested).decode('utf-8')
-    elif t == LIST_TYPE or t == TUPLE_TYPE:
+    elif t == LIST_TYPE or t == TUPLE_TYPE or t == SET_TYPE:
       vlen = stream.read_var_int64()
       vlist = [self.decode_from_stream(stream, True) for _ in range(vlen)]
       if t == LIST_TYPE:
         return vlist
-      else:
+      elif t == TUPLE_TYPE:
         return tuple(vlist)
+      else:
+        return set(vlist)
     elif t == DICT_TYPE:
       vlen = stream.read_var_int64()
       v = {}
@@ -271,6 +279,8 @@ class FastPrimitivesCoderImpl(StreamCoderImpl):
         k = self.decode_from_stream(stream, True)
         v[k] = self.decode_from_stream(stream, True)
       return v
+    elif t == BOOL_TYPE:
+      return not not stream.read_byte()
     else:
       return self.fallback_coder_impl.decode_from_stream(stream, nested)
 
@@ -489,6 +499,13 @@ class TupleSequenceCoderImpl(SequenceCoderImpl):
     return tuple(components)
 
 
+class IterableCoderImpl(SequenceCoderImpl):
+  """A coder for homogeneous iterable objects."""
+
+  def _construct_from_sequence(self, components):
+    return components
+
+
 class WindowedValueCoderImpl(StreamCoderImpl):
   """A coder for windowed values."""
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9dd16455/sdks/python/apache_beam/coders/coders.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index 11964b0..da06db1 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -520,6 +520,29 @@ class TupleSequenceCoder(FastCoder):
     return 'TupleSequenceCoder[%r]' % self._elem_coder
 
 
+class IterableCoder(FastCoder):
+  """Coder of iterables of homogeneous objects."""
+
+  def __init__(self, elem_coder):
+    self._elem_coder = elem_coder
+
+  def _create_impl(self):
+    return coder_impl.IterableCoderImpl(self._elem_coder.get_impl())
+
+  def is_deterministic(self):
+    return self._elem_coder.is_deterministic()
+
+  @staticmethod
+  def from_type_hint(typehint, registry):
+    return IterableCoder(registry.get_coder(typehint.inner_type))
+
+  def _get_component_coders(self):
+    return (self._elem_coder,)
+
+  def __repr__(self):
+    return 'IterableCoder[%r]' % self._elem_coder
+
+
 class WindowCoder(PickleCoder):
   """Coder for windows in windowed values."""
 

http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/9dd16455/sdks/python/apache_beam/coders/coders_test_common.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coders_test_common.py b/sdks/python/apache_beam/coders/coders_test_common.py
index dd4a873..008fa9f 100644
--- a/sdks/python/apache_beam/coders/coders_test_common.py
+++ b/sdks/python/apache_beam/coders/coders_test_common.py
@@ -124,6 +124,8 @@ class CodersTest(unittest.TestCase):
     self.check_coder(coder, (), (1, 2, 3))
     self.check_coder(coder, [], [1, 2, 3])
     self.check_coder(coder, dict(), {'a': 'b'}, {0: dict(), 1: len})
+    self.check_coder(coder, set(), {'a', 'b'})
+    self.check_coder(coder, True, False)
     self.check_coder(coder, len)
     self.check_coder(coders.TupleCoder((coder,)), ('a',), (1,))
 
@@ -193,6 +195,14 @@ class CodersTest(unittest.TestCase):
   def test_utf8_coder(self):
     self.check_coder(coders.StrUtf8Coder(), 'a', u'ab\u00FF', u'\u0101\0')
 
+  def test_iterable_coder(self):
+    self.check_coder(coders.IterableCoder(coders.VarIntCoder()),
+                     [1], [-1, 0, 100])
+    self.check_coder(
+        coders.TupleCoder((coders.VarIntCoder(),
+                           coders.IterableCoder(coders.VarIntCoder()))),
+        (1, [1, 2, 3]))
+
   def test_nested_observables(self):
     class FakeObservableIterator(observable.ObservableMixin):