You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ro...@apache.org on 2016/10/18 17:54:15 UTC
[1/3] incubator-beam git commit: Fix tests unnecessarily using
windowed side inputs
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk 24b7bcc26 -> afa5ebc70
Fix tests unnecessarily using windowed side inputs
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/5d9506df
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/5d9506df
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/5d9506df
Branch: refs/heads/python-sdk
Commit: 5d9506df44e5b2cd703702d4188c25c5bbb16e7f
Parents: 552f6d7
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Fri Oct 7 16:17:47 2016 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Tue Oct 18 10:48:16 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/pipeline_test.py | 1 +
sdks/python/apache_beam/transforms/util.py | 7 +++++--
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5d9506df/sdks/python/apache_beam/pipeline_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/pipeline_test.py b/sdks/python/apache_beam/pipeline_test.py
index 8c1b3ba..dedd732 100644
--- a/sdks/python/apache_beam/pipeline_test.py
+++ b/sdks/python/apache_beam/pipeline_test.py
@@ -239,6 +239,7 @@ class PipelineTest(unittest.TestCase):
'oom:combine/GroupByKey/group_by_key': 1,
('oom:check', None): 1,
'assert_that/singleton': 1,
+ ('assert_that/WindowInto', None): 1,
('assert_that/Map(match)', None): 1,
('oom:combine/GroupByKey/group_by_window', None): 1,
('oom:combine/Combine/ParDo(CombineValuesDoFn)', None): 1})
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/5d9506df/sdks/python/apache_beam/transforms/util.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py
index 4564cf9..aeac0d9 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -21,6 +21,8 @@
from __future__ import absolute_import
from apache_beam.pvalue import AsIter as AllOf
+from apache_beam.transforms import core
+from apache_beam.transforms import window
from apache_beam.transforms.core import CombinePerKey, Create, Flatten, GroupByKey, Map
from apache_beam.transforms.ptransform import PTransform
from apache_beam.transforms.ptransform import ptransform_fn
@@ -220,8 +222,9 @@ def assert_that(actual, matcher, label='assert_that'):
class AssertThat(PTransform):
def apply(self, pipeline):
- return pipeline | 'singleton' >> Create([None]) | Map(match,
- AllOf(actual))
+ return pipeline | 'singleton' >> Create([None]) | Map(
+ match,
+ AllOf(actual | core.WindowInto(window.GlobalWindows())))
def default_label(self):
return label
[3/3] incubator-beam git commit: Closes #1070
Posted by ro...@apache.org.
Closes #1070
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/afa5ebc7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/afa5ebc7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/afa5ebc7
Branch: refs/heads/python-sdk
Commit: afa5ebc70aef3a7ad20b4e70a62e23a90bd7547a
Parents: 24b7bcc 5d9506d
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Tue Oct 18 10:53:59 2016 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Tue Oct 18 10:53:59 2016 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/pipeline_test.py | 1 +
.../python/apache_beam/transforms/sideinputs.py | 4 ++
.../apache_beam/transforms/sideinputs_test.py | 40 ++++++++++++++++++++
sdks/python/apache_beam/transforms/util.py | 7 +++-
4 files changed, 50 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
[2/3] incubator-beam git commit: Dissallow (unimplemented) windowed
side inputs.
Posted by ro...@apache.org.
Dissallow (unimplemented) windowed side inputs.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/552f6d7b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/552f6d7b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/552f6d7b
Branch: refs/heads/python-sdk
Commit: 552f6d7baa6e6205b290d21e642911a04ca259ec
Parents: 24b7bcc
Author: Robert Bradshaw <ro...@gmail.com>
Authored: Fri Oct 7 15:28:50 2016 -0700
Committer: Robert Bradshaw <ro...@gmail.com>
Committed: Tue Oct 18 10:48:16 2016 -0700
----------------------------------------------------------------------
.../python/apache_beam/transforms/sideinputs.py | 4 ++
.../apache_beam/transforms/sideinputs_test.py | 40 ++++++++++++++++++++
2 files changed, 44 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/552f6d7b/sdks/python/apache_beam/transforms/sideinputs.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/sideinputs.py b/sdks/python/apache_beam/transforms/sideinputs.py
index 6484a7c..6c698da 100644
--- a/sdks/python/apache_beam/transforms/sideinputs.py
+++ b/sdks/python/apache_beam/transforms/sideinputs.py
@@ -50,6 +50,10 @@ class CreatePCollectionView(PTransform):
return input_type
def apply(self, pcoll):
+ if not pcoll.windowing.is_default():
+ raise ValueError(
+ "Side inputs only supported for global windows, default triggering. "
+ "Found %s" % pcoll.windowing)
return self.view
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/552f6d7b/sdks/python/apache_beam/transforms/sideinputs_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py
new file mode 100644
index 0000000..8e292e3
--- /dev/null
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -0,0 +1,40 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""Unit tests for side inputs."""
+
+import logging
+import unittest
+
+import apache_beam as beam
+from apache_beam.transforms import window
+
+
+class SideInputsTest(unittest.TestCase):
+
+ # TODO(BEAM-733): Actually support this.
+ def test_no_sideinput_windowing(self):
+ p = beam.Pipeline('DirectPipelineRunner')
+ pc = p | beam.Create([0, 1]) | beam.WindowInto(window.FixedWindows(10))
+ with self.assertRaises(ValueError):
+ # pylint: disable=expression-not-assigned
+ pc | beam.Map(lambda x, side: None, side=beam.pvalue.AsIter(pc))
+
+
+if __name__ == '__main__':
+ logging.getLogger().setLevel(logging.DEBUG)
+ unittest.main()