You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by bc...@apache.org on 2016/11/22 00:31:55 UTC
[1/2] incubator-beam git commit: Closes #1395
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk 50afc87cd -> b4fc20e44
Closes #1395
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/b4fc20e4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/b4fc20e4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/b4fc20e4
Branch: refs/heads/python-sdk
Commit: b4fc20e4432d2b956a0333d8e0c6b9aeb472a804
Parents: 50afc87 c127f7a
Author: bchambers <bc...@google.com>
Authored: Mon Nov 21 16:21:26 2016 -0800
Committer: bchambers <bc...@google.com>
Committed: Mon Nov 21 16:21:26 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/internal/apiclient.py | 2 +-
.../apache_beam/internal/apiclient_test.py | 1 +
sdks/python/apache_beam/transforms/display.py | 27 ++++++++++++++++++++
.../apache_beam/transforms/display_test.py | 17 ++++++++++++
4 files changed, 46 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[2/2] incubator-beam git commit: Fixing error with PipelineOptions
DisplayData of lists
Posted by bc...@apache.org.
Fixing error with PipelineOptions DisplayData of lists
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/c127f7a9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/c127f7a9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/c127f7a9
Branch: refs/heads/python-sdk
Commit: c127f7a9e6f9885bc595e1462b27b6961c0f465d
Parents: 50afc87
Author: Pablo <pa...@google.com>
Authored: Mon Nov 21 11:15:37 2016 -0800
Committer: bchambers <bc...@google.com>
Committed: Mon Nov 21 16:21:26 2016 -0800
----------------------------------------------------------------------
sdks/python/apache_beam/internal/apiclient.py | 2 +-
.../apache_beam/internal/apiclient_test.py | 1 +
sdks/python/apache_beam/transforms/display.py | 27 ++++++++++++++++++++
.../apache_beam/transforms/display_test.py | 17 ++++++++++++
4 files changed, 46 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/internal/apiclient.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/apiclient.py b/sdks/python/apache_beam/internal/apiclient.py
index 8992ec3..5612631 100644
--- a/sdks/python/apache_beam/internal/apiclient.py
+++ b/sdks/python/apache_beam/internal/apiclient.py
@@ -242,7 +242,7 @@ class Environment(object):
dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(
key='options', value=to_json_value(options_dict)))
- dd = DisplayData.create_from(options)
+ dd = DisplayData.create_from_options(options)
items = [item.get_dict() for item in dd.items]
self.proto.sdkPipelineOptions.additionalProperties.append(
dataflow.Environment.SdkPipelineOptionsValue.AdditionalProperty(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/internal/apiclient_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/internal/apiclient_test.py b/sdks/python/apache_beam/internal/apiclient_test.py
index 8fddae7..66cc8db 100644
--- a/sdks/python/apache_beam/internal/apiclient_test.py
+++ b/sdks/python/apache_beam/internal/apiclient_test.py
@@ -31,5 +31,6 @@ class UtilTest(unittest.TestCase):
pipeline_options,
DataflowPipelineRunner.BATCH_ENVIRONMENT_MAJOR_VERSION)
+
if __name__ == '__main__':
unittest.main()
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/transforms/display.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/display.py b/sdks/python/apache_beam/transforms/display.py
index 365abaf..c38fd9b 100644
--- a/sdks/python/apache_beam/transforms/display.py
+++ b/sdks/python/apache_beam/transforms/display.py
@@ -108,6 +108,33 @@ class DisplayData(object):
key=key))
@classmethod
+ def create_from_options(cls, pipeline_options):
+ """ Creates DisplayData from a PipelineOptions instance.
+
+ When creating DisplayData, this method will convert the value of any
+ item of a non-supported type to its string representation.
+ The normal DisplayData.create_from method rejects those items.
+
+ Returns:
+ A DisplayData instance with populated items.
+
+ Raises:
+ ValueError: If the has_display_data argument is not an instance of
+ HasDisplayData.
+ """
+ from apache_beam.utils.options import PipelineOptions
+ if not isinstance(pipeline_options, PipelineOptions):
+ raise ValueError(
+ 'Element of class {}.{} does not subclass PipelineOptions'
+ .format(pipeline_options.__module__,
+ pipeline_options.__class__.__name__))
+
+ items = {k: (v if DisplayDataItem._get_value_type(v) is not None
+ else str(v))
+ for k, v in pipeline_options.display_data().items()}
+ return cls(pipeline_options._namespace(), items)
+
+ @classmethod
def create_from(cls, has_display_data):
""" Creates DisplayData from a HasDisplayData instance.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/c127f7a9/sdks/python/apache_beam/transforms/display_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/display_test.py b/sdks/python/apache_beam/transforms/display_test.py
index d514065..fc50abe 100644
--- a/sdks/python/apache_beam/transforms/display_test.py
+++ b/sdks/python/apache_beam/transforms/display_test.py
@@ -29,6 +29,7 @@ import apache_beam as beam
from apache_beam.transforms.display import HasDisplayData
from apache_beam.transforms.display import DisplayData
from apache_beam.transforms.display import DisplayDataItem
+from apache_beam.utils.options import PipelineOptions
class DisplayDataItemMatcher(BaseMatcher):
@@ -105,6 +106,22 @@ class DisplayDataTest(unittest.TestCase):
self.assertTrue(isinstance(display_dofn, HasDisplayData))
self.assertEqual(display_dofn.display_data(), {})
+ def test_unsupported_type_display_data(self):
+ class MyDisplayComponent(HasDisplayData):
+ def display_data(self):
+ return {'item_key': 'item_value'}
+
+ with self.assertRaises(ValueError):
+ DisplayData.create_from_options(MyDisplayComponent())
+
+ def test_create_list_display_data(self):
+ flags = ['--extra_package', 'package1', '--extra_package', 'package2']
+ pipeline_options = PipelineOptions(flags=flags)
+ items = DisplayData.create_from_options(pipeline_options).items
+ hc.assert_that(items, hc.contains_inanyorder(
+ DisplayDataItemMatcher('extra_packages',
+ str(['package1', 'package2']))))
+
def test_base_cases(self):
""" Tests basic display data cases (key:value, key:dict)
It does not test subcomponent inclusion