You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2019/08/31 00:32:35 UTC
[beam] branch master updated: [BEAM-7389] Update to use
util.ToString transform
This is an automated email from the ASF dual-hosted git repository.
altay pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 0c61e3c [BEAM-7389] Update to use util.ToString transform
new 445d6ba Merge pull request #9433 from davidcavazos/tostring-sample
0c61e3c is described below
commit 0c61e3ceeb574a80da76b98ebdd6ce6304fad3cd
Author: David Cavazos <dc...@google.com>
AuthorDate: Mon Aug 26 13:10:01 2019 -0700
[BEAM-7389] Update to use util.ToString transform
---
.../snippets/transforms/element_wise/to_string.py | 62 +++++++++++++---
.../transforms/element_wise/to_string_test.py | 82 +++++++++++++++++-----
2 files changed, 119 insertions(+), 25 deletions(-)
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string.py b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string.py
index 7a64bfa..9edda77 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string.py
@@ -20,23 +20,67 @@ from __future__ import absolute_import
from __future__ import print_function
-def to_string(test=None):
- # [START to_string]
+def to_string_kvs(test=None):
+ # [START to_string_kvs]
import apache_beam as beam
with beam.Pipeline() as pipeline:
plants = (
pipeline
| 'Garden plants' >> beam.Create([
- ('Strawberry', 'perennial'),
- ('Carrot', 'biennial'),
- ('Eggplant', 'perennial'),
- ('Tomato', 'annual'),
- ('Potato', 'perennial'),
+ ('🍓', 'Strawberry'),
+ ('🥕', 'Carrot'),
+ ('🍆', 'Eggplant'),
+ ('🍅', 'Tomato'),
+ ('🥔', 'Potato'),
])
- | 'To string' >> beam.Map(lambda plant: str(plant))
+ | 'To string' >> beam.ToString.Kvs()
| beam.Map(print)
)
- # [END to_string]
+ # [END to_string_kvs]
if test:
test(plants)
+
+
+def to_string_element(test=None):
+ # [START to_string_element]
+ import apache_beam as beam
+
+ with beam.Pipeline() as pipeline:
+ plant_lists = (
+ pipeline
+ | 'Garden plants' >> beam.Create([
+ ['🍓', 'Strawberry', 'perennial'],
+ ['🥕', 'Carrot', 'biennial'],
+ ['🍆', 'Eggplant', 'perennial'],
+ ['🍅', 'Tomato', 'annual'],
+ ['🥔', 'Potato', 'perennial'],
+ ])
+ | 'To string' >> beam.ToString.Element()
+ | beam.Map(print)
+ )
+ # [END to_string_element]
+ if test:
+ test(plant_lists)
+
+
+def to_string_iterables(test=None):
+ # [START to_string_iterables]
+ import apache_beam as beam
+
+ with beam.Pipeline() as pipeline:
+ plants_csv = (
+ pipeline
+ | 'Garden plants' >> beam.Create([
+ ['🍓', 'Strawberry', 'perennial'],
+ ['🥕', 'Carrot', 'biennial'],
+ ['🍆', 'Eggplant', 'perennial'],
+ ['🍅', 'Tomato', 'annual'],
+ ['🥔', 'Potato', 'perennial'],
+ ])
+ | 'To string' >> beam.ToString.Iterables()
+ | beam.Map(print)
+ )
+ # [END to_string_iterables]
+ if test:
+ test(plants_csv)
diff --git a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string_test.py b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string_test.py
index 5fcf83c..a5ff5b2 100644
--- a/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string_test.py
+++ b/sdks/python/apache_beam/examples/snippets/transforms/element_wise/to_string_test.py
@@ -19,36 +19,86 @@
from __future__ import absolute_import
from __future__ import print_function
+import sys
import unittest
import mock
-from apache_beam.examples.snippets.transforms.element_wise.to_string import *
from apache_beam.testing.test_pipeline import TestPipeline
from apache_beam.testing.util import assert_that
from apache_beam.testing.util import equal_to
+from . import to_string
+
+
+def check_plants(actual):
+ # [START plants]
+ plants = [
+ '🍓,Strawberry',
+ '🥕,Carrot',
+ '🍆,Eggplant',
+ '🍅,Tomato',
+ '🥔,Potato',
+ ]
+ # [END plants]
+ assert_that(actual, equal_to(plants))
+
+
+def check_plant_lists(actual):
+ # [START plant_lists]
+ plant_lists = [
+ "['🍓', 'Strawberry', 'perennial']",
+ "['🥕', 'Carrot', 'biennial']",
+ "['🍆', 'Eggplant', 'perennial']",
+ "['🍅', 'Tomato', 'annual']",
+ "['🥔', 'Potato', 'perennial']",
+ ]
+ # [END plant_lists]
+
+ # Some unicode characters become escaped with double backslashes.
+ import apache_beam as beam
+
+ def normalize_escaping(elem):
+ # In Python 2 all utf-8 characters are escaped with double backslashes.
+ # TODO: Remove this after Python 2 deprecation.
+ # https://issues.apache.org/jira/browse/BEAM-8124
+ if sys.version_info.major == 2:
+ return elem.decode('string-escape')
+
+ # In Python 3.5 some utf-8 characters are escaped with double backslashes.
+ if '\\' in elem:
+ return bytes(elem, 'utf-8').decode('unicode-escape')
+ return elem
+ actual = actual | beam.Map(normalize_escaping)
+ assert_that(actual, equal_to(plant_lists))
+
+
+def check_plants_csv(actual):
+ # [START plants_csv]
+ plants_objects = [
+ '🍓,Strawberry,perennial',
+ '🥕,Carrot,biennial',
+ '🍆,Eggplant,perennial',
+ '🍅,Tomato,annual',
+ '🥔,Potato,perennial',
+ ]
+ # [END plants_csv]
+ assert_that(actual, equal_to(plants_objects))
+
@mock.patch('apache_beam.Pipeline', TestPipeline)
# pylint: disable=line-too-long
@mock.patch('apache_beam.examples.snippets.transforms.element_wise.to_string.print', lambda elem: elem)
# pylint: enable=line-too-long
class ToStringTest(unittest.TestCase):
- def __init__(self, methodName):
- super(ToStringTest, self).__init__(methodName)
- # [START plants]
- plants = [
- "('Strawberry', 'perennial')",
- "('Carrot', 'biennial')",
- "('Eggplant', 'perennial')",
- "('Tomato', 'annual')",
- "('Potato', 'perennial')",
- ]
- # [END plants]
- self.plants_test = lambda actual: assert_that(actual, equal_to(plants))
-
- def test_to_string(self):
- to_string(self.plants_test)
+ def test_to_string_kvs(self):
+ to_string.to_string_kvs(check_plants)
+
+ def test_to_string_element(self):
+ to_string.to_string_element(check_plant_lists)
+
+ def test_to_string_iterables(self):
+ to_string.to_string_iterables(check_plants_csv)
if __name__ == '__main__':