You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2019/06/12 18:12:04 UTC
[beam] branch master updated: [BEAM-7510] Fixing fileio tests
checking JSON serialization (flaky) (#8795)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 2798d62 [BEAM-7510] Fixing fileio tests checking JSON serialization (flaky) (#8795)
2798d62 is described below
commit 2798d626507c7f5c8699096b3a188200ac64a299
Author: Pablo <pa...@users.noreply.github.com>
AuthorDate: Wed Jun 12 11:11:49 2019 -0700
[BEAM-7510] Fixing fileio tests checking JSON serialization (flaky) (#8795)
* Fixing test
* Unskip test
---
sdks/python/apache_beam/io/fileio_test.py | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index 98d6033..f87aa2f 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -296,10 +296,11 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
result = (p
| fileio.MatchFiles(FileSystems.join(dir, '*'))
| fileio.ReadMatches()
- | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n')))
+ | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n'))
+ | beam.Map(json.loads))
assert_that(result,
- equal_to([json.dumps(row) for row in self.SIMPLE_COLLECTION]))
+ equal_to([row for row in self.SIMPLE_COLLECTION]))
def test_write_to_different_file_types_some_spilling(self):
@@ -321,7 +322,8 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
cncf_res = (p
| fileio.MatchFiles(FileSystems.join(dir, 'cncf*'))
| fileio.ReadMatches()
- | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n')))
+ | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n'))
+ | beam.Map(json.loads))
apache_res = (p
| "MatchApache" >> fileio.MatchFiles(
@@ -331,7 +333,7 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
lambda rf: csv.reader(_get_file_reader(rf))))
assert_that(cncf_res,
- equal_to([json.dumps(row)
+ equal_to([row
for row in self.SIMPLE_COLLECTION
if row['foundation'] == 'cncf']),
label='verifyCNCF')
@@ -379,7 +381,6 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
found_files = [m.path for m in leftovers[0].metadata_list]
self.assertListEqual(found_files, [f2])
- @unittest.skip('BEAM-7510 - test is flaky')
def test_write_to_different_file_types(self):
dir = self._new_tempdir()
@@ -399,7 +400,8 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
cncf_res = (p
| fileio.MatchFiles(FileSystems.join(dir, 'cncf*'))
| fileio.ReadMatches()
- | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n')))
+ | beam.FlatMap(lambda f: f.read_utf8().strip().split('\n'))
+ | beam.Map(json.loads))
apache_res = (p
| "MatchApache" >> fileio.MatchFiles(
@@ -409,7 +411,7 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
lambda rf: csv.reader(_get_file_reader(rf))))
assert_that(cncf_res,
- equal_to([json.dumps(row)
+ equal_to([row
for row in self.SIMPLE_COLLECTION
if row['foundation'] == 'cncf']),
label='verifyCNCF')