You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2023/06/08 02:48:42 UTC
[beam] branch master updated: reads all files from metadata list (#27059)
This is an automated email from the ASF dual-hosted git repository.
yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 74bad1add9c reads all files from metadata list (#27059)
74bad1add9c is described below
commit 74bad1add9cb6954c5f62aff81371c327546cc89
Author: Nathaniel Young <ny...@users.noreply.github.com>
AuthorDate: Wed Jun 7 19:48:33 2023 -0700
reads all files from metadata list (#27059)
---
sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py b/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py
index 508ab159513..55b3ba1695a 100644
--- a/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py
+++ b/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py
@@ -196,11 +196,16 @@ class AvroNycTripsIT(unittest.TestCase):
avro_nyc_trips.run(test_pipeline.get_full_options_as_args(**extra_opts))
# load result avro file and compare
- metadata = FileSystems.match([f'{test_output}*'])[0].metadata_list[0]
- with FileSystems.open(metadata.path) as f:
- avro_reader = fastavro.reader(f)
+ metadata_list = FileSystems.match([f'{test_output}*'])[0].metadata_list
+ result = []
- result = sorted(avro_reader, key=lambda x: x['service'])
+ for metadata in metadata_list:
+ with FileSystems.open(metadata.path) as f:
+ avro_reader = fastavro.reader(f)
+
+ result.extend(avro_reader)
+
+ result.sort(key=lambda x: x['service'])
self.assertEqual(self.EXPECTED, result)