You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by yh...@apache.org on 2023/06/08 02:48:42 UTC

[beam] branch master updated: reads all files from metadata list (#27059)

This is an automated email from the ASF dual-hosted git repository.

yhu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git


The following commit(s) were added to refs/heads/master by this push:
     new 74bad1add9c reads all files from metadata list (#27059)
74bad1add9c is described below

commit 74bad1add9cb6954c5f62aff81371c327546cc89
Author: Nathaniel Young <ny...@users.noreply.github.com>
AuthorDate: Wed Jun 7 19:48:33 2023 -0700

    reads all files from metadata list (#27059)
---
 sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py b/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py
index 508ab159513..55b3ba1695a 100644
--- a/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py
+++ b/sdks/python/apache_beam/examples/avro_nyc_trips_it_test.py
@@ -196,11 +196,16 @@ class AvroNycTripsIT(unittest.TestCase):
     avro_nyc_trips.run(test_pipeline.get_full_options_as_args(**extra_opts))
 
     # load result avro file and compare
-    metadata = FileSystems.match([f'{test_output}*'])[0].metadata_list[0]
-    with FileSystems.open(metadata.path) as f:
-      avro_reader = fastavro.reader(f)
+    metadata_list = FileSystems.match([f'{test_output}*'])[0].metadata_list
+    result = []
 
-      result = sorted(avro_reader, key=lambda x: x['service'])
+    for metadata in metadata_list:
+      with FileSystems.open(metadata.path) as f:
+        avro_reader = fastavro.reader(f)
+
+        result.extend(avro_reader)
+
+    result.sort(key=lambda x: x['service'])
 
     self.assertEqual(self.EXPECTED, result)