You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2019/06/18 05:09:59 UTC
[beam] branch master updated: Fixing file naming for windows (#8870)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new fe6891a Fixing file naming for windows (#8870)
fe6891a is described below
commit fe6891ad0dd181b1df641e4df6671858598fac6c
Author: Pablo <pa...@users.noreply.github.com>
AuthorDate: Tue Jun 18 07:09:46 2019 +0200
Fixing file naming for windows (#8870)
* Fixing file naming for windows
* Adding to fix.
* trailing whitespace aiaiai
---
sdks/python/apache_beam/io/fileio_test.py | 29 +++++++++++++++++++----------
1 file changed, 19 insertions(+), 10 deletions(-)
diff --git a/sdks/python/apache_beam/io/fileio_test.py b/sdks/python/apache_beam/io/fileio_test.py
index f87aa2f..75e4175 100644
--- a/sdks/python/apache_beam/io/fileio_test.py
+++ b/sdks/python/apache_beam/io/fileio_test.py
@@ -437,7 +437,7 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
# streaming mode.
WriteFilesTest.all_records = []
- dir = self._new_tempdir()
+ dir = '%s%s' % (self._new_tempdir(), os.sep)
# Setting up the input (TestStream)
ts = TestStream().advance_watermark_to(0)
@@ -450,6 +450,10 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
ts.advance_processing_time(5)
ts.advance_watermark_to(timestamp)
+ def no_colon_file_naming(*args):
+ file_name = fileio.destination_prefix_naming()(*args)
+ return file_name.replace(':', '_')
+
# The pipeline that we are testing
options = PipelineOptions()
options.view_as(StandardOptions).streaming = True
@@ -467,6 +471,7 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
_ = (res
| beam.io.fileio.WriteToFiles(path=dir,
+ file_naming=no_colon_file_naming,
max_writers_per_bundle=0)
| beam.Map(lambda fr: FileSystems.join(dir, fr.file_name))
| beam.ParDo(self.record_dofn()))
@@ -505,6 +510,10 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
.add_elements([next(input), next(input)])
.advance_watermark_to(40))
+ def no_colon_file_naming(*args):
+ file_name = fileio.destination_prefix_naming()(*args)
+ return file_name.replace(':', '_')
+
with TestPipeline() as p:
_ = (p
| ts
@@ -515,7 +524,7 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
sink=lambda dest: (
WriteFilesTest.CsvSink(WriteFilesTest.CSV_HEADERS)
if dest == 'apache' else WriteFilesTest.JsonSink()),
- file_naming=fileio.destination_prefix_naming(),
+ file_naming=no_colon_file_naming,
max_writers_per_bundle=0,
))
@@ -534,25 +543,25 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
stringmatches.matches_regexp(
FileSystems.join(
dir,
- 'cncf-1970-01-01T00:00:00-1970-01-01T00:00:10--.*'
+ 'cncf-1970-01-01T00_00_00-1970-01-01T00_00_10--.*'
)
),
stringmatches.matches_regexp(
FileSystems.join(
dir,
- 'cncf-1970-01-01T00:00:10-1970-01-01T00:00:20--.*'
+ 'cncf-1970-01-01T00_00_10-1970-01-01T00_00_20--.*'
)
),
stringmatches.matches_regexp(
FileSystems.join(
dir,
- 'cncf-1970-01-01T00:00:20-1970-01-01T00:00:30--.*'
+ 'cncf-1970-01-01T00_00_20-1970-01-01T00_00_30--.*'
)
),
stringmatches.matches_regexp(
FileSystems.join(
dir,
- 'cncf-1970-01-01T00:00:30-1970-01-01T00:00:40--.*'
+ 'cncf-1970-01-01T00_00_30-1970-01-01T00_00_40--.*'
)
)
]),
@@ -562,19 +571,19 @@ class WriteFilesTest(_TestCaseWithTempDirCleanUp):
matches_all([
stringmatches.matches_regexp(FileSystems.join(
dir,
- 'apache-1970-01-01T00:00:00-1970-01-01T00:00:10--.*')
+ 'apache-1970-01-01T00_00_00-1970-01-01T00_00_10--.*')
),
stringmatches.matches_regexp(FileSystems.join(
dir,
- 'apache-1970-01-01T00:00:10-1970-01-01T00:00:20--.*')
+ 'apache-1970-01-01T00_00_10-1970-01-01T00_00_20--.*')
),
stringmatches.matches_regexp(FileSystems.join(
dir,
- 'apache-1970-01-01T00:00:20-1970-01-01T00:00:30--.*')
+ 'apache-1970-01-01T00_00_20-1970-01-01T00_00_30--.*')
),
stringmatches.matches_regexp(FileSystems.join(
dir,
- 'apache-1970-01-01T00:00:30-1970-01-01T00:00:40--.*')
+ 'apache-1970-01-01T00_00_30-1970-01-01T00_00_40--.*')
)
]),
label='verifyApacheFiles')