You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by pa...@apache.org on 2018/04/30 23:15:26 UTC
[beam] branch master updated (8faad95 -> 9d75d06)
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git.
from 8faad95 Merge pull request #5243: [BEAM-4175] Fix Go ParDo direct output pardo issue
new dabffc2 Creation of utils.py whit CountingSource class on it.
new 9d75d06 Changed import from examples.snippets to io.utils
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
sdks/python/apache_beam/io/utils.py | 65 ++++++++++++++++++++++
.../apache_beam/transforms/ptransform_test.py | 4 +-
2 files changed, 67 insertions(+), 2 deletions(-)
create mode 100644 sdks/python/apache_beam/io/utils.py
--
To stop receiving notification emails like this one, please contact
pabloem@apache.org.
[beam] 01/02: Creation of utils.py whit CountingSource class on it.
Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit dabffc280db2a4c461df3ae5b3e75242b7b682d6
Author: Javier Antonio Gonzalez Trejo <ja...@gmail.com>
AuthorDate: Thu Apr 26 23:05:47 2018 -0500
Creation of utils.py whit CountingSource class on it.
---
sdks/python/apache_beam/io/utils.py | 65 +++++++++++++++++++++++++++++++++++++
1 file changed, 65 insertions(+)
diff --git a/sdks/python/apache_beam/io/utils.py b/sdks/python/apache_beam/io/utils.py
new file mode 100644
index 0000000..d6b312d
--- /dev/null
+++ b/sdks/python/apache_beam/io/utils.py
@@ -0,0 +1,65 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""Utils for the io library.
+* CountingSource: Subclass of iobase.BoundedSource. Used
+on transforms.ptransform_test.test_read_metrics.
+"""
+
+from apache_beam.io import iobase
+from apache_beam.io.range_trackers import OffsetRangeTracker
+from apache_beam.metrics import Metrics
+
+
+class CountingSource(iobase.BoundedSource):
+
+ def __init__(self, count):
+ self.records_read = Metrics.counter(self.__class__, 'recordsRead')
+ self._count = count
+
+ def estimate_size(self):
+ return self._count
+
+ def get_range_tracker(self, start_position, stop_position):
+ if start_position is None:
+ start_position = 0
+ if stop_position is None:
+ stop_position = self._count
+
+ return OffsetRangeTracker(start_position, stop_position)
+
+ def read(self, range_tracker):
+ for i in range(self._count):
+ if not range_tracker.try_claim(i):
+ return
+ self.records_read.inc()
+ yield i
+
+ def split(self, desired_bundle_size, start_position=None,
+ stop_position=None):
+ if start_position is None:
+ start_position = 0
+ if stop_position is None:
+ stop_position = self._count
+
+ bundle_start = start_position
+ while bundle_start < self._count:
+ bundle_stop = max(self._count, bundle_start + desired_bundle_size)
+ yield iobase.SourceBundle(weight=(bundle_stop - bundle_start),
+ source=self,
+ start_position=bundle_start,
+ stop_position=bundle_stop)
+ bundle_start = bundle_stop
--
To stop receiving notification emails like this one, please contact
pabloem@apache.org.
[beam] 02/02: Changed import from examples.snippets to io.utils
Posted by pa...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
pabloem pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
commit 9d75d06643f0d443ede4d172cca2c5d8b3c5ef65
Author: Javier Antonio Gonzalez Trejo <ja...@gmail.com>
AuthorDate: Thu Apr 26 23:34:39 2018 -0500
Changed import from examples.snippets to io.utils
---
sdks/python/apache_beam/transforms/ptransform_test.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index 350a19f..e23fad7 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -182,7 +182,7 @@ class PTransformTest(unittest.TestCase):
@attr('ValidatesRunner')
def test_read_metrics(self):
- from apache_beam.examples.snippets.snippets import CountingSource
+ from apache_beam.io.utils import CountingSource
class CounterDoFn(beam.DoFn):
def __init__(self):
@@ -197,7 +197,7 @@ class PTransformTest(unittest.TestCase):
(pipeline | Read(CountingSource(100)) | beam.ParDo(CounterDoFn()))
res = pipeline.run()
res.wait_until_finish()
- # This counter is defined in snippets.CountingSource.
+ # This counter is defined in utils.CountingSource.
metric_results = res.metrics().query(MetricsFilter()
.with_name('recordsRead'))
outputs_counter = metric_results['counters'][0]
--
To stop receiving notification emails like this one, please contact
pabloem@apache.org.