You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/06/27 19:30:20 UTC
[2/2] incubator-beam git commit: Pylint integration for Python SDK
Pylint integration for Python SDK
Runs pylint on modified files under sdks/python/**/*.py.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/f71c1ddd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/f71c1ddd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/f71c1ddd
Branch: refs/heads/python-sdk
Commit: f71c1dddb10d4a8b0f8c8ed8722e6662781d19f9
Parents: 21c819f
Author: Ahmet Altay <al...@google.com>
Authored: Fri Jun 24 14:04:27 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Mon Jun 27 12:30:02 2016 -0700
----------------------------------------------------------------------
sdks/python/.pylintrc | 174 +++++++++++++++++++
sdks/python/apache_beam/coders/coders.py | 2 +-
sdks/python/apache_beam/coders/stream_test.py | 2 +-
.../apache_beam/examples/streaming_wordcount.py | 2 +-
sdks/python/apache_beam/examples/wordcount.py | 4 +-
.../apache_beam/examples/wordcount_debugging.py | 6 +-
sdks/python/apache_beam/io/bigquery.py | 2 +-
.../apache_beam/transforms/combiners_test.py | 7 +-
sdks/python/apache_beam/transforms/core.py | 17 +-
.../apache_beam/transforms/ptransform_test.py | 43 +++--
.../apache_beam/transforms/trigger_test.py | 25 +--
.../apache_beam/typehints/trivial_inference.py | 3 +-
sdks/python/apache_beam/typehints/typehints.py | 5 +-
sdks/python/run_pylint.sh | 48 +++++
sdks/python/tox.ini | 2 +
15 files changed, 292 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/.pylintrc
----------------------------------------------------------------------
diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc
new file mode 100644
index 0000000..edf13ee
--- /dev/null
+++ b/sdks/python/.pylintrc
@@ -0,0 +1,174 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+[MASTER]
+# Ignore auto-generated files.
+ignore=clients,windmill_pb2.py,windmill_service_pb2.py
+
+[BASIC]
+# Regular expression which should only match the name
+# of functions or classes which do not require a docstring.
+no-docstring-rgx=(__.*__|main)
+
+# Min length in lines of a function that requires a docstring.
+docstring-min-length=10
+
+# Regular expression which should only match correct module names. The
+# leading underscore is sanctioned for private modules by Google's style
+# guide.
+#
+# There are exceptions to the basic rule (_?[a-z][a-z0-9_]*) to cover
+# requirements of Python's module system and of the presubmit framework.
+module-rgx=^(_?[a-z][a-z0-9_]*)|__init__|PRESUBMIT|PRESUBMIT_unittest$
+
+# Regular expression which should only match correct module level names
+const-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression which should only match correct class attribute
+class-attribute-rgx=^(_?[A-Z][A-Z0-9_]*|__[a-z0-9_]+__|_?[a-z][a-z0-9_]*)$
+
+# Regular expression which should only match correct class names
+class-rgx=^_?[A-Z][a-zA-Z0-9]*$
+
+# Regular expression which should only match correct function names.
+# 'camel_case' and 'snake_case' group names are used for consistency of naming
+# styles across functions and methods.
+function-rgx=^(?:(?P<camel_case>_?[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_?[a-z][a-z0-9_]*))$
+
+# Regular expression which should only match correct method names.
+# 'camel_case' and 'snake_case' group names are used for consistency of naming
+# styles across functions and methods. 'exempt' indicates a name which is
+# consistent with all naming styles.
+method-rgx=^(?:(?P<exempt>__[a-z0-9_]+__|next)|(?P<camel_case>_{0,2}[A-Z][a-zA-Z0-9]*)|(?P<snake_case>_{0,2}[a-z][a-z0-9_]*))$
+
+# Regular expression which should only match correct instance attribute names
+attr-rgx=^_{0,2}[a-z][a-z0-9_]*$
+
+# Regular expression which should only match correct argument names
+argument-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression which should only match correct variable names
+variable-rgx=^[a-z][a-z0-9_]*$
+
+# Regular expression which should only match correct list comprehension /
+# generator expression variable names
+inlinevar-rgx=^[a-z][a-z0-9_]*$
+
+# List of builtins function names that should not be used, separated by a comma
+bad-functions=input,apply,reduce
+
+# Good variable names which should always be accepted, separated by a comma
+good-names=main,_
+
+# Bad variable names which should always be refused, separated by a comma
+bad-names=
+
+[MESSAGES CONTROL]
+disable =
+ abstract-method,
+ arguments-differ,
+ attribute-defined-outside-init,
+ bad-builtin,
+ bad-option-value,
+ bad-super-call,
+ broad-except,
+ consider-using-enumerate,
+ cyclic-import,
+ design,
+ expression-not-assigned,
+ fixme,
+ function-redefined,
+ global-statement,
+ import-error,
+ import-self,
+ invalid-name,
+ locally-disabled,
+ locally-enabled,
+ misplaced-bare-raise,
+ missing-docstring,
+ multiple-statements,
+ no-member,
+ no-name-in-module,
+ no-self-argument,
+ no-self-use,
+ no-value-for-parameter,
+ not-callable,
+ pointless-statement,
+ protected-access,
+ raising-non-exception,
+ redefined-builtin,
+ redefined-outer-name,
+ redefined-variable-type,
+ redundant-keyword-arg,
+ reimported,
+ relative-import,
+ similarities,
+ simplifiable-if-statement,
+ super-init-not-called,
+ undefined-variable,
+ unexpected-keyword-arg,
+ ungrouped-imports,
+ unidiomatic-typecheck,
+ unnecessary-lambda,
+ unneeded-not,
+ unused-argument,
+ unused-import,
+ unused-variable,
+ unused-wildcard-import,
+ used-before-assignment,
+ wildcard-import,
+ wrong-import-order,
+ wrong-import-position,
+
+
+[REPORTS]
+# Tells whether to display a full report or only the messages
+reports=no
+
+[CLASSES]
+# List of method names used to declare (i.e. assign) instance attributes.
+defining-attr-methods=__init__,__new__,setUp
+
+# "class_" is also a valid for the first argument to a class method.
+valid-classmethod-first-arg=cls,class_
+
+[FORMAT]
+# Maximum number of characters on a single line.
+max-line-length=80
+
+# Maximum number of lines in a module
+max-module-lines=99999
+
+# String used as indentation unit. (2 spaces.)
+indent-string=' '
+
+# Number of spaces of indent required.
+indent-after-paren=4
+
+# Regexp for a line that is allowed to be longer than the limit.
+# Long import lines or URLs in comments or pydocs.
+ignore-long-lines=(?x)
+ (^\s*(import|from)\s
+ |^\s*(\#\ )?<?(https?|ftp):\/\/[^\s\/$.?#].[^\s]*>?$
+ )
+
+[VARIABLES]
+# Tells whether we should check for unused import in __init__ files.
+init-import=no
+
+# A regular expression matching names used for dummy variables (i.e. not used).
+dummy-variables-rgx=^\*{0,2}(_$|unused_|dummy_|args|kwargs)
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/coders/coders.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/coders.py b/sdks/python/apache_beam/coders/coders.py
index 24a1f3e..6d5b10a 100644
--- a/sdks/python/apache_beam/coders/coders.py
+++ b/sdks/python/apache_beam/coders/coders.py
@@ -152,7 +152,7 @@ class Coder(object):
def __eq__(self, other):
# pylint: disable=protected-access
return (self.__class__ == other.__class__
- and self._dict_without_impl() == other._dict_without_impl())
+ and self._dict_without_impl() == other._dict_without_impl())
# pylint: enable=protected-access
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/coders/stream_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/coders/stream_test.py b/sdks/python/apache_beam/coders/stream_test.py
index 463ccf7..f2163ca 100644
--- a/sdks/python/apache_beam/coders/stream_test.py
+++ b/sdks/python/apache_beam/coders/stream_test.py
@@ -83,7 +83,7 @@ class StreamTest(unittest.TestCase):
base = -1.7
self.run_read_write_var_int64(
[int(base**pow)
- for pow in range(1, int(63 * math.log(2) / math.log(-base)))])
+ for pow in range(1, int(63 * math.log(2) / math.log(-base)))])
def test_large_var_int64(self):
self.run_read_write_var_int64([0, 2**63 - 1, -2**63, 2**63 - 3])
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/examples/streaming_wordcount.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/streaming_wordcount.py b/sdks/python/apache_beam/examples/streaming_wordcount.py
index 685e742..eda74dd 100644
--- a/sdks/python/apache_beam/examples/streaming_wordcount.py
+++ b/sdks/python/apache_beam/examples/streaming_wordcount.py
@@ -53,7 +53,7 @@ def run(argv=None):
# Capitalize the characters in each line.
transformed = (lines
| (beam.FlatMap('split',
- lambda x: re.findall(r'[A-Za-z\']+', x))
+ lambda x: re.findall(r'[A-Za-z\']+', x))
.with_output_types(unicode))
| beam.Map('pair_with_one', lambda x: (x, 1))
| beam.WindowInto(window.FixedWindows(15, 0))
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/examples/wordcount.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/wordcount.py b/sdks/python/apache_beam/examples/wordcount.py
index b369294..fc02e91 100644
--- a/sdks/python/apache_beam/examples/wordcount.py
+++ b/sdks/python/apache_beam/examples/wordcount.py
@@ -28,8 +28,8 @@ import apache_beam as beam
empty_line_aggregator = beam.Aggregator('emptyLines')
average_word_size_aggregator = beam.Aggregator('averageWordLength',
- beam.combiners.MeanCombineFn(),
- float)
+ beam.combiners.MeanCombineFn(),
+ float)
class WordExtractingDoFn(beam.DoFn):
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/examples/wordcount_debugging.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/wordcount_debugging.py b/sdks/python/apache_beam/examples/wordcount_debugging.py
index c06ab04..c989b6a 100644
--- a/sdks/python/apache_beam/examples/wordcount_debugging.py
+++ b/sdks/python/apache_beam/examples/wordcount_debugging.py
@@ -122,7 +122,8 @@ def run(argv=None):
# each word and filter by a list of words.
filtered_words = (
p | beam.io.Read('read', beam.io.TextFileSource(known_args.input))
- | CountWords() | beam.ParDo('FilterText', FilterTextFn('Flourish|stomach')))
+ | CountWords()
+ | beam.ParDo('FilterText', FilterTextFn('Flourish|stomach')))
# assert_that is a convenient PTransform that checks a PCollection has an
# expected value. Asserts are best used in unit tests with small data sets but
@@ -132,7 +133,8 @@ def run(argv=None):
# of the Pipeline implies that the expectations were met. Learn more at
# https://cloud.google.com/dataflow/pipelines/testing-your-pipeline on how to
# test your pipeline.
- beam.assert_that(filtered_words, beam.equal_to([('Flourish', 3), ('stomach', 1)]))
+ beam.assert_that(
+ filtered_words, beam.equal_to([('Flourish', 3), ('stomach', 1)]))
# Format the counts into a PCollection of strings and write the output using a
# "Write" transform that has side effects.
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/io/bigquery.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/io/bigquery.py b/sdks/python/apache_beam/io/bigquery.py
index 030f821..9d33134 100644
--- a/sdks/python/apache_beam/io/bigquery.py
+++ b/sdks/python/apache_beam/io/bigquery.py
@@ -46,7 +46,7 @@ of the side table. The execution framework may use some caching techniques to
share the side inputs between calls in order to avoid excessive reading::
main_table = pipeline | beam.io.Read(beam.io.BigQuerySource('very_big_table')
- side_table = pipeline | beam.io.Read(beam.io.BigQuerySource('not_so_big_table')
+ side_table = pipeline | beam.io.Read(beam.io.BigQuerySource('not_big_table')
results = (
main_table
| beam.Map('process data',
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/combiners_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/combiners_test.py b/sdks/python/apache_beam/transforms/combiners_test.py
index b753a17..112c591 100644
--- a/sdks/python/apache_beam/transforms/combiners_test.py
+++ b/sdks/python/apache_beam/transforms/combiners_test.py
@@ -109,7 +109,8 @@ class CombineTest(unittest.TestCase):
pcoll = pipeline | Create(
'start-perkey', [('a', x) for x in [6, 3, 1, 1, 9, 1, 5, 2, 0, 6]])
result_ktop = pcoll | beam.CombinePerKey('top-perkey', combiners.Largest(5))
- result_kbot = pcoll | beam.CombinePerKey('bot-perkey', combiners.Smallest(4))
+ result_kbot = pcoll | beam.CombinePerKey(
+ 'bot-perkey', combiners.Smallest(4))
assert_that(result_ktop, equal_to([('a', [9, 6, 6, 5, 3])]), label='k:top')
assert_that(result_kbot, equal_to([('a', [0, 1, 1, 1])]), label='k:bot')
pipeline.run()
@@ -158,8 +159,8 @@ class CombineTest(unittest.TestCase):
p
| Create([('a', 100, 0.0), ('b', 10, -1), ('c', 1, 100)])
| beam.CombineGlobally(combine.TupleCombineFn(max,
- combine.MeanCombineFn(),
- sum)).without_defaults())
+ combine.MeanCombineFn(),
+ sum)).without_defaults())
assert_that(result, equal_to([('c', 111.0 / 3, 99.0)]))
p.run()
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/core.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/core.py b/sdks/python/apache_beam/transforms/core.py
index 3a0eb30..445367f 100644
--- a/sdks/python/apache_beam/transforms/core.py
+++ b/sdks/python/apache_beam/transforms/core.py
@@ -437,7 +437,7 @@ class CallableWrapperCombineFn(CombineFn):
'All functions for a Combine PTransform must accept a '
'single argument compatible with: Iterable[Any]. '
'Instead a function with input type: %s was received.'
- % input_args[0])
+ % input_args[0])
input_args = (element_type(input_args[0]),) + input_args[1:]
# TODO(robertwb): Assert output type is consistent with input type?
hints = fn_hints.copy()
@@ -813,10 +813,12 @@ class CombineGlobally(PTransform):
return transform
combined = (pcoll
- | add_input_types(Map('KeyWithVoid', lambda v: (None, v))
- .with_output_types(KV[None, pcoll.element_type]))
- | CombinePerKey('CombinePerKey', self.fn, *self.args, **self.kwargs)
- | Map('UnKey', lambda (k, v): v))
+ | add_input_types(Map('KeyWithVoid', lambda v: (None, v))
+ .with_output_types(
+ KV[None, pcoll.element_type]))
+ | CombinePerKey(
+ 'CombinePerKey', self.fn, *self.args, **self.kwargs)
+ | Map('UnKey', lambda (k, v): v))
if not self.has_defaults and not self.as_view:
return combined
@@ -884,7 +886,7 @@ class CombineValues(PTransformWithSideInputs):
def apply(self, pcoll):
args, kwargs = util.insert_values_in_args(
- self.args, self.kwargs, self.side_inputs)
+ self.args, self.kwargs, self.side_inputs)
input_type = pcoll.element_type
key_type = None
@@ -1027,7 +1029,8 @@ class GroupByKey(PTransform):
'GroupByKey operation "%s"' % self.label)
reify_output_type = KV[key_type, typehints.WindowedValue[value_type]]
- gbk_input_type = KV[key_type, Iterable[typehints.WindowedValue[value_type]]]
+ gbk_input_type = (
+ KV[key_type, Iterable[typehints.WindowedValue[value_type]]])
gbk_output_type = KV[key_type, Iterable[value_type]]
return (pcoll
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/ptransform_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index 3244473..8ae7a37 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -286,7 +286,7 @@ class PTransformTest(unittest.TestCase):
vals_2 = [2, 4, 6, 8, 10, 12, 14]
pipeline = Pipeline('DirectPipelineRunner')
pcoll = pipeline | beam.Create('start', ([('a', x) for x in vals_1] +
- [('b', x) for x in vals_2]))
+ [('b', x) for x in vals_2]))
result = pcoll | beam.CombinePerKey('mean', self._MeanCombineFn())
assert_that(result, equal_to([('a', sum(vals_1) / len(vals_1)),
('b', sum(vals_2) / len(vals_2))]))
@@ -297,7 +297,7 @@ class PTransformTest(unittest.TestCase):
vals_2 = [2, 4, 6, 8, 10, 12, 14]
pipeline = Pipeline('DirectPipelineRunner')
pcoll = pipeline | beam.Create('start', ([('a', x) for x in vals_1] +
- [('b', x) for x in vals_2]))
+ [('b', x) for x in vals_2]))
result = pcoll | beam.CombinePerKey(sum)
assert_that(result, equal_to([('a', sum(vals_1)), ('b', sum(vals_2))]))
pipeline.run()
@@ -307,7 +307,7 @@ class PTransformTest(unittest.TestCase):
vals_2 = [2, 4, 6, 8, 10, 12, 14]
pipeline = Pipeline('DirectPipelineRunner')
pcoll = pipeline | beam.Create('start', ([('a', x) for x in vals_1] +
- [('b', x) for x in vals_2]))
+ [('b', x) for x in vals_2]))
divisor = pipeline | beam.Create('divisor', [2])
result = pcoll | beam.CombinePerKey(
lambda vals, d: max(v for v in vals if v % d == 0),
@@ -790,9 +790,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
d = (self.p
| beam.Create('s', ['t', 'e', 's', 't']).with_output_types(str)
| beam.FlatMap('dup', lambda x: [x + x])
- .with_input_types(str).with_output_types(str)
+ .with_input_types(str).with_output_types(str)
| beam.FlatMap('upper', lambda x: [x.upper()])
- .with_input_types(str).with_output_types(str))
+ .with_input_types(str).with_output_types(str))
assert_that(d, equal_to(['TT', 'EE', 'SS', 'TT']))
self.p.run()
@@ -803,7 +803,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
with self.assertRaises(typehints.TypeCheckError) as e:
d = (self.p
| beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
- | beam.Map('upper', lambda x: x.upper()).with_input_types(str).with_output_types(str))
+ | beam.Map('upper', lambda x: x.upper())
+ .with_input_types(str).with_output_types(str))
self.assertEqual("Type hint violation for 'upper': "
"requires <type 'str'> but got <type 'int'> for x",
@@ -813,7 +814,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# No error should be raised if this type-checks properly.
d = (self.p
| beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
- | beam.Map('to_str', lambda x: str(x)).with_input_types(int).with_output_types(str))
+ | beam.Map('to_str', lambda x: str(x))
+ .with_input_types(int).with_output_types(str))
assert_that(d, equal_to(['1', '2', '3', '4']))
self.p.run()
@@ -853,7 +855,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
with self.assertRaises(typehints.TypeCheckError) as e:
(self.p
| beam.Create('strs', ['1', '2', '3', '4', '5']).with_output_types(str)
- | beam.Map('lower', lambda x: x.lower()).with_input_types(str).with_output_types(str)
+ | beam.Map('lower', lambda x: x.lower())
+ .with_input_types(str).with_output_types(str)
| beam.Filter('below 3', lambda x: x < 3).with_input_types(int))
self.assertEqual("Type hint violation for 'below 3': "
@@ -864,7 +867,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# No error should be raised if this type-checks properly.
d = (self.p
| beam.Create('strs', ['1', '2', '3', '4', '5']).with_output_types(str)
- | beam.Map('to int', lambda x: int(x)).with_input_types(str).with_output_types(int)
+ | beam.Map('to int', lambda x: int(x))
+ .with_input_types(str).with_output_types(int)
| beam.Filter('below 3', lambda x: x < 3).with_input_types(int))
assert_that(d, equal_to([1, 2]))
self.p.run()
@@ -894,7 +898,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
(self.p
| beam.Create('str', range(5)).with_output_types(int)
| beam.Filter('half', half)
- | beam.Map('to bool', lambda x: bool(x)).with_input_types(int).with_output_types(bool))
+ | beam.Map('to bool', lambda x: bool(x))
+ .with_input_types(int).with_output_types(bool))
def test_group_by_key_only_output_type_deduction(self):
d = (self.p
@@ -982,7 +987,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# checking was disabled above.
(self.p
| beam.Create('t', [1, 2, 3]).with_output_types(int)
- | beam.Map('lower', lambda x: x.lower()).with_input_types(str).with_output_types(str))
+ | beam.Map('lower', lambda x: x.lower())
+ .with_input_types(str).with_output_types(str))
def test_run_time_type_checking_enabled_type_violation(self):
self.p.options.view_as(TypeOptions).pipeline_type_check = False
@@ -1020,7 +1026,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# Pipeline checking is off, but the above function should satisfy types at
# run-time.
result = (self.p
- | beam.Create('t', ['t', 'e', 's', 't', 'i', 'n', 'g']).with_output_types(str)
+ | beam.Create('t', ['t', 'e', 's', 't', 'i', 'n', 'g'])
+ .with_output_types(str)
| beam.Map('gen keys', group_with_upper_ord)
| beam.GroupByKey('O'))
@@ -1232,9 +1239,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
self.assertEqual(
"All functions for a Combine PTransform must accept a "
- "single argument compatible with: Iterable[Any]. "
- "Instead a function with input type: <type 'int'> was received.",
- e.exception.message)
+ "single argument compatible with: Iterable[Any]. "
+ "Instead a function with input type: <type 'int'> was received.",
+ e.exception.message)
def test_combine_pipeline_type_propagation_using_decorators(self):
@with_output_types(int)
@@ -1532,7 +1539,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
d = (self.p
| beam.Create('c', ['t', 'e', 's', 't']).with_output_types(str)
- | beam.Map('dup key', lambda x: (x, x)).with_output_types(typehints.KV[str, str])
+ | beam.Map('dup key', lambda x: (x, x))
+ .with_output_types(typehints.KV[str, str])
| combine.Count.PerKey('count dups'))
self.assertCompatible(typehints.KV[str, int], d.element_type)
@@ -1565,7 +1573,8 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
self.p.options.view_as(TypeOptions).runtime_type_check = True
d = (self.p
- | beam.Create('w', [True, True, False, True, True]).with_output_types(bool)
+ | beam.Create('w', [True, True, False, True, True])
+ .with_output_types(bool)
| combine.Count.PerElement('count elems'))
self.assertCompatible(typehints.KV[bool, int], d.element_type)
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/transforms/trigger_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index 5d4d7a8..fee9b1b 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -175,17 +175,18 @@ class TriggerTest(unittest.TestCase):
late=AfterCount(1)),
AccumulationMode.ACCUMULATING,
[(1, 'a'), (15, 'b'), (7, 'c'), (30, 'd')],
- {IntervalWindow(1, 25): [
- set('abc'), # early
- set('abc'), # on time
- set('abcxy') # late
- ],
- IntervalWindow(30, 40): [
- set('d'), # on time
- ],
- IntervalWindow(1, 40): [
- set('abcdxyz') # late
- ],
+ {
+ IntervalWindow(1, 25): [
+ set('abc'), # early
+ set('abc'), # on time
+ set('abcxy') # late
+ ],
+ IntervalWindow(30, 40): [
+ set('d'), # on time
+ ],
+ IntervalWindow(1, 40): [
+ set('abcdxyz') # late
+ ],
},
2,
late_data=[(1, 'x'), (2, 'y'), (21, 'z')])
@@ -374,7 +375,7 @@ class TriggerPipelineTest(unittest.TestCase):
| beam.FlatMap(lambda t: [('A', t), ('B', t + 5)])
| beam.Map(lambda (k, t): TimestampedValue((k, t), t))
| beam.WindowInto(FixedWindows(10), trigger=AfterCount(3),
- accumulation_mode=AccumulationMode.DISCARDING)
+ accumulation_mode=AccumulationMode.DISCARDING)
| beam.GroupByKey()
| beam.Map(lambda (k, v): ('%s-%s' % (k, len(v)), set(v))))
assert_that(result, equal_to(
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/typehints/trivial_inference.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py
index 55d580d..82ce765 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -237,7 +237,8 @@ def infer_return_type(c, input_types, debug=False, depth=5):
return infer_return_type_func(c.im_func, input_types, debug, depth)
elif isinstance(c, BoundMethod):
input_types = [c.unbound.im_class] + input_types
- return infer_return_type_func(c.unbound.im_func, input_types, debug, depth)
+ return infer_return_type_func(
+ c.unbound.im_func, input_types, debug, depth)
elif isinstance(c, (type, types.ClassType)):
if c in typehints.DISALLOWED_PRIMITIVE_TYPES:
return {
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/apache_beam/typehints/typehints.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py
index c109dc6..5e31fd1 100644
--- a/sdks/python/apache_beam/typehints/typehints.py
+++ b/sdks/python/apache_beam/typehints/typehints.py
@@ -15,7 +15,7 @@
# limitations under the License.
#
-"""Syntax and semantics for type-hinting custom-functions/PTransforms in the SDK.
+"""Syntax & semantics for type-hinting custom-functions/PTransforms in the SDK.
This module defines type-hinting objects and the corresponding syntax for
type-hinting function arguments, function return types, or PTransform object
@@ -322,7 +322,8 @@ def _unified_repr(o):
Returns:
A qualified name for the passed Python object fit for string formatting.
"""
- return repr(o) if isinstance(o, (TypeConstraint, types.NoneType)) else o.__name__
+ return repr(o) if isinstance(
+ o, (TypeConstraint, types.NoneType)) else o.__name__
def check_constraint(type_constraint, object_instance):
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/run_pylint.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh
new file mode 100755
index 0000000..e8062c0
--- /dev/null
+++ b/sdks/python/run_pylint.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# This script will run pylint on files that changed compared to the current
+# HEAD of the branch.
+#
+# Use "pylint apache_beam" to run pylint all files.
+#
+# The exit-code of the script indicates success or a failure.
+
+BASE_BRANCH=python-sdk
+
+set -e
+set -o pipefail
+
+# Retrieve base branch for comparison. Travis does not fetch it by default.
+git remote set-branches --add origin $BASE_BRANCH
+git fetch
+
+# Get the name of the files that changed compared to the HEAD of the branch.
+# Filter the output to .py files only. Rewrite the paths relative to the
+# sdks/python folder.
+CHANGED_FILES=$(git diff --name-only origin/$BASE_BRANCH apache_beam \
+ | { grep ".py$" || true; } \
+ | sed 's/sdks\/python\///g')
+
+if test "$CHANGED_FILES"; then
+ echo "Running pylint on changed files:"
+ echo "$CHANGED_FILES"
+ pylint $CHANGED_FILES
+else
+ echo "Not running pylint. No eligible files."
+fi
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/f71c1ddd/sdks/python/tox.ini
----------------------------------------------------------------------
diff --git a/sdks/python/tox.ini b/sdks/python/tox.ini
index c0208b5..356de57 100644
--- a/sdks/python/tox.ini
+++ b/sdks/python/tox.ini
@@ -19,6 +19,8 @@
envlist = py27
[testenv:py27]
+deps=pylint
commands =
python setup.py test
+ {toxinidir}/run_pylint.sh
passenv = TRAVIS*