You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/06/30 17:32:45 UTC
[1/2] incubator-beam git commit: Fix expression-not-assigned and
unused-variable lint warnings.
Repository: incubator-beam
Updated Branches:
refs/heads/python-sdk 5434e599b -> bff980178
Fix expression-not-assigned and unused-variable lint warnings.
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/7bedbcc4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/7bedbcc4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/7bedbcc4
Branch: refs/heads/python-sdk
Commit: 7bedbcc4a468b1201378b824987aedb6352ee8d2
Parents: 5434e59
Author: Ahmet Altay <al...@google.com>
Authored: Wed Jun 29 17:33:23 2016 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Wed Jun 29 17:45:44 2016 -0700
----------------------------------------------------------------------
sdks/python/.pylintrc | 2 -
sdks/python/apache_beam/dataflow_test.py | 4 +-
sdks/python/apache_beam/runners/runner_test.py | 6 +-
.../apache_beam/transforms/aggregator_test.py | 2 +-
.../apache_beam/transforms/ptransform_test.py | 72 ++++++++++----------
.../apache_beam/transforms/trigger_test.py | 1 -
sdks/python/apache_beam/transforms/util.py | 2 +-
.../apache_beam/typehints/trivial_inference.py | 1 -
sdks/python/apache_beam/typehints/typehints.py | 2 +-
.../apache_beam/typehints/typehints_test.py | 20 +++---
sdks/python/run_pylint.sh | 7 +-
11 files changed, 58 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/.pylintrc
----------------------------------------------------------------------
diff --git a/sdks/python/.pylintrc b/sdks/python/.pylintrc
index efe8147..b283f04 100644
--- a/sdks/python/.pylintrc
+++ b/sdks/python/.pylintrc
@@ -88,7 +88,6 @@ disable =
consider-using-enumerate,
cyclic-import,
design,
- expression-not-assigned,
fixme,
function-redefined,
global-statement,
@@ -126,7 +125,6 @@ disable =
unneeded-not,
unused-argument,
unused-import,
- unused-variable,
unused-wildcard-import,
used-before-assignment,
wildcard-import,
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/dataflow_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/dataflow_test.py b/sdks/python/apache_beam/dataflow_test.py
index 5e0222c..d3721ee 100644
--- a/sdks/python/apache_beam/dataflow_test.py
+++ b/sdks/python/apache_beam/dataflow_test.py
@@ -181,8 +181,8 @@ class DataflowTest(unittest.TestCase):
pipeline = Pipeline('DirectPipelineRunner')
pcol = pipeline | Create('start', [1, 2])
side = pipeline | Create('side', [3, 4]) # 2 values in side input.
- pcol | FlatMap('compute', lambda x, s: [x * s], AsSingleton(side))
- with self.assertRaises(ValueError) as e:
+ pcol | FlatMap('compute', lambda x, s: [x * s], AsSingleton(side)) # pylint: disable=expression-not-assigned
+ with self.assertRaises(ValueError):
pipeline.run()
def test_default_value_singleton_side_input(self):
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/runners/runner_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/runners/runner_test.py b/sdks/python/apache_beam/runners/runner_test.py
index bffb2ff..20a7259 100644
--- a/sdks/python/apache_beam/runners/runner_test.py
+++ b/sdks/python/apache_beam/runners/runner_test.py
@@ -58,9 +58,9 @@ class RunnerTest(unittest.TestCase):
'--no_auth=True'
]))
- res = (p | ptransform.Create('create', [1, 2, 3])
- | ptransform.FlatMap('do', lambda x: [(x, x)])
- | ptransform.GroupByKey('gbk'))
+ (p | ptransform.Create('create', [1, 2, 3]) # pylint: disable=expression-not-assigned
+ | ptransform.FlatMap('do', lambda x: [(x, x)])
+ | ptransform.GroupByKey('gbk'))
remote_runner.job = apiclient.Job(p.options)
super(DataflowPipelineRunner, remote_runner).run(p)
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/transforms/aggregator_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/aggregator_test.py b/sdks/python/apache_beam/transforms/aggregator_test.py
index 308d909..0b3a20b 100644
--- a/sdks/python/apache_beam/transforms/aggregator_test.py
+++ b/sdks/python/apache_beam/transforms/aggregator_test.py
@@ -64,7 +64,7 @@ class AggregatorTest(unittest.TestCase):
context.aggregate_to(a, context.element)
p = beam.Pipeline('DirectPipelineRunner')
- p | beam.Create([0, 1, 2, 3]) | beam.ParDo(UpdateAggregators())
+ p | beam.Create([0, 1, 2, 3]) | beam.ParDo(UpdateAggregators()) # pylint: disable=expression-not-assigned
res = p.run()
for (_, _, expected), a in zip(counter_types, aggeregators):
actual = res.aggregated_values(a).values()[0]
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/transforms/ptransform_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index d6ee18a..c98a945 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -697,9 +697,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
return [prefix + context.element.upper()]
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('t', [1, 2, 3]).with_output_types(int)
- | beam.ParDo('upper', ToUpperCaseWithPrefix(), 'hello'))
+ (self.p
+ | beam.Create('t', [1, 2, 3]).with_output_types(int)
+ | beam.ParDo('upper', ToUpperCaseWithPrefix(), 'hello'))
self.assertEqual("Type hint violation for 'upper': "
"requires <type 'str'> but got <type 'int'> for context",
@@ -731,9 +731,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
return [context.element + num]
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('t', ['1', '2', '3']).with_output_types(str)
- | beam.ParDo('add', AddWithNum(), 5))
+ (self.p
+ | beam.Create('t', ['1', '2', '3']).with_output_types(str)
+ | beam.ParDo('add', AddWithNum(), 5))
self.p.run()
self.assertEqual("Type hint violation for 'add': "
@@ -749,9 +749,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# The function above is expecting an int for its only parameter. However, it
# will receive a str instead, which should result in a raised exception.
with self.assertRaises(typehints.TypeCheckError) as e:
- c = (self.p
- | beam.Create('s', ['b', 'a', 'r']).with_output_types(str)
- | beam.FlatMap('to str', int_to_str))
+ (self.p
+ | beam.Create('s', ['b', 'a', 'r']).with_output_types(str)
+ | beam.FlatMap('to str', int_to_str))
self.assertEqual("Type hint violation for 'to str': "
"requires <type 'int'> but got <type 'str'> for a",
@@ -805,10 +805,10 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# The transform before 'Map' has indicated that it outputs PCollections with
# int's, while Map is expecting one of str.
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
- | beam.Map('upper', lambda x: x.upper())
- .with_input_types(str).with_output_types(str))
+ (self.p
+ | beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
+ | beam.Map('upper', lambda x: x.upper())
+ .with_input_types(str).with_output_types(str))
self.assertEqual("Type hint violation for 'upper': "
"requires <type 'str'> but got <type 'int'> for x",
@@ -832,9 +832,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# Hinted function above expects a str at pipeline construction.
# However, 'Map' should detect that Create has hinted an int instead.
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
- | beam.Map('upper', upper))
+ (self.p
+ | beam.Create('s', [1, 2, 3, 4]).with_output_types(int)
+ | beam.Map('upper', upper))
self.assertEqual("Type hint violation for 'upper': "
"requires <type 'str'> but got <type 'int'> for s",
@@ -884,9 +884,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# Func above was hinted to only take a float, yet an int will be passed.
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('ints', [1, 2, 3, 4]).with_output_types(int)
- | beam.Filter('half', more_than_half))
+ (self.p
+ | beam.Create('ints', [1, 2, 3, 4]).with_output_types(int)
+ | beam.Filter('half', more_than_half))
self.assertEqual("Type hint violation for 'half': "
"requires <type 'float'> but got <type 'int'> for a",
@@ -932,9 +932,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def test_group_by_key_only_does_not_type_check(self):
# GBK will be passed raw int's here instead of some form of KV[A, B].
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('s', [1, 2, 3]).with_output_types(int)
- | beam.GroupByKeyOnly('F'))
+ (self.p
+ | beam.Create('s', [1, 2, 3]).with_output_types(int)
+ | beam.GroupByKeyOnly('F'))
self.assertEqual("Input type hint violation at F: "
"expected Tuple[TypeVariable[K], TypeVariable[V]], "
@@ -945,10 +945,10 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# Create is returning a List[int, str], rather than a KV[int, str] that is
# aliased to Tuple[int, str].
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | (beam.Create('s', range(5))
- .with_output_types(typehints.Iterable[int]))
- | beam.GroupByKey('T'))
+ (self.p
+ | (beam.Create('s', range(5))
+ .with_output_types(typehints.Iterable[int]))
+ | beam.GroupByKey('T'))
self.assertEqual("Input type hint violation at T: "
"expected Tuple[TypeVariable[K], TypeVariable[V]], "
@@ -1388,9 +1388,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def test_mean_globally_pipeline_checking_violated(self):
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('c', ['test']).with_output_types(str)
- | combine.Mean.Globally('mean'))
+ (self.p
+ | beam.Create('c', ['test']).with_output_types(str)
+ | combine.Mean.Globally('mean'))
self.assertEqual("Type hint violation for 'ParDo(CombineValuesDoFn)': "
"requires Tuple[TypeVariable[K], "
@@ -1608,10 +1608,10 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def test_per_key_pipeline_checking_violated(self):
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('n', range(100)).with_output_types(int)
- | beam.Map('num + 1', lambda x: x + 1).with_output_types(int)
- | combine.Top.PerKey('top mod', 1, lambda a, b: a < b))
+ (self.p
+ | beam.Create('n', range(100)).with_output_types(int)
+ | beam.Map('num + 1', lambda x: x + 1).with_output_types(int)
+ | combine.Top.PerKey('top mod', 1, lambda a, b: a < b))
self.assertEqual("Input type hint violation at GroupByKey: "
"expected Tuple[TypeVariable[K], TypeVariable[V]], "
@@ -1742,9 +1742,9 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def test_to_dict_pipeline_check_violated(self):
with self.assertRaises(typehints.TypeCheckError) as e:
- d = (self.p
- | beam.Create('d', [1, 2, 3, 4]).with_output_types(int)
- | combine.ToDict('to dict'))
+ (self.p
+ | beam.Create('d', [1, 2, 3, 4]).with_output_types(int)
+ | combine.ToDict('to dict'))
self.assertEqual("Type hint violation for 'ParDo(CombineValuesDoFn)': "
"requires Tuple[TypeVariable[K], "
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/transforms/trigger_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/trigger_test.py b/sdks/python/apache_beam/transforms/trigger_test.py
index 2cdd1e3..a3ad8d8 100644
--- a/sdks/python/apache_beam/transforms/trigger_test.py
+++ b/sdks/python/apache_beam/transforms/trigger_test.py
@@ -495,7 +495,6 @@ class TranscriptTest(unittest.TestCase):
AccumulationMode, spec.get('accumulation_mode', 'ACCUMULATING').upper())
output_time_fn = getattr(
OutputTimeFn, spec.get('output_time_fn', 'OUTPUT_AT_EOW').upper())
- allowed_lateness = float(spec.get('allowed_lateness', '-inf'))
driver = GeneralTriggerDriver(
Windowing(window_fn, trigger_fn, accumulation_mode, output_time_fn))
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/transforms/util.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/util.py b/sdks/python/apache_beam/transforms/util.py
index b8380a0..87990a3 100644
--- a/sdks/python/apache_beam/transforms/util.py
+++ b/sdks/python/apache_beam/transforms/util.py
@@ -228,4 +228,4 @@ def assert_that(actual, matcher, label='assert_that'):
def default_label(self):
return label
- actual.pipeline | AssertThat()
+ actual.pipeline | AssertThat() # pylint: disable=expression-not-assigned
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/typehints/trivial_inference.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/typehints/trivial_inference.py b/sdks/python/apache_beam/typehints/trivial_inference.py
index 82ce765..c8b2d72 100644
--- a/sdks/python/apache_beam/typehints/trivial_inference.py
+++ b/sdks/python/apache_beam/typehints/trivial_inference.py
@@ -356,7 +356,6 @@ def infer_return_type_func(f, input_types, debug=False, depth=0):
if var_args or kw_args:
state.stack[-1] = Any
state.stack[-var_args - kw_args] = Any
- inputs = [] if pop_count == 1 else state.stack[1 - pop_count:]
return_type = infer_return_type(state.stack[-pop_count].value,
state.stack[1 - pop_count:],
debug=debug,
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/typehints/typehints.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/typehints/typehints.py b/sdks/python/apache_beam/typehints/typehints.py
index 5e31fd1..20f471c 100644
--- a/sdks/python/apache_beam/typehints/typehints.py
+++ b/sdks/python/apache_beam/typehints/typehints.py
@@ -989,7 +989,7 @@ class WindowedTypeConstraint(TypeConstraint):
try:
check_constraint(self.inner_type, instance.value)
- except (CompositeTypeHintError, SimpleTypeHintError) as e:
+ except (CompositeTypeHintError, SimpleTypeHintError):
raise CompositeTypeHintError(
'%s hint type-constraint violated. The type of element in '
'is incorrect. Expected an instance of type %s, '
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/apache_beam/typehints/typehints_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/typehints/typehints_test.py b/sdks/python/apache_beam/typehints/typehints_test.py
index aa04fe2..5c6fc77 100644
--- a/sdks/python/apache_beam/typehints/typehints_test.py
+++ b/sdks/python/apache_beam/typehints/typehints_test.py
@@ -400,7 +400,7 @@ class TupleHintTestCase(TypeHintTestCase):
class ListHintTestCase(TypeHintTestCase):
def test_getitem_invalid_composite_type_param(self):
- with self.assertRaises(TypeError) as e:
+ with self.assertRaises(TypeError):
typehints.List[4]
def test_list_constraint_compatibility(self):
@@ -503,11 +503,11 @@ class DictHintTestCase(TypeHintTestCase):
e.exception.message)
def test_key_type_must_be_valid_composite_param(self):
- with self.assertRaises(TypeError) as e:
+ with self.assertRaises(TypeError):
typehints.Dict[list, int]
def test_value_type_must_be_valid_composite_param(self):
- with self.assertRaises(TypeError) as e:
+ with self.assertRaises(TypeError):
typehints.Dict[str, 5]
def test_compatibility(self):
@@ -692,7 +692,7 @@ class IterableHintTestCase(TypeHintTestCase):
def test_type_check_violation_invalid_composite_type(self):
hint = typehints.Iterable[typehints.List[int]]
l = ([['t', 'e'], ['s', 't']])
- with self.assertRaises(TypeError) as e:
+ with self.assertRaises(TypeError):
hint.type_check(l)
def test_type_check_violation_valid_composite_type(self):
@@ -773,7 +773,7 @@ class TakesDecoratorTestCase(TypeHintTestCase):
t = [1, 2]
@with_input_types(a=t)
- def foo(a):
+ def unused_foo(a):
pass
self.assertEqual('All type hint arguments must be a non-sequence, a '
@@ -786,7 +786,7 @@ class TakesDecoratorTestCase(TypeHintTestCase):
@check_type_hints
@with_input_types(a=t)
- def foo(a):
+ def unused_foo(a):
pass
self.assertEqual('All type hint arguments must be a non-sequence, a type, '
@@ -874,24 +874,24 @@ class ReturnsDecoratorTestCase(TypeHintTestCase):
def test_no_kwargs_accepted(self):
with self.assertRaises(ValueError):
@with_output_types(m=int)
- def foo():
+ def unused_foo():
return 5
def test_must_be_primitive_type_or_type_constraint(self):
with self.assertRaises(TypeError):
@with_output_types(5)
- def foo():
+ def unused_foo():
pass
with self.assertRaises(TypeError):
@with_output_types([1, 2])
- def foo():
+ def unused_foo():
pass
def test_must_be_single_return_type(self):
with self.assertRaises(ValueError):
@with_output_types(int, str)
- def foo():
+ def unused_foo():
return 4, 'f'
def test_type_check_violation(self):
http://git-wip-us.apache.org/repos/asf/incubator-beam/blob/7bedbcc4/sdks/python/run_pylint.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_pylint.sh b/sdks/python/run_pylint.sh
index 4e0b129..8cc7fef 100755
--- a/sdks/python/run_pylint.sh
+++ b/sdks/python/run_pylint.sh
@@ -34,9 +34,10 @@ git remote set-branches --add origin $BASE_BRANCH
git fetch
# Get the name of the files that changed compared to the HEAD of the branch.
-# Filter the output to .py files only. Rewrite the paths relative to the
-# sdks/python folder.
-CHANGED_FILES=$(git diff --name-only origin/$BASE_BRANCH apache_beam \
+# Use diff-filter to exclude deleted files. (i.e. Do not try to lint files that
+# does not exist any more.) Filter the output to .py files only. Rewrite the
+# paths relative to the sdks/python folder.
+CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRTUXB origin/$BASE_BRANCH apache_beam \
| { grep ".py$" || true; } \
| sed 's/sdks\/python\///g')
[2/2] incubator-beam git commit: Closes #566
Posted by dh...@apache.org.
Closes #566
Project: http://git-wip-us.apache.org/repos/asf/incubator-beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-beam/commit/bff98017
Tree: http://git-wip-us.apache.org/repos/asf/incubator-beam/tree/bff98017
Diff: http://git-wip-us.apache.org/repos/asf/incubator-beam/diff/bff98017
Branch: refs/heads/python-sdk
Commit: bff98017888e79ab22bdda11cef4c8508c9b8dfe
Parents: 5434e59 7bedbcc
Author: Dan Halperin <dh...@google.com>
Authored: Thu Jun 30 10:32:25 2016 -0700
Committer: Dan Halperin <dh...@google.com>
Committed: Thu Jun 30 10:32:25 2016 -0700
----------------------------------------------------------------------
sdks/python/.pylintrc | 2 -
sdks/python/apache_beam/dataflow_test.py | 4 +-
sdks/python/apache_beam/runners/runner_test.py | 6 +-
.../apache_beam/transforms/aggregator_test.py | 2 +-
.../apache_beam/transforms/ptransform_test.py | 72 ++++++++++----------
.../apache_beam/transforms/trigger_test.py | 1 -
sdks/python/apache_beam/transforms/util.py | 2 +-
.../apache_beam/typehints/trivial_inference.py | 1 -
sdks/python/apache_beam/typehints/typehints.py | 2 +-
.../apache_beam/typehints/typehints_test.py | 20 +++---
sdks/python/run_pylint.sh | 7 +-
11 files changed, 58 insertions(+), 61 deletions(-)
----------------------------------------------------------------------