You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by jr...@apache.org on 2024/11/11 15:25:20 UTC
(beam) branch revert-32880-typecheck_errmsg created (now c17e964d2af)
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a change to branch revert-32880-typecheck_errmsg
in repository https://gitbox.apache.org/repos/asf/beam.git
at c17e964d2af Revert "Update typecheck err msg (#32880)"
This branch includes the following new commits:
new c17e964d2af Revert "Update typecheck err msg (#32880)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(beam) 01/01: Revert "Update typecheck err msg (#32880)"
Posted by jr...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jrmccluskey pushed a commit to branch revert-32880-typecheck_errmsg
in repository https://gitbox.apache.org/repos/asf/beam.git
commit c17e964d2afb341807432c6c2544a938ffbf7910
Author: Jack McCluskey <34...@users.noreply.github.com>
AuthorDate: Mon Nov 11 10:25:09 2024 -0500
Revert "Update typecheck err msg (#32880)"
This reverts commit 76c5d56ea95ee1497cb80dfe1de1d1abd8d34b15.
---
sdks/python/apache_beam/io/gcp/pubsub_test.py | 3 +-
sdks/python/apache_beam/transforms/ptransform.py | 25 +-----
.../apache_beam/transforms/ptransform_test.py | 90 +++++++++++++++-------
.../apache_beam/typehints/decorators_test.py | 4 +-
.../apache_beam/typehints/typed_pipeline_test.py | 30 ++++----
5 files changed, 86 insertions(+), 66 deletions(-)
diff --git a/sdks/python/apache_beam/io/gcp/pubsub_test.py b/sdks/python/apache_beam/io/gcp/pubsub_test.py
index 73ba8d6abdb..2e3e9b30161 100644
--- a/sdks/python/apache_beam/io/gcp/pubsub_test.py
+++ b/sdks/python/apache_beam/io/gcp/pubsub_test.py
@@ -901,8 +901,7 @@ class TestWriteToPubSub(unittest.TestCase):
options = PipelineOptions([])
options.view_as(StandardOptions).streaming = True
- with self.assertRaisesRegex(Exception,
- r'requires.*PubsubMessage.*applied.*str'):
+ with self.assertRaisesRegex(Exception, r'Type hint violation'):
with TestPipeline(options=options) as p:
_ = (
p
diff --git a/sdks/python/apache_beam/transforms/ptransform.py b/sdks/python/apache_beam/transforms/ptransform.py
index 4848dc4aade..6ec74170537 100644
--- a/sdks/python/apache_beam/transforms/ptransform.py
+++ b/sdks/python/apache_beam/transforms/ptransform.py
@@ -497,12 +497,13 @@ class PTransform(WithTypeHints, HasDisplayData, Generic[InputT, OutputT]):
at_context = ' %s %s' % (input_or_output, context) if context else ''
raise TypeCheckError(
'{type} type hint violation at {label}{context}: expected {hint}, '
- 'got {actual_type}'.format(
+ 'got {actual_type}\nFull type hint:\n{debug_str}'.format(
type=input_or_output.title(),
label=self.label,
context=at_context,
hint=hint,
- actual_type=pvalue_.element_type))
+ actual_type=pvalue_.element_type,
+ debug_str=type_hints.debug_str()))
def _infer_output_coder(self, input_type=None, input_coder=None):
# type: (...) -> Optional[coders.Coder]
@@ -938,25 +939,7 @@ class PTransformWithSideInputs(PTransform):
bindings = getcallargs_forhints(argspec_fn, *arg_types, **kwargs_types)
hints = getcallargs_forhints(
argspec_fn, *input_types[0], **input_types[1])
-
- # First check the main input.
- arg_hints = iter(hints.items())
- element_arg, element_hint = next(arg_hints)
- if not typehints.is_consistent_with(
- bindings.get(element_arg, typehints.Any), element_hint):
- transform_nest_level = self.label.count("/")
- split_producer_label = pvalueish.producer.full_label.split("/")
- producer_label = "/".join(
- split_producer_label[:transform_nest_level + 1])
- raise TypeCheckError(
- f"The transform '{self.label}' requires "
- f"PCollections of type '{element_hint}' "
- f"but was applied to a PCollection of type"
- f" '{bindings[element_arg]}' "
- f"(produced by the transform '{producer_label}'). ")
-
- # Now check the side inputs.
- for arg, hint in arg_hints:
+ for arg, hint in hints.items():
if arg.startswith('__unknown__'):
continue
if hint is None:
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index 7db017a5915..0acea547ccd 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -1298,13 +1298,17 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def process(self, element, prefix):
return [prefix + element.upper()]
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'Upper.*requires.*str.*applied.*int'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'T' >> beam.Create([1, 2, 3]).with_output_types(int)
| 'Upper' >> beam.ParDo(ToUpperCaseWithPrefix(), 'hello'))
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'Upper': "
+ "requires {} but got {} for element".format(str, int))
+
def test_do_fn_pipeline_runtime_type_check_satisfied(self):
self.p._options.view_as(TypeOptions).runtime_type_check = True
@@ -1331,14 +1335,18 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def process(self, element, num):
return [element + num]
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'Add.*requires.*int.*applied.*str'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'T' >> beam.Create(['1', '2', '3']).with_output_types(str)
| 'Add' >> beam.ParDo(AddWithNum(), 5))
self.p.run()
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'Add': "
+ "requires {} but got {} for element".format(int, str))
+
def test_pardo_does_not_type_check_using_type_hint_decorators(self):
@with_input_types(a=int)
@with_output_types(typing.List[str])
@@ -1347,13 +1355,17 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# The function above is expecting an int for its only parameter. However, it
# will receive a str instead, which should result in a raised exception.
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'ToStr.*requires.*int.*applied.*str'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'S' >> beam.Create(['b', 'a', 'r']).with_output_types(str)
| 'ToStr' >> beam.FlatMap(int_to_str))
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'ToStr': "
+ "requires {} but got {} for a".format(int, str))
+
def test_pardo_properly_type_checks_using_type_hint_decorators(self):
@with_input_types(a=str)
@with_output_types(typing.List[str])
@@ -1375,8 +1387,7 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def test_pardo_does_not_type_check_using_type_hint_methods(self):
# The first ParDo outputs pcoll's of type int, however the second ParDo is
# expecting pcoll's of type str instead.
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'Upper.*requires.*str.*applied.*int'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'S' >> beam.Create(['t', 'e', 's', 't']).with_output_types(str)
@@ -1387,6 +1398,11 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
'Upper' >> beam.FlatMap(lambda x: [x.upper()]).with_input_types(
str).with_output_types(str)))
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'Upper': "
+ "requires {} but got {} for x".format(str, int))
+
def test_pardo_properly_type_checks_using_type_hint_methods(self):
# Pipeline should be created successfully without an error
d = (
@@ -1403,14 +1419,18 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def test_map_does_not_type_check_using_type_hints_methods(self):
# The transform before 'Map' has indicated that it outputs PCollections with
# int's, while Map is expecting one of str.
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'Upper.*requires.*str.*applied.*int'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'S' >> beam.Create([1, 2, 3, 4]).with_output_types(int)
| 'Upper' >> beam.Map(lambda x: x.upper()).with_input_types(
str).with_output_types(str))
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'Upper': "
+ "requires {} but got {} for x".format(str, int))
+
def test_map_properly_type_checks_using_type_hints_methods(self):
# No error should be raised if this type-checks properly.
d = (
@@ -1429,13 +1449,17 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
# Hinted function above expects a str at pipeline construction.
# However, 'Map' should detect that Create has hinted an int instead.
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'Upper.*requires.*str.*applied.*int'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'S' >> beam.Create([1, 2, 3, 4]).with_output_types(int)
| 'Upper' >> beam.Map(upper))
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'Upper': "
+ "requires {} but got {} for s".format(str, int))
+
def test_map_properly_type_checks_using_type_hints_decorator(self):
@with_input_types(a=bool)
@with_output_types(int)
@@ -1453,8 +1477,7 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
def test_filter_does_not_type_check_using_type_hints_method(self):
# Filter is expecting an int but instead looks to the 'left' and sees a str
# incoming.
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'Below 3.*requires.*int.*applied.*str'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'Strs' >> beam.Create(['1', '2', '3', '4', '5'
@@ -1463,6 +1486,11 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
str).with_output_types(str)
| 'Below 3' >> beam.Filter(lambda x: x < 3).with_input_types(int))
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'Below 3': "
+ "requires {} but got {} for x".format(int, str))
+
def test_filter_type_checks_using_type_hints_method(self):
# No error should be raised if this type-checks properly.
d = (
@@ -1480,13 +1508,17 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
return a > 0.50
# Func above was hinted to only take a float, yet a str will be passed.
- with self.assertRaisesRegex(typehints.TypeCheckError,
- r'Half.*requires.*float.*applied.*str'):
+ with self.assertRaises(typehints.TypeCheckError) as e:
(
self.p
| 'Ints' >> beam.Create(['1', '2', '3', '4']).with_output_types(str)
| 'Half' >> beam.Filter(more_than_half))
+ self.assertStartswith(
+ e.exception.args[0],
+ "Type hint violation for 'Half': "
+ "requires {} but got {} for a".format(float, str))
+
def test_filter_type_checks_using_type_hints_decorator(self):
@with_input_types(b=int)
def half(b):
@@ -2096,10 +2128,14 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
self.p
| 'C' >> beam.Create(['test']).with_output_types(str)
| 'Mean' >> combine.Mean.Globally())
- err_msg = e.exception.args[0]
- assert "CombinePerKey" in err_msg
- assert "Tuple[TypeVariable[K]" in err_msg
- assert "Tuple[None, <class 'str'>" in err_msg
+
+ expected_msg = \
+ "Type hint violation for 'CombinePerKey': " \
+ "requires Tuple[TypeVariable[K], Union[<class 'float'>, <class 'int'>, " \
+ "<class 'numpy.float64'>, <class 'numpy.int64'>]] " \
+ "but got Tuple[None, <class 'str'>] for element"
+
+ self.assertStartswith(e.exception.args[0], expected_msg)
def test_mean_globally_runtime_checking_satisfied(self):
self.p._options.view_as(TypeOptions).runtime_type_check = True
@@ -2159,12 +2195,14 @@ class PTransformTypeCheckTestCase(TypeHintTestCase):
typing.Tuple[str, str]))
| 'EvenMean' >> combine.Mean.PerKey())
self.p.run()
- err_msg = e.exception.args[0]
- assert "CombinePerKey(MeanCombineFn)" in err_msg
- assert "requires" in err_msg
- assert "Tuple[TypeVariable[K]" in err_msg
- assert "applied" in err_msg
- assert "Tuple[<class 'str'>, <class 'str'>]" in err_msg
+
+ expected_msg = \
+ "Type hint violation for 'CombinePerKey(MeanCombineFn)': " \
+ "requires Tuple[TypeVariable[K], Union[<class 'float'>, <class 'int'>, " \
+ "<class 'numpy.float64'>, <class 'numpy.int64'>]] " \
+ "but got Tuple[<class 'str'>, <class 'str'>] for element"
+
+ self.assertStartswith(e.exception.args[0], expected_msg)
def test_mean_per_key_runtime_checking_satisfied(self):
self.p._options.view_as(TypeOptions).runtime_type_check = True
diff --git a/sdks/python/apache_beam/typehints/decorators_test.py b/sdks/python/apache_beam/typehints/decorators_test.py
index 71edc75f31a..3baf9fa8322 100644
--- a/sdks/python/apache_beam/typehints/decorators_test.py
+++ b/sdks/python/apache_beam/typehints/decorators_test.py
@@ -409,7 +409,7 @@ class DecoratorsTest(unittest.TestCase):
return a
with self.assertRaisesRegex(TypeCheckError,
- r'requires .*int.* but was applied .*str'):
+ r'requires .*int.* but got .*str'):
_ = ['a', 'b', 'c'] | Map(fn)
# Same pipeline doesn't raise without annotations on fn.
@@ -423,7 +423,7 @@ class DecoratorsTest(unittest.TestCase):
_ = [1, 2, 3] | Map(fn) # Doesn't raise - correct types.
with self.assertRaisesRegex(TypeCheckError,
- r'requires .*int.* but was applied .*str'):
+ r'requires .*int.* but got .*str'):
_ = ['a', 'b', 'c'] | Map(fn)
@decorators.no_annotations
diff --git a/sdks/python/apache_beam/typehints/typed_pipeline_test.py b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
index 57e7f44f692..72aed46f5e7 100644
--- a/sdks/python/apache_beam/typehints/typed_pipeline_test.py
+++ b/sdks/python/apache_beam/typehints/typed_pipeline_test.py
@@ -88,11 +88,11 @@ class MainInputTest(unittest.TestCase):
self.assertEqual(['1', '2', '3'], sorted(result))
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
['a', 'b', 'c'] | beam.ParDo(MyDoFn())
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
[1, 2, 3] | (beam.ParDo(MyDoFn()) | 'again' >> beam.ParDo(MyDoFn()))
def test_typed_dofn_method(self):
@@ -104,11 +104,11 @@ class MainInputTest(unittest.TestCase):
self.assertEqual(['1', '2', '3'], sorted(result))
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
_ = ['a', 'b', 'c'] | beam.ParDo(MyDoFn())
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
_ = [1, 2, 3] | (beam.ParDo(MyDoFn()) | 'again' >> beam.ParDo(MyDoFn()))
def test_typed_dofn_method_with_class_decorators(self):
@@ -124,12 +124,12 @@ class MainInputTest(unittest.TestCase):
with self.assertRaisesRegex(
typehints.TypeCheckError,
- r'requires.*Tuple\[<class \'int\'>, <class \'int\'>\].*applied.*str'):
+ r'requires.*Tuple\[<class \'int\'>, <class \'int\'>\].*got.*str'):
_ = ['a', 'b', 'c'] | beam.ParDo(MyDoFn())
with self.assertRaisesRegex(
typehints.TypeCheckError,
- r'requires.*Tuple\[<class \'int\'>, <class \'int\'>\].*applied.*int'):
+ r'requires.*Tuple\[<class \'int\'>, <class \'int\'>\].*got.*int'):
_ = [1, 2, 3] | (beam.ParDo(MyDoFn()) | 'again' >> beam.ParDo(MyDoFn()))
def test_typed_callable_iterable_output(self):
@@ -156,11 +156,11 @@ class MainInputTest(unittest.TestCase):
self.assertEqual(['1', '2', '3'], sorted(result))
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
_ = ['a', 'b', 'c'] | beam.ParDo(my_do_fn)
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
_ = [1, 2, 3] | (beam.ParDo(my_do_fn) | 'again' >> beam.ParDo(my_do_fn))
def test_typed_callable_instance(self):
@@ -177,11 +177,11 @@ class MainInputTest(unittest.TestCase):
self.assertEqual(['1', '2', '3'], sorted(result))
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
_ = ['a', 'b', 'c'] | pardo
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'requires.*int.*applied.*str'):
+ r'requires.*int.*got.*str'):
_ = [1, 2, 3] | (pardo | 'again' >> pardo)
def test_filter_type_hint(self):
@@ -430,7 +430,7 @@ class MainInputTest(unittest.TestCase):
return pcoll | beam.ParDo(fn)
with self.assertRaisesRegex(typehints.TypeCheckError,
- r'ParDo.*requires.*float.*applied.*str'):
+ r'ParDo.*requires.*float.*got.*str'):
_ = ['1', '2', '3'] | MyMap()
with self.assertRaisesRegex(typehints.TypeCheckError,
r'MyMap.*expected.*str.*got.*bytes'):
@@ -632,14 +632,14 @@ class MainInputTest(unittest.TestCase):
return e
@typehints.with_input_types(int)
- def accepts_int(e):
+ def requires_int(e):
return e
class MyPTransform(beam.PTransform):
def expand(self, pcoll):
unknowns = pcoll | beam.Map(produces_unkown)
ints = pcoll | beam.Map(int)
- return (unknowns, ints) | beam.Flatten() | beam.Map(accepts_int)
+ return (unknowns, ints) | beam.Flatten() | beam.Map(requires_int)
_ = [1, 2, 3] | MyPTransform()
@@ -761,8 +761,8 @@ class SideInputTest(unittest.TestCase):
with self.assertRaisesRegex(
typehints.TypeCheckError,
- r'requires.*Tuple\[Union\[<class \'int\'>, <class \'str\'>\], ...\].*'
- r'applied.*Tuple\[Union\[<class \'float\'>, <class \'int\'>\], ...\]'):
+ r'requires Tuple\[Union\[<class \'int\'>, <class \'str\'>\], ...\] but '
+ r'got Tuple\[Union\[<class \'float\'>, <class \'int\'>\], ...\]'):
_ = [1.2] | beam.Map(lambda *_: 'a', 5).with_input_types(int, str)
def test_var_keyword_side_input_hint(self):