You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@wayang.apache.org by be...@apache.org on 2022/04/08 17:16:45 UTC
[incubator-wayang] 24/32: [WAYANG-#8] add types Tests and small corrections
This is an automated email from the ASF dual-hosted git repository.
bertty pushed a commit to branch python-platform
in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit 220590b861ac8d118451bb59e6afa3ed57d41762
Author: Bertty Contreras-Rojas <be...@databloom.ai>
AuthorDate: Thu Apr 7 20:22:31 2022 +0200
[WAYANG-#8] add types Tests and small corrections
Signed-off-by: bertty <be...@apache.org>
---
python/src/pywy/dataquanta.py | 8 +-
python/src/pywy/exception.py | 3 +
python/src/pywy/operators/unary.py | 22 +--
python/src/pywy/tests/unit/types_test.py | 259 ++++++++++++++++++++++++++++++-
python/src/pywy/types.py | 64 ++++----
5 files changed, 303 insertions(+), 53 deletions(-)
diff --git a/python/src/pywy/dataquanta.py b/python/src/pywy/dataquanta.py
index 0b9483d3..283286e5 100644
--- a/python/src/pywy/dataquanta.py
+++ b/python/src/pywy/dataquanta.py
@@ -1,7 +1,7 @@
from typing import Set, List, cast
from pywy.core import Translator
-from pywy.types import (GenericTco, Predicate, Function, FlatmapFunction, IterableO, T, I, O)
+from pywy.types import (GenericTco, Predicate, Function, FlatmapFunction, IterableOut, T, In, Out)
from pywy.operators import *
from pywy.core import PywyPlan
from pywy.core import Plugin
@@ -57,13 +57,13 @@ class DataQuanta(GenericTco):
def filter(self: "DataQuanta[T]", p: Predicate) -> "DataQuanta[T]":
return DataQuanta(self.context, self._connect(FilterOperator(p)))
- def map(self: "DataQuanta[I]", f: Function) -> "DataQuanta[O]":
+ def map(self: "DataQuanta[In]", f: Function) -> "DataQuanta[Out]":
return DataQuanta(self.context, self._connect(MapOperator(f)))
- def flatmap(self: "DataQuanta[I]", f: FlatmapFunction) -> "DataQuanta[IterableO]":
+ def flatmap(self: "DataQuanta[In]", f: FlatmapFunction) -> "DataQuanta[IterableOut]":
return DataQuanta(self.context, self._connect(FlatmapOperator(f)))
- def store_textfile(self: "DataQuanta[I]", path: str):
+ def store_textfile(self: "DataQuanta[In]", path: str):
last: List[SinkOperator] = [cast(SinkOperator, self._connect(TextFileSink(path)))]
plan = PywyPlan(self.context.plugins, last)
diff --git a/python/src/pywy/exception.py b/python/src/pywy/exception.py
new file mode 100644
index 00000000..a6f59b75
--- /dev/null
+++ b/python/src/pywy/exception.py
@@ -0,0 +1,3 @@
+
+class PywyException(Exception):
+ pass
\ No newline at end of file
diff --git a/python/src/pywy/operators/unary.py b/python/src/pywy/operators/unary.py
index 5b4b8b0a..198a9fbf 100644
--- a/python/src/pywy/operators/unary.py
+++ b/python/src/pywy/operators/unary.py
@@ -1,14 +1,14 @@
from itertools import chain
from pywy.operators.base import PywyOperator
from pywy.types import (
- GenericTco,
- GenericUco,
- Predicate,
- getTypePredicate,
- Function,
- getTypeFunction,
- FlatmapFunction,
- getTypeFlatmapFunction
+ GenericTco,
+ GenericUco,
+ Predicate,
+ get_type_predicate,
+ Function,
+ get_type_function,
+ FlatmapFunction,
+ get_type_flatmap_function
)
@@ -34,7 +34,7 @@ class FilterOperator(UnaryToUnaryOperator):
predicate: Predicate
def __init__(self, predicate: Predicate):
- type = getTypePredicate(predicate) if predicate else None
+ type = get_type_predicate(predicate) if predicate else None
super().__init__("Filter", type, type)
self.predicate = predicate
@@ -49,7 +49,7 @@ class MapOperator(UnaryToUnaryOperator):
function: Function
def __init__(self, function: Function):
- types = getTypeFunction(function) if function else (None, None)
+ types = get_type_function(function) if function else (None, None)
super().__init__("Map", types[0], types[1])
self.function = function
@@ -71,7 +71,7 @@ class FlatmapOperator(UnaryToUnaryOperator):
fmfunction: FlatmapFunction
def __init__(self, fmfunction: FlatmapFunction):
- types = getTypeFlatmapFunction(fmfunction) if fmfunction else (None, None)
+ types = get_type_flatmap_function(fmfunction) if fmfunction else (None, None)
super().__init__("Flatmap", types[0], types[1])
self.fmfunction = fmfunction
diff --git a/python/src/pywy/tests/unit/types_test.py b/python/src/pywy/tests/unit/types_test.py
index d1e6774f..bb3783e8 100644
--- a/python/src/pywy/tests/unit/types_test.py
+++ b/python/src/pywy/tests/unit/types_test.py
@@ -1,16 +1,259 @@
+import inspect
import unittest
+from unittest.mock import Mock
+from pywy.exception import PywyException
+from pywy.types import get_type_function, get_type_bifunction, get_type_flatmap_function, get_type_predicate
-class TestUnitTypes(unittest.TestCase):
+empty_type = inspect._empty
+
+
+class TestUnitTypesPredicate(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ def test_predicate_without_parameters(self):
+ def pred() -> bool:
+ return True
+
+ try:
+ get_type_predicate(pred)
+ self.fail("The predicates parameters are mandatory")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Predicate are distinct than one," in str(ex))
+
+ def test_predicate_with_one_parameter_no_type(self):
+ def pred(x) -> bool:
+ return True
+
+ try:
+ pred_type = get_type_predicate(pred)
+ self.assertEqual(pred_type, empty_type)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_predicate_with_one_parameter_with_basic_type(self):
+ def pred(x: int) -> bool:
+ return True
+
+ try:
+ pred_type = get_type_predicate(pred)
+ self.assertEqual(pred_type, int)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_predicate_with_one_parameter_with_obe_type(self):
+ def pred(x: Mock) -> bool:
+ return True
+
+ try:
+ pred_type = get_type_predicate(pred)
+ self.assertEqual(pred_type, Mock)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_predicate_with_two_parameters(self):
+ def pred(x: Mock, y: Mock) -> bool:
+ return True
+
+ try:
+ get_type_predicate(pred)
+ self.fail("the predicate can have just one input")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Predicate are distinct than one" in str(ex))
+
+
+class TestUnitTypesFunction(unittest.TestCase):
def setUp(self):
pass
- def test_TO_REMOVE(self):
- """
- TODO REMOVE THIS TEST, IT JUST TO VALIDATE THAT EVERYTHING IS CORRECT IN TERMS OF ENVIRONMENT
- Returns
- -------
+ def test_function_without_parameters_no_return(self):
+ def func():
+ return
+
+ try:
+ get_type_function(func)
+ self.fail("The function parameters are mandatory")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Function are distinct than one," in str(ex))
+
+ def test_function_with_one_parameter_no_type_no_return(self):
+ def func(x):
+ return
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, empty_type)
+ self.assertEqual(output_type, empty_type)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_one_parameter_with_basic_type_no_return(self):
+ def func(x: int):
+ return
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, int)
+ self.assertEqual(output_type, empty_type)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_one_parameter_with_obj_type_no_return(self):
+ def func(x: Mock):
+ return
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, Mock)
+ self.assertEqual(output_type, empty_type)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_two_parameters_no_return(self):
+ def func(x: Mock, y: Mock):
+ return
+
+ try:
+ get_type_function(func)
+ self.fail("the function can have just one input")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Function are distinct than one" in str(ex))
+
+ def test_function_without_parameters_basic_return(self):
+ def func() -> int:
+ return 0
+
+ try:
+ get_type_function(func)
+ self.fail("The function parameters are mandatory")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Function are distinct than one," in str(ex))
+
+ def test_function_with_one_parameter_no_type_basic_return(self):
+ def func(x) -> int:
+ return 0
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, empty_type)
+ self.assertEqual(output_type, int)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_one_parameter_with_basic_type_basic_return(self):
+ def func(x: int) -> int:
+ return 0
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, int)
+ self.assertEqual(output_type, int)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_one_parameter_with_obj_type_basic_return(self):
+ def func(x: Mock) -> int:
+ return 0
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, Mock)
+ self.assertEqual(output_type, int)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_two_parameters_basic_return(self):
+ def func(x: Mock, y: Mock) -> int:
+ return 0
+
+ try:
+ get_type_function(func)
+ self.fail("the function can have just one input")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Function are distinct than one" in str(ex))
+
+ def test_function_without_parameters_obj_return(self):
+ def func() -> Mock:
+ return Mock()
+
+ try:
+ get_type_function(func)
+ self.fail("The function parameters are mandatory")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Function are distinct than one," in str(ex))
+
+ def test_function_with_one_parameter_no_type_obj_return(self):
+ def func(x) -> Mock:
+ return Mock()
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, empty_type)
+ self.assertEqual(output_type, Mock)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_one_parameter_with_basic_type_basic_return(self):
+ def func(x: int) -> Mock:
+ return Mock()
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, int)
+ self.assertEqual(output_type, Mock)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_one_parameter_with_obe_type_basic_return(self):
+ def func(x: Mock) -> Mock:
+ return Mock()
+
+ try:
+ input_type, output_type = get_type_function(func)
+ self.assertEqual(input_type, Mock)
+ self.assertEqual(output_type, Mock)
+ except PywyException as ex:
+ self.fail(str(ex))
+
+ def test_function_with_two_parameters_basic_return(self):
+ def func(x: Mock, y: Mock) -> Mock:
+ return Mock()
+
+ try:
+ get_type_function(func)
+ self.fail("the function can have just one input")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the Function are distinct than one" in str(ex))
+
+
+class TestUnitTypesBiFunction(unittest.TestCase):
+ def setUp(self):
+ pass
+
+ # TODO add the missing test for get_type_bifunction
+ def test_bifunction_without_parameters_no_return(self):
+ def func():
+ return
+
+ try:
+ get_type_bifunction(func)
+ self.fail("The bifunction parameters are mandatory")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the BiFunction are distinct than two," in str(ex))
+
+
+class TestUnitTypesFlatmapFunction(unittest.TestCase):
+ def setUp(self):
+ pass
- """
- self.assertEqual("a", "a")
+ # TODO add the missing test for get_type_flatmap_function
+ def test_flatmapfunction_without_parameters_no_return(self):
+ def func():
+ return
+ try:
+ get_type_flatmap_function(func)
+ self.fail("The bifunction parameters are mandatory")
+ except PywyException as ex:
+ self.assertTrue("the parameters for the FlatmapFunction are distinct than one," in str(ex))
diff --git a/python/src/pywy/types.py b/python/src/pywy/types.py
index 4a0979f5..e1413457 100644
--- a/python/src/pywy/types.py
+++ b/python/src/pywy/types.py
@@ -1,13 +1,15 @@
-from typing import ( Generic, TypeVar, Callable, Hashable, Iterable )
+from typing import (Generic, TypeVar, Callable, Hashable, Iterable)
from inspect import signature
+from pywy.exception import PywyException
-T = TypeVar("T") # Type
-I = TypeVar("I") # Input Type number 1
-I2 = TypeVar("I2") # Input Type number 2
-O = TypeVar("O") # Output Type
+T = TypeVar("T") # Type
+In = TypeVar("In") # Input Type number 1
+In2 = TypeVar("In2") # Input Type number 2
+Out = TypeVar("Out") # Output Type
-IterableT = Iterable[T] # Iterable of type 'T'
-IterableO = Iterable[O] # Iterable of type 'O'
+IterableT = Iterable[T] # Iterable of type 'T'
+IterableOut = Iterable[Out] # Iterable of type 'O'
+IterableIn = Iterable[In] # Iterable of type 'O'
T_co = TypeVar("T_co", covariant=True)
U_co = TypeVar("U_co", covariant=True)
@@ -17,41 +19,43 @@ GenericTco = Generic[T_co]
GenericUco = Generic[U_co]
Predicate = Callable[[T], bool]
-Function = Callable[[I], O]
-BiFunction = Callable[[I, I2], O]
-Function = Callable[[I], O]
+Function = Callable[[In], Out]
+BiFunction = Callable[[In, In2], Out]
-FlatmapFunction = Callable[[T], IterableO]
+FlatmapFunction = Callable[[T], IterableOut]
-def getTypePredicate(callable: Predicate) -> Generic :
- sig = signature(callable)
- if(len(sig.parameters) != 1):
- raise Exception("the parameters for the Predicate are distinct than one, {}".format(str(sig.parameters)))
+def get_type_predicate(call: Predicate) -> type:
+ sig = signature(call)
+ if len(sig.parameters) != 1:
+ raise PywyException("the parameters for the Predicate are distinct than one, {}".format(str(sig.parameters)))
keys = list(sig.parameters.keys())
return sig.parameters[keys[0]].annotation
-def getTypeFunction(callable: Function) -> Generic :
- sig = signature(callable)
- if(len(sig.parameters) != 1):
- raise Exception("the parameters for the Function are distinct than one, {}".format(str(sig.parameters)))
+
+def get_type_function(call: Function) -> (type, type):
+ sig = signature(call)
+ if len(sig.parameters) != 1:
+ raise PywyException("the parameters for the Function are distinct than one, {}".format(str(sig.parameters)))
keys = list(sig.parameters.keys())
- return (sig.parameters[keys[0]].annotation, sig.return_annotation)
+ return sig.parameters[keys[0]].annotation, sig.return_annotation
+
-def getTypeBiFunction(callable: BiFunction) -> (Generic, Generic, Generic) :
- sig = signature(callable)
- if(len(sig.parameters) != 2):
- raise Exception("the parameters for the BiFunction are distinct than two, {}".format(str(sig.parameters)))
+def get_type_bifunction(call: BiFunction) -> (type, type, type):
+ sig = signature(call)
+ if len(sig.parameters) != 2:
+ raise PywyException("the parameters for the BiFunction are distinct than two, {}".format(str(sig.parameters)))
keys = list(sig.parameters.keys())
- return (sig.parameters[keys[0]].annotation, sig.parameters[keys[1]].annotation, sig.return_annotation)
+ return sig.parameters[keys[0]].annotation, sig.parameters[keys[1]].annotation, sig.return_annotation
+
-def getTypeFlatmapFunction(callable: FlatmapFunction) -> (Generic, Generic) :
- sig = signature(callable)
- if(len(sig.parameters) != 1):
- raise Exception("the parameters for the FlatmapFunction are distinct than one, {}".format(str(sig.parameters)))
+def get_type_flatmap_function(call: FlatmapFunction) -> (type, type):
+ sig = signature(call)
+ if len(sig.parameters) != 1:
+ raise PywyException("the parameters for the FlatmapFunction are distinct than one, {}".format(str(sig.parameters)))
keys = list(sig.parameters.keys())
- return (sig.parameters[keys[0]].annotation, sig.return_annotation)
\ No newline at end of file
+ return sig.parameters[keys[0]].annotation, sig.return_annotation