You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2021/03/09 07:04:56 UTC
[flink] branch master updated: [FLINK-21641][python] Support
'useModules' and 'listFullModules' in PyFlink (#15101)
This is an automated email from the ASF dual-hosted git repository.
dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 77f7db7 [FLINK-21641][python] Support 'useModules' and 'listFullModules' in PyFlink (#15101)
77f7db7 is described below
commit 77f7db7f1c0e6f4249b78abc7c96c6a66752c9b4
Author: Jane <55...@users.noreply.github.com>
AuthorDate: Tue Mar 9 15:04:34 2021 +0800
[FLINK-21641][python] Support 'useModules' and 'listFullModules' in PyFlink (#15101)
---
flink-python/pyflink/table/__init__.py | 3 +-
flink-python/pyflink/table/module.py | 36 ++++++-
flink-python/pyflink/table/table_environment.py | 30 +++++-
.../table/tests/test_environment_completeness.py | 4 -
.../table/tests/test_table_environment_api.py | 103 ++++++++++++++++++++-
5 files changed, 165 insertions(+), 11 deletions(-)
diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py
index b082331..75436c9 100644
--- a/flink-python/pyflink/table/__init__.py
+++ b/flink-python/pyflink/table/__init__.py
@@ -72,7 +72,7 @@ from pyflink.table.data_view import DataView, ListView, MapView
from pyflink.table.environment_settings import EnvironmentSettings
from pyflink.table.explain_detail import ExplainDetail
from pyflink.table.expression import Expression
-from pyflink.table.module import Module
+from pyflink.table.module import Module, ModuleEntry
from pyflink.table.result_kind import ResultKind
from pyflink.table.sinks import CsvTableSink, TableSink, WriteMode
from pyflink.table.sources import CsvTableSource, TableSource
@@ -105,6 +105,7 @@ __all__ = [
'ListView',
'MapView',
'Module',
+ 'ModuleEntry',
'OverWindowedTable',
'ResultKind',
'Row',
diff --git a/flink-python/pyflink/table/module.py b/flink-python/pyflink/table/module.py
index 52ba22a..f4cdc2e 100644
--- a/flink-python/pyflink/table/module.py
+++ b/flink-python/pyflink/table/module.py
@@ -17,7 +17,7 @@
################################################################################
from pyflink.java_gateway import get_gateway
-__all__ = ['Module', 'HiveModule']
+__all__ = ['HiveModule', 'Module', 'ModuleEntry']
class Module(object):
@@ -49,3 +49,37 @@ class HiveModule(Module):
else:
j_hive_module = gateway.jvm.org.apache.flink.table.module.hive.HiveModule(hive_version)
super(HiveModule, self).__init__(j_hive_module)
+
+
+class ModuleEntry(object):
+ """
+ A POJO to represent a module's name and use status.
+ """
+
+ def __init__(self, name: str, used: bool, j_module_entry=None):
+ if j_module_entry is None:
+ gateway = get_gateway()
+ self._j_module_entry = gateway.jvm.org.apache.flink.table.module.ModuleEntry(name, used)
+ else:
+ self._j_module_entry = j_module_entry
+
+ def name(self) -> str:
+ return self._j_module_entry.name()
+
+ def used(self) -> bool:
+ return self._j_module_entry.used()
+
+ def __repr__(self):
+ return self._j_module_entry.toString()
+
+ def __eq__(self, other):
+ return isinstance(other, self.__class__) and self._j_module_entry == other._j_module_entry
+
+ def __hash__(self):
+ return self._j_module_entry.hashCode()
+
+ def __ne__(self, other):
+ return not self.__eq__(other)
+
+ def __lt__(self, other):
+ return not self.__eq__(other)
diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py
index 29b6763..920086e 100644
--- a/flink-python/pyflink/table/table_environment.py
+++ b/flink-python/pyflink/table/table_environment.py
@@ -33,7 +33,8 @@ from pyflink.common import JobExecutionResult
from pyflink.dataset import ExecutionEnvironment
from pyflink.java_gateway import get_gateway
from pyflink.serializers import BatchedSerializer, PickleSerializer
-from pyflink.table import Table, EnvironmentSettings, Module, Expression, ExplainDetail, TableSink
+from pyflink.table import Table, EnvironmentSettings, Expression, ExplainDetail, \
+ Module, ModuleEntry, TableSink
from pyflink.table.catalog import Catalog
from pyflink.table.descriptors import StreamTableDescriptor, BatchTableDescriptor, \
ConnectorDescriptor, ConnectTableDescriptor
@@ -49,7 +50,7 @@ from pyflink.table.udf import UserDefinedFunctionWrapper, AggregateFunction, uda
from pyflink.table.utils import to_expression_jarray
from pyflink.util import utils
from pyflink.util.utils import get_j_env_configuration, is_local_deployment, load_java_class, \
- to_j_explain_detail_arr
+ to_j_explain_detail_arr, to_jarray
__all__ = [
'BatchTableEnvironment',
@@ -189,6 +190,18 @@ class TableEnvironment(object):
"""
self._j_tenv.unloadModule(module_name)
+ def use_modules(self, *module_names: str):
+ """
+ Use an array of :class:`~pyflink.table.Module` with given names.
+ ValidationException is thrown when there is duplicate name or no module with the given name.
+
+ :param module_names: Names of the modules to be used.
+
+ .. versionadded:: 1.13.0
+ """
+ j_module_names = to_jarray(get_gateway().jvm.String, module_names)
+ self._j_tenv.useModules(j_module_names)
+
def create_java_temporary_system_function(self, name: str, function_class_name: str):
"""
Registers a java user defined function class as a temporary system function.
@@ -590,7 +603,7 @@ class TableEnvironment(object):
def list_modules(self) -> List[str]:
"""
- Gets the names of all modules registered in this environment.
+ Gets the names of all modules used in this environment.
:return: List of module names.
@@ -599,6 +612,17 @@ class TableEnvironment(object):
j_module_name_array = self._j_tenv.listModules()
return [item for item in j_module_name_array]
+ def list_full_modules(self) -> List[ModuleEntry]:
+ """
+ Gets the names and statuses of all modules loaded in this environment.
+
+ :return: List of module names and use statuses.
+
+ .. versionadded:: 1.13.0
+ """
+ j_module_entry_array = self._j_tenv.listFullModules()
+ return [ModuleEntry(entry.name(), entry.used()) for entry in j_module_entry_array]
+
def list_databases(self) -> List[str]:
"""
Gets the names of all databases in the current catalog.
diff --git a/flink-python/pyflink/table/tests/test_environment_completeness.py b/flink-python/pyflink/table/tests/test_environment_completeness.py
index a9fec44..e941f5a 100644
--- a/flink-python/pyflink/table/tests/test_environment_completeness.py
+++ b/flink-python/pyflink/table/tests/test_environment_completeness.py
@@ -41,10 +41,6 @@ class EnvironmentAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTest
'getCompletionHints',
'fromValues',
'create',
- 'loadModule',
- 'unloadModule',
- 'useModules',
- 'listFullModules',
'createTemporarySystemFunction',
'dropTemporarySystemFunction',
'createFunction',
diff --git a/flink-python/pyflink/table/tests/test_table_environment_api.py b/flink-python/pyflink/table/tests/test_table_environment_api.py
index 911544c..c6f841f 100644
--- a/flink-python/pyflink/table/tests/test_table_environment_api.py
+++ b/flink-python/pyflink/table/tests/test_table_environment_api.py
@@ -32,7 +32,7 @@ from pyflink.datastream.tests.test_util import DataStreamTestSinkFunction
from pyflink.find_flink_home import _find_flink_source_root
from pyflink.java_gateway import get_gateway
from pyflink.table import DataTypes, CsvTableSink, StreamTableEnvironment, EnvironmentSettings, \
- Module, ResultKind
+ Module, ResultKind, ModuleEntry
from pyflink.table.descriptors import FileSystem, OldCsv, Schema
from pyflink.table.explain_detail import ExplainDetail
from pyflink.table.expressions import col
@@ -43,7 +43,7 @@ from pyflink.testing import source_sink_utils
from pyflink.testing.test_case_utils import PyFlinkOldStreamTableTestCase, \
PyFlinkOldBatchTableTestCase, PyFlinkBlinkBatchTableTestCase, PyFlinkBlinkStreamTableTestCase, \
PyFlinkLegacyBlinkBatchTableTestCase, PyFlinkLegacyFlinkStreamTableTestCase, \
- PyFlinkLegacyBlinkStreamTableTestCase
+ PyFlinkLegacyBlinkStreamTableTestCase, _load_specific_flink_module_jars
from pyflink.util.utils import get_j_env_configuration
@@ -97,6 +97,52 @@ class TableEnvironmentTest(object):
expected = ['python_scalar_func', 'scalar_func', 'agg_func', 'table_func']
self.assert_equals(actual, expected)
+ def test_load_module_twice(self):
+ t_env = self.t_env
+ self.check_list_modules('core')
+ self.check_list_full_modules(1, 'core')
+ self.assertRaisesRegex(
+ Py4JJavaError, "A module with name 'core' already exists",
+ t_env.load_module, 'core', Module(
+ get_gateway().jvm.org.apache.flink.table.module.CoreModule.INSTANCE))
+
+ def test_unload_module_twice(self):
+ t_env = self.t_env
+ t_env.unload_module('core')
+ self.check_list_modules()
+ self.check_list_full_modules(0)
+ self.assertRaisesRegex(
+ Py4JJavaError, "No module with name 'core' exists",
+ t_env.unload_module, 'core')
+
+ def test_use_modules(self):
+ # please do not change this order since ModuleMock depends on FunctionDefinitionMock
+ _load_specific_flink_module_jars('/flink-table/flink-table-common')
+ _load_specific_flink_module_jars('/flink-table/flink-table-api-java')
+
+ t_env = self.t_env
+ t_env.load_module('x', Module(
+ get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("x")
+ ))
+ t_env.load_module('y', Module(
+ get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("y")
+ ))
+ self.check_list_modules('core', 'x', 'y')
+ self.check_list_full_modules(3, 'core', 'x', 'y')
+
+ t_env.use_modules('y', 'core')
+ self.check_list_modules('y', 'core')
+ self.check_list_full_modules(2, 'y', 'core', 'x')
+
+ def check_list_modules(self, *expected_used_modules: str):
+ self.assert_equals(self.t_env.list_modules(), list(expected_used_modules))
+
+ def check_list_full_modules(self, used_module_cnt: int, *expected_loaded_modules: str):
+ self.assert_equals(self.t_env.list_full_modules(),
+ [ModuleEntry(module,
+ expected_loaded_modules.index(module) < used_module_cnt)
+ for module in expected_loaded_modules])
+
def test_unload_and_load_module(self):
t_env = self.t_env
t_env.unload_module('core')
@@ -943,6 +989,59 @@ class BlinkBatchTableEnvironmentTests(PyFlinkBlinkBatchTableTestCase):
expected = ['scalar_func', 'agg_func', 'table_func']
self.assert_equals(actual, expected)
+ def test_load_module_twice(self):
+ self.check_list_modules('core')
+ self.check_list_full_modules(1, 'core')
+ self.assertRaisesRegex(
+ Py4JJavaError, "A module with name 'core' already exists",
+ self.t_env.load_module, 'core', Module(
+ get_gateway().jvm.org.apache.flink.table.module.CoreModule.INSTANCE))
+
+ def test_unload_module_twice(self):
+ self.t_env.unload_module('core')
+ self.check_list_modules()
+ self.check_list_full_modules(0)
+ self.assertRaisesRegex(
+ Py4JJavaError, "No module with name 'core' exists",
+ self.t_env.unload_module, 'core')
+
+ def test_use_duplicated_modules(self):
+ self.assertRaisesRegex(
+ Py4JJavaError, "Module 'core' appears more than once",
+ self.t_env.use_modules, 'core', 'core')
+
+ def test_use_nonexistent_module(self):
+ self.assertRaisesRegex(
+ Py4JJavaError, "No module with name 'dummy' exists",
+ self.t_env.use_modules, 'core', 'dummy')
+
+ def test_use_modules(self):
+ # please do not change this order since ModuleMock depends on FunctionDefinitionMock
+ _load_specific_flink_module_jars('/flink-table/flink-table-common')
+ _load_specific_flink_module_jars('/flink-table/flink-table-api-java')
+
+ self.t_env.load_module('x', Module(
+ get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("x")
+ ))
+ self.t_env.load_module('y', Module(
+ get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("y")
+ ))
+ self.check_list_modules('core', 'x', 'y')
+ self.check_list_full_modules(3, 'core', 'x', 'y')
+
+ self.t_env.use_modules('y', 'core')
+ self.check_list_modules('y', 'core')
+ self.check_list_full_modules(2, 'y', 'core', 'x')
+
+ def check_list_modules(self, *expected_used_modules: str):
+ self.assert_equals(self.t_env.list_modules(), list(expected_used_modules))
+
+ def check_list_full_modules(self, used_module_cnt: int, *expected_loaded_modules: str):
+ self.assert_equals(self.t_env.list_full_modules(),
+ [ModuleEntry(module,
+ expected_loaded_modules.index(module) < used_module_cnt)
+ for module in expected_loaded_modules])
+
def test_unload_and_load_module(self):
t_env = self.t_env
t_env.unload_module('core')