You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by di...@apache.org on 2021/03/09 07:04:56 UTC

[flink] branch master updated: [FLINK-21641][python] Support 'useModules' and 'listFullModules' in PyFlink (#15101)

This is an automated email from the ASF dual-hosted git repository.

dianfu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git


The following commit(s) were added to refs/heads/master by this push:
     new 77f7db7  [FLINK-21641][python] Support 'useModules' and 'listFullModules' in PyFlink (#15101)
77f7db7 is described below

commit 77f7db7f1c0e6f4249b78abc7c96c6a66752c9b4
Author: Jane <55...@users.noreply.github.com>
AuthorDate: Tue Mar 9 15:04:34 2021 +0800

    [FLINK-21641][python] Support 'useModules' and 'listFullModules' in PyFlink (#15101)
---
 flink-python/pyflink/table/__init__.py             |   3 +-
 flink-python/pyflink/table/module.py               |  36 ++++++-
 flink-python/pyflink/table/table_environment.py    |  30 +++++-
 .../table/tests/test_environment_completeness.py   |   4 -
 .../table/tests/test_table_environment_api.py      | 103 ++++++++++++++++++++-
 5 files changed, 165 insertions(+), 11 deletions(-)

diff --git a/flink-python/pyflink/table/__init__.py b/flink-python/pyflink/table/__init__.py
index b082331..75436c9 100644
--- a/flink-python/pyflink/table/__init__.py
+++ b/flink-python/pyflink/table/__init__.py
@@ -72,7 +72,7 @@ from pyflink.table.data_view import DataView, ListView, MapView
 from pyflink.table.environment_settings import EnvironmentSettings
 from pyflink.table.explain_detail import ExplainDetail
 from pyflink.table.expression import Expression
-from pyflink.table.module import Module
+from pyflink.table.module import Module, ModuleEntry
 from pyflink.table.result_kind import ResultKind
 from pyflink.table.sinks import CsvTableSink, TableSink, WriteMode
 from pyflink.table.sources import CsvTableSource, TableSource
@@ -105,6 +105,7 @@ __all__ = [
     'ListView',
     'MapView',
     'Module',
+    'ModuleEntry',
     'OverWindowedTable',
     'ResultKind',
     'Row',
diff --git a/flink-python/pyflink/table/module.py b/flink-python/pyflink/table/module.py
index 52ba22a..f4cdc2e 100644
--- a/flink-python/pyflink/table/module.py
+++ b/flink-python/pyflink/table/module.py
@@ -17,7 +17,7 @@
 ################################################################################
 from pyflink.java_gateway import get_gateway
 
-__all__ = ['Module', 'HiveModule']
+__all__ = ['HiveModule', 'Module', 'ModuleEntry']
 
 
 class Module(object):
@@ -49,3 +49,37 @@ class HiveModule(Module):
         else:
             j_hive_module = gateway.jvm.org.apache.flink.table.module.hive.HiveModule(hive_version)
         super(HiveModule, self).__init__(j_hive_module)
+
+
+class ModuleEntry(object):
+    """
+    A POJO to represent a module's name and use status.
+    """
+
+    def __init__(self, name: str, used: bool, j_module_entry=None):
+        if j_module_entry is None:
+            gateway = get_gateway()
+            self._j_module_entry = gateway.jvm.org.apache.flink.table.module.ModuleEntry(name, used)
+        else:
+            self._j_module_entry = j_module_entry
+
+    def name(self) -> str:
+        return self._j_module_entry.name()
+
+    def used(self) -> bool:
+        return self._j_module_entry.used()
+
+    def __repr__(self):
+        return self._j_module_entry.toString()
+
+    def __eq__(self, other):
+        return isinstance(other, self.__class__) and self._j_module_entry == other._j_module_entry
+
+    def __hash__(self):
+        return self._j_module_entry.hashCode()
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __lt__(self, other):
+        return not self.__eq__(other)
diff --git a/flink-python/pyflink/table/table_environment.py b/flink-python/pyflink/table/table_environment.py
index 29b6763..920086e 100644
--- a/flink-python/pyflink/table/table_environment.py
+++ b/flink-python/pyflink/table/table_environment.py
@@ -33,7 +33,8 @@ from pyflink.common import JobExecutionResult
 from pyflink.dataset import ExecutionEnvironment
 from pyflink.java_gateway import get_gateway
 from pyflink.serializers import BatchedSerializer, PickleSerializer
-from pyflink.table import Table, EnvironmentSettings, Module, Expression, ExplainDetail, TableSink
+from pyflink.table import Table, EnvironmentSettings, Expression, ExplainDetail, \
+    Module, ModuleEntry, TableSink
 from pyflink.table.catalog import Catalog
 from pyflink.table.descriptors import StreamTableDescriptor, BatchTableDescriptor, \
     ConnectorDescriptor, ConnectTableDescriptor
@@ -49,7 +50,7 @@ from pyflink.table.udf import UserDefinedFunctionWrapper, AggregateFunction, uda
 from pyflink.table.utils import to_expression_jarray
 from pyflink.util import utils
 from pyflink.util.utils import get_j_env_configuration, is_local_deployment, load_java_class, \
-    to_j_explain_detail_arr
+    to_j_explain_detail_arr, to_jarray
 
 __all__ = [
     'BatchTableEnvironment',
@@ -189,6 +190,18 @@ class TableEnvironment(object):
         """
         self._j_tenv.unloadModule(module_name)
 
+    def use_modules(self, *module_names: str):
+        """
+        Use an array of :class:`~pyflink.table.Module` with given names.
+        ValidationException is thrown when there is duplicate name or no module with the given name.
+
+        :param module_names: Names of the modules to be used.
+
+        .. versionadded:: 1.13.0
+        """
+        j_module_names = to_jarray(get_gateway().jvm.String, module_names)
+        self._j_tenv.useModules(j_module_names)
+
     def create_java_temporary_system_function(self, name: str, function_class_name: str):
         """
         Registers a java user defined function class as a temporary system function.
@@ -590,7 +603,7 @@ class TableEnvironment(object):
 
     def list_modules(self) -> List[str]:
         """
-        Gets the names of all modules registered in this environment.
+        Gets the names of all modules used in this environment.
 
         :return: List of module names.
 
@@ -599,6 +612,17 @@ class TableEnvironment(object):
         j_module_name_array = self._j_tenv.listModules()
         return [item for item in j_module_name_array]
 
+    def list_full_modules(self) -> List[ModuleEntry]:
+        """
+        Gets the names and statuses of all modules loaded in this environment.
+
+        :return: List of module names and use statuses.
+
+        .. versionadded:: 1.13.0
+        """
+        j_module_entry_array = self._j_tenv.listFullModules()
+        return [ModuleEntry(entry.name(), entry.used()) for entry in j_module_entry_array]
+
     def list_databases(self) -> List[str]:
         """
         Gets the names of all databases in the current catalog.
diff --git a/flink-python/pyflink/table/tests/test_environment_completeness.py b/flink-python/pyflink/table/tests/test_environment_completeness.py
index a9fec44..e941f5a 100644
--- a/flink-python/pyflink/table/tests/test_environment_completeness.py
+++ b/flink-python/pyflink/table/tests/test_environment_completeness.py
@@ -41,10 +41,6 @@ class EnvironmentAPICompletenessTests(PythonAPICompletenessTestCase, PyFlinkTest
             'getCompletionHints',
             'fromValues',
             'create',
-            'loadModule',
-            'unloadModule',
-            'useModules',
-            'listFullModules',
             'createTemporarySystemFunction',
             'dropTemporarySystemFunction',
             'createFunction',
diff --git a/flink-python/pyflink/table/tests/test_table_environment_api.py b/flink-python/pyflink/table/tests/test_table_environment_api.py
index 911544c..c6f841f 100644
--- a/flink-python/pyflink/table/tests/test_table_environment_api.py
+++ b/flink-python/pyflink/table/tests/test_table_environment_api.py
@@ -32,7 +32,7 @@ from pyflink.datastream.tests.test_util import DataStreamTestSinkFunction
 from pyflink.find_flink_home import _find_flink_source_root
 from pyflink.java_gateway import get_gateway
 from pyflink.table import DataTypes, CsvTableSink, StreamTableEnvironment, EnvironmentSettings, \
-    Module, ResultKind
+    Module, ResultKind, ModuleEntry
 from pyflink.table.descriptors import FileSystem, OldCsv, Schema
 from pyflink.table.explain_detail import ExplainDetail
 from pyflink.table.expressions import col
@@ -43,7 +43,7 @@ from pyflink.testing import source_sink_utils
 from pyflink.testing.test_case_utils import PyFlinkOldStreamTableTestCase, \
     PyFlinkOldBatchTableTestCase, PyFlinkBlinkBatchTableTestCase, PyFlinkBlinkStreamTableTestCase, \
     PyFlinkLegacyBlinkBatchTableTestCase, PyFlinkLegacyFlinkStreamTableTestCase, \
-    PyFlinkLegacyBlinkStreamTableTestCase
+    PyFlinkLegacyBlinkStreamTableTestCase, _load_specific_flink_module_jars
 from pyflink.util.utils import get_j_env_configuration
 
 
@@ -97,6 +97,52 @@ class TableEnvironmentTest(object):
         expected = ['python_scalar_func', 'scalar_func', 'agg_func', 'table_func']
         self.assert_equals(actual, expected)
 
+    def test_load_module_twice(self):
+        t_env = self.t_env
+        self.check_list_modules('core')
+        self.check_list_full_modules(1, 'core')
+        self.assertRaisesRegex(
+            Py4JJavaError, "A module with name 'core' already exists",
+            t_env.load_module, 'core', Module(
+                get_gateway().jvm.org.apache.flink.table.module.CoreModule.INSTANCE))
+
+    def test_unload_module_twice(self):
+        t_env = self.t_env
+        t_env.unload_module('core')
+        self.check_list_modules()
+        self.check_list_full_modules(0)
+        self.assertRaisesRegex(
+            Py4JJavaError, "No module with name 'core' exists",
+            t_env.unload_module, 'core')
+
+    def test_use_modules(self):
+        # please do not change this order since ModuleMock depends on FunctionDefinitionMock
+        _load_specific_flink_module_jars('/flink-table/flink-table-common')
+        _load_specific_flink_module_jars('/flink-table/flink-table-api-java')
+
+        t_env = self.t_env
+        t_env.load_module('x', Module(
+            get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("x")
+        ))
+        t_env.load_module('y', Module(
+            get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("y")
+        ))
+        self.check_list_modules('core', 'x', 'y')
+        self.check_list_full_modules(3, 'core', 'x', 'y')
+
+        t_env.use_modules('y', 'core')
+        self.check_list_modules('y', 'core')
+        self.check_list_full_modules(2, 'y', 'core', 'x')
+
+    def check_list_modules(self, *expected_used_modules: str):
+        self.assert_equals(self.t_env.list_modules(), list(expected_used_modules))
+
+    def check_list_full_modules(self, used_module_cnt: int, *expected_loaded_modules: str):
+        self.assert_equals(self.t_env.list_full_modules(),
+                           [ModuleEntry(module,
+                                        expected_loaded_modules.index(module) < used_module_cnt)
+                            for module in expected_loaded_modules])
+
     def test_unload_and_load_module(self):
         t_env = self.t_env
         t_env.unload_module('core')
@@ -943,6 +989,59 @@ class BlinkBatchTableEnvironmentTests(PyFlinkBlinkBatchTableTestCase):
         expected = ['scalar_func', 'agg_func', 'table_func']
         self.assert_equals(actual, expected)
 
+    def test_load_module_twice(self):
+        self.check_list_modules('core')
+        self.check_list_full_modules(1, 'core')
+        self.assertRaisesRegex(
+            Py4JJavaError, "A module with name 'core' already exists",
+            self.t_env.load_module, 'core', Module(
+                get_gateway().jvm.org.apache.flink.table.module.CoreModule.INSTANCE))
+
+    def test_unload_module_twice(self):
+        self.t_env.unload_module('core')
+        self.check_list_modules()
+        self.check_list_full_modules(0)
+        self.assertRaisesRegex(
+            Py4JJavaError, "No module with name 'core' exists",
+            self.t_env.unload_module, 'core')
+
+    def test_use_duplicated_modules(self):
+        self.assertRaisesRegex(
+            Py4JJavaError, "Module 'core' appears more than once",
+            self.t_env.use_modules, 'core', 'core')
+
+    def test_use_nonexistent_module(self):
+        self.assertRaisesRegex(
+            Py4JJavaError, "No module with name 'dummy' exists",
+            self.t_env.use_modules, 'core', 'dummy')
+
+    def test_use_modules(self):
+        # please do not change this order since ModuleMock depends on FunctionDefinitionMock
+        _load_specific_flink_module_jars('/flink-table/flink-table-common')
+        _load_specific_flink_module_jars('/flink-table/flink-table-api-java')
+
+        self.t_env.load_module('x', Module(
+            get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("x")
+        ))
+        self.t_env.load_module('y', Module(
+            get_gateway().jvm.org.apache.flink.table.utils.ModuleMock("y")
+        ))
+        self.check_list_modules('core', 'x', 'y')
+        self.check_list_full_modules(3, 'core', 'x', 'y')
+
+        self.t_env.use_modules('y', 'core')
+        self.check_list_modules('y', 'core')
+        self.check_list_full_modules(2, 'y', 'core', 'x')
+
+    def check_list_modules(self, *expected_used_modules: str):
+        self.assert_equals(self.t_env.list_modules(), list(expected_used_modules))
+
+    def check_list_full_modules(self, used_module_cnt: int, *expected_loaded_modules: str):
+        self.assert_equals(self.t_env.list_full_modules(),
+                           [ModuleEntry(module,
+                                        expected_loaded_modules.index(module) < used_module_cnt)
+                            for module in expected_loaded_modules])
+
     def test_unload_and_load_module(self):
         t_env = self.t_env
         t_env.unload_module('core')