You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/09/11 03:39:03 UTC
(paimon-python) branch main updated: java_based_implementation packages java-bridge jar by default (#13)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-python.git
The following commit(s) were added to refs/heads/main by this push:
new 165ad9b java_based_implementation packages java-bridge jar by default (#13)
165ad9b is described below
commit 165ad9b0bd7b130e7fc55ffc8a6eefaa5212fcba
Author: yuzelin <33...@users.noreply.github.com>
AuthorDate: Wed Sep 11 11:38:59 2024 +0800
java_based_implementation packages java-bridge jar by default (#13)
---
README.md | 14 +--
dev/build-wheels.sh | 2 +-
dev/lint-python.sh | 3 +
java_based_implementation/gateway_server.py | 7 +-
.../tests/test_write_and_read.py | 12 ++-
java_based_implementation/tests/utils.py | 25 -----
java_based_implementation/util/constants.py | 1 +
java_based_implementation/util/setup_utils.py | 89 +++++++++++++++++
setup.py | 109 ++++++++++++++-------
9 files changed, 184 insertions(+), 78 deletions(-)
diff --git a/README.md b/README.md
index 58e0c67..f605935 100644
--- a/README.md
+++ b/README.md
@@ -28,14 +28,6 @@ and running "External Tools" → "flake8".
We can use `py4j` to leverage Java code to read Paimon data. This section describes how to use this implementation.
-### Build paimon-python-java-bridge
-
-```bash
-cd java_based_implementation/paimon-python-java-bridge/
-mvn clean install -DskipTests
-```
-The built target is java-based-implementation/paimon-python-java-bridge/target/paimon-python-java-bridge-<version>.jar
-
### Set Environment Variables
`py4j` need to access a JVM, so we should set JVM arguments (optional) and Java classpath. A convenient way is using
@@ -44,10 +36,14 @@ The built target is java-based-implementation/paimon-python-java-bridge/target/p
```python
import os
-os.environ['PYPAIMON_JAVA_CLASSPATH'] = '/path/to/paimon-python-java-bridge-<version>.jar'
+os.environ['PYPAIMON_JAVA_CLASSPATH'] = '/path/to/dependent_jars/*'
os.environ['_PYPAIMON_JVM_ARGS'] = 'jvm_arg1 jvm_arg2 ...'
```
+NOTE: the package has set paimon core and hadoop dependencies. If you just test in local or run code in hadoop, you doesn't
+need to set classpath. If you need other dependencies such as OSS/S3 filesystem jars, or special catalog which isn't implemented
+in paimon core, please download jars and set classpath.
+
# API Reference
TODO
diff --git a/dev/build-wheels.sh b/dev/build-wheels.sh
index 537706b..404c42e 100755
--- a/dev/build-wheels.sh
+++ b/dev/build-wheels.sh
@@ -33,7 +33,7 @@ for ((i=0;i<${#py_env[@]};i++)) do
# force the linker to use the older glibc version in Linux
export CFLAGS="-I. -include dev/glibc_version_fix.h"
fi
- ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py bdist_wheel
+ ${PY_ENV_DIR}/${py_env[i]}/bin/python setup.py clean bdist_wheel
done
## 4. convert linux_x86_64 wheel to manylinux1 wheel in Linux
diff --git a/dev/lint-python.sh b/dev/lint-python.sh
index a287853..ea2edc5 100755
--- a/dev/lint-python.sh
+++ b/dev/lint-python.sh
@@ -578,6 +578,9 @@ function tox_check() {
# Ensure the permission of the scripts set correctly
chmod +x $PAIMON_PYTHON_DIR/dev/*
+ # tox runs codes in virtual env, set var to avoid error
+ export _PYPAIMON_TOX_TEST="true"
+
if [[ ${BUILD_REASON} = 'IndividualCI' ]]; then
# Only run test in latest python version triggered by a Git push
$TOX_PATH -vv -c $PAIMON_PYTHON_DIR/tox.ini -e ${LATEST_PYTHON} --recreate 2>&1 | tee -a $LOG_FILE
diff --git a/java_based_implementation/gateway_server.py b/java_based_implementation/gateway_server.py
index 933669a..60940a5 100644
--- a/java_based_implementation/gateway_server.py
+++ b/java_based_implementation/gateway_server.py
@@ -21,8 +21,9 @@ import platform
import signal
from subprocess import Popen, PIPE
-from java_based_implementation.util.constants import (PYPAIMON_JVM_ARGS, PYPAIMON_JAVA_CLASSPATH,
- PYPAIMON_MAIN_ARGS, PYPAIMON_MAIN_CLASS)
+from java_based_implementation.util.constants import (PYPAIMON_JVM_ARGS, PYPAIMON_MAIN_ARGS,
+ PYPAIMON_MAIN_CLASS)
+from java_based_implementation.util.setup_utils import get_classpath
def on_windows():
@@ -47,7 +48,7 @@ def launch_gateway_server_process(env):
# TODO construct Java module log settings
log_settings = []
jvm_args = env.get(PYPAIMON_JVM_ARGS, '').split()
- classpath = env.get(PYPAIMON_JAVA_CLASSPATH)
+ classpath = get_classpath(env)
main_args = env.get(PYPAIMON_MAIN_ARGS, '').split()
command = [
java_executable,
diff --git a/java_based_implementation/tests/test_write_and_read.py b/java_based_implementation/tests/test_write_and_read.py
index 02ead23..4511825 100644
--- a/java_based_implementation/tests/test_write_and_read.py
+++ b/java_based_implementation/tests/test_write_and_read.py
@@ -16,7 +16,6 @@
# limitations under the License.
################################################################################
-import os
import tempfile
import unittest
import pandas as pd
@@ -24,8 +23,8 @@ import pyarrow as pa
from java_based_implementation.api_impl import Catalog, Table
from java_based_implementation.java_gateway import get_gateway
-from java_based_implementation.tests.utils import set_bridge_jar, create_simple_table
-from java_based_implementation.util import constants, java_utils
+from java_based_implementation.tests.utils import create_simple_table
+from java_based_implementation.util import java_utils, setup_utils
from py4j.protocol import Py4JJavaError
@@ -33,10 +32,13 @@ class TableWriteReadTest(unittest.TestCase):
@classmethod
def setUpClass(cls):
- classpath = set_bridge_jar()
- os.environ[constants.PYPAIMON_JAVA_CLASSPATH] = classpath
+ setup_utils.setup_java_bridge()
cls.warehouse = tempfile.mkdtemp()
+ @classmethod
+ def tearDownClass(cls):
+ setup_utils.clean()
+
def testReadEmptyAppendTable(self):
create_simple_table(self.warehouse, 'default', 'empty_append_table', False)
catalog = Catalog.create({'warehouse': self.warehouse})
diff --git a/java_based_implementation/tests/utils.py b/java_based_implementation/tests/utils.py
index f600376..e709e0a 100644
--- a/java_based_implementation/tests/utils.py
+++ b/java_based_implementation/tests/utils.py
@@ -16,35 +16,10 @@
# limitations under the License.
################################################################################
-import os
-import shutil
-import subprocess
-import tempfile
-
from java_based_implementation.java_gateway import get_gateway
from java_based_implementation.util.java_utils import to_j_catalog_context
-def set_bridge_jar() -> str:
- current_file_path = os.path.abspath(__file__)
- current_dir = os.path.dirname(current_file_path)
- parent_dir = os.path.dirname(current_dir)
- java_module = os.path.join(parent_dir, 'paimon-python-java-bridge')
- # build paimon-python-java-bridge
- subprocess.run(
- ["mvn", "clean", "package"],
- cwd=java_module,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE
- )
- jar_name = 'paimon-python-java-bridge-0.9-SNAPSHOT.jar'
- jar_file = os.path.join(java_module, 'target', jar_name)
- # move to temp dir
- temp_dir = tempfile.mkdtemp()
- shutil.move(jar_file, temp_dir)
- return os.path.join(temp_dir, jar_name)
-
-
def create_simple_table(warehouse, database, table_name, has_pk, options=None):
if options is None:
options = {
diff --git a/java_based_implementation/util/constants.py b/java_based_implementation/util/constants.py
index 19f6eaf..71ca0e6 100644
--- a/java_based_implementation/util/constants.py
+++ b/java_based_implementation/util/constants.py
@@ -22,3 +22,4 @@ PYPAIMON_JVM_ARGS = '_PYPAIMON_JVM_ARGS'
PYPAIMON_JAVA_CLASSPATH = '_PYPAIMON_JAVA_CLASSPATH'
PYPAIMON_MAIN_CLASS = 'org.apache.paimon.python.PythonGatewayServer'
PYPAIMON_MAIN_ARGS = '_PYPAIMON_MAIN_ARGS'
+PYPAIMON_TOX_TEST = '_PYPAIMON_TOX_TEST'
diff --git a/java_based_implementation/util/setup_utils.py b/java_based_implementation/util/setup_utils.py
new file mode 100644
index 0000000..ec5e367
--- /dev/null
+++ b/java_based_implementation/util/setup_utils.py
@@ -0,0 +1,89 @@
+################################################################################
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import importlib
+import os
+import shutil
+import subprocess
+
+from java_based_implementation.util import constants
+from java_based_implementation.util.constants import PYPAIMON_JAVA_CLASSPATH
+
+_JAVA_IMPL_MODULE = 'java_based_implementation'
+_JAVA_DEPS = 'java_dependencies'
+_JAVA_BRIDGE = 'paimon-python-java-bridge'
+# TODO configure version
+_JAVA_BRIDGE_VERSION = '0.9-SNAPSHOT'
+
+
+def get_package_data():
+ is_tox_test = os.environ.get(constants.PYPAIMON_TOX_TEST)
+ if is_tox_test and is_tox_test.lower() == "true":
+ return ['']
+
+ setup_java_bridge()
+ return [os.path.join(_JAVA_DEPS, '*')]
+
+
+def clean():
+ java_deps_dir = os.path.join(_find_java_impl_dir(), _JAVA_DEPS)
+ if os.path.exists(java_deps_dir):
+ shutil.rmtree(java_deps_dir)
+
+
+def get_classpath(env):
+ user_defined = env.get(PYPAIMON_JAVA_CLASSPATH)
+
+ module = importlib.import_module(_JAVA_IMPL_MODULE)
+ builtin_java_bridge = os.path.join(*module.__path__, _JAVA_DEPS, _JAVA_BRIDGE + '.jar')
+
+ if user_defined is None:
+ return builtin_java_bridge
+ else:
+ return os.pathsep.join([builtin_java_bridge, user_defined])
+
+
+def setup_java_bridge():
+ java_impl_dir = _find_java_impl_dir()
+
+ java_deps_dir = os.path.join(java_impl_dir, _JAVA_DEPS)
+ if not os.path.exists(java_deps_dir):
+ os.mkdir(java_deps_dir)
+
+ java_bridge_dst = os.path.join(java_deps_dir, _JAVA_BRIDGE + '.jar')
+ if os.path.exists(java_bridge_dst):
+ return
+
+ java_bridge_module = os.path.join(java_impl_dir, _JAVA_BRIDGE)
+ subprocess.run(
+ ["mvn", "clean", "package"],
+ cwd=java_bridge_module,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE
+ )
+
+ shutil.copy(
+ os.path.join(java_bridge_module, 'target/{}-{}.jar'
+ .format(_JAVA_BRIDGE, _JAVA_BRIDGE_VERSION)),
+ java_bridge_dst
+ )
+
+
+def _find_java_impl_dir():
+ abspath = os.path.abspath(__file__)
+ return os.path.dirname(os.path.dirname(abspath))
diff --git a/setup.py b/setup.py
index 6f062be..de23c21 100644
--- a/setup.py
+++ b/setup.py
@@ -16,38 +16,77 @@
# limitations under the License.
################################################################################
-from setuptools import setup
-
-PACKAGES = [
- 'paimon_python_api',
- 'java_based_implementation',
- 'java_based_implementation.util'
-]
-
-install_requires = [
- 'py4j==0.10.9.7',
- 'python-dateutil>=2.8.0,<3',
- 'pytz>=2018.3',
- 'numpy>=1.22.4',
- 'pandas>=1.3.0',
- 'pyarrow>=5.0.0'
-]
-
-setup(
- name='paimon_python',
- version='0.9.0.dev1',
- packages=PACKAGES,
- install_requires=install_requires,
- description='Apache Paimon Python API',
- author='Apache Software Foundation',
- author_email='dev@paimon.apache.org',
- url='https://paimon.apache.org',
- classifiers=[
- 'Development Status :: 5 - Production/Stable',
- 'License :: OSI Approved :: Apache Software License',
- 'Programming Language :: Python :: 3.8',
- 'Programming Language :: Python :: 3.9',
- 'Programming Language :: Python :: 3.10',
- 'Programming Language :: Python :: 3.11'],
- python_requires='>=3.8'
-)
+import fnmatch
+import java_based_implementation.util.setup_utils
+import os
+import shutil
+
+from setuptools import Command, setup
+
+
+class CleanCommand(Command):
+ description = 'Clean up temporary files and directories of last build.'
+ user_options = []
+
+ def initialize_options(self):
+ pass
+
+ def finalize_options(self):
+ pass
+
+ def run(self):
+ directories_to_delete = ['build', 'dist', '*.egg-info']
+
+ for directory in directories_to_delete:
+ if '*' in directory:
+ for matched_dir in filter(lambda x: fnmatch.fnmatch(x, directory), os.listdir('.')):
+ if os.path.isdir(matched_dir):
+ shutil.rmtree(matched_dir)
+ else:
+ if os.path.exists(directory):
+ shutil.rmtree(directory)
+
+
+try:
+ PACKAGES = [
+ 'paimon_python_api',
+ 'java_based_implementation',
+ 'java_based_implementation.util'
+ ]
+
+ PACKAGE_DATA = {
+ 'java_based_implementation': java_based_implementation.util.setup_utils.get_package_data()
+ }
+
+ install_requires = [
+ 'py4j==0.10.9.7',
+ 'python-dateutil>=2.8.0,<3',
+ 'pytz>=2018.3',
+ 'numpy>=1.22.4',
+ 'pandas>=1.3.0',
+ 'pyarrow>=5.0.0'
+ ]
+
+ setup(
+ name='paimon_python',
+ version='0.9.0.dev1',
+ packages=PACKAGES,
+ include_package_data=True,
+ package_data=PACKAGE_DATA,
+ cmdclass={'clean': CleanCommand},
+ install_requires=install_requires,
+ description='Apache Paimon Python API',
+ author='Apache Software Foundation',
+ author_email='dev@paimon.apache.org',
+ url='https://paimon.apache.org',
+ classifiers=[
+ 'Development Status :: 5 - Production/Stable',
+ 'License :: OSI Approved :: Apache Software License',
+ 'Programming Language :: Python :: 3.8',
+ 'Programming Language :: Python :: 3.9',
+ 'Programming Language :: Python :: 3.10',
+ 'Programming Language :: Python :: 3.11'],
+ python_requires='>=3.8'
+ )
+finally:
+ java_based_implementation.util.setup_utils.clean()