You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2023/12/12 05:52:19 UTC
(phoenix) branch 5.1 updated: PHOENIX-7143 Detect JVM version and add the necessary module flags in startup scripts
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch 5.1
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/5.1 by this push:
new c3714c1777 PHOENIX-7143 Detect JVM version and add the necessary module flags in startup scripts
c3714c1777 is described below
commit c3714c1777b237efb2f49f0124f0f992d0dd7dab
Author: Istvan Toth <st...@apache.org>
AuthorDate: Mon Dec 11 14:08:57 2023 +0100
PHOENIX-7143 Detect JVM version and add the necessary module flags in startup scripts
---
bin/performance.py | 43 ++++-----------------------
bin/pherf-standalone.py | 37 ++---------------------
bin/phoenix_utils.py | 78 +++++++++++++++++++++++++++++++++++++++++++++++++
bin/psql.py | 38 ++----------------------
bin/sqlline.py | 35 +---------------------
5 files changed, 90 insertions(+), 141 deletions(-)
diff --git a/bin/performance.py b/bin/performance.py
index 212a1f659d..5b09a29c93 100755
--- a/bin/performance.py
+++ b/bin/performance.py
@@ -63,44 +63,13 @@ statements = ""
phoenix_utils.setPath()
-# HBase configuration folder path (where hbase-site.xml reside) for
-# HBase/Phoenix client side property override
-hbase_config_path = phoenix_utils.hbase_conf_dir
-
-java_home = os.getenv('JAVA_HOME')
-
-# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
-hbase_env_path = None
-hbase_env_cmd = None
-if os.name == 'posix':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
- hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
-elif os.name == 'nt':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
- hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
-if not hbase_env_path or not hbase_env_cmd:
- sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name, os.linesep))
- sys.exit(-1)
-
-hbase_env = {}
-if os.path.isfile(hbase_env_path):
- p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
- for x in p.stdout:
- (k, _, v) = x.decode().partition('=')
- hbase_env[k.strip()] = v.strip()
-
-if 'JAVA_HOME' in hbase_env:
- java_home = hbase_env['JAVA_HOME']
-
-if java_home:
- java_cmd = os.path.join(java_home, 'bin', 'java')
-else:
- java_cmd = 'java'
-
-execute = ('%s $PHOENIX_OPTS -cp "%s%s%s%s%s" -Dlog4j.configuration=file:' +
- os.path.join(phoenix_utils.current_dir, "log4j.properties") +
+java_cmd = phoenix_utils.java
+
+execute = ('%s %s $PHOENIX_OPTS -cp "%s%s%s%s%s"-Dlog4j2.configurationFile=file:' +
+ os.path.join(phoenix_utils.current_dir, "log4j2.properties") +
' org.apache.phoenix.util.PhoenixRuntime -t %s %s ') % \
- (java_cmd, hbase_config_path, os.pathsep, phoenix_utils.slf4j_backend_jar, os.pathsep,
+ (java_cmd, phoenix_utils.jvm_module_flags, hbase_config_path, os.pathsep,
+ phoenix_utils.slf4j_backend_jar, os.pathsep,
phoenix_utils.logging_jar, os.pathsep, phoenix_utils.phoenix_client_embedded_jar,
table, zookeeper)
diff --git a/bin/pherf-standalone.py b/bin/pherf-standalone.py
index 16617d6b6a..ddd7804913 100755
--- a/bin/pherf-standalone.py
+++ b/bin/pherf-standalone.py
@@ -30,41 +30,8 @@ phoenix_utils.setPath()
args = phoenix_utils.shell_quote(sys.argv[1:])
-# HBase configuration folder path (where hbase-site.xml reside) for
-# HBase/Phoenix client side property override
-hbase_config_path = phoenix_utils.hbase_conf_dir
-
-java_home = os.getenv('JAVA_HOME')
-
-# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
-hbase_env_path = None
-hbase_env_cmd = None
-if os.name == 'posix':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
- hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
-elif os.name == 'nt':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
- hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
-if not hbase_env_path or not hbase_env_cmd:
- sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name, os.linesep))
- sys.exit(-1)
-
-hbase_env = {}
-if os.path.isfile(hbase_env_path):
- p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
- for x in p.stdout:
- (k, _, v) = tryDecode(x).partition('=')
- hbase_env[k.strip()] = v.strip()
-
-if 'JAVA_HOME' in hbase_env:
- java_home = hbase_env['JAVA_HOME']
-
-if java_home:
- java = os.path.join(java_home, 'bin', 'java')
-else:
- java = 'java'
-
-java_cmd = java +' -Xms512m -Xmx3072m -cp "' + \
+java_cmd = phoenix_utils.java + ' ' + phoenix_utils.jvm_module_flags + \
+ ' -Xms512m -Xmx3072m -cp "' + \
phoenix_utils.pherf_conf_path + os.pathsep + \
phoenix_utils.hbase_conf_dir + os.pathsep + \
phoenix_utils.slf4j_backend_jar + os.pathsep + \
diff --git a/bin/phoenix_utils.py b/bin/phoenix_utils.py
index 3ae92fd5ed..1161b9e715 100755
--- a/bin/phoenix_utils.py
+++ b/bin/phoenix_utils.py
@@ -22,6 +22,7 @@
from __future__ import print_function
import os
import fnmatch
+import re
import subprocess
def find(pattern, classPaths):
@@ -185,6 +186,8 @@ def setPath():
if logging_jar is None or logging_jar == "":
logging_jar = findFileInPathWithoutRecursion(LOGGING_JAR_PATTERN, os.path.join(current_dir, "..","lib"))
+ __set_java_home()
+ __set_jvm_flags()
return ""
def shell_quote(args):
@@ -202,6 +205,78 @@ def shell_quote(args):
import pipes
return " ".join([pipes.quote(tryDecode(v)) for v in args])
+
+def __set_java_home():
+ global java_home
+ global java
+ java_home = os.getenv('JAVA_HOME')
+ java = 'java'
+
+ # HBase configuration folder path (where hbase-site.xml reside) for
+ # HBase/Phoenix client side property override
+ hbase_config_path = hbase_conf_dir
+
+ # load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
+ hbase_env_path = None
+ hbase_env_cmd = None
+ if os.name == 'posix':
+ hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
+ hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
+ elif os.name == 'nt':
+ hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
+ hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
+ if not hbase_env_path or not hbase_env_cmd:
+ sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name, os.linesep))
+ sys.exit(-1)
+
+ hbase_env = {}
+ if os.path.isfile(hbase_env_path):
+ p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
+ for x in p.stdout:
+ (k, _, v) = tryDecode(x).partition('=')
+ hbase_env[k.strip()] = v.strip()
+
+ if 'JAVA_HOME' in hbase_env:
+ java_home = hbase_env['JAVA_HOME']
+
+ if java_home:
+ java = os.path.join(java_home, 'bin', 'java')
+
+ return ""
+
+
+def __set_jvm_flags():
+ global jvm_module_flags
+ jvm_module_flags = ""
+ # This should be ASCII
+ version_output = subprocess.check_output([java, "-version"], stderr=subprocess.STDOUT).decode()
+ version_output = tryDecode(version_output)
+ m = re.search(r'version\s"(\d+)\.(\d+)', version_output)
+ if (m is None):
+ # Could not find version
+ return ""
+ major = m.group(1)
+ minor = m.group(2)
+ if (major is None or minor is None):
+ #Could not identify version
+ return ""
+ if (minor == "1"):
+ major = minor
+ if (int(major) >= 11):
+ # Copied from hbase startup script
+ jvm_module_flags = "-Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true \
+--add-modules jdk.unsupported \
+--add-opens java.base/java.nio=ALL-UNNAMED \
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED \
+--add-opens java.base/java.lang=ALL-UNNAMED \
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED \
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED \
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED \
+--add-exports java.security.jgss/sun.security.krb5=ALL-UNNAMED \
+--add-exports java.base/sun.net.dns=ALL-UNNAMED \
+--add-exports java.base/sun.net.util=ALL-UNNAMED"
+ return ""
+
def common_sqlline_args(parser):
parser.add_argument('-v', '--verbose', help='Verbosity on sqlline.', action="store_true")
parser.add_argument('-c', '--color', help='Color setting for sqlline.', action="store_true")
@@ -222,3 +297,6 @@ if __name__ == "__main__":
print("sqlline_with_deps_jar:", sqlline_with_deps_jar)
print("slf4j_backend_jar:", slf4j_backend_jar)
print("logging_jar:", logging_jar)
+ print("java_home:", java_home)
+ print("java:", java)
+ print("jvm_module_flags:", jvm_module_flags)
diff --git a/bin/psql.py b/bin/psql.py
index 6c63a21a03..58c04ddf8f 100755
--- a/bin/psql.py
+++ b/bin/psql.py
@@ -30,41 +30,7 @@ phoenix_utils.setPath()
args = phoenix_utils.shell_quote(sys.argv[1:])
-# HBase configuration folder path (where hbase-site.xml reside) for
-# HBase/Phoenix client side property override
-hbase_config_path = phoenix_utils.hbase_conf_dir
-
-java_home = os.getenv('JAVA_HOME')
-
-# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
-hbase_env_path = None
-hbase_env_cmd = None
-if os.name == 'posix':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
- hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
-elif os.name == 'nt':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
- hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
-if not hbase_env_path or not hbase_env_cmd:
- sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name, os.linesep))
- sys.exit(-1)
-
-hbase_env = {}
-if os.path.isfile(hbase_env_path):
- p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
- for x in p.stdout:
- (k, _, v) = tryDecode(x).partition('=')
- hbase_env[k.strip()] = v.strip()
-
-if 'JAVA_HOME' in hbase_env:
- java_home = hbase_env['JAVA_HOME']
-
-if java_home:
- java = os.path.join(java_home, 'bin', 'java')
-else:
- java = 'java'
-
-java_cmd = java + ' $PHOENIX_OPTS ' + \
+java_cmd = phoenix_utils.java + ' ' + phoenix_utils.jvm_module_flags + ' $PHOENIX_OPTS ' + \
' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + phoenix_utils.hadoop_conf + \
os.pathsep + phoenix_utils.slf4j_backend_jar + \
os.pathsep + phoenix_utils.logging_jar + \
@@ -72,4 +38,6 @@ java_cmd = java + ' $PHOENIX_OPTS ' + \
os.path.join(phoenix_utils.current_dir, "log4j.properties") + \
" org.apache.phoenix.util.PhoenixRuntime " + args
+print(java_cmd)
+
os.execl("/bin/sh", "/bin/sh", "-c", java_cmd)
diff --git a/bin/sqlline.py b/bin/sqlline.py
index e454ab7fbd..291fc21568 100755
--- a/bin/sqlline.py
+++ b/bin/sqlline.py
@@ -71,9 +71,6 @@ else:
jdbc_url = 'jdbc:phoenix:' + zookeeper
sqlfile = tryDecode(args.sqlfile)
-# HBase configuration folder path (where hbase-site.xml reside) for
-# HBase/Phoenix client side property override
-hbase_config_path = phoenix_utils.hbase_conf_dir
if sqlfile and not os.path.isfile(sqlfile):
parser.print_help()
@@ -82,36 +79,6 @@ if sqlfile and not os.path.isfile(sqlfile):
if sqlfile:
sqlfile = "--run=" + phoenix_utils.shell_quote([sqlfile])
-java_home = os.getenv('JAVA_HOME')
-
-# load hbase-env.??? to extract JAVA_HOME, HBASE_PID_DIR, HBASE_LOG_DIR
-hbase_env_path = None
-hbase_env_cmd = None
-if os.name == 'posix':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.sh')
- hbase_env_cmd = ['bash', '-c', 'source %s && env' % hbase_env_path]
-elif os.name == 'nt':
- hbase_env_path = os.path.join(hbase_config_path, 'hbase-env.cmd')
- hbase_env_cmd = ['cmd.exe', '/c', 'call %s & set' % hbase_env_path]
-if not hbase_env_path or not hbase_env_cmd:
- sys.stderr.write("hbase-env file unknown on platform {}{}".format(os.name, os.linesep))
- sys.exit(-1)
-
-hbase_env = {}
-if os.path.isfile(hbase_env_path):
- p = subprocess.Popen(hbase_env_cmd, stdout = subprocess.PIPE)
- for x in p.stdout:
- (k, _, v) = x.decode().partition('=')
- hbase_env[k.strip()] = v.strip()
-
-if 'JAVA_HOME' in hbase_env:
- java_home = hbase_env['JAVA_HOME']
-
-if java_home:
- java = os.path.join(java_home, 'bin', 'java')
-else:
- java = 'java'
-
colorSetting = tryDecode(args.color)
# disable color setting for windows OS
if os.name == 'nt':
@@ -123,7 +90,7 @@ if os.uname()[4].startswith('ppc'):
else:
disable_jna = ""
-java_cmd = java + ' $PHOENIX_OPTS ' + \
+java_cmd = phoenix_utils.java + ' ' + phoenix_utils.jvm_module_flags + ' $PHOENIX_OPTS ' + \
' -cp "' + phoenix_utils.hbase_conf_dir + os.pathsep + \
phoenix_utils.hadoop_conf + os.pathsep + \
phoenix_utils.sqlline_with_deps_jar + os.pathsep + \