You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by sw...@apache.org on 2016/10/18 22:57:09 UTC
ambari git commit: AMBARI-18307. Add sqlline capability to AMS
collector to help debugging. (swagle)
Repository: ambari
Updated Branches:
refs/heads/branch-2.5 ac32c2885 -> db4aef04e
AMBARI-18307. Add sqlline capability to AMS collector to help debugging. (swagle)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/db4aef04
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/db4aef04
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/db4aef04
Branch: refs/heads/branch-2.5
Commit: db4aef04e99697d4681e592294889c925bbd64ce
Parents: ac32c28
Author: Siddharth Wagle <sw...@hortonworks.com>
Authored: Tue Oct 18 15:56:11 2016 -0700
Committer: Siddharth Wagle <sw...@hortonworks.com>
Committed: Tue Oct 18 15:56:30 2016 -0700
----------------------------------------------------------------------
ambari-metrics/ambari-metrics-assembly/pom.xml | 13 ++
.../conf/unix/sqlline/log4j.properties | 76 ++++++++
.../conf/unix/sqlline/phoenix_utils.py | 193 +++++++++++++++++++
.../conf/unix/sqlline/sqlline.py | 105 ++++++++++
.../ambari-metrics-timelineservice/pom.xml | 13 ++
5 files changed, 400 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-assembly/pom.xml b/ambari-metrics/ambari-metrics-assembly/pom.xml
index f6cb361..a55ae57 100644
--- a/ambari-metrics/ambari-metrics-assembly/pom.xml
+++ b/ambari-metrics/ambari-metrics-assembly/pom.xml
@@ -309,6 +309,19 @@
</sources>
</mapping>
<mapping>
+ <directory>/usr/lib/ambari-metrics-collector/bin</directory>
+ <filemode>755</filemode>
+ <username>root</username>
+ <groupname>root</groupname>
+ <directoryIncluded>false</directoryIncluded>
+ <sources>
+ <source>
+ <location>${collector.dir}/conf/unix/sqlline</location>
+ <filter>false</filter>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
<directory>/etc/ambari-metrics-collector/conf</directory>
<configuration>true</configuration>
<sources>
http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties
new file mode 100644
index 0000000..3f7760a
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/log4j.properties
@@ -0,0 +1,76 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+#
+
+# Define some default values that can be overridden by system properties
+psql.root.logger=WARN,console
+psql.log.dir=.
+psql.log.file=psql.log
+hadoop.log.dir=.
+
+# Define the root logger to the system property "plsql.root.logger".
+log4j.rootLogger=${psql.root.logger}
+
+# Logging Threshold to INFO for query-server. root logger still at WARN for sqlline clients.
+log4j.threshold=INFO
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${psql.log.dir}/${psql.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+
+# Pattern format: Date LogLevel LoggerName LogMessage
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n
+
+# Debugging Pattern format
+#log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
+
+#
+# Null Appender
+#
+log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n
+
+# Custom Logging levels
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper=ERROR
+log4j.logger.org.apache.hadoop.hbase.zookeeper.ZKUtil=ERROR
+log4j.logger.org.apache.hadoop.hbase.HBaseConfiguration=ERROR
+
+# query server packages
+log4j.logger.org.apache.calcite.avatica=INFO
+log4j.logger.org.apache.phoenix.queryserver.server=INFO
+log4j.logger.org.eclipse.jetty.server=INFO
http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py
new file mode 100644
index 0000000..6d9b791
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/phoenix_utils.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+############################################################################
+
+import os
+import fnmatch
+import subprocess
+
+def find(pattern, classPaths):
+ paths = classPaths.split(os.pathsep)
+
+ # for each class path
+ for path in paths:
+ # remove * if it's at the end of path
+ if ((path is not None) and (len(path) > 0) and (path[-1] == '*')) :
+ path = path[:-1]
+
+ for root, dirs, files in os.walk(path):
+ # sort the file names so *-client always precedes *-thin-client
+ files.sort()
+ for name in files:
+ if fnmatch.fnmatch(name, pattern):
+ return os.path.join(root, name)
+
+ return ""
+
+def findFileInPathWithoutRecursion(pattern, path):
+ if not os.path.exists(path):
+ return ""
+ files = [f for f in os.listdir(path) if os.path.isfile(os.path.join(path,f))]
+ # sort the file names so *-client always precedes *-thin-client
+ files.sort()
+ for name in files:
+ if fnmatch.fnmatch(name, pattern):
+ return os.path.join(path, name)
+
+ return ""
+
+def which(file):
+ for path in os.environ["PATH"].split(os.pathsep):
+ if os.path.exists(os.path.join(path, file)):
+ return os.path.join(path, file)
+ return None
+
+def findClasspath(file):
+ aPath = which(file)
+ command = "%s%s" %(aPath, ' classpath')
+ return subprocess.Popen(command, shell=True, stdout=subprocess.PIPE).stdout.read()
+
+def setPath():
+ PHOENIX_CLIENT_JAR_PATTERN = "phoenix-*-client.jar"
+ PHOENIX_THIN_CLIENT_JAR_PATTERN = "phoenix-*-thin-client.jar"
+ PHOENIX_QUERYSERVER_JAR_PATTERN = "phoenix-server-*-runnable.jar"
+ PHOENIX_TESTS_JAR_PATTERN = "phoenix-core-*-tests*.jar"
+
+ # Backward support old env variable PHOENIX_LIB_DIR replaced by PHOENIX_CLASS_PATH
+ global phoenix_class_path
+ phoenix_class_path = os.getenv('PHOENIX_LIB_DIR','')
+ if phoenix_class_path == "":
+ phoenix_class_path = os.getenv('PHOENIX_CLASS_PATH','')
+
+ global hbase_conf_dir
+ # if HBASE_CONF_DIR set explicitly, use that
+ hbase_conf_dir = os.getenv('HBASE_CONF_DIR', os.getenv('HBASE_CONF_PATH'))
+ if not hbase_conf_dir:
+ # else fall back to HBASE_HOME
+ if os.getenv('HBASE_HOME'):
+ hbase_conf_dir = os.path.join(os.getenv('HBASE_HOME'), "conf")
+ elif os.name == 'posix':
+ # default to the bigtop configuration dir
+ hbase_conf_dir = '/etc/hbase/conf'
+ else:
+ # Try to provide something valid
+ hbase_conf_dir = '.'
+ global hbase_conf_path # keep conf_path around for backward compatibility
+ hbase_conf_path = hbase_conf_dir
+
+ global current_dir
+ current_dir = os.path.dirname(os.path.abspath(__file__))
+
+ global phoenix_jar_path
+ phoenix_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*")
+
+ global phoenix_client_jar
+ phoenix_client_jar = find("phoenix-*-client.jar", phoenix_jar_path)
+ if phoenix_client_jar == "":
+ phoenix_client_jar = findFileInPathWithoutRecursion(PHOENIX_CLIENT_JAR_PATTERN, os.path.join(current_dir, ".."))
+ if phoenix_client_jar == "":
+ phoenix_client_jar = find(PHOENIX_CLIENT_JAR_PATTERN, phoenix_class_path)
+
+ global phoenix_test_jar_path
+ phoenix_test_jar_path = os.path.join(current_dir, "..", "phoenix-core", "target","*")
+
+ global hadoop_conf
+ hadoop_conf = os.getenv('HADOOP_CONF_DIR', None)
+ if not hadoop_conf:
+ if os.name == 'posix':
+ # Try to provide a sane configuration directory for Hadoop if not otherwise provided.
+ # If there's no jaas file specified by the caller, this is necessary when Kerberos is enabled.
+ hadoop_conf = '/etc/hadoop/conf'
+ else:
+ # Try to provide something valid..
+ hadoop_conf = '.'
+
+ global hadoop_classpath
+ if (os.name != 'nt'):
+ hadoop_classpath = findClasspath('hadoop')
+ else:
+ hadoop_classpath = os.getenv('HADOOP_CLASSPATH', '')
+
+ global hadoop_common_jar_path
+ hadoop_common_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*")
+
+ global hadoop_common_jar
+ hadoop_common_jar = find("hadoop-common*.jar", hadoop_common_jar_path)
+
+ global hadoop_hdfs_jar_path
+ hadoop_hdfs_jar_path = os.path.join(current_dir, "..", "phoenix-assembly", "target","*")
+
+ global hadoop_hdfs_jar
+ hadoop_hdfs_jar = find("hadoop-hdfs*.jar", hadoop_hdfs_jar_path)
+
+ global testjar
+ testjar = find(PHOENIX_TESTS_JAR_PATTERN, phoenix_test_jar_path)
+ if testjar == "":
+ testjar = findFileInPathWithoutRecursion(PHOENIX_TESTS_JAR_PATTERN, os.path.join(current_dir, ".."))
+ if testjar == "":
+ testjar = find(PHOENIX_TESTS_JAR_PATTERN, phoenix_class_path)
+
+ global phoenix_queryserver_jar
+ phoenix_queryserver_jar = find(PHOENIX_QUERYSERVER_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-server", "target", "*"))
+ if phoenix_queryserver_jar == "":
+ phoenix_queryserver_jar = findFileInPathWithoutRecursion(PHOENIX_QUERYSERVER_JAR_PATTERN, os.path.join(current_dir, "..", "lib"))
+ if phoenix_queryserver_jar == "":
+ phoenix_queryserver_jar = findFileInPathWithoutRecursion(PHOENIX_QUERYSERVER_JAR_PATTERN, os.path.join(current_dir, ".."))
+
+
+ global phoenix_thin_client_jar
+ phoenix_thin_client_jar = find(PHOENIX_THIN_CLIENT_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-server-client", "target", "*"))
+ if phoenix_thin_client_jar == "":
+ phoenix_thin_client_jar = findFileInPathWithoutRecursion(PHOENIX_THIN_CLIENT_JAR_PATTERN, os.path.join(current_dir, ".."))
+
+ return ""
+
+def shell_quote(args):
+ """
+ Return the platform specific shell quoted string. Handles Windows and *nix platforms.
+
+ :param args: array of shell arguments
+ :return: shell quoted string
+ """
+ if os.name == 'nt':
+ import subprocess
+ return subprocess.list2cmdline(args)
+ else:
+ # pipes module isn't available on Windows
+ import pipes
+ return " ".join([pipes.quote(v) for v in args])
+
+if __name__ == "__main__":
+ setPath()
+ print "phoenix_class_path:", phoenix_class_path
+ print "hbase_conf_dir:", hbase_conf_dir
+ print "hbase_conf_path:", hbase_conf_path
+ print "current_dir:", current_dir
+ print "phoenix_jar_path:", phoenix_jar_path
+ print "phoenix_client_jar:", phoenix_client_jar
+ print "phoenix_test_jar_path:", phoenix_test_jar_path
+ print "hadoop_common_jar_path:", hadoop_common_jar_path
+ print "hadoop_common_jar:", hadoop_common_jar
+ print "hadoop_hdfs_jar_path:", hadoop_hdfs_jar_path
+ print "hadoop_hdfs_jar:", hadoop_hdfs_jar
+ print "testjar:", testjar
+ print "phoenix_queryserver_jar:", phoenix_queryserver_jar
+ print "phoenix_thin_client_jar:", phoenix_thin_client_jar
+ print "hadoop_classpath:", hadoop_classpath
http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py
new file mode 100644
index 0000000..852f264
--- /dev/null
+++ b/ambari-metrics/ambari-metrics-timelineservice/conf/unix/sqlline/sqlline.py
@@ -0,0 +1,105 @@
+#!/usr/bin/env python
+############################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+############################################################################
+
+import os
+import subprocess
+import sys
+import phoenix_utils
+import atexit
+import fnmatch
+
+# Set JAVA_HOME or JDK_HOME before running sqlline.py
+# Example: ./sqlline.py localhost:61181:/ams-hbase-unsecure
+
+global childProc
+childProc = None
+def kill_child():
+ if childProc is not None:
+ childProc.terminate()
+ childProc.kill()
+ if os.name != 'nt':
+ os.system("reset")
+atexit.register(kill_child)
+
+def find_java():
+ try:
+ java = os.environ.get('JAVA_HOME')
+ if not java:
+ java = os.environ.get('JDK_HOME')
+ except:
+ java = None
+ if java and os.path.exists(java):
+ if java.endswith('bin') or java.endswith('bin/'):
+ return os.path.join(java, 'java')
+ if java.find('/bin/java') == -1:
+ return os.path.join(java, 'bin', 'java')
+ else:
+ return 'java'
+
+#phoenix_utils.setPath()
+
+if len(sys.argv) < 2:
+ print "Zookeeper not specified. \nUsage: sqlline.py <zookeeper> \
+<optional_sql_file> \nExample: \n 1. sqlline.py localhost:2181:/hbase \n 2. sqlline.py \
+localhost:2181:/hbase ../examples/stock_symbol.sql"
+ sys.exit()
+
+sqlfile = ""
+
+if len(sys.argv) > 2:
+ sqlfile = "--run=" + phoenix_utils.shell_quote([sys.argv[2]])
+
+colorSetting = "true"
+# disable color setting for windows OS
+if os.name == 'nt':
+ colorSetting = "false"
+
+# Set custom path vars
+hbase_conf_dir = os.environ.get('HBASE_CONF_DIR', '/etc/ams-hbase/conf')
+hadoop_conf = os.environ.get('HADOOP_CONF_DIR', '/etc/ams-hbase/conf')
+hadoop_classpath = os.environ.get('HADOOP_CLASSPATH', '/usr/lib/ams-hbase/lib')
+
+PATH_PREFIX = '/usr/lib/ambari-metrics-collector/'
+if not os.path.exists(PATH_PREFIX):
+ print 'Fatal Error: ' + PATH_PREFIX + ' not a valid path.'
+ sys.exit(1)
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+phoenix_client_path = PATH_PREFIX + '*'
+
+java_cmd = find_java() + ' -cp "' + phoenix_client_path + \
+ os.pathsep + hadoop_conf + '" -Dlog4j.configuration=file:' + \
+ os.path.join(current_dir, "log4j.properties") + \
+ " sqlline.SqlLine -d org.apache.phoenix.jdbc.PhoenixDriver \
+-u jdbc:phoenix:" + phoenix_utils.shell_quote([sys.argv[1]]) + \
+ " -n none -p none --color=" + colorSetting + " --fastConnect=false --verbose=true \
+--isolation=TRANSACTION_READ_COMMITTED " + sqlfile
+
+print 'java command: %s' % str(java_cmd)
+
+childProc = subprocess.Popen(java_cmd, shell=True)
+#Wait for child process exit
+(output, error) = childProc.communicate()
+returncode = childProc.returncode
+childProc = None
+# Propagate Java return code to this script
+if returncode is not None:
+ sys.exit(returncode)
http://git-wip-us.apache.org/repos/asf/ambari/blob/db4aef04/ambari-metrics/ambari-metrics-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-metrics/ambari-metrics-timelineservice/pom.xml b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
index 1144573..fdbfdcb 100644
--- a/ambari-metrics/ambari-metrics-timelineservice/pom.xml
+++ b/ambari-metrics/ambari-metrics-timelineservice/pom.xml
@@ -182,6 +182,19 @@
</sources>
</mapping>
<mapping>
+ <directory>/usr/lib/ambari-metrics-collector/bin</directory>
+ <filemode>755</filemode>
+ <username>root</username>
+ <groupname>root</groupname>
+ <directoryIncluded>false</directoryIncluded>
+ <sources>
+ <source>
+ <location>conf/unix/sqlline</location>
+ <filter>false</filter>
+ </source>
+ </sources>
+ </mapping>
+ <mapping>
<directory>/etc/ambari-metrics-collector/conf</directory>
<configuration>true</configuration>
<sources>