You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by wl...@apache.org on 2017/06/13 02:07:34 UTC
incubator-hawq git commit: HAWQ-1480 - Added feature for packing a
core file
Repository: incubator-hawq
Updated Branches:
refs/heads/master b7f9f36f5 -> ca53f37df
HAWQ-1480 - Added feature for packing a core file
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/ca53f37d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/ca53f37d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/ca53f37d
Branch: refs/heads/master
Commit: ca53f37df70a383fc7f107ac75de1015d9e22467
Parents: b7f9f36
Author: Shubham Sharma <sh...@gmail.com>
Authored: Mon Jun 5 21:32:15 2017 -0700
Committer: Wen Lin <wl...@pivotal.io>
Committed: Tue Jun 13 10:05:52 2017 +0800
----------------------------------------------------------------------
tools/sbin/packcore | 261 +++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 261 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/ca53f37d/tools/sbin/packcore
----------------------------------------------------------------------
diff --git a/tools/sbin/packcore b/tools/sbin/packcore
new file mode 100755
index 0000000..6c04672
--- /dev/null
+++ b/tools/sbin/packcore
@@ -0,0 +1,261 @@
+#!/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Merged from https://github.com/greenplum-db/gpdb/blob/master/gpMgmt/sbin/packcore
+
+'''
+USAGE: ./packcore -b|--binary BINARY COREFILE
+ where BINARY is the path to postgres binary
+ where COREFILE is the core file
+'''
+
+import glob
+import os
+import re
+import string
+import shutil
+import stat
+import sys
+from optparse import OptionParser
+from subprocess import Popen, PIPE
+
+
+def _getPlatformInfo():
+ if which('lsb_release') is None:
+ for file in glob.glob('/etc/*release'):
+ shutil.copy(file, '.')
+ else:
+ Popen('/usr/bin/lsb_release -a > ./lsb_release.out', shell=True)
+
+ Popen('uname -r > uname.out', shell=True)
+
+
+def _getFileInfo(coreFile):
+ cmd = Popen('/usr/bin/file ' + coreFile, shell=True, stdout=PIPE)
+ return cmd.communicate()[0]
+
+
+def _isCore(fileCmdOutput):
+ if fileCmdOutput.find('LSB core file') is -1:
+ return False
+ return True
+
+
+def _findBinary(fileCmdOutput):
+ start = string.find(fileCmdOutput, "'") + 1
+ end = string.find(fileCmdOutput, "'", start)
+ cmd = fileCmdOutput[start:end].split()[0].translate(None, string.punctuation)
+ if os.path.isabs(cmd):
+ return cmd
+ return which(cmd)
+
+
+def _getLibraryListWithLDD(binary):
+ # We manually seed this with few libraries that are missed
+ # This may not be needed for all processes, but will round out the
+ # postgres binary debugging
+ # TODO: Identify methods to distinguish a 32 vs. 64 bit executable
+ libraries = ['/lib64/libgcc_s.so.1', '/lib64/libnss_files.so.2', '/lib/libgcc_s.so.1', '/lib/libnss_files.so.2']
+ ldd_output = Popen('ldd `which postgres`', shell=True, stdout=PIPE)
+ for line in ldd_output.stdout:
+ match = re.search(r'(\S+) \(0x', line)
+ if match and match.group(1):
+ libraries.append(match.group(1).strip())
+ return libraries
+
+
+def _getLibraryListWithGDB(coreFile, binary):
+ gdb = which('gdb')
+ if gdb is None:
+ return False
+
+ libraries = []
+ # fix for issues with PYTHONPATH and PYTHONHOME
+ environ = os.environ.copy()
+ for key in ('PYTHONHOME', 'PYTHONPATH', 'LD_LIBRARY_PATH'):
+ if key in environ:
+ del environ[key]
+ cmd = Popen(gdb + ' --eval-command="quit" ' + binary + ' -c ' + coreFile, shell=True, stdout=PIPE, stderr=PIPE)
+ result = cmd.communicate()[0]
+
+ for line in result.splitlines():
+ if line.find('Reading symbols') is 0:
+ end = line.find('...')
+ libraries.append(line[21:end])
+
+ return libraries
+
+
+def _copyFilePath(src, dst):
+ srcDir = os.path.dirname(src)
+ if srcDir.find('/') is 0:
+ srcDir = srcDir[1:]
+ dstDir = os.path.join(dst, srcDir)
+ if not os.path.exists(dstDir):
+ os.makedirs(dstDir)
+ shutil.copy(src, dstDir)
+
+
+def _generateGDBScript(b, c):
+ with open('runGDB.sh', 'w') as f1:
+ print >>f1, '#!/bin/bash'
+ print >>f1, 'unset PYTHONHOME'
+ print >>f1, 'unset PYTHONPATH'
+ print >>f1, 'curDIR=`pwd`'
+ print >>f1, '/usr/bin/gdb --eval-command="set sysroot $curDIR" --eval-command="core %s" %s' % (c, b)
+ os.chmod('runGDB.sh', 0755)
+
+
+# This is taken from Python 3.3:
+def which(cmd, mode=os.F_OK | os.X_OK, path=None):
+ """Given a command, mode, and a PATH string, return the path which
+ conforms to the given mode on the PATH, or None if there is no such
+ file.
+
+ `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result
+ of os.environ.get("PATH"), or can be overridden with a custom search
+ path.
+
+ """
+ # Check that a given file can be accessed with the correct mode.
+ # Additionally check that `file` is not a directory, as on Windows
+ # directories pass the os.access check.
+ def _access_check(fn, mode):
+ return (os.path.exists(fn) and os.access(fn, mode)
+ and not os.path.isdir(fn))
+
+ # If we're given a path with a directory part, look it up directly rather
+ # than referring to PATH directories. This includes checking relative to the
+ # current directory, e.g. ./script
+ if os.path.dirname(cmd):
+ if _access_check(cmd, mode):
+ return cmd
+ return None
+
+ if path is None:
+ path = os.environ.get("PATH", os.defpath)
+ if not path:
+ return None
+ path = path.split(os.pathsep)
+
+ if sys.platform == "win32":
+ # The current directory takes precedence on Windows.
+ if not os.curdir in path:
+ path.insert(0, os.curdir)
+
+ # PATHEXT is necessary to check on Windows.
+ pathext = os.environ.get("PATHEXT", "").split(os.pathsep)
+ # See if the given file matches any of the expected path extensions.
+ # This will allow us to short circuit when given "python.exe".
+ # If it does match, only test that one, otherwise we have to try
+ # others.
+ if any(cmd.lower().endswith(ext.lower()) for ext in pathext):
+ files = [cmd]
+ else:
+ files = [cmd + ext for ext in pathext]
+ else:
+ # On other platforms you don't have things like PATHEXT to tell you
+ # what file suffixes are executable, so just pass on cmd as-is.
+ files = [cmd]
+
+ seen = set()
+ for dir in path:
+ normdir = os.path.normcase(dir)
+ if not normdir in seen:
+ seen.add(normdir)
+ for thefile in files:
+ name = os.path.join(dir, thefile)
+ if _access_check(name, mode):
+ return name
+ return None
+
+
+def packCoreFile(coreFile, binary):
+ packDir = './packcore-' + os.path.basename(coreFile)
+ oldDir = os.getcwd()
+ try:
+ os.mkdir(packDir)
+ os.chdir(packDir)
+ shutil.copy(coreFile, '.')
+ _getPlatformInfo()
+ shutil.copy(binary, '.')
+
+ libraries = _getLibraryListWithGDB(coreFile, binary)
+
+ if libraries is False:
+ libraries = _getLibraryListWithLDD(binary)
+
+ for lib in libraries:
+ try:
+ _copyFilePath(lib, '.')
+ except IOError:
+ continue
+
+ _generateGDBScript(os.path.basename(binary), os.path.basename(coreFile))
+ os.chdir(oldDir)
+ cmd = Popen('tar zcf packcore-' + os.path.basename(coreFile) + '.tgz ' + packDir, shell=True)
+ cmd.wait()
+ finally:
+ os.chdir(oldDir)
+ Popen('rm -rf ' + packDir, shell=True)
+
+
+def parseArgs():
+ u = '''%prog [options] core_file
+This will create an archive with the core file and all required
+libraries for analysis. The preference is to use GDB so that we can
+resolve dependencies for extensions.'''
+
+ parser = OptionParser(version='%prog: 0.4beta', usage=u)
+ parser.add_option('-b', '--binary', action='store', type='string', dest='binary', metavar='PROGRAMME', help='The full path to the binary that created the core file. Used when packcore cannot determine the source binary')
+ (option, args) = parser.parse_args()
+ if len(args) != 1:
+ parser.error('Please specify a core file')
+ sys.exit(1)
+ return (option, args)
+
+
+def main():
+ # Check python vesion
+ if sys.hexversion < 0x020600f0:
+ sys.stderr.write('packcore requires a minimum python version of 2.6. Current version is:\n' + sys.version)
+ sys.exit(1)
+
+ (options, args) = parseArgs()
+
+ coreFile = os.path.abspath(args[0])
+ fileCmd = _getFileInfo(coreFile)
+ if not _isCore(fileCmd):
+ sys.stderr.write(args[0] + ' is not a valid core file\n')
+ sys.exit(1)
+
+ if options.binary:
+ binary = which(options.binary)
+ else:
+ binary = _findBinary(fileCmd)
+
+ if not binary:
+ sys.stderr.write("Unable to find full path to binary for core file\n")
+ sys.exit(1)
+
+ packCoreFile(coreFile, binary)
+ sys.exit(0)
+
+
+if __name__ == "__main__":
+ main()