You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@geode.apache.org by ds...@apache.org on 2015/07/06 23:46:59 UTC
[2/2] incubator-geode git commit: GEODE-79: Imported
logcollectionutil from geode-1.0.0-SNAPSHOT-2.src.tar
GEODE-79: Imported logcollectionutil from geode-1.0.0-SNAPSHOT-2.src.tar
Project: http://git-wip-us.apache.org/repos/asf/incubator-geode/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-geode/commit/e189c21a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-geode/tree/e189c21a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-geode/diff/e189c21a
Branch: refs/heads/feature/GEODE-79
Commit: e189c21a64769f2df17b8f133ebfec52b308184f
Parents: 11d0367
Author: Darrel Schneider <ds...@pivotal.io>
Authored: Mon Jul 6 10:47:31 2015 -0700
Committer: Darrel Schneider <ds...@pivotal.io>
Committed: Mon Jul 6 10:47:31 2015 -0700
----------------------------------------------------------------------
logcollectionutil/README | 42 ++
logcollectionutil/REMOVED_JARS.txt | 11 +
logcollectionutil/gfe-sendlogs.py | 327 ++++++++++++++
logcollectionutil/sendlogs-java/.classpath | 13 +
logcollectionutil/sendlogs-java/README | 103 +++++
logcollectionutil/sendlogs-java/TODO | 38 ++
logcollectionutil/sendlogs-java/build.xml | 45 ++
...Pivotal_GemFire_LogCollectionUtility_1.0.txt | 450 +++++++++++++++++++
.../sendlogs-java/scripts/logcollect | 22 +
.../sendlogs/driver/CollectLogsFromFileMap.java | 68 +++
.../pivotal/gemfire/sendlogs/driver/Driver.java | 268 +++++++++++
.../gemfire/sendlogs/driver/GemFireHost.java | 174 +++++++
.../gemfire/sendlogs/remote/FTPUploader.java | 71 +++
.../gemfire/sendlogs/remote/RemoteCommands.java | 186 ++++++++
.../gemfire/sendlogs/remote/RemoteHost.java | 204 +++++++++
.../gemfire/sendlogs/utilities/CliParser.java | 225 ++++++++++
.../sendlogs/utilities/CloseGracefully.java | 82 ++++
.../sendlogs/utilities/CompressArtifacts.java | 80 ++++
.../gemfire/sendlogs/utilities/FileSystem.java | 66 +++
.../pivotal/gemfire/sendlogs/utilities/Log.java | 37 ++
.../utilities/SendlogsExceptionHandler.java | 29 ++
.../gemfire/sendlogs/utilities/Time.java | 23 +
.../gemfire/sendlogs/utilities/Usage.java | 36 ++
.../gemfire/sendlogs/utilities/Version.java | 25 ++
.../driver/CollectLogsFromFileMapTest.java | 78 ++++
.../sendlogs/driver/GemFireHostTest.java | 61 +++
.../sendlogs/remote/FTPUploaderTest.java | 49 ++
.../gemfire/sendlogs/remote/RemoteHostTest.java | 35 ++
.../sendlogs/utilities/CliParserTest.java | 41 ++
.../utilities/CompressArtifactsTest.java | 30 ++
.../gemfire/sendlogs/utilities/UsageTest.java | 37 ++
31 files changed, 2956 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/README
----------------------------------------------------------------------
diff --git a/logcollectionutil/README b/logcollectionutil/README
new file mode 100644
index 0000000..405d64d
--- /dev/null
+++ b/logcollectionutil/README
@@ -0,0 +1,42 @@
+gfe-sendlogs.py
+===============
+This is the initial version of the gemfire support utility to collect customer
+log files. This script requires an active GemFire cluster as it uses gfsh
+to collect logs, stack traces, and configuration. This python script uses
+core python 2 libraries which should be installed by default on linux hosts.
+
+
+The current usage
+=================
+Usage: gfe-sendlogs.py -c <company> -o <output-dir> [OPTION]
+
+Required arguments
+ -c, --company=COMPANY company name
+ -o, --output-dir=DIR directory to stage logs
+
+Optional arguments
+ -l, --locator=LOCATOR locator to connect to (defaults to localhost)
+ -p, --port=PORT locator port to connect to (defaults to 10334)
+ -t, --ticket=TICKET_NUMBER ticket number
+ -u, --upload Upload zip file to pivotal FTP server for the GemFire support team.
+ -d, --do-not-remove-temp Don't cleanup the logs stored in the temporary location.
+ -f, --ftp-server=FTPSERVER Specify an FTP server (defaults to ftp.gemstone.com)
+ -v, --version Print version of this script.
+
+
+Known Limitations
+==================
+1. The FTP upload action does not support proxies.
+ - This is a limitation of the core python FTP libraries. In order to use a
+ proxy we would need to resort to "non-standard" libraries which the end user would
+ need to install.
+
+2. This requires an active GemFire cluster.
+ - This script is basically just a wrapper to gfsh and therefore requires that
+ the gemfire cluster be up and running and responding to gfsh queries.
+
+3. This will not gather statistic files.
+ - There is currently no way to get statistic files from a cluster via gfsh.
+ Support for this may be added with a new version of this script that operates
+ outside of GemFire and works at the operating system level or as this feature
+ is implemented in gfsh.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/REMOVED_JARS.txt
----------------------------------------------------------------------
diff --git a/logcollectionutil/REMOVED_JARS.txt b/logcollectionutil/REMOVED_JARS.txt
new file mode 100644
index 0000000..4473966
--- /dev/null
+++ b/logcollectionutil/REMOVED_JARS.txt
@@ -0,0 +1,11 @@
+The following jars were removed from this source code. These should be
+downloaded from maven central as part of the build.
+
+sendlogs-java/lib/commons-compress-1.6.jar
+sendlogs-java/lib/commons-io-2.4.jar
+sendlogs-java/lib/commons-net-3.3.jar
+sendlogs-java/lib/ganymed-ssh2-261.jar
+sendlogs-java/lib/hamcrest-core-1.3.jar
+sendlogs-java/lib/jopt-simple-4.6.jar
+sendlogs-java/lib/junit-4.11.jar
+sendlogs-java/lib/log4j-1.2.17.jar
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/gfe-sendlogs.py
----------------------------------------------------------------------
diff --git a/logcollectionutil/gfe-sendlogs.py b/logcollectionutil/gfe-sendlogs.py
new file mode 100755
index 0000000..95e28e8
--- /dev/null
+++ b/logcollectionutil/gfe-sendlogs.py
@@ -0,0 +1,327 @@
+#!/usr/bin/env python
+import commands
+import ftplib
+import time
+import getopt
+import sys
+import os
+import zipfile
+
+__version__ = .01
+
+DEBUG = False
+TIMESTAMP = time.strftime("%m-%d-%Y-%H%M%S")
+# Default options
+info_container = {"locator": "localhost",
+ "port": "10334",
+ "ftp-server": "ftp.gemstone.com"
+ }
+
+
+def usage():
+ """
+ Print usage and exit.
+ """
+ print "Usage: %s -c <company> -o <output-dir> [OPTION]\n" % os.path.basename(sys.argv[0])
+ print "Required arguments"
+ print "\t-c, --company=COMPANY\t\tcompany name "
+ print "\t-o, --output-dir=DIR\t\tdirectory to stage logs"
+
+ print "\nOptional arguments"
+ print "\t-l, --locator=LOCATOR\t\tlocator to connect to (defaults to localhost)"
+ print "\t-p, --port=PORT\t\t\tlocator port to connect to (defaults to 10334)"
+ print "\t-t, --ticket=TICKET_NUMBER\tticket number"
+ print "\t-u, --upload\t\t\tUpload zip file to pivotal FTP server for the GemFire support team."
+ print "\t-d, --do-not-remove-temp\t\tDon't cleanup the logs stored in the temporary location."
+ print "\t-f, --ftp-server=FTPSERVER\t\tSpecify an FTP server (defaults to ftp.gemstone.com)"
+ print "\t-v, --version\t\t\tPrint version of this script."
+ sys.exit(1)
+
+
+def print_version():
+ """
+ Print version and exit.
+ """
+ print "%s version %s" % (os.path.basename(sys.argv[0]), __version__)
+ sys.exit(1)
+
+
+def handle_arguments(opt):
+ """
+ Populate the info_container dict with all of the command line options.
+ """
+ for option, argument in opt:
+ if option == "help" or option == "-h":
+ usage()
+ # Required arguments
+ elif option == "--company" or option == "-c":
+ info_container["company"] = argument
+ elif option == "--output-dir" or option == "-o":
+ info_container["output-dir"] = os.path.abspath(argument)
+ # Optional arguments
+ elif option == "--locator" or option == "-l":
+ info_container["locator"] = argument
+ elif option == "--port" or option == "-p":
+ info_container["port"] = argument
+ elif option == "--upload" or option == "-u":
+ info_container["upload"] = False
+ elif option == "--ticket" or option == "-t":
+ info_container["ticket"] = argument
+ elif option == "--ftp-server" or option == "-f":
+ info_container["ftp-server"] = argument
+ elif option == "--do-not-remove-temp" or option == "-d":
+ info_container["do-not-remove-temp"] = True
+ elif option == "--version" or option == "-v":
+ print_version()
+ if not "company" in info_container or not "output-dir" in info_container:
+ usage()
+
+
+def parse_options():
+ """
+ Check for valid input from the CLI and pass on for processing.
+ """
+ try:
+ long_options = ["help", "version", "company=", "locator=", "port=", "do-not-upload", "ticket",
+ "output-dir", "do-not-remove-temp", "ftp-server"]
+ opt, args = getopt.getopt(sys.argv[1:], "hvrduc:l:p:t:o:f:", long_options)
+ handle_arguments(opt)
+ except getopt.GetoptError, error:
+ print "%s\n" % error
+ usage()
+ sys.exit(1)
+
+
+def check_for_gfsh():
+ """
+ Ensure that 'gfsh' is in the user's PATH and save that location.
+ """
+ gfsh_command = "gfsh"
+ found_gfsh = False
+ for path in os.environ["PATH"].split(":"):
+ gfsh_test_location = "%s/%s" % (path, gfsh_command)
+ if os.path.isfile(gfsh_test_location):
+ found_gfsh = True
+ info_container["gfsh_location"] = gfsh_test_location
+ if DEBUG:
+ print "Using GFSH at: %s" % info_container["gfsh_location"]
+ break
+ if not found_gfsh:
+ print "Please ensure that the gfsh command is available in your PATH"
+ sys.exit(1)
+
+
+def check_valid_output_dir():
+ """
+ Ensure that the requested output directory is valid.
+ """
+ if not os.path.isdir(info_container["output-dir"]):
+ print "'%s' is not a valid directory." % info_container["output-dir"]
+ sys.exit(1)
+ if DEBUG:
+ print "output dir is valid: %s" % info_container["output-dir"]
+
+
+def create_tmp_dir():
+ """
+ Create the temporary directory to store all of the output log files.
+ """
+ directory_name = "%s/%s" % (info_container["output-dir"], TIMESTAMP)
+ try:
+ os.mkdir(directory_name)
+ info_container["output-dir"] = directory_name
+ except Exception, e:
+ print "Unable to create directory: %s" % directory_name
+ print e
+ sys.exit(1)
+
+
+class Gfsh:
+ def __init__(self):
+ """
+ The GFSH class handles the interactions with the gfsh cli.
+ """
+ self.locator = info_container["locator"]
+ self.locator_port = info_container["port"]
+ self.output_dir = info_container["output-dir"]
+ self.zip_filename = ""
+ self.gfsh_log_dir = "-Dgfsh.log-dir=%s" % self.output_dir
+
+ def export_logs(self):
+ """
+ Use the gfsh "export logs" command to grab all of the log files from a cluster.
+ """
+ command = self._connect_to_locator()
+ command += ' -e "export logs --dir=%s" ' % self.output_dir
+ self._execute(command)
+
+ def export_stack_traces(self):
+ """
+ Use the gfsh "export stack-traces" command to get stack traces from a cluster.
+ """
+ command = self._connect_to_locator()
+ command += ' -e "export stack-traces --file=%s/stack_traces.txt" ' % self.output_dir
+ self._execute(command)
+
+ def export_config(self):
+ """
+ Use the gfsh "export config" command to get the configs from all members.
+ """
+ command = self._connect_to_locator()
+ command += ' -e "export config --dir=%s" ' % self.output_dir
+ self._execute(command)
+
+ def create_zip_file(self):
+ """
+ Create the zip file from all of the files in the output directory.
+ """
+ zipf = self._open_zip()
+ try:
+ for root, dirs, files in os.walk(self.output_dir):
+ for f in files:
+ if not f.endswith(".zip"):
+ zipf.write(os.path.join(root, f), f)
+ except Exception, e:
+ print "Error creating zip file."
+ print e
+ sys.exit(1)
+
+ def send_zip_to_support(self):
+ """
+ If the user requests, automatically update the zip file that was generated to the
+ FTP_SERVER. Restrictions on the FTP_SERVER ensure that once this file is uploaded only
+ the Pivotal support teams can download this file even though it's uploaded as an
+ anonymous user.
+ """
+ ftp_command = 'STOR /incoming/%s' % os.path.basename(self.zip_filename)
+ try:
+ ftp = self._get_ftp_connection()
+ ftp.storbinary(ftp_command, open(self.zip_filename, 'rb'))
+ print "%s successfully uploaded to %s " % (self.zip_filename, info_container["ftp-server"])
+ except Exception, e:
+ print "Unable upload file. Please provide this information to support."
+ print e
+ sys.exit(1)
+
+ def do_cleanup(self):
+ """
+ Go through various cleanup tasks. Delete all of the temporary files, if the user asks.
+
+ This does not delete the created zip file.
+ """
+ if "upload" in info_container:
+ self.send_zip_to_support()
+ else:
+ print "Zip file created at: %s" % self._get_zipfile_location()
+ if not "do-not-remove-temp" in info_container:
+ print "Cleaning up temporary files in %s " % self.output_dir
+ if os.path.isdir(self.output_dir):
+ for f in os.listdir(self.output_dir):
+ if self._check_file_extension(f):
+ os.remove("%s/%s" % (self.output_dir, f))
+
+ @staticmethod
+ def _check_file_extension(filename):
+ """
+ When cleaning up, ensure that we only clean log files that we've generated.
+ :return: true if filename ends with one of our defined extension types.
+ """
+ if filename.endswith(".log"):
+ return True
+ elif filename.endswith(".gfs"):
+ return True
+ elif filename.endswith(".xml"):
+ return True
+ elif filename.endswith(".properties"):
+ return True
+ elif filename.endswith(".txt"):
+ return True
+ return False
+
+ @staticmethod
+ def _get_ftp_connection():
+ """
+ Try and connect to the FTP_SERVER.
+ TODO: Implement proxy support
+ """
+ ftp = ftplib.FTP(info_container["ftp-server"])
+ try:
+ ftp.login()
+ except Exception, e:
+ print "Unable to connect to the ftp server. Please try again later"
+ print e
+ sys.exit(1)
+ return ftp
+
+ def _get_zipfile_location(self):
+ """
+ Return the full path to the zip file that was created.
+ """
+ return self.zip_filename
+
+ def _open_zip(self):
+ """
+ Define the name of the zip file and create the ZipFile object.
+ :return: ZipFile object
+ """
+ self.zip_filename = "%s/%s" % (info_container["output-dir"], info_container["company"])
+ if "ticket" in info_container:
+ self.zip_filename += "_%s" % info_container["ticket"]
+ self.zip_filename += "_%s.zip" % TIMESTAMP
+ zipf = zipfile.ZipFile(self.zip_filename, 'w', zipfile.ZIP_DEFLATED)
+ if DEBUG:
+ print "zip filename: %s" % self.zip_filename
+ return zipf
+
+ def _connect_to_locator(self):
+ """
+ Defines the common command for connecting gfsh to the locator. This command is used in conjunction
+ with other commands that require gfsh being connected to a locator.
+ :return: command string for connecting to the locator
+ """
+ command = "JAVA_ARGS=%s " % self.gfsh_log_dir
+ command += info_container["gfsh_location"]
+ command += ' -e "connect '
+ command += ' --locator=%s[%s]"' % (self.locator, self.locator_port)
+ return command
+
+ def _execute(self, command):
+ """
+ Execute the actual gfsh command.
+ """
+ results = commands.getstatusoutput(command)
+
+ # gfsh seems to always exit 0...
+ if results[0] != 0 or self._check_for_errors(results[1]):
+ print "Something went wrong..."
+ print results[1]
+ sys.exit(1)
+ if DEBUG:
+ print "gfsh command: %s " % command
+ print "gfsh command status:%s " % results[1]
+
+ @staticmethod
+ def _check_for_errors(results):
+ error_strings = ["Could not connect to", "is not valid or is currently not available"]
+ for error in error_strings:
+ if error in results:
+ return True
+ return False
+
+
+def main():
+ parse_options()
+ check_valid_output_dir()
+ check_for_gfsh()
+ create_tmp_dir()
+
+ g = Gfsh()
+ g.export_logs()
+ g.export_config()
+ g.export_stack_traces()
+ g.create_zip_file()
+ g.do_cleanup()
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/.classpath
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/.classpath b/logcollectionutil/sendlogs-java/.classpath
new file mode 100644
index 0000000..cb39306
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/.classpath
@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="src"/>
+ <classpathentry kind="src" path="tests"/>
+ <classpathentry kind="lib" path="lib/commons-compress-1.6.jar"/>
+ <classpathentry kind="lib" path="lib/commons-io-2.4.jar"/>
+ <classpathentry kind="lib" path="lib/commons-net-3.3.jar"/>
+ <classpathentry kind="lib" path="lib/ganymed-ssh2-261.jar"/>
+ <classpathentry kind="lib" path="lib/hamcrest-core-1.3.jar"/>
+ <classpathentry kind="lib" path="lib/jopt-simple-4.6.jar"/>
+ <classpathentry kind="lib" path="lib/junit-4.11.jar"/>
+ <classpathentry kind="output" path="bin"/>
+</classpath>
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/README
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/README b/logcollectionutil/sendlogs-java/README
new file mode 100644
index 0000000..edfad03
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/README
@@ -0,0 +1,103 @@
+Pivotal GemFire Log Collection Utility
+======================================
+
+This utility is used to gather log files from a GemFire cluster. In the default mode
+this requires that a GemFire process be running on each machine that this tool is to gather
+logs from. If you would like to collect log files from machines in which a GemFire process
+is not running, you can use the "Static Copy Mode". See the "Current Usage" section for
+details on using "Static Copy Mode".
+
+This utility provides an alternative to using gfsh and doesn't require GemFire members to
+be in a healthy state as it doesn't use the internal GemFire communication protocols to
+complete operations. Unlike gfsh this log collection utility is also not interactive and
+focused at the host level to gather all logs and stats from members.
+
+The gfsh command list for 7.0.X can be reviewed at the following URL which includes
+the "export stack-traces" and "export logs" command which overlap in terms of functinality.
+
+See section on "GemFire Monitoring":
+http://gemfire702.docs.pivotal.io/7.0.2/userguide/tools_modules/gfsh/quick_ref_commands_by_area.html
+
+Default Usage
+=============
+java -jar gfe-logcollect.jar -c <company> -o <output dir> [OPTIONS]
+
+Required arguments:
+ -c company name to append to output filename
+ -o output directory to store all collected log files
+
+Optional arguments:
+ -a comma separated list of hosts with no spaces. EG. host1,host2,host3 (defaults to localhost)
+ -u username to use to connect via ssh (defaults to current user)
+ -i identity file to use for PKI based ssh (defaults to ~/.ssh/id_[dsa|rsa]
+ -p prompt for a password to use for ssh connections
+ -t ticket number to append to created zip file
+ -d don't clean up collected log files after the zip has been created
+ -s send the zip file to Pivotal support
+ -f ftp server to upload collected logs to. Defaults to ftp.gemstone.com
+ -v print version of this utility
+ -h print this help information
+
+Static Copy Mode
+================
+java -jar gfe-logcollect.jar -c <company> -o <output dir> -m <filename> [OPTIONS]
+
+Required arguments:
+ -c company name to append to output filename
+ -o output directory to store all collected log files
+ -m <file> Use a file with log locations instead of scanning for logs.
+ Entries should be in the format hostname:/log/location
+ -a should not be used with the -m option
+ All other commandline options work as expected
+
+Known Limitations
+=================
+1. Only supports Linux hosts.
+2. Requires SSH access between machines.
+3. Requires that the username be the same for each host that this app scans.
+ EG. you can't specify user@host1, anotherUser@host2, etc.
+4. Requires that SSH access is available across all hosts using either the same
+ password or the same public key.
+5. In order to get stacks using jstack, this process must be ran as the same user
+ who owns the GFE process.
+6. Requires 'jps' (typically in $JAVA_HOME/bin) to be in the user's PATH on each
+ machine.
+7. Requires 'jstack' (typically in $JAVA_HOME/bin) to be in the users PATH on each
+ machine.
+8. Requires Java 7 on the host running the utility.
+
+Additional Information on Usage
+===============================
+The default way that log collector works is by using ssh and then issuing a series of
+shell commands to collect data about the processes running on each host listed via the
+"-a hostname,hostname...". The utility initially logs into each host via ssh and
+attempts to identify the running GemFire processes and PID's by issuing the "jps -l"
+command and looking for Java runtimes with the sub strings "com.vmware","com.gemstone",
+"io.pivotal", "com.pivotal" or "gemfire" in their names. If zero possible GemFire processes
+are found, a second attampt will be made via "ps -fa | grep java" and looking for
+any process with "classpath" and "gemfire.jar" that are listed. Once the PID's have
+been identified, the utility issues via ssh the command "jstack -l <pid>" to each
+process to dump a thread stack which gets collected. In the next step the utility issues
+the command "ls -l /proc/<pid>/fd" listing the process file handles in which to look for
+log files. From the possible list of directories found via the file handles, the utility
+copies any file with the extensions ".log", ".err", ".cfg", ".gfs", ".stack", ".xml",
+".properties" and ".txt" to the output directory. If the "-s" option is used then the
+zip file created in the output directory is sent to the ftp server at ftp.gemstone.com
+and placed in the public/incoming directory for the support team. There is no notification
+on the uploaded files and so the user will need to create a support ticket and in the
+ticket should reference uploaded files.
+
+The static copy mode will not attempt to collect stack traces and differs by fetching the files
+only from the hosts and directories specified in the file provided via the "-m" option. As
+such no GemFire processes need to be running for the collection of available artifacts to occur.
+
+It is recommended that users review and inspect any artifacts collected to insure that
+these do not violate company policies in regards to sharing data with outside parties including
+Pivotal support.
+
+Copyright
+=========
+2015 Pivotal Software, Inc. All Rights Reserved. This
+product is protected by U.S. and international copyright and
+intellectual property laws. Pivotal products are covered by one
+or more patents listed at http://www.pivotal.io/patents.
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/TODO
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/TODO b/logcollectionutil/sendlogs-java/TODO
new file mode 100644
index 0000000..8a422d8
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/TODO
@@ -0,0 +1,38 @@
+Needs better build script to compile utility and package for distribution (gradle or maven)
+
+Packaging for distribution needs to be fixed so that bin directory is clean of extra
+files not required to use tool or build artifacts
+
+Open source the tool
+
+Need test to validate handling of bad data in CollectLogsFromFileMap.java
+and -m <filename> option (blank lines, bad text, incorrect format)
+
+Add command line option to not zip results
+
+Add command line option to only collect thread dumps
+
+Add command line option to collect heap histogram (jmap -histo:live <pid> )
+
+Add command line option to send generic commands ( similar to parallel ssh shells)
+
+Improve documentation to include what and how the tool works
+alone with additional best practices
+
+Create example/quickstart
+
+Command line short options (-a) should have corresponding long values (--hostnames)
+
+Need to update build script to run junit tests
+
+Need test to validate handling of bad data in CollectLogsFromFileMap.java
+and -m <filename> option (blank lines, bad text, incorrect format)
+
+Update package from com.pivotal.gemfire to io.pivotal.gemfire
+
+Need to reformat code for consisent readability
+
+
+
+
+
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/build.xml
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/build.xml b/logcollectionutil/sendlogs-java/build.xml
new file mode 100644
index 0000000..66c019e
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/build.xml
@@ -0,0 +1,45 @@
+<?xml version="1.0"?>
+<project name="logcollectionutil" basedir="." default="jar">
+ <property name="src" value="src"/>
+ <property name="output" value="bin"/>
+
+ <target name="compile" depends="create">
+ <javac destdir="bin">
+ <src path="${src}"/>
+ <classpath refid="java"/>
+ </javac>
+ </target>
+
+ <target name="jar" depends="compile">
+ <jar destfile="${output}/gfe-logcollect.jar">
+ <fileset dir="${output}"/>
+ <manifest>
+ <attribute name="Main-Class"
+ value="com.pivotal.gemfire.sendlogs.driver.Driver" />
+ <attribute name="Class-Path" value="../lib/commons-compress-1.6.jar ../lib/commons-net-3.3.jar ../lib/hamcrest-core-1.3.jar ../lib/commons-io-2.4.jar ../lib/ganymed-ssh2-261.jar ../lib/jopt-simple-4.6.jar ../lib/log4j-1.2.17.jar ../lib/junit-4.11.jar"/>
+ </manifest>
+ </jar>
+ </target>
+
+
+ <target name="clean">
+ <delete dir="${output}"/>
+ <delete dir="logs"/>
+ <delete file="${output}/gfe-logcollect.jar"/>
+ </target>
+
+ <target name="create" depends="clean">
+ <mkdir dir="${output}"/>
+ </target>
+
+ <target name="run" depends="jar">
+ <java jar="${output}/gfe-logcollect.jar" fork="true"/>
+ </target>
+
+
+ <path id="java">
+ <fileset dir="./lib">
+ <include name="*.jar"/>
+ </fileset>
+ </path>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/open_source_licenses-Pivotal_GemFire_LogCollectionUtility_1.0.txt
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/open_source_licenses-Pivotal_GemFire_LogCollectionUtility_1.0.txt b/logcollectionutil/sendlogs-java/open_source_licenses-Pivotal_GemFire_LogCollectionUtility_1.0.txt
new file mode 100755
index 0000000..b87e02f
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/open_source_licenses-Pivotal_GemFire_LogCollectionUtility_1.0.txt
@@ -0,0 +1,450 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+=======================================================================
+
+Pivotal GemFire LogCollectionUtility 1.0 GA
+
+Pivotal GemFire LogCollectionUtility : includes a number of subcomponents with
+separate copyright notices and license terms. The product that
+includes this file does not necessarily use all the open source
+subcomponents referred to below. Your use of the source
+code for the these subcomponents is subject to the terms and
+conditions of the following license
+
+SECTION 1: BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES
+
+ >>> ganymed-ssh2-build261
+ >>> jopt-simple-4.6
+
+
+
+SECTION 2: Apache License, V2.0
+
+ >>> commons-compress-1.6
+ >>> commons-io-2.4
+ >>> commons-net-3.3
+ >>> log4j-1.2.17
+
+
+
+--------------- SECTION 1: BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES ----------
+
+BSD-STYLE, MIT-STYLE, OR SIMILAR STYLE LICENSES are applicable to the following component(s).
+
+
+>>> ganymed-ssh2-build261
+
+Copyright (c) 2006 - 2013 Christian Plattner. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+a.) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+b.) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+c.) Neither the name of Christian Plattner nor the names of its contributors may
+ be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+This software includes work that was released under the following license:
+
+Copyright (c) 2005 - 2006 Swiss Federal Institute of Technology (ETH Zurich),
+ Department of Computer Science (http://www.inf.ethz.ch),
+ Christian Plattner. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+a.) Redistributions of source code must retain the above copyright
+ notice, this list of conditions and the following disclaimer.
+b.) Redistributions in binary form must reproduce the above copyright
+ notice, this list of conditions and the following disclaimer in the
+ documentation and/or other materials provided with the distribution.
+c.) Neither the name of ETH Zurich nor the names of its contributors may
+ be used to endorse or promote products derived from this software
+ without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGE.
+
+
+The Java implementations of the AES, Blowfish and 3DES ciphers have been
+taken (and slightly modified) from the cryptography package released by
+"The Legion Of The Bouncy Castle".
+
+Their license states the following:
+
+Copyright (c) 2000 - 2004 The Legion Of The Bouncy Castle
+(http://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
+
+>>> jopt-simple-4.6
+
+The MIT License
+
+ Copyright (c) 2004-2014 Paul R. Holser, Jr.
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be
+ included in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+--------------- SECTION 2: Apache License, V2.0 ----------
+
+Apache License, V2.0 is applicable to the following component(s).
+
+
+>>> commons-compress-1.6
+
+Apache Commons Compress
+Copyright 2002-2013 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+The files in the package org.apache.commons.compress.archivers.sevenz
+were derived from the LZMA SDK, version 9.20 (C/ and CPP/7zip/),
+which has been placed in the public domain:
+
+"LZMA SDK is placed in the public domain." (http://www.7-zip.org/sdk.html)
+
+
+>>> commons-io-2.4
+
+Apache Commons IO
+Copyright 2002-2012 The Apache Software Foundation
+
+This product includes software developed by
+The Apache Software Foundation (http://www.apache.org/).
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+>>> commons-net-3.3
+
+Apache Commons Net
+Copyright 2001-2013 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
+>>> log4j-1.2.17
+
+Apache log4j
+Copyright 2010 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+
+log4j-1.2.17.jar
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+===========================================================================
+
+To the extent any open source components are licensed under the
+GPL and/or LGPL, or other similar licenses that require the
+source code and/or modifications to source code to be made
+available (as would be noted above), you may obtain a copy of
+the source code corresponding to the binaries for such open
+source components and modifications thereto, if any, (the
+"Source Files"), by downloading the Source Files from Pivotal website at
+(http://www.gopivotal.com/open-source),or by sending a request,
+with your name and address to: Pivotal Software Inc.3496 Deer Creek Rd,
+Palo Alto, CA 94304,Attention: General Counsel.
+All such requests should clearly specify: OPEN SOURCE FILES REQUEST,
+Attention General Counsel. Pivotal shall mail a copy of the
+Source Files to you on a CD or equivalent physical medium. This
+offer to obtain a copy of the Source Files is valid for three
+years from the date you acquired this Software product.
+Alternatively, the Source Files may accompany the Pivotal product.
+
+[GEMFIRELOGCOLLECTIONUTILITY10GASS030614]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/scripts/logcollect
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/scripts/logcollect b/logcollectionutil/sendlogs-java/scripts/logcollect
new file mode 100755
index 0000000..f5fcb80
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/scripts/logcollect
@@ -0,0 +1,22 @@
+#!/bin/bash
+# If the 'java' command is in $PATH, use it first.
+# If not, fall back to using $JAVA_HOME/bin/java
+
+command -v java > /dev/null 2>&1
+if [[ $? != 0 ]]; then
+ if [[ -n $JAVA_HOME ]]; then
+ command -v $JAVA_HOME/bin/java > /dev/null 2>&1
+ if [[ $? == 0 ]]; then
+ JAVA=$JAVA_HOME/bin/java
+ fi
+ fi
+else
+ JAVA=java
+fi
+
+if [[ -z $JAVA ]]; then
+ echo "Can't find a valid 'java' executable."
+ exit 1
+else
+ $JAVA -jar gfe-logcollect.jar $@
+fi
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/CollectLogsFromFileMap.java
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/CollectLogsFromFileMap.java b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/CollectLogsFromFileMap.java
new file mode 100644
index 0000000..8c980a3
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/CollectLogsFromFileMap.java
@@ -0,0 +1,68 @@
+package com.pivotal.gemfire.sendlogs.driver;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.log4j.Logger;
+
+import com.pivotal.gemfire.sendlogs.utilities.CloseGracefully;
+import com.pivotal.gemfire.sendlogs.utilities.SendlogsExceptionHandler;
+
+public class CollectLogsFromFileMap {
+ private final Logger logger = Logger.getLogger(CollectLogsFromFileMap.class.getName());
+ private final File configFile;
+
+ /**
+ * Parse user configuration.
+ * @param configurationFile User defined configuration file mapping hostnames to directories.
+ */
+ public CollectLogsFromFileMap(File configurationFile) {
+ this.configFile = configurationFile;
+ }
+
+ /**
+ * Get a list of hostnames and directories from user created file.
+ * This should be in the format:
+ * hostname:/dir/to/grab/logs/from
+ */
+ public HashMap<String, HashSet<String>> collectLogLocations() {
+ BufferedReader br = null;
+
+ HashMap<String, HashSet<String>> hostAndFile = new HashMap<String, HashSet<String>>();
+ String line;
+ try {
+ br = new BufferedReader(new FileReader(configFile));
+ while ((line = br.readLine()) != null) {
+ String[] hostStringSplit = line.split(":");
+ if (hostStringSplit.length < 2 ) {
+ if(!(line.equals("") || line.length()==0)){
+ logger.warn("Encountered invalid line: " + line);
+ }
+ continue;
+ }
+
+ // If this host has already had a directory defined, add the
+ // new one to the list of directories
+ HashSet<String> al = hostAndFile.get(hostStringSplit[0]);
+ if (al == null) {
+ hostAndFile.put(hostStringSplit[0], new HashSet<String>());
+ hostAndFile.get(hostStringSplit[0]).add(hostStringSplit[1]);
+ } else {
+ al.add(hostStringSplit[1]);
+ }
+ }
+ } catch (IOException e) {
+ SendlogsExceptionHandler.handleException(e);
+ } finally {
+ if (br != null) {
+ CloseGracefully.close(br);
+ }
+ }
+ return hostAndFile;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/Driver.java
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/Driver.java b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/Driver.java
new file mode 100644
index 0000000..c155046
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/Driver.java
@@ -0,0 +1,268 @@
+package com.pivotal.gemfire.sendlogs.driver;
+
+import java.io.Console;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+
+import com.pivotal.gemfire.sendlogs.remote.FTPUploader;
+import com.pivotal.gemfire.sendlogs.remote.RemoteHost;
+import com.pivotal.gemfire.sendlogs.utilities.CliParser;
+import com.pivotal.gemfire.sendlogs.utilities.CloseGracefully;
+import com.pivotal.gemfire.sendlogs.utilities.CompressArtifacts;
+import com.pivotal.gemfire.sendlogs.utilities.FileSystem;
+import com.pivotal.gemfire.sendlogs.utilities.Log;
+import com.pivotal.gemfire.sendlogs.utilities.SendlogsExceptionHandler;
+import com.pivotal.gemfire.sendlogs.utilities.Usage;
+
+/**
+ * Primary driver class for the entire application. Does some initial verification and setup
+ * of the environment and then uses the {@link com.pivotal.gemfire.sendlogs.driver.GemFireHost} class to do all of the actual
+ * work.
+ *
+ * TODO: This should really be broken out into different classes.
+ *
+ * @author ablakeman
+ */
+public class Driver {
+ static CliParser options = null;
+ private static String passwordFromUser = null;
+ private enum AuthMethod { PASSWORD, IDENTITY, DEFAULT }
+ private static File pubkeyLocation;
+ private static String baseDir;
+ private static AuthMethod authMethod = null;
+ private static ArrayList<String> hosts;
+ private static String zipFilename;
+ private static boolean haveFilesBeenCollected = false;
+ private static final Logger logger = Logger.getLogger(Driver.class.getName());
+ private static final String OUTPUTLOGFILENAME = "sendlog.out";
+ private static final long MAXZIPFILESIZEINBYTES = 2L * 1024 * 1024 * 1024; /* 2GB */
+ private static String username;
+ private static CompressArtifacts zipFile;
+
+
+ public static void main(String args[]) {
+ if (args.length < 1) {
+ Usage.printUsage();
+ CloseGracefully.exitApp();
+ }
+ options = new CliParser(args);
+ baseDir = createAndGetBaseDir();
+ setupLogging(getLogFilename());
+ setAuthMethod();
+ username = options.getUsername();
+ hosts = new ArrayList<String>();
+ zipFile = new CompressArtifacts(baseDir, options.getCustomerName(), options.getTicketNumber());
+ zipFilename = zipFile.getZipFilename();
+
+ /* If the user has provided a config file with hosts and directories use that*/
+ if (options.staticCopyMode()) {
+ collectLogs();
+ } else {
+ /* If not then proceed with the default scanning for PIDs and getting directories */
+ findAndCollectLogs();
+ }
+
+ zipFile.addFileToZip(getLogFilename(), "sendlogs");
+ zipFile.close();
+
+ if (haveFilesBeenCollected) {
+ if (options.shouldRemoveTempFiles()) {
+ cleanupTmpFiles();
+ }
+ if (options.shouldUploadZipToPivotal()) {
+ if (!sendZipToPivotal()) {
+ logger.error("Unable to upload the file to "+ options.getFtpServer() +".");
+ } else {
+ logger.info("Upload complete.");
+ }
+ }
+ logger.info("Logs available at: " + zipFilename);
+ } else {
+ logger.error("Unable to find or access any gemfire processes on the given hosts.");
+ }
+ }
+
+ /**
+ * Proceed in the default manner of going to each host, looking for a gemfire process, and finding the
+ * logs associated with that process.
+ */
+ private static void findAndCollectLogs() {
+ for (String host : options.getAddresses()) {
+ hosts.add(host);
+ logger.info("Collecting logs from: " + host);
+
+ GemFireHost gfh = new GemFireHost(getRemoteHost(host), baseDir, zipFile);
+ gfh.findAndRetrieveLogs();
+
+ if (gfh.haveFilesBeenCollected()) {
+ haveFilesBeenCollected = true;
+ } else {
+ logger.warn("No Gemfire proceses found on " + host);
+ }
+ gfh.close();
+ }
+ }
+
+ /**
+ * Collect logs from user defined locations. In this situation the user has chosen to
+ * provide a file with a mapping of hosts:/log/directories instead of allowing this
+ * utility to scan for PIDs and get log files.
+ */
+ private static void collectLogs() {
+ File logFileFromUser = new File(options.getStaticFileLocation());
+
+ if (!logFileFromUser.exists()) {
+ CloseGracefully.exitApp("Unable to open configuration file: " + logFileFromUser.getPath());
+ }
+
+ CollectLogsFromFileMap logsFromFile = new CollectLogsFromFileMap(logFileFromUser);
+ HashMap<String, HashSet<String>> hostsAndDirectories = logsFromFile.collectLogLocations();
+
+ for (String host : hostsAndDirectories.keySet()) {
+ hosts.add(host);
+ logger.info("Collecting logs from: " + host);
+
+ GemFireHost gfh = new GemFireHost(getRemoteHost(host), baseDir, zipFile);
+ gfh.retrieveLogs(hostsAndDirectories.get(host));
+
+ if (gfh.haveFilesBeenCollected() && gfh.getMissedDirectories().isEmpty()) {
+ haveFilesBeenCollected = true;
+ } else {
+ logger.warn("Unable to collect logs from the following locations on " + host);
+ for (String dir : gfh.getMissedDirectories()) {
+ logger.warn(dir);
+ }
+ }
+ gfh.close();
+ }
+ }
+
+
+ /**
+ * Get {@link RemoteHost} object for specified authentication method
+ * @param host - hostname of the machine
+ * @return {@link RemoteHost}
+ */
+ private static RemoteHost getRemoteHost(String host) {
+ RemoteHost rh;
+ switch(authMethod) {
+ case PASSWORD: rh = new RemoteHost(username, host, passwordFromUser);
+ break;
+ case IDENTITY: rh = new RemoteHost(username, host, pubkeyLocation);
+ break;
+ default: rh = new RemoteHost(username, host);
+ break;
+ }
+ return rh;
+ }
+
+ /**
+ * Set up logging for the app.
+ */
+ private static void setupLogging(String logFilename) {
+ if (options.debugMode()) {
+ Log.setupLogging(logFilename, Level.DEBUG);
+ } else {
+ Log.setupLogging(logFilename, Level.INFO);
+ }
+ }
+
+ /**
+ * Return the output log file name to store results of this run to send to support.
+ */
+ private static String getLogFilename() {
+ return baseDir + "/" + OUTPUTLOGFILENAME;
+ }
+
+ /**
+ * Send the zip file with all of the logs to the pivotal support FTP server.
+ */
+ private static boolean sendZipToPivotal() {
+ FTPUploader ftpU = new FTPUploader(options.getFtpServer());
+ File zipFileToUpload = new File(zipFilename);
+ logger.debug("max zip file size: " + MAXZIPFILESIZEINBYTES);
+ logger.debug("zip length: " + zipFileToUpload.length());
+ if (zipFileToUpload.length() > MAXZIPFILESIZEINBYTES) {
+ logger.error("Unable to upload zip file as it's larger than 2GB. Please contact support.");
+ return false;
+ }
+ logger.info("Uploading logs to " + options.getFtpServer());
+
+ return ftpU.sendFile(zipFilename);
+ }
+
+ /**
+ * Deletes all of the files copied over, unless the user passed the option that
+ * says not to.
+ */
+ private static void cleanupTmpFiles() {
+ /* If the baseDirectory isn't an actual directory return. This ends up being redundant but is safer. */
+ if (!(new File(baseDir).isDirectory())) return;
+
+ for (String host : hosts) {
+ /* If somehow host is null, some sort of whitespace, or empty, return to be safe. */
+ if (host == null || host.trim().equals("")) return;
+ File hostDir = new File(baseDir + "/" + host);
+ if (hostDir.isDirectory()) {
+ try {
+ FileUtils.deleteDirectory(hostDir);
+ } catch (IOException ioe) {
+ try {
+ // Try once more in case the first attempt didn't finish because of open files.
+ FileUtils.deleteDirectory(hostDir);
+ } catch (IOException e) {
+ SendlogsExceptionHandler.handleException(e);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Figure out what Authentication method should be used for SSH connections. Either use a password,
+ * user specified identity file, or standard identity files.
+ */
+ private static void setAuthMethod() {
+ /* Get password from user if that option was specified */
+ if (options.promptForPassword()) {
+ passwordFromUser = getPasswordFromUser();
+ authMethod = AuthMethod.PASSWORD;
+ } else if (options.hasIdentityFileFromUser()) {
+ pubkeyLocation = options.getIdentityFile();
+ if (pubkeyLocation == null) CloseGracefully.exitApp("Unable to read specified identityfile.");
+ authMethod = AuthMethod.IDENTITY;
+ } else {
+ authMethod = AuthMethod.DEFAULT;
+ }
+ }
+
+ /**
+ * Retrieve password from user.
+ * @return String that the user entered at the prompt.
+ */
+ private static String getPasswordFromUser() {
+ Console console = System.console();
+ return new String(console.readPassword("Enter password for SSH: "));
+ }
+
+ /**
+ * Create a temporary directory in which to store all of the collected log files, etc.
+ * @return Full pathname of the created directory.
+ */
+ private static String createAndGetBaseDir() {
+ FileSystem fs = new FileSystem(options.getOutputDir());
+ fs.validateOrCreateBaseDirectory();
+ if (!fs.createTemporaryDirectory()) {
+ CloseGracefully.exitApp("Unable to create directory temporary directory: "
+ + fs.getTempLogDir() + " Check permissions.");
+ }
+ return fs.getTempLogDir();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/GemFireHost.java
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/GemFireHost.java b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/GemFireHost.java
new file mode 100644
index 0000000..aef9071
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/driver/GemFireHost.java
@@ -0,0 +1,174 @@
+package com.pivotal.gemfire.sendlogs.driver;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+
+import com.pivotal.gemfire.sendlogs.remote.RemoteCommands;
+import com.pivotal.gemfire.sendlogs.remote.RemoteHost;
+import com.pivotal.gemfire.sendlogs.utilities.CompressArtifacts;
+import com.pivotal.gemfire.sendlogs.utilities.FileSystem;
+import com.pivotal.gemfire.sendlogs.utilities.SendlogsExceptionHandler;
+/**
+ * This class handles the actual collection of logs from GemFire hosts. There
+ * should be one instance of this class per GemFire host.
+ *
+ * @author ablakema
+ */
+public class GemFireHost {
+ private final RemoteHost rh;
+ private final String workingDir;
+ private final RemoteCommands rc;
+ private final String hostname;
+ private final CompressArtifacts zipFile;
+ private ArrayList<String> missedDirs = new ArrayList<String>();
+ private boolean haveFilesBeenCollected = false;
+ private static final Logger logger = Logger.getLogger(GemFireHost.class.getName());
+
+
+ /**
+ * Sets up the required environment to perform operations on a remote host
+ * and collect files.
+ *
+ * @param rh {@link com.pivotal.gemfire.sendlogs.remote.RemoteHost} object for remote host actions.
+ * @param topLevelDir Top level working directory
+ * @param zipFile {@link com.pivotal.gemfire.sendlogs.utilities.CompressArtifacts} to add copied files to.
+ */
+ public GemFireHost(RemoteHost rh, String topLevelDir, CompressArtifacts zipFile) {
+ this.rh = rh;
+ this.zipFile = zipFile;
+ this.hostname = rh.getHostname();
+ workingDir = String.format("%s/%s", topLevelDir, hostname);
+ rc = new RemoteCommands(rh);
+ createWorkingDirectory(workingDir);
+ }
+
+ /**
+ * Does all of the actual work for this class. SCP's log files,
+ * and adds each file to the zip file.
+ *
+ * This *could* probably be written to redirect the SCPInputStream directly to the ZipFile.
+ *
+ */
+ public void findAndRetrieveLogs() {
+ ArrayList<String> pidsFromHost = rc.getGemFirePids();
+ /* Verify that we have some PIDs to work with */
+ if (pidsFromHost == null || pidsFromHost.isEmpty()) return;
+
+ for (String pid : pidsFromHost) {
+ dumpStacksToFile(pid);
+ collectLogFilesFromPidAndWriteToZip(pid);
+ }
+ }
+
+ /**
+ * @return An ArrayList containing directories that either didn't exist or
+ * directories in which we didn't find any files we are interested in.
+ */
+ public ArrayList<String> getMissedDirectories() {
+ return missedDirs;
+ }
+
+ /**
+ * Retrieve files from the specified directory
+ */
+ public void retrieveLogs(HashSet<String> directories) {
+ for (String dirName: directories) {
+ collectLogFilesFromDirAndWriteToZip(dirName);
+ }
+ }
+
+ /**
+ * @return True if log files have been found and collected.
+ */
+ public boolean haveFilesBeenCollected() {
+ return haveFilesBeenCollected;
+ }
+
+ /**
+ * Close the SSH connection to the remote host.
+ */
+ public void close() {
+ rh.closeConnection();
+ }
+
+ /**
+ * Copy the log files from the remote host and then add them to the zip file.
+ * @param pid PID of the GemFire process.
+ */
+ private void collectLogFilesFromPidAndWriteToZip(String pid) {
+ String workingDirForPid = workingDir + "/" + pid;
+ createWorkingDirectory(workingDirForPid);
+
+ HashSet<String> logFileNames = rc.getLogFileNamesFromPid(pid);
+ if (logFileNames != null && !logFileNames.isEmpty()) {
+ for (String logFile : logFileNames){
+ File remoteFile = new File(logFile);
+ String localFilename = workingDirForPid + "/" + remoteFile.getName();
+ rh.getRemoteFile(remoteFile.getAbsolutePath(), workingDirForPid);
+ zipFile.addFileToZip(localFilename, hostname + "/" + pid );
+ /* TODO: This hasn't actually verified that the above methods were successful */
+ haveFilesBeenCollected = true;
+ }
+ }
+ }
+
+ /**
+ * Copy the log files from the remote host and then add them to the zip file.
+ * @param baseDirectory Base working directory for log copies and such.
+ */
+ private void collectLogFilesFromDirAndWriteToZip(String baseDirectory) {
+ boolean foundLogsInThisDir = false;
+ String workingDirForBaseDir = workingDir + "/" + baseDirectory;
+ createWorkingDirectory(workingDirForBaseDir);
+ logger.debug("Working directory for " + hostname + " " + workingDirForBaseDir);
+
+ HashSet<String> logFileNames = rc.getLogFileNamesFromDirectory(baseDirectory);
+ if (logFileNames != null && !logFileNames.isEmpty()) {
+ for (String logFile : logFileNames){
+ File lf = new File(logFile);
+ logger.debug("Trying to get " + logFile + " from " + hostname);
+ rh.getRemoteFile(logFile, workingDirForBaseDir);
+ logger.debug("Trying to zip: " + workingDirForBaseDir + "/" + lf.getName());
+ zipFile.addFileToZip(workingDirForBaseDir + "/" + lf.getName(), hostname + "/" + lf.getParent());
+ /* TODO: This hasn't actually verified that the above methods were successful */
+ haveFilesBeenCollected = true;
+ foundLogsInThisDir = true;
+ }
+ }
+
+ if (Boolean.FALSE.equals(foundLogsInThisDir)) {
+ missedDirs.add(baseDirectory);
+ }
+ }
+
+ /**
+ * Writes the stack of a given PID to a file. Filename is in the format pid.stack
+ * @param pid PID of the GemFire process.
+ */
+ private void dumpStacksToFile(String pid) {
+ String stackFileName = workingDir + "/" + pid + ".stack";
+ File stackFile = new File(stackFileName);
+ try {
+ FileUtils.writeStringToFile(stackFile, rc.getStackTrace(pid));
+ zipFile.addFileToZip(stackFile.getAbsolutePath(), hostname);
+ /* TODO: This hasn't actually verified that the above methods were successful */
+ haveFilesBeenCollected = true;
+ } catch (IOException e) {
+ SendlogsExceptionHandler.handleException(e);
+ }
+ }
+
+ /**
+ * Creates a working directory for this host. The base level working directory is
+ * related to the hostname of the server. Additional directories for each PID are then
+ * created. This is is all relative to the working directory defined in {@link Driver}
+ */
+ private void createWorkingDirectory(String dirName) {
+ FileSystem.createDirectory(dirName);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/FTPUploader.java
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/FTPUploader.java b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/FTPUploader.java
new file mode 100644
index 0000000..a3b88a1
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/FTPUploader.java
@@ -0,0 +1,71 @@
+package com.pivotal.gemfire.sendlogs.remote;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.commons.net.ftp.FTPClient;
+
+import com.pivotal.gemfire.sendlogs.utilities.SendlogsExceptionHandler;
+
+/**
+ * Uploads a file to the GemStone FTP server.
+ *
+ * TODO: 1. Finish proxy support.
+ *
+ * @author ablakema
+ */
+public class FTPUploader {
+ private String ftpServer;
+ private final FTPClient ftp;
+
+ public FTPUploader(String ftpServer) {
+ int FTPTIMEOUTINMS = 30000;
+
+ this.ftpServer = ftpServer;
+ ftp = new FTPClient();
+ ftp.setConnectTimeout(FTPTIMEOUTINMS);
+ }
+
+ /**
+ * Upload file to remote FTP server.
+ *
+ * @param filename
+ * Filename of the file to upload
+ * @return True if the file was successfully uploaded.
+ */
+ public boolean sendFile(String filename) {
+ File zipFile = new File(filename);
+ if (connect()) {
+ try {
+ return ftp.storeFile("incoming/" + zipFile.getName(),
+ new FileInputStream(filename));
+ } catch (IOException e) {
+ SendlogsExceptionHandler.handleException(e);
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Check for a valid connection or connect.
+ *
+ * @return True if there is already a valid connection or a new connection
+ * was successfully made.
+ */
+ private boolean connect() {
+ boolean isConnected = false;
+ try {
+ if (!ftp.isConnected()) {
+ ftp.connect(ftpServer);
+ ftp.login("anonymous", "");
+ }
+ isConnected = true;
+ } catch (IOException e) {
+ SendlogsExceptionHandler.handleException(e);
+ }
+ return isConnected;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteCommands.java
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteCommands.java b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteCommands.java
new file mode 100644
index 0000000..9551708
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteCommands.java
@@ -0,0 +1,186 @@
+package com.pivotal.gemfire.sendlogs.remote;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.HashSet;
+
+import org.apache.log4j.Logger;
+
+/**
+ *
+ * This relies on the {@link RemoteHost} class to do the actual work.
+ *
+ * @author ablakeman
+ *
+ */
+public class RemoteCommands {
+ private final RemoteHost rh;
+ private static final Logger logger = Logger.getLogger(RemoteCommands.class.getName());
+
+ public RemoteCommands(RemoteHost rh) {
+ this.rh = rh;
+ }
+
+ /**
+ * @return List of all pids matching a subset of package names.
+ */
+ public ArrayList<String> getGemFirePids() {
+ ArrayList<String> pids = new ArrayList<String>();
+ String[] output = rh.executeCommand("jps -l").split("\n");
+
+ for (String line : output) {
+ if (line.contains("com.vmware") || line.contains("com.gemstone") || line.contains("com.pivotal")
+ || line.contains("io.pivotal") || line.contains("gemfire") || line.contains("com.gopivotal")) {
+ if (!(line.contains("com.pivotal.gemfire.sendlogs")
+ || line.contains("io.pivotal.gemfire.sendlogs")) ) {
+ String foundGfePid = line.split("\\s+")[0];
+ logger.info("Found gemfire pid: " + foundGfePid);
+ pids.add(foundGfePid);
+ }
+ }
+ }
+
+ getGemFirePidsFromProcessTable(pids);
+
+ return pids.isEmpty() ? null : pids;
+ }
+
+ /**
+ * Try to find possible GemFire processes using the ps command
+ * looks for gemfire.jar and classpath in output of ps.
+ *
+ * @param pids
+ */
+
+ protected void getGemFirePidsFromProcessTable(ArrayList<String> pids){
+ //Looking for possible PID's via the process classpath that contain gemfire.jar
+ //We should consider making this optional since many add everything including
+ //the kitchen sink to the classpath even when not required
+ String[] output;
+ output=rh.executeCommand("ps -fa | grep java").split("\n");
+ for (String line : output){
+ if ( line.contains("classpath") && line.contains("gemfire.jar") ){
+ if(!(line.contains("com.pivotal.gemfire.sendlogs")||
+ line.contains("io.pivotal.gemfire.sendlogs"))){
+ String foundGfePid = line.split("\\s+")[1];
+ if(!pids.contains(foundGfePid)){
+ pids.add(foundGfePid);
+ logger.info("Found gemfire pid: " + foundGfePid);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * @param pid PID of GemFire process.
+ * @return String containing the entire stack trace. This is newline delimited.
+ */
+ public String getStackTrace(String pid) {
+ return rh.executeCommand("jstack -l " + pid);
+ }
+
+ /**
+ * Get all of the log files associated with the remote PID. This first reads /proc/$PID/fd.
+ * From there it will look for base directories of files open from /proc/$PID/fd and then
+ * look for valid logs in each of the directories found in this manner.
+ * @param pid PID of GemFire process.
+ * @return A HashSet containing the full filenames of all found logs or null if nothing was found.
+ */
+ public HashSet<String> getLogFileNamesFromPid(String pid) {
+ HashSet<String> allFiles = new HashSet<String>();
+ HashSet<String> filesFromLs = new HashSet<String>();
+
+ /* First get files with open file descriptors */
+ String[] linesFromRemoteFdFolder = rh.executeCommand(String.format("ls -l /proc/%s/fd", pid)).split("\n");
+ if (linesFromRemoteFdFolder.length > 0) {
+ for (String s : linesFromRemoteFdFolder) {
+ /* TODO: Figure out a better way to do this */
+ if (checkFileType(s)) {
+ String[] lsOutputSplit = s.split("\\s+");
+ String filename = lsOutputSplit[lsOutputSplit.length -1];
+ filesFromLs.add(filename);
+ allFiles.add(filename);
+ }
+ }
+ /* Then check the directories that those files were in for other logs */
+ for (String extraLogFile : getlogFilesFromFdOutput(filesFromLs)) {
+ allFiles.add(extraLogFile);
+ }
+ }
+ return allFiles.isEmpty() ? null : allFiles;
+ }
+
+ /**
+ * This method should be re-written to provide the functionality of
+ * {@link com.pivotal.gemfire.sendlogs.remote.RemoteCommands#getLogFileNamesFromPid(String)}. Given the a base
+ * directory name return a HashSet<String> of files that we're interested in collecting.
+ * @param dirName Directory name in which to look for interesting log files.
+ * @return HashSet<String> that includes all of the files that were matched in {@link com.pivotal.gemfire.sendlogs.remote.RemoteCommands#checkFileType}
+ */
+ public HashSet<String> getLogFileNamesFromDirectory(String dirName) {
+ HashSet<String> allFiles = new HashSet<String>();
+ String[] dirContents = rh.executeCommand(String.format("ls -1 " + dirName)).split("\n");
+ if (dirContents.length > 0) {
+ for (String filename : dirContents) {
+ if (checkFileType(filename)) {
+ allFiles.add(dirName + "/" + filename);
+ }
+ }
+ }
+ return allFiles.isEmpty() ? null : allFiles;
+ }
+
+ /**
+ * Based on the extension of the file, determine whether or not it should be collected to send to support.
+ * @param fullPath Full path of file to determine whether or not this is an interesting file that should be
+ * collected.
+ * @return True if the file is of an interesting type to support.
+ */
+ public boolean checkFileType(String fullPath) {
+ return fullPath.toLowerCase().endsWith(".log")
+ || fullPath.toLowerCase().endsWith(".err")
+ || fullPath.toLowerCase().endsWith(".cfg")
+ || fullPath.toLowerCase().endsWith(".gfs")
+ || fullPath.toLowerCase().endsWith(".stack")
+ || fullPath.toLowerCase().endsWith(".xml")
+ || fullPath.toLowerCase().endsWith(".properties")
+ || fullPath.toLowerCase().endsWith(".txt");
+ }
+
+ /**
+ * @param fullFileName Complete filename of a file including it's path. EG. /export/foo/bar/test.txt
+ * @return String containing the base path of the file passed in. EG. passing in /export/foo/bar/baz/test.txt
+ * will return /export/foo/bar/baz
+ */
+ public String getBaseDir(String fullFileName) {
+ File f = new File(fullFileName);
+ return f.getParent();
+ }
+
+ /**
+ * Used by {@link com.pivotal.gemfire.sendlogs.remote.RemoteCommands#getLogFileNamesFromPid} to get log files (and
+ * anything else defined by {@link com.pivotal.gemfire.sendlogs.remote.RemoteCommands#checkFileType})
+ *
+ * @param filesFromLs Files from the LS output of the 'fd' folder for a given PID. This takes that 'ls' outpoint which
+ * will have a symlink to the original log folder and gets the base directory name from that
+ * symlink.
+ * @return HashSet containing a list of all of the files found that aren't currently open but are in a "log" directory.
+ */
+ private HashSet<String> getlogFilesFromFdOutput(HashSet<String> filesFromLs) {
+ HashSet<String> nonOpenLogs = new HashSet<String>();
+ if (!filesFromLs.isEmpty()) {
+ for (String fullFileName : filesFromLs) {
+ String baseDirOfFile = getBaseDir(fullFileName);
+ String[] linesFromRemoteFdFolder = rh.executeCommand("ls " + baseDirOfFile).split("\n");
+ for (String line: linesFromRemoteFdFolder) {
+ if (checkFileType(line)) {
+ /* the LS returns just the filename so re-append the full PATH to the file */
+ nonOpenLogs.add(baseDirOfFile + "/"+ line);
+ }
+ }
+ }
+ }
+ return nonOpenLogs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-geode/blob/e189c21a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteHost.java
----------------------------------------------------------------------
diff --git a/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteHost.java b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteHost.java
new file mode 100644
index 0000000..a4d9c4e
--- /dev/null
+++ b/logcollectionutil/sendlogs-java/src/com/pivotal/gemfire/sendlogs/remote/RemoteHost.java
@@ -0,0 +1,204 @@
+package com.pivotal.gemfire.sendlogs.remote;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+
+import com.pivotal.gemfire.sendlogs.utilities.CloseGracefully;
+import com.pivotal.gemfire.sendlogs.utilities.SendlogsExceptionHandler;
+
+import ch.ethz.ssh2.Connection;
+import ch.ethz.ssh2.SCPClient;
+import ch.ethz.ssh2.SCPInputStream;
+import ch.ethz.ssh2.Session;
+import ch.ethz.ssh2.StreamGobbler;
+
+/**
+ * Maintains the connection to the remote hosts. Provides the "lower level" interfaces to working with the
+ * remote host. EG. logging in, executing commands, copying files, etc.
+ * @author ablakema
+ */
+public class RemoteHost {
+ private final String username;
+ private final String hostname;
+ private final AuthType authType;
+ private final Logger logger = Logger.getLogger(RemoteHost.class.getName());
+
+ private Connection conn = null;
+ private Session sess = null;
+ private String password = null;
+ private File sshIdentityFileLocation = null;
+
+ private enum AuthType {
+ PASSWORD, PUBKEY
+ }
+
+ /**
+ * @param username Username used for SSH communications.
+ * @param hostname Remote hostname
+ * @param password Obtained from the user via STDIN.
+ */
+ public RemoteHost(String username, String hostname, String password) {
+ this.username = username;
+ this.password = password;
+ this.hostname = hostname;
+ this.authType = AuthType.PASSWORD;
+ }
+
+ /**
+ * @param username Username used for SSH communications.
+ * @param hostname Remote hostname
+ * @param pubkeyLocation Obtained from the user via command line option.
+ */
+ public RemoteHost(String username, String hostname, File pubkeyLocation) {
+ this.username = username;
+ this.hostname = hostname;
+ this.sshIdentityFileLocation = pubkeyLocation;
+
+ this.authType = AuthType.PUBKEY;
+ }
+
+ /**
+ * Uses the system default public keys to try authentication.
+ * @param username Username used for SSH communications.
+ * @param hostname Remote hostname
+ */
+ public RemoteHost(String username, String hostname) {
+ this.username = username;
+ this.hostname = hostname;
+
+ this.authType = AuthType.PUBKEY;
+
+ sshIdentityFileLocation = getValidSshPrivKey();
+ }
+
+ /**
+ * @return Location of first available ssh key.
+ */
+ private File getValidSshPrivKey() {
+ /* TODO: Implement a way of *trying* both of these */
+ String homeDir = System.getProperty("user.home");
+ File rsaFile = new File(String.format("%s/.ssh/id_rsa", homeDir));
+ File dsaFile = new File(String.format("%s/.ssh/id_dsa", homeDir));
+
+ /* Default to checking for id_rsa because it's better. */
+ if (rsaFile.isFile()) {
+ return rsaFile;
+ } else if (dsaFile.isFile()) {
+ return dsaFile;
+ }
+ return null;
+ }
+
+ /**
+ * Tries to connect to the remote host via ssh.
+ *
+ * This currently has a long timeout ~45 seconds. This may need to be reduced.
+ * @return true if the connection was successful.
+ */
+ private boolean connect() {
+ boolean isAuthenticated = false;
+ /* If we've already connected use that connection */
+ if (conn != null && conn.isAuthenticationComplete()) return true;
+
+ try {
+ conn = new Connection(hostname);
+ conn.connect();
+
+ if (authType == AuthType.PASSWORD) {
+ isAuthenticated = conn.authenticateWithPassword(username,
+ password);
+ } else if (sshIdentityFileLocation != null) {
+ isAuthenticated = conn.authenticateWithPublicKey(username,
+ sshIdentityFileLocation, "ignoreBecauseNotImplemented");
+ } else {
+ logger.warn("No valid credentials provided for " + hostname);
+ }
+
+ if (Boolean.FALSE.equals(isAuthenticated)) {
+ logger.error("Authentication failed for " + hostname);
+ } else {
+ return true;
+ }
+ } catch (Exception e) {
+ SendlogsExceptionHandler.handleException(e);
+ conn.close();
+ }
+ return isAuthenticated;
+ }
+
+ /**
+ * Copies over remote file to the local working directory.
+ *
+ * @param remoteFilename Full path to the remote file.
+ * @param baseDir Base directory where the file is to be copied.
+ */
+ public void getRemoteFile(String remoteFilename, String baseDir) {
+ SCPInputStream scpIs = null;
+ try {
+ if (connect()) {
+ String localFilename = baseDir + "/" + new File(remoteFilename).getName();
+ SCPClient scp = conn.createSCPClient();
+ FileOutputStream fos = new FileOutputStream(localFilename);
+ scpIs = scp.get(remoteFilename);
+ IOUtils.copy(scpIs, fos);
+ }
+ } catch (IOException e) {
+ SendlogsExceptionHandler.handleException(e);
+ }finally {
+ if (scpIs != null ) CloseGracefully.close(scpIs);
+ }
+ }
+
+ /**
+ *
+ * @param command Executes specified command on the remote host that this instance is connected to.
+ * @return String with the results of command. This is newline delimited.
+ */
+ public String executeCommand(String command) {
+ logger.debug("Executing command: " + command + " on host " + hostname);
+ BufferedReader br = null;
+ StringBuilder data = new StringBuilder();
+ try {
+ if (connect()) {
+ sess = conn.openSession();
+ sess.execCommand(command);
+ InputStream stdout = new StreamGobbler(sess.getStdout());
+ br = new BufferedReader(new InputStreamReader(stdout));
+ while (true) {
+ String line = br.readLine();
+ if (line == null)
+ break;
+ data.append(line);
+ data.append("\n");
+ }
+ }
+ } catch (Exception e) {
+ SendlogsExceptionHandler.handleException(e);
+ } finally {
+ if (sess != null) sess.close();
+ if (br != null) CloseGracefully.close(br);
+ }
+ return data.toString();
+ }
+
+ /**
+ * @return The hostname defined for this instance of {@link RemoteHost}.
+ */
+ public String getHostname() {
+ return hostname;
+ }
+
+ /**
+ * Close the connection to the SSH server.
+ */
+ public void closeConnection() {
+ conn.close();
+ }
+}