You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by rl...@apache.org on 2015/11/05 04:10:04 UTC
[8/8] incubator-hawq git commit: HAWQ-121. Remove legacy command line
tools.
HAWQ-121. Remove legacy command line tools.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/9932786b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/9932786b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/9932786b
Branch: refs/heads/master
Commit: 9932786b8351733847b4a404decfe5345880cb81
Parents: 4e39237
Author: rlei <rl...@pivotal.io>
Authored: Wed Nov 4 10:47:24 2015 +0800
Committer: rlei <rl...@pivotal.io>
Committed: Thu Nov 5 10:59:11 2015 +0800
----------------------------------------------------------------------
tools/Makefile | 23 -
tools/bin/README | 205 --
tools/bin/gpconfig | 374 ---
tools/bin/gpexpand | 2931 ---------------------
tools/bin/gpfilespace | 1515 -----------
tools/bin/gpmigrator | 3797 ---------------------------
tools/bin/gprecoverseg | 20 -
tools/bin/gpstart | 853 -------
tools/bin/gpstate | 16 -
tools/bin/gpstop | 674 -----
tools/doc/gpconfig_help | 158 --
tools/doc/gpexpand_help | 196 --
tools/doc/gpfilespace_help | 196 --
tools/doc/gpmigrator_help | 145 --
tools/doc/gpmigrator_mirror_help | 141 -
tools/doc/gprecoverseg_help | 157 --
tools/doc/gpstart_help | 155 --
tools/doc/gpstate_help | 203 --
tools/doc/gpstop_help | 189 --
tools/sbin/gprepairmirrorseg.py | 539 ----
tools/sbin/gpsegstart.py | 555 ----
tools/sbin/gpsegstop.py | 155 --
tools/sbin/gpsegtoprimaryormirror.py | 179 --
tools/sbin/gpstandbystart.sh | 33 -
tools/sbin/gpupgrademirror.py | 3968 -----------------------------
25 files changed, 17377 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9932786b/tools/Makefile
----------------------------------------------------------------------
diff --git a/tools/Makefile b/tools/Makefile
index 8eb25f9..bc2d5bd 100644
--- a/tools/Makefile
+++ b/tools/Makefile
@@ -27,32 +27,15 @@ clean distclean:
#---------------------------------------------------------------------
SET_VERSION_SCRIPTS = \
- bin/gpaddmirrors \
- bin/gpbitmapreindex \
bin/gpcheck \
bin/gpcheckperf \
bin/gpcrondump.py \
- bin/gpdbrestore \
- bin/gpdeletesystem \
- bin/gpexpand \
- bin/gpextract \
- bin/gpfilespace \
bin/gpload.py \
bin/gplogfilter \
- bin/gpmigrator \
- bin/gpmigrator_mirror \
- bin/gpmovemirrors \
- bin/gprebuildsystem \
- bin/gprecoverseg \
bin/hawq \
bin/gpscp \
- bin/gpsizecalc \
- bin/gpskew \
bin/gpssh \
bin/gpssh-exkeys \
- bin/gpstart \
- bin/gpstate \
- bin/gpstop \
bin/gpsys1 \
bin/lib/gpcheckcat \
sbin/gpaddconfig.py \
@@ -61,13 +44,7 @@ SET_VERSION_SCRIPTS = \
sbin/gpcleansegmentdir.py \
sbin/gpfixuserlimts \
sbin/gpgetstatususingtransition.py \
- sbin/gprepairmirrorseg.py \
- sbin/gpsegstart.py \
- sbin/gpsegstop.py \
- sbin/gpsegtoprimaryormirror.py \
sbin/gpsetdbid.py \
- sbin/gpsuspend.py \
- sbin/gpupgrademirror.py \
lib/python/gppylib/programs/clsAddMirrors.py \
lib/python/gppylib/programs/clsHostCacheLookup.py \
lib/python/gppylib/programs/clsInjectFault.py \
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9932786b/tools/bin/README
----------------------------------------------------------------------
diff --git a/tools/bin/README b/tools/bin/README
deleted file mode 100644
index 34430c2..0000000
--- a/tools/bin/README
+++ /dev/null
@@ -1,205 +0,0 @@
-
-
-=-=-=-=-=-=-=-=-=-=-=- PYTHON =-=-=-=-=-=-=-=-=-=-=-=
-
-For Developers
-==============
-
-If you'd like to run the python scripts directly from this
-directory you will need to modify your PYTHONPATH variable to add the "ext"
-directory. so something like:
-
- export PYTHONPATH=~/dev/cdb2/gpMgmt/bin/ext
-
-should do the trick.
-
-
-Python Version
---------------
-
-* The current utilities bundle python 2.5.1. We are planning on moving
- forward to 2.6.1 in the near future.
-
-
-Where Things Go
----------------
-
-* We are trying to eliminate the "lib" directory here. If you are adding a GP
- module please add it to the gppylib dir. If you are adding a 3rd party
- module please add it to the ext dir. Please follow this as removing
- the use of lib/gplib.py and lib/gpmlib.py can be difficult.
-
-
-List of Management Scripts Written in Perl
-------------------------------------------
-bin/gpugpart.pl - Upgrades postgres style partition to Greenplum partitioning
-run_operator_tests.pl - ???
-
-List of Management Scripts Written in Bash
-------------------------------------------
-bin/gpcheckos.bash - Like gpcheckos, but in bash
-bin/gpcrondump - Dumps a database
-bin/gpdbrestore - Restores a database dumped from gpcrondump
-bin/gpinitsystem - Creates a new Greenplum Database
-bin/gpload - Sets env variables and calls gpload.py
-bin/gpprintdbsizes - Removed in 4.0
-bin/gprebuildsystem - Deprecated in 3.4
-
-
-List of Management Scripts Written in Python (no libraries)
------------------------------------------------------------
-bin/gpdetective - Extracts useful information from a Greenplum Database
-bin/gpload.py - Loads data into a Greenplum Database
-bin/gpmigrator - Upgrades from previous versions
-bin/gpsys1 - Print system information on a host (???)
-
-
-List of Management Scripts Written in Python (gpmlib - old libraries)
----------------------------------------------------------------------
-bin/gpaddmirrors - Adds mirrors to an array (needs rewrite)
-bin/gprecoverseg - Recovers a failed segment (needs rewrite)
-bin/gpchecknet - Checks network performance
-bin/gpcheckperf - Checks the hardware for Greenplum Database
-bin/gpscp - Copies files to many hosts
-bin/gpssh - Remote shell to many hosts
-bin/gpssh-exkeys - Exchange ssh keys between many hosts
-
-
-List of Management Scripts Written in Python (gppylib - current libraries)
---------------------------------------------------------------------------
-bin/gpactivatestandby - Activates the Standby Master
-bin/gpaddconfig - Edits postgresql.conf file for all segments
-bin/gpcheckos - Check operating system settings
-bin/gpdeletesystem - Deletes a Greenplum Database
-bin/gpexpand - Adds additional segments to a Greenplum Database
-bin/gpfilespace - Adds a filespace to a Greenplum Database (partial use of libraries)
-bin/gpinitstandby - Initializes standby master
-bin/gplogfilter - Filters log files
-bin/gpstart - Start a Greenplum Database
-bin/gpstop - Stop a Greenplum Database
-
-bin/gpfaultinjector - Cause a host or segment to fail
-
-sbin/gpaddconfig.py - Helper script for gpaddconfig
-sbin/gpfaultinjector.py - Helper script for gpfaultinjector
-sbin/gpsegcopy - Helper script for gpexpand
-sbin/gpsegstart.py - Helper script for gpstart
-sbin/gpsegstop.py - Helper script for gpstop
-sbin/gpsegtoprimaryormirror.py - Helper script for failover
-sbin/gpstandbystart.sh - Removed in 4.0
-sbin/gpage.py - Removed in 4.0 due to query prioritization
-sbin/gpsuspend.py - Unused Helper script for ??? (unused)
-
-
-Overview of gppylib
--------------------
-
-dattimeutils.py - Several utility functions for dealing with date/time data
-
-gparray.py
- |
- +- GpDB - Configuration information for a single dbid
- |
- +- GpSegment - Configuration information for a single content id
- | \- Contains multiple GpDB objects
- |
- +- GpArray - Configuartion information for a Greenplum Database
- \- Contains mulptile GpSegment objects
-
-gpcheckos.py - Old dead code? Not called by gpcheckos.
-stringUtil.py - weird gpcheckos foo ???
-commands/gpcheckosUnix.py - commands for gpcheckos
-gpcheckosCmd.py - Should move to commands/gp.py
-gpcheckosDa.py - ???
-gpcheckosXml.py - Should be modified to be less gpcheckos specific
- |
- +- GpXml - mostly generic XML file wrapper
-
-gphostcache.py
- |
- +- GpHost - Information about a single Host
- | \- Contains all the GpDB objects on the host
- |
- +- GpHostCache - Helps resolve interface->hostname lookups
- \- Contains all the GpHosts referenced in a Greenplum Database
-
-gplog.py - Utility functions to assist in Greenplum standard logging
-
-gpparseopts.py - Wrapper around optparse library to aid in locating help files
-
-gpsubprocess.py - Wrapper around python subprocess (?)
- \- Used by commands/base.py
- - Should move to the commands submodule?
-
-logfilter.py - Contains numerous odd utility functions mostly not specific to logfilter
-
-pgconf.py - Contains helper functions for reading postgresql.conf files
- |
- +- gucdict - dictionary of guc->value pairs
- | \- Contains setting objects
- |
- +- setting - the setting of a single guc and some type coercion funcs
- |
- +- ConfigurationError - subclass of EnvironmentError, raised by type coercion functions
-
-segcopy.py - code for copying a segment from one location to another
- \- should be subclass of command ???
-
-userinput.py - wrapper functions around raw_input
-
-commands/base.py - Core of commands submodule (could use some work)
- |
- +- WorkerPool - Multithreading to execute multiple Command objects
- | \- Spawns multiple Worker objects
- |
- +- Worker - A single thread used to execute Command objects
- |
- +- CommandResult - Packages results of a Command object
- |
- +- ExecutionError - subclass of Exception
- |
- +- ExecutionContext - Abstract class
- | |
- | +- LocalExecutionContext - execute a command locally
- | |
- | +- RemoteExecutionContext - execute a command remotely
- |
- +- Command - abstract class for executing (shell level) commands
- |
- +- SQLCommand - abstract class for executing SQL commands
-
-commands/clsInjectFault - binary for filerep fault injection
- \- doesn't belong as a part of commands submodule
-
-commands/gp.py - Implements lots of subclasses of Command for various tasks
-commands/pg.py - Like gp.py, not clear what the separation is, if any.
-commands/unix.py - Platform information + more subclasses of Command
-commands/san.py - SAN related commands
-commands/test_pg - some tests for commands/pg.py
-
-db/catalog.py - Wrappers for executing certain queries
- \- also contains some goofy wrappers for catalog tables
- - GpConfigurationTable no longer actually looks at gp_configuration... (bad design?)
-db/dbconn.py - Connections to the database
- |
- +- ConnectionError - subclass of a StandardError (unused?)
- |
- +- Pgpass - wrapper for handling a .pgpass file
- |
- +- DbURL - descriptor of how to connect to a database
- |
- +- functions for returning a pygresql.connection object
- |
- +- Should have a wrapper class around a pygresql connection object!
-
-util/san_utils.py - SAN related utility functions that are not Commands
-util/gp_utils.py - Greenplum related utility functions that are not Commands
-util/ssh_session.py - SSH and SCP related utility functions brought in from gpmlib.py/gplib.py
- that are used by gpssh, gpscp and gpssh-exkeys
-
-
-
-
-
-
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/9932786b/tools/bin/gpconfig
----------------------------------------------------------------------
diff --git a/tools/bin/gpconfig b/tools/bin/gpconfig
deleted file mode 100755
index d7115f6..0000000
--- a/tools/bin/gpconfig
+++ /dev/null
@@ -1,374 +0,0 @@
-#!/usr/bin/env python
-#
-# Copyright (c) Greenplum Inc 2009. All Rights Reserved.
-
-import os, sys, re
-
-try:
- from optparse import Option, OptionParser
- from gppylib.gpparseopts import OptParser, OptChecker
- from gppylib.gparray import GpArray
- from gppylib.gphostcache import *
- from gppylib.gplog import *
- from gppylib.commands.unix import *
- from gppylib.commands.gp import *
- from gppylib.db import dbconn
- from gppylib.userinput import *
- from pygresql.pg import DatabaseError
- from gppylib.gpcoverage import GpCoverage
-except ImportError, e:
- sys.exit('Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e))
-
-EXECNAME = os.path.split(__file__)[-1]
-
-prohibitedGucs = set(["port", "listen_addresses"])
-longShow = set(["port"])
-
-def parseargs():
-
- parser = OptParser(option_class=OptChecker)
- parser.remove_option('-h')
- parser.add_option('-h', '-?', '--help', action='help')
- parser.add_option('--verbose', action='store_true')
- parser.add_option('--skipvalidation', action='store_true')
- parser.add_option('--masteronly', action='store_true')
- parser.add_option('--debug', action='store_true')
- parser.add_option('-c', '--change', type='string')
- parser.add_option('-r', '--remove', type='string')
- parser.add_option('-s', '--show', type='string')
- parser.add_option('-v', '--value', type='string')
- parser.add_option('-m', '--mastervalue', type='string')
- parser.add_option('-l', '--list', action='store_true')
- parser.add_option('-P', '--primaryvalue', type='string')
- parser.add_option('-M', '--mirrorvalue', type='string')
- (options, args) = parser.parse_args()
-
- USER=os.getenv('USER')
- if USER is None or USER is ' ':
- logger.error('USER environment variable must be set.')
- parser.exit()
-
- if options.list:
- doList(options.skipvalidation)
- parser.exit()
-
- if options.show:
-
- if options.skipvalidation:
- logger.error('--skipvalidation can not be combined with --show')
- elif options.show in longShow:
- doShow(options.show, longform=True)
- else:
- doShow(options.show, longform=False)
-
- parser.exit()
-
- if options.change and options.remove:
- logger.error("Multiple actions specified. See the --help info.")
- parser.exit()
-
- if options.change:
- options.entry = options.change
- elif options.remove:
- options.entry = options.remove
- options.remove = True
- else:
- logger.error("No action specified. See the --help info.")
- parser.exit()
-
- if options.remove and (options.value or options.primaryvalue or options.mirrorvalue or options.mastervalue):
- logger.error("remove action does not take a value, primary value, mirror value or master value parameter")
- parser.exit()
-
- if options.change and (not options.value and (not options.mirrorvalue and not options.primaryvalue)):
- logger.error("change requested but value not specified")
- parser.exit()
-
- if options.change and options.mastervalue and options.masteronly:
- logger.error("when changing a parameter on the master only specify the --value not --mastervalue")
- parser.exit()
-
- if options.change and (options.value and (options.primaryvalue or options.mirrorvalue)):
- logger.error("cannot use both value option and primaryvalue/mirrorvalue option")
- parser.exit()
-
- if options.value and (not options.mastervalue):
- options.mastervalue = options.value
-
- return options
-
-class JetPackQuery:
- def __init__(self, name):
- self.query = """select -1::int, '%s'::text, current_setting('%s')::text
- union all
- select gp_segment_id::int, name, current_setting(name)
- from gp_dist_random('pg_settings_gpsql') where name = '%s'""" % (name, name, name)
-
-class JetPackGuc:
- def __init__(self,row):
- self.context = row[0]
- self.name = row[1]
- self.value = row[2]
-
- def Print(self):
- print "[context: %s] [name: %s] [value: %s]" % (self.context, self.name, self.value)
-
-class GucQuery:
- def __init__(self, name=None):
- self.query = 'SELECT name, setting, unit, short_desc, context, vartype, min_val, max_val FROM pg_settings'
- if name:
- self.query = self.query + " where name = '" + name + "'"
-
-class Guc:
- def __init__(self,row):
- self.name = row[0]
- self.setting = row[1]
- self.unit = row[2]
- self.short_desc = row[3]
- self.context = row[4]
- self.vartype = row[5]
- self.min_val = row[6]
- self.max_val = row[7]
-
- def validate(self, newval, newmasterval):
- # todo add code here...
- # be careful 128KB in postgresql.conf is translated into 32KB units
-
- if self.name == "max_connections" and (not options.masteronly):
- try:
- s = int(newval)
- m = int(newmasterval)
- if s <= m:
- return "the value of max_connections must be greater on the segments than on the master"
- except:
- return "invalid value for max_connections"
-
- if self.name == 'unix_socket_permissions':
- if newval[0] != '0':
- logger.warn('Permission not entered in octal format.It was interpreted as Decimal. %s in Octal = 0%s' % (newval, int(newval, 8)))
-
- return "ok"
-
- def Print(self):
- print "[name: %s] [unit: %s] [context: %s] [vartype: %s] [min_val: %s] [max_val: %s]" % (self.name, self.unit, self.context, self.vartype, self.min_val, self.max_val)
-
-def userConfirm():
- if not ask_yesno('', "Are you sure you want to ignore unreachable hosts?",'N'):
- logger.info("User Aborted. Exiting...")
- sys.exit(0)
-
-def verbosePrint(options, normalized_hostname, hostname, directory):
- if options.verbose:
- msg = "normalized_host=%s host=%s dir=%s" % (normalized_hostname, hostname, directory)
- logger.info(msg)
-
-def doList(skipvalidation):
- try:
- dburl = dbconn.DbURL()
- conn = dbconn.connect(dburl, True)
- rows = dbconn.execSQL(conn, GucQuery().query)
-
- for row in rows:
- guc = Guc(row)
- if skipvalidation or (guc.name not in prohibitedGucs):
- guc.Print()
-
- conn.close()
-
- except DatabaseError, ex:
- logger.error('Failed to connect to database, this script can only be run when the database is up.')
-
-def doShow(gucname, longform):
- try:
- dburl = dbconn.DbURL()
- conn = dbconn.connect(dburl, False)
- query = JetPackQuery(gucname).query
- rows = dbconn.execSQL(conn, query)
-
- mastervalue = None
- value = None
- valid = True
-
- if longform:
-
- print "GUC : %s" % gucname
- for row in rows:
- guc = JetPackGuc(row)
- print "Context: %5s Value: %s" % (guc.context, guc.value)
-
- else:
- for row in rows:
- guc = JetPackGuc(row)
-
- if guc.context == -1:
- mastervalue = guc.value
-
- elif not value:
- value = guc.value
-
- elif value == guc.value:
- pass
-
- else:
- valid = False
- break
-
- if valid:
- print "Values on all segments are consistent"
- print "GUC : %s" % gucname
- print "Master value: %s" % mastervalue
- print "Segment value: %s" % value
-
- else:
- print "WARNING: GUCS ARE OUT OF SYNC: "
-
- rows = dbconn.execSQL(conn, query)
- for row in rows:
- guc = JetPackGuc(row)
- guc.Print()
-
- conn.close()
-
- except DatabaseError, ex:
-
- if re.search("unrecognized configuration parameter", ex.__str__()):
- logger.error('Failed to retrieve GUC information, guc does not exist: ' + gucname)
- elif re.search("could not connect to server", ex.__str__()):
- logger.error('Failed to retrieve GUC information, the database is not accesible')
- else:
- logger.error('Failed to retrieve GUC information: ' + ex.__str__())
-
-
-def doAddConfigScript(pool, hostname, segs, value):
- directory_string = None
-
- for seg in segs:
-
- if directory_string:
- directory_string = directory_string + "\n" + seg.datadir
- else:
- directory_string = seg.datadir
-
- verbosePrint(options, hostname, seg.hostname, seg.datadir)
-
- cmd = GpAddConfigScript(hostname, directory_string, options.entry, value, options.remove, ctxt=REMOTE, remoteHost=hostname)
- pool.addCommand(cmd)
-
-#------------------------------- Mainline --------------------------------
-
-coverage = GpCoverage()
-coverage.start()
-
-logger = get_default_logger()
-setup_tool_logging(EXECNAME,getLocalHostname(),getUserName())
-
-options = parseargs()
-
-if options.debug:
- enable_verbose_logging()
-
-try:
- dburl = dbconn.DbURL()
-
- gparray = GpArray.initFromCatalog(dburl,utility=True)
-
- if not options.skipvalidation:
-
- conn = dbconn.connect(dburl, True)
-
- rows = dbconn.execSQL(conn, GucQuery(options.entry).query)
-
- guc = None
-
- for row in rows:
- if guc:
- logger.fatal("more than 1 GUC matches: " + options.entry)
- sys.exit(1)
-
- guc = Guc(row)
-
- if not guc:
- logger.fatal("not a valid GUC: " + options.entry)
- sys.exit(1)
-
- conn.close()
-
- if options.entry in prohibitedGucs:
- logger.fatal("The parameter '%s' is not modifiable with this tool" % options.entry)
- sys.exit(1)
-
- if options.value:
- msg = guc.validate(options.value, options.mastervalue)
- if msg != "ok":
- logger.fatal("new GUC value failed validation: " + msg )
- sys.exit(1)
-except DatabaseError, ex:
- logger.error(ex.__str__())
- logger.error('Failed to connect to database, exiting without action. This script can only be run when the database is up.')
- sys.exit(1)
-
-pool = WorkerPool()
-
-hostCache = GpHostCache(gparray, pool)
-failedPings = hostCache.ping_hosts(pool)
-
-if len(failedPings):
- for i in failedPings:
- logger.warning('unreachable host: ' + i.hostname)
- userConfirm()
-
-try:
- # do the segments
- if not options.masteronly:
- if not gparray.allSegmentsAlive():
- logger.error("Can not change guc because some of segments are down")
- sys.exit(2)
-
- for h in hostCache.get_hosts():
- directory_string = None
-
- if options.primaryvalue:
- doAddConfigScript(pool, h.hostname, [seg for seg in h.dbs if seg.isSegmentPrimary()], options.primaryvalue)
-
- if options.mirrorvalue:
- doAddConfigScript(pool, h.hostname, [seg for seg in h.dbs if seg.isSegmentMirror()], options.mirrorvalue)
-
- if not options.primaryvalue and not options.mirrorvalue:
- doAddConfigScript(pool, h.hostname, h.dbs, options.value)
-
- # do the master
- if options.mastervalue or options.remove:
- verbosePrint(options, gparray.master.hostname, gparray.master.hostname, gparray.master.datadir)
- cmd = GpAddConfigScript("master", gparray.master.datadir, options.entry, options.mastervalue, options.remove, ctxt=REMOTE, remoteHost=gparray.master.hostname)
- pool.addCommand(cmd)
-
- # do the standby master
- if gparray.standbyMaster:
- verbosePrint(options, gparray.standbyMaster.hostname, gparray.standbyMaster.hostname, gparray.standbyMaster.datadir)
- cmd = GpAddConfigScript("standbymaster", gparray.standbyMaster.datadir, options.entry, options.mastervalue, options.remove, ctxt=REMOTE, remoteHost=gparray.standbyMaster.hostname)
- pool.addCommand(cmd)
-
- pool.join()
- items = pool.getCompletedItems()
- failure = False
- for i in items:
- if not i.was_successful():
- logger.error('failed updating the postgresql.conf files on host: ' + i.remoteHost)
- failure = True
-
- pool.check_results()
-except Exception, e:
- logger.error('errors in job:')
- logger.error(e.__str__())
- logger.error('exiting early')
-
-pool.haltWork()
-pool.joinWorkers()
-
-if failure:
- logger.error('finished with errors')
-else:
- logger.info("completed successfully")
-
-coverage.stop()
-coverage.generate_report()