You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by od...@apache.org on 2016/05/03 02:33:17 UTC
[01/13] incubator-hawq git commit: HAWQ-687. The segment resource
increase pending counter is not adjusted correctly when failing to increase
resource quota
Repository: incubator-hawq
Updated Branches:
refs/heads/HAWQ-703 6717e7d66 -> b1db3991c
HAWQ-687. The segment resource increase pending counter is not adjusted correctly when failing to increase resource quota
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1469782e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1469782e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1469782e
Branch: refs/heads/HAWQ-703
Commit: 1469782edd0552e1635952a9677ac38241665257
Parents: 649828f
Author: YI JIN <yj...@pivotal.io>
Authored: Tue Apr 19 13:51:36 2016 +1000
Committer: YI JIN <yj...@pivotal.io>
Committed: Tue Apr 19 13:51:36 2016 +1000
----------------------------------------------------------------------
.../resourcemanager/communication/rmcomm_RM2RMSEG.c | 13 ++++++++-----
1 file changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1469782e/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
----------------------------------------------------------------------
diff --git a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
index a726723..a79bd6c 100644
--- a/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
+++ b/src/backend/resourcemanager/communication/rmcomm_RM2RMSEG.c
@@ -627,6 +627,11 @@ void processContainersAfterIncreaseMemoryQuota(GRMContainerSet ctns, bool accept
PRESPOOL->AddPendingContainerCount--;
elog(LOG, "AddPendingContainerCount minus 1, current value %d",
PRESPOOL->AddPendingContainerCount);
+
+ /* This container can not generate additional increase pending */
+ minusResourceBundleData(&(ctn->Resource->IncPending),
+ ctn->MemoryMB,
+ ctn->Core);
/*
* Add container to ToKickContainers if lifetime is not too long.
* If the resource manager is not in clean up phase, directly drop
@@ -635,16 +640,14 @@ void processContainersAfterIncreaseMemoryQuota(GRMContainerSet ctns, bool accept
if( !isCleanGRMResourceStatus() &&
ctn->Life < RESOURCE_CONTAINER_MAX_LIFETIME )
{
- /* This container can not generate additional increase pending */
- minusResourceBundleData(&(ctn->Resource->IncPending),
- ctn->MemoryMB,
- ctn->Core);
addGRMContainerToToBeAccepted(ctn);
}
/* Add container to KickedContainers if lifetime is long enough */
else
{
- removePendingResourceRequestInRootQueue(ctn->MemoryMB, ctn->Core, false);
+ removePendingResourceRequestInRootQueue(ctn->MemoryMB,
+ ctn->Core,
+ false);
addGRMContainerToKicked(ctn);
}
}
[05/13] incubator-hawq git commit: HAWQ-688. Fix infinite loop when
hawq fail to register itself to YARN
Posted by od...@apache.org.
HAWQ-688. Fix infinite loop when hawq fail to register itself to YARN
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/59f54323
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/59f54323
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/59f54323
Branch: refs/heads/HAWQ-703
Commit: 59f5432373aaa58f32fb6d9299eab62744879ac5
Parents: 76658bc
Author: Wen Lin <wl...@pivotal.io>
Authored: Wed Apr 20 14:35:44 2016 +0800
Committer: Wen Lin <wl...@pivotal.io>
Committed: Wed Apr 20 14:35:44 2016 +0800
----------------------------------------------------------------------
depends/libyarn/src/CMakeLists.txt | 2 +-
depends/libyarn/src/libyarnclient/LibYarnClient.cpp | 6 +++++-
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/59f54323/depends/libyarn/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/depends/libyarn/src/CMakeLists.txt b/depends/libyarn/src/CMakeLists.txt
index c2a3cff..cafe999 100644
--- a/depends/libyarn/src/CMakeLists.txt
+++ b/depends/libyarn/src/CMakeLists.txt
@@ -2,7 +2,7 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
SET(libyarn_VERSION_MAJOR 0)
SET(libyarn_VERSION_MINOR 1)
-SET(libyarn_VERSION_PATCH 16)
+SET(libyarn_VERSION_PATCH 17)
SET(libyarn_VERSION_STRING "${libyarn_VERSION_MAJOR}.${libyarn_VERSION_MINOR}.${libyarn_VERSION_PATCH}")
SET(libyarn_VERSION_API 1)
SET(libyarn_ROOT_SOURCES_DIR ${CMAKE_SOURCE_DIR}/src)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/59f54323/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
----------------------------------------------------------------------
diff --git a/depends/libyarn/src/libyarnclient/LibYarnClient.cpp b/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
index 3c76f92..0cf90fa 100644
--- a/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
+++ b/depends/libyarn/src/libyarnclient/LibYarnClient.cpp
@@ -184,15 +184,19 @@ int LibYarnClient::createJob(string &jobName, string &queue,string &jobId) {
//3. wait util AM is ACCEPTED and return the AMRMToken
ApplicationReport report;
- while (true) {
+ int retry = 10;
+ while (retry > 0) {
report = applicationClient->getApplicationReport(appId);
LOG(INFO,"LibYarnClient::createJob, appId[cluster_timestamp:%lld,id:%d], appState:%d",
appId.getClusterTimestamp(), appId.getId(), report.getYarnApplicationState());
if ((report.getAMRMToken().getPassword() != "") && report.getYarnApplicationState() == YarnApplicationState::ACCEPTED) {
break;
} else {
+ retry--;
usleep(1000000L);
}
+ if (retry == 0)
+ THROW(AccessControlException, "can not register application to YARN");
}
clientAppId = appId;
[03/13] incubator-hawq git commit: HAWQ-668. Fix hawq check run
without HADOOP_HOME
Posted by od...@apache.org.
HAWQ-668. Fix hawq check run without HADOOP_HOME
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/95197ad7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/95197ad7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/95197ad7
Branch: refs/heads/HAWQ-703
Commit: 95197ad7edd64fba5ad210c0944c4fc99ba90438
Parents: e74109b
Author: rlei <rl...@pivotal.io>
Authored: Wed Apr 20 00:08:05 2016 +0800
Committer: Ruilong Huo <rh...@pivotal.io>
Committed: Wed Apr 20 10:54:15 2016 +0800
----------------------------------------------------------------------
tools/bin/gpcheck | 13 ++++++++++---
tools/doc/gpcheck_help | 44 ++++++++++++++++++++++++++++++++++++++++++--
2 files changed, 52 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/95197ad7/tools/bin/gpcheck
----------------------------------------------------------------------
diff --git a/tools/bin/gpcheck b/tools/bin/gpcheck
index 1d0019c..6099867 100755
--- a/tools/bin/gpcheck
+++ b/tools/bin/gpcheck
@@ -736,6 +736,9 @@ def testHAWQconfig(host):
if hawq is None:
return # skip HAWQ test when hawq is None
+ if hdfs is None:
+ return # skip HAWQ test when hdfs is None
+
if options.verbose:
logger.info("-- test HAWQ config")
@@ -743,7 +746,9 @@ def testHAWQconfig(host):
checkFailed(host.hostname, "collect HAWQ configuration error: %s" % hawq.errormsg)
return
- datanode_list = parse_host_list_file("%s/etc/hadoop/slaves" % HADOOP_HOME)
+ datanode_list = list()
+ if HADOOP_HOME:
+ datanode_list = parse_host_list_file("%s/etc/hadoop/slaves" % HADOOP_HOME)
is_datanode = False
if host.hostname in datanode_list:
is_datanode = True
@@ -819,7 +824,7 @@ def testHAWQconfig(host):
else:
checkFailed(host.hostname, "HAWQ configuration dfs.domain.socket.path: %s, does not exist on %s" % (actual_config['dfs.domain.socket.path'], host.hostname))
- if 'output.replace-datanode-on-failure' in actual_config:
+ if 'output.replace-datanode-on-failure' in actual_config and len(datanode_list) > 0:
if len(datanode_list) < 4:
if actual_config['output.replace-datanode-on-failure'] == 'true':
checkFailed(host.hostname, "HAWQ configuration: output.replace-datanode-on-failure expect false, current is true")
@@ -926,7 +931,9 @@ def testHDFSConfig(host):
# Check if nodemanager direcotries exists
directory_check_list = []
- datanode_list = parse_host_list_file("%s/etc/hadoop/slaves" % HADOOP_HOME)
+ datanode_list = list()
+ if HADOOP_HOME:
+ datanode_list = parse_host_list_file("%s/etc/hadoop/slaves" % HADOOP_HOME)
is_datanode = False
if host.hostname in datanode_list:
is_datanode = True
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/95197ad7/tools/doc/gpcheck_help
----------------------------------------------------------------------
diff --git a/tools/doc/gpcheck_help b/tools/doc/gpcheck_help
index f57765e..587f32f 100644
--- a/tools/doc/gpcheck_help
+++ b/tools/doc/gpcheck_help
@@ -6,7 +6,8 @@ Verifies and validates HAWQ platform settings.
SYNOPSIS
*****************************************************
-hawq check -f <hostfile_hawq_check> [--hadoop <hadoop_home>]
+hawq check -f <hostfile_hawq_check>
+ [--hadoop | --hadoop-home <hadoop_home>]
[--stdout | --zipout] [--config <config_file>]
hawq check --zipin <hawq_check_zipfile>
@@ -68,13 +69,38 @@ OPTIONS
contain a single host name for all hosts in your HAWQ system
(master, standby master, and segments).
---hadoop hadoop_home
+--hadoop/--hadoop-home hadoop_home
Use this option to specify your hadoop installation location so that
hawq check can validate HDFS settings. This option is not needed when
HADOOP_HOME environment variable is set.
+--kerberos
+
+ Use this option to specify HDFS/YARN enables Kerberos mode, so
+ that hawq check can validate HDFS/YARN settings with Kerberos
+ enabled.
+
+
+--hdfs-ha
+
+ Use this option to specify HDFS HA mode enabled, so that hawq check
+ can validate HDFS settings with HA enabled.
+
+
+--yarn
+
+ Use this option to specify HAWQ enables YARN mode, so that hawq check
+ can validate HAWQ/YARN settings.
+
+
+--yarn-ha
+
+ Use this option to specify HAWQ enables YARN HA mode, so that
+ hawq check can validate HAWQ/YARN settings with YARN-HA enabled.
+
+
--stdout
Display collected host information from hawq check. No checks or
@@ -115,6 +141,20 @@ entering a host file and specifying the hadoop location:
# hawq check -f hostfile_hawq_check --hadoop ~/hadoop-2.0.0/
+Verify and validate the HAWQ platform settings with HDFS HA
+enabled, YARN HA enabled and Kerberos enabled:
+
+ # hawq check -f hostfile_hawq_check --hadoop ~/hadoop-2.0.0/
+ --hdfs-ha --yarn-ha --kerberos
+
+
+Verify and validate the HAWQ platform settings with HDFS HA
+enabled, and Kerberos enabled:
+
+ # hawq check -f hostfile_hawq_check --hadoop ~/hadoop-2.0.0/
+ --hdfs-ha --kerberos
+
+
Save HAWQ platform settings to a zip file, when HADOOP_HOME
environment variable is set:
[12/13] incubator-hawq git commit: Revert "HAWQ-711. Integrate
libhdfs3 and libyarn makefile into hawq"
Posted by od...@apache.org.
Revert "HAWQ-711. Integrate libhdfs3 and libyarn makefile into hawq"
This reverts commit 1b0e6e2d728b5f4f9da09ecc19dc32c87a9fdc81.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b357dbcd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b357dbcd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b357dbcd
Branch: refs/heads/HAWQ-703
Commit: b357dbcd9417c120d7bedda020771dd3d941d519
Parents: 6209c42
Author: Lei Chang <ch...@gmail.com>
Authored: Thu Apr 28 17:59:48 2016 +0800
Committer: Lei Chang <ch...@gmail.com>
Committed: Thu Apr 28 17:59:48 2016 +0800
----------------------------------------------------------------------
GNUmakefile.in | 8 ----
configure | 66 ++++++++++++++++++++++++--------
configure.in | 5 +--
depends/libhdfs3/.gitignore | 2 -
depends/libhdfs3/Makefile | 33 ----------------
depends/libhdfs3/Makefile.global.in | 21 ----------
depends/libyarn/.gitignore | 2 -
depends/libyarn/Makefile | 33 ----------------
depends/libyarn/Makefile.global.in | 21 ----------
9 files changed, 51 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/GNUmakefile.in
----------------------------------------------------------------------
diff --git a/GNUmakefile.in b/GNUmakefile.in
index 5523967..f20ee2c 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -10,8 +10,6 @@ include $(top_builddir)/src/Makefile.global
all:
# $(MAKE) -C doc all
- $(MAKE) -C depends/libhdfs3 all
- $(MAKE) -C depends/libyarn all
$(MAKE) -C src all
$(MAKE) -C config all
$(MAKE) -C contrib/formatter_fixedwidth all
@@ -24,8 +22,6 @@ all:
install:
# $(MAKE) -C doc $@
- $(MAKE) -C depends/libhdfs3 $@
- $(MAKE) -C depends/libyarn $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
@@ -38,8 +34,6 @@ install:
installdirs uninstall:
# $(MAKE) -C doc $@
- $(MAKE) -C depends/libhdfs3 $@
- $(MAKE) -C depends/libyarn $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
@@ -59,8 +53,6 @@ distprep:
clean:
# $(MAKE) -C doc $@
$(MAKE) -C contrib $@
- $(MAKE) -C depends/libhdfs3 $@
- $(MAKE) -C depends/libyarn $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 5e65a48..76842d4 100755
--- a/configure
+++ b/configure
@@ -2,7 +2,7 @@
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for HAWQ 8.2.15.
#
-# Report bugs to <https://issues.apache.org/jira/browse/HAWQ>.
+# Report bugs to https://issues.apache.org/jira/browse/HAWQ
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -269,10 +269,11 @@ fi
$as_echo "$0: be upgraded to zsh 4.3.4 or later."
else
$as_echo "$0: Please tell bug-autoconf@gnu.org and
-$0: https://issues.apache.org/jira/browse/HAWQ about your
-$0: system, including any error possibly output before this
-$0: message. Then install a modern shell, or manually run
-$0: the script under such a shell if you do have one."
+$0: https://issues.apache.org/jira/browse/HAWQ
+$0: about your system, including any
+$0: error possibly output before this message. Then install
+$0: a modern shell, or manually run the script under such a
+$0: shell if you do have one."
fi
exit 1
fi
@@ -840,6 +841,7 @@ enable_cassert
enable_forceplan
enable_debugbreak
enable_debugntuplestore
+enable_mdverinstrumentation
enable_testutils
enable_snmp
enable_gphdfs
@@ -1514,6 +1516,7 @@ build with coverage testing instrumentation
--enable-forceplan enable forcing of plans via EXPLAIN FORCE (for debugging)
--enable-debugbreak enable debug_break and debug_break_n (for debugging)
--enable-ntuplestore enable debug_ntuplestore (for debugging)
+ --enable-mdverinstrumentation enable metadata versioning instrumentation
--enable-testutils enable testing utilities
--enable-snmp enable snmp for MIB and alerts via TRAP/INFORM
--enable-gphdfs enables GPHDFS support
@@ -1576,7 +1579,7 @@ Some influential environment variables:
Use these variables to override the choices made by `configure' or to help
it to find libraries and programs with nonstandard names/locations.
-Report bugs to <https://issues.apache.org/jira/browse/HAWQ>.
+Report bugs to https://issues.apache.org/jira/browse/HAWQ
_ACEOF
ac_status=$?
fi
@@ -1846,9 +1849,9 @@ $as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;}
$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;}
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5
$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;}
-( $as_echo "## --------------------------------------------------------- ##
+( $as_echo "## ------------------------------------ ##
## Report this to https://issues.apache.org/jira/browse/HAWQ ##
-## --------------------------------------------------------- ##"
+## ------------------------------------ ##"
) | sed "s/^/$as_me: WARNING: /" >&2
;;
esac
@@ -2085,9 +2088,9 @@ $as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;}
$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;}
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5
$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;}
-( $as_echo "## --------------------------------------------------------- ##
+( $as_echo "## --------------------------------------------------- ##
## Report this to https://issues.apache.org/jira/browse/HAWQ ##
-## --------------------------------------------------------- ##"
+## -------------------------------------------------- ##"
) | sed "s/^/$as_me: WARNING: /" >&2
;;
esac
@@ -5333,6 +5336,34 @@ fi
+# Enable metadata versioning instrumentation
+
+pgac_args="$pgac_args enable_mdverinstrumentation"
+
+# Check whether --enable-mdverinstrumentation was given.
+if test "${enable_mdverinstrumentation+set}" = set; then :
+ enableval=$enable_mdverinstrumentation;
+ case $enableval in
+ yes)
+
+$as_echo "#define MD_VERSIONING_INSTRUMENTATION 1" >>confdefs.h
+
+ ;;
+ no)
+ :
+ ;;
+ *)
+ as_fn_error $? "no argument expected for --enable-mdverinstrumentation option" "$LINENO" 5
+ ;;
+ esac
+
+else
+ enable_mdverinstrumentation=no
+
+fi
+
+
+
#
# Enable testing utilities
#
@@ -5501,6 +5532,9 @@ fi
+
+
+
#
# Include directories
#
@@ -11091,10 +11125,10 @@ fi
for ac_header in json-c/json.h
do :
- ac_fn_c_check_header_mongrel "$LINENO" "json-c/json.h" "ac_cv_header_json_c_json_h" "$ac_includes_default"
-if test "x$ac_cv_header_json_c_json_h" = xyes; then :
+ ac_fn_c_check_header_mongrel "$LINENO" "json-c/json.h" "ac_cv_header_json_json_h" "$ac_includes_default"
+if test "x$ac_cv_header_json_json_h" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_JSON_C_JSON_H 1
+#define HAVE_JSON_JSON_H 1
_ACEOF
else
@@ -16506,7 +16540,7 @@ fi
-ac_config_files="$ac_config_files GNUmakefile src/VERSIONS.mk depends/libhdfs3/Makefile.global depends/libyarn/Makefile.global src/Makefile.global src/pl/pljava/src/java/Makefile.global"
+ac_config_files="$ac_config_files GNUmakefile src/VERSIONS.mk src/Makefile.global src/pl/pljava/src/java/Makefile.global"
ac_config_links="$ac_config_links src/backend/port/dynloader.c:src/backend/port/dynloader/${template}.c src/backend/port/pg_sema.c:${SEMA_IMPLEMENTATION} src/backend/port/pg_shmem.c:${SHMEM_IMPLEMENTATION} src/include/dynloader.h:src/backend/port/dynloader/${template}.h src/include/pg_config_os.h:src/include/port/${template}.h src/Makefile.port:src/makefiles/Makefile.${template}"
@@ -17094,7 +17128,7 @@ $config_links
Configuration commands:
$config_commands
-Report bugs to <https://issues.apache.org/jira/browse/HAWQ>."
+Report bugs to https://issues.apache.org/jira/browse/HAWQ"
_ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
@@ -17226,8 +17260,6 @@ do
"src/backend/port/tas.s") CONFIG_LINKS="$CONFIG_LINKS src/backend/port/tas.s:src/backend/port/tas/${tas_file}" ;;
"GNUmakefile") CONFIG_FILES="$CONFIG_FILES GNUmakefile" ;;
"src/VERSIONS.mk") CONFIG_FILES="$CONFIG_FILES src/VERSIONS.mk" ;;
- "depends/libhdfs3/Makefile.global") CONFIG_FILES="$CONFIG_FILES depends/libhdfs3/Makefile.global" ;;
- "depends/libyarn/Makefile.global") CONFIG_FILES="$CONFIG_FILES depends/libyarn/Makefile.global" ;;
"src/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/Makefile.global" ;;
"src/pl/pljava/src/java/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/pl/pljava/src/java/Makefile.global" ;;
"src/backend/port/dynloader.c") CONFIG_LINKS="$CONFIG_LINKS src/backend/port/dynloader.c:src/backend/port/dynloader/${template}.c" ;;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index f1fdc02..a5b322b 100644
--- a/configure.in
+++ b/configure.in
@@ -17,7 +17,7 @@ dnl Read the Autoconf manual for details.
dnl
m4_pattern_forbid(^PGAC_)dnl to catch undefined macros
-AC_INIT([HAWQ], [8.2.15], [https://issues.apache.org/jira/browse/HAWQ])
+AC_INIT([HAWQ], [8.2.15], [support@greenplum.com])
AC_PREREQ(2.63)
dnl m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.63], [], [m4_fatal([Autoconf version 2.63 is required.
@@ -310,7 +310,6 @@ elif test "$GCC" = yes; then
CFLAGS="-O0"
else
CFLAGS="-O3"
- fi
else
# if the user selected debug mode, don't use -O
if test "$enable_debug" != yes; then
@@ -2056,7 +2055,7 @@ fi
AC_SUBST(vpath_build)
-AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk depends/libhdfs3/Makefile.global depends/libyarn/Makefile.global src/Makefile.global src/pl/pljava/src/java/Makefile.global])
+AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk src/Makefile.global src/pl/pljava/src/java/Makefile.global])
AC_CONFIG_LINKS([
src/backend/port/dynloader.c:src/backend/port/dynloader/${template}.c
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/depends/libhdfs3/.gitignore
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/.gitignore b/depends/libhdfs3/.gitignore
deleted file mode 100644
index 8341b2e..0000000
--- a/depends/libhdfs3/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-build/
-Makefile.global
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/depends/libhdfs3/Makefile
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/Makefile b/depends/libhdfs3/Makefile
deleted file mode 100644
index e509659..0000000
--- a/depends/libhdfs3/Makefile
+++ /dev/null
@@ -1,33 +0,0 @@
-# -*-makefile-*-
-#------------------------------------------------------------------------------
-# A makefile that integrate building this module with hawq
-#------------------------------------------------------------------------------
-subdir = depends/libhdfs3
-top_builddir = ../../
-include Makefile.global
-
-PRE_CFG_ARG =
-# get argument for running ../boostrap
-ifeq ($(enable_debug), yes)
- PRE_CFG_ARG += --enable-debug
-endif # enable_debug
-
-ifeq ($(enable_coverage), yes)
- PRE_CFG_ARG += --enable-coverage
-endif # enable_coverage
-
-##########################################################################
-#
-.PHONY: all install make pre-config
-
-all : make
-
-install: make
- cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8 install
-
-make: pre-config
- cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8
-
-pre-config:
- cd $(top_srcdir)/$(subdir)/ && mkdir -p build && cd build && ../bootstrap --prefix=$(prefix) $(PRE_CFG_ARG)
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/depends/libhdfs3/Makefile.global.in
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/Makefile.global.in b/depends/libhdfs3/Makefile.global.in
deleted file mode 100644
index 12dd05c..0000000
--- a/depends/libhdfs3/Makefile.global.in
+++ /dev/null
@@ -1,21 +0,0 @@
-# -*-makefile-*-
-#------------------------------------------------------------------------------
-# A makefile that integrate building this module with hawq
-#------------------------------------------------------------------------------
-
-prefix := @prefix@
-enable_debug = @enable_debug@
-enable_coverage = @enable_coverage@
-
-# Support for VPATH builds
-vpath_build = @vpath_build@
-abs_top_srcdir = @abs_top_srcdir@
-
-ifneq ($(vpath_build),yes)
-top_srcdir = $(top_builddir)
-srcdir = .
-else # vpath_build = yes
-top_srcdir = $(abs_top_srcdir)
-srcdir = $(top_srcdir)/$(subdir)
-VPATH = $(srcdir)
-endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/depends/libyarn/.gitignore
----------------------------------------------------------------------
diff --git a/depends/libyarn/.gitignore b/depends/libyarn/.gitignore
deleted file mode 100644
index 8341b2e..0000000
--- a/depends/libyarn/.gitignore
+++ /dev/null
@@ -1,2 +0,0 @@
-build/
-Makefile.global
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/depends/libyarn/Makefile
----------------------------------------------------------------------
diff --git a/depends/libyarn/Makefile b/depends/libyarn/Makefile
deleted file mode 100644
index 3e17622..0000000
--- a/depends/libyarn/Makefile
+++ /dev/null
@@ -1,33 +0,0 @@
-# -*-makefile-*-
-#------------------------------------------------------------------------------
-# A makefile that integrate building this module with hawq
-#------------------------------------------------------------------------------
-subdir = depends/libyarn
-top_builddir = ../../
-include Makefile.global
-
-PRE_CFG_ARG =
-# get argument for running ../boostrap
-ifeq ($(enable_debug), yes)
- PRE_CFG_ARG += --enable-debug
-endif # enable_debug
-
-ifeq ($(enable_coverage), yes)
- PRE_CFG_ARG += --enable-coverage
-endif # enable_coverage
-
-##########################################################################
-#
-.PHONY: all install make pre-config
-
-all : make
-
-install: make
- cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8 install
-
-make: pre-config
- cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8
-
-pre-config:
- cd $(top_srcdir)/$(subdir)/ && mkdir -p build && cd build && ../bootstrap --prefix=$(prefix) $(PRE_CFG_ARG)
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b357dbcd/depends/libyarn/Makefile.global.in
----------------------------------------------------------------------
diff --git a/depends/libyarn/Makefile.global.in b/depends/libyarn/Makefile.global.in
deleted file mode 100644
index 12dd05c..0000000
--- a/depends/libyarn/Makefile.global.in
+++ /dev/null
@@ -1,21 +0,0 @@
-# -*-makefile-*-
-#------------------------------------------------------------------------------
-# A makefile that integrate building this module with hawq
-#------------------------------------------------------------------------------
-
-prefix := @prefix@
-enable_debug = @enable_debug@
-enable_coverage = @enable_coverage@
-
-# Support for VPATH builds
-vpath_build = @vpath_build@
-abs_top_srcdir = @abs_top_srcdir@
-
-ifneq ($(vpath_build),yes)
-top_srcdir = $(top_builddir)
-srcdir = .
-else # vpath_build = yes
-top_srcdir = $(abs_top_srcdir)
-srcdir = $(top_srcdir)/$(subdir)
-VPATH = $(srcdir)
-endif
[10/13] incubator-hawq git commit: Fix Orca error message: Orca
supports Rollup #117665973
Posted by od...@apache.org.
Fix Orca error message: Orca supports Rollup #117665973
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/3be28ec5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/3be28ec5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/3be28ec5
Branch: refs/heads/HAWQ-703
Commit: 3be28ec5f537a619d70de0a7cbca4161f7cbaebb
Parents: 1b0e6e2
Author: Venkatesh (Venky) Raghavan <vr...@pivotal.io>
Authored: Tue Apr 26 11:18:08 2016 -0700
Committer: Ming LI <ml...@pivotal.io>
Committed: Wed Apr 27 18:28:12 2016 +0800
----------------------------------------------------------------------
src/backend/gpopt/translate/CTranslatorUtils.cpp | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/3be28ec5/src/backend/gpopt/translate/CTranslatorUtils.cpp
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/translate/CTranslatorUtils.cpp b/src/backend/gpopt/translate/CTranslatorUtils.cpp
index 4c47455..2b12f1b 100644
--- a/src/backend/gpopt/translate/CTranslatorUtils.cpp
+++ b/src/backend/gpopt/translate/CTranslatorUtils.cpp
@@ -1571,7 +1571,9 @@ CTranslatorUtils::PdrgpbsGroupBy
}
// grouping sets
- if (1 != gpdb::UlListLength(plGroupClause))
+ const ULONG ulGroupClause = gpdb::UlListLength(plGroupClause);
+ GPOS_ASSERT(0 < ulGroupClause);
+ if (1 < ulGroupClause)
{
GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT("Multiple grouping sets specifications"));
}
@@ -1583,9 +1585,9 @@ CTranslatorUtils::PdrgpbsGroupBy
return PdrgpbsRollup(pmp, pgrcl, ulCols, phmululGrpColPos, pbsGrpCols);
}
- if (GROUPINGTYPE_GROUPING_SETS != pgrcl->groupType)
+ if (GROUPINGTYPE_CUBE == pgrcl->groupType)
{
- GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT("Rollup and cube"));
+ GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT(\u201cCube\u201d));
}
DrgPbs *pdrgpbs = GPOS_NEW(pmp) DrgPbs(pmp);
@@ -1606,7 +1608,7 @@ CTranslatorUtils::PdrgpbsGroupBy
}
else if (IsA(pnodeGroupingSet, GroupingClause))
{
- GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT("Rollup and cube"));
+ GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT("Multiple grouping sets specifications"));
}
else
{
[13/13] incubator-hawq git commit: HAWQ-703. Serialize HCatalog
Complex Types to plain text (as Hive profile).
Posted by od...@apache.org.
HAWQ-703. Serialize HCatalog Complex Types to plain text (as Hive profile).
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/b1db3991
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/b1db3991
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/b1db3991
Branch: refs/heads/HAWQ-703
Commit: b1db3991cc9be1536a17b93953d81f29c8dfa323
Parents: 6717e7d b357dbc
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Mon May 2 13:00:44 2016 -0700
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Mon May 2 17:33:09 2016 -0700
----------------------------------------------------------------------
depends/libyarn/src/CMakeLists.txt | 2 +-
.../libyarn/src/libyarnclient/LibYarnClient.cpp | 6 +-
pxf/Makefile | 5 +-
pxf/build.gradle | 5 +
.../org/apache/hawq/pxf/api/package-info.java | 23 ++
.../hawq/pxf/api/utilities/Utilities.java | 11 +-
.../hawq/pxf/plugins/hbase/package-info.java | 23 ++
.../hawq/pxf/plugins/hdfs/AvroResolver.java | 4 +-
.../hawq/pxf/plugins/hdfs/package-info.java | 23 ++
.../hawq/pxf/plugins/hive/package-info.java | 23 ++
.../hive/utilities/EnumHiveToHawqType.java | 2 +-
.../hive/utilities/HiveUtilitiesTest.java | 8 +-
.../pxf/service/MetadataResponseFormatter.java | 1 +
.../apache/hawq/pxf/service/package-info.java | 23 ++
.../pxf/service/utilities/ProtocolData.java | 2 +-
src/backend/executor/execMain.c | 2 +-
.../gpopt/translate/CTranslatorUtils.cpp | 10 +-
.../communication/rmcomm_RM2RMSEG.c | 13 +-
src/backend/utils/misc/etc/gpcheck.cnf | 48 ++-
src/bin/psql/describe.c | 2 +
tools/bin/gpcheck | 372 ++++++++++++++++++-
tools/bin/gppylib/gpcheckutil.py | 17 +-
tools/bin/hawqpylib/hawqlib.py | 18 +
tools/doc/gpcheck_help | 44 ++-
tools/sbin/gpcheck_hostdump | 50 ++-
25 files changed, 684 insertions(+), 53 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
----------------------------------------------------------------------
diff --cc pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
index 13d402c,0000000..a9ff877
mode 100644,000000..100644
--- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/utilities/EnumHiveToHawqType.java
@@@ -1,112 -1,0 +1,112 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hawq.pxf.plugins.hive.utilities;
+
+import org.apache.hawq.pxf.api.utilities.EnumHawqType;
+import org.apache.hawq.pxf.api.UnsupportedTypeException;
+
+/**
+ *
+ * Hive types, which are supported by plugin, mapped to HAWQ's types
+ * @see EnumHawqType
+ */
+public enum EnumHiveToHawqType {
+
+ TinyintType("tinyint", EnumHawqType.Int2Type),
+ SmallintType("smallint", EnumHawqType.Int2Type),
+ IntType("int", EnumHawqType.Int4Type),
+ BigintType("bigint", EnumHawqType.Int8Type),
+ BooleanType("boolean", EnumHawqType.BoolType),
+ FloatType("float", EnumHawqType.Float4Type),
+ DoubleType("double", EnumHawqType.Float8Type),
+ StringType("string", EnumHawqType.TextType),
+ BinaryType("binary", EnumHawqType.ByteaType),
+ TimestampType("timestamp", EnumHawqType.TimestampType),
+ DateType("date", EnumHawqType.DateType),
+ DecimalType("decimal", EnumHawqType.NumericType, "[(,)]"),
+ VarcharType("varchar", EnumHawqType.VarcharType, "[(,)]"),
+ CharType("char", EnumHawqType.BpcharType, "[(,)]"),
+ ArrayType("array", EnumHawqType.TextType, "[<,>]"),
+ MapType("map", EnumHawqType.TextType, "[<,>]"),
+ StructType("struct", EnumHawqType.TextType, "[<,>]"),
+ UnionType("uniontype", EnumHawqType.TextType, "[<,>]");
+
+ private String typeName;
+ private EnumHawqType hawqType;
+ private String splitExpression;
+
+ EnumHiveToHawqType(String typeName, EnumHawqType hawqType) {
+ this.typeName = typeName;
+ this.hawqType = hawqType;
+ }
+
+ EnumHiveToHawqType(String typeName, EnumHawqType hawqType, String splitExpression) {
+ this(typeName, hawqType);
+ this.splitExpression = splitExpression;
+ }
+
+ /**
+ *
+ * @return name of type
+ */
+ public String getTypeName() {
+ return this.typeName;
+ }
+
+ /**
+ *
+ * @return corresponding HAWQ type
+ */
+ public EnumHawqType getHawqType() {
+ return this.hawqType;
+ }
+
+ /**
+ *
+ * @return split by expression
+ */
+ public String getSplitExpression() {
+ return this.splitExpression;
+ }
+
+ /**
+ * Returns Hive to HAWQ type mapping entry for given Hive type
+ *
+ * @param hiveType full Hive type with modifiers, for example - decimal(10, 0), char(5), binary, array<string>, map<string,float> etc
+ * @return corresponding Hive to HAWQ type mapping entry
+ * @throws UnsupportedTypeException if there is no corresponding HAWQ type
+ */
+ public static EnumHiveToHawqType getHiveToHawqType(String hiveType) {
+ for (EnumHiveToHawqType t : values()) {
+ String hiveTypeName = hiveType;
+ if (t.getSplitExpression() != null) {
+ String[] tokens = hiveType.split(t.getSplitExpression());
+ hiveTypeName = tokens[0];
+ }
+
- if (t.getTypeName().equals(hiveTypeName)) {
++ if (t.getTypeName().toLowerCase().equals(hiveTypeName.toLowerCase())) {
+ return t;
+ }
+ }
+ throw new UnsupportedTypeException("Unable to map Hive's type: "
+ + hiveType + " to HAWQ's type");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
----------------------------------------------------------------------
diff --cc pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
index 1054d0d,466dedb..e9b024a
--- a/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
+++ b/pxf/pxf-hive/src/test/java/org/apache/hawq/pxf/plugins/hive/utilities/HiveUtilitiesTest.java
@@@ -54,12 -54,6 +54,13 @@@ public class HiveUtilitiesTest
{"char(40)", "bpchar", "40"},
};
+ static String[][] complexTypes = {
- {"array<string>", "text"},
- {"map<string,float>", "text"},
- {"struct<street:string,city:string,state:string,zip:int>", "text"},
++ {"ArraY<string>", "text"},
++ {"MaP<stRing, float>", "text"},
++ {"Struct<street:string, city:string, state:string, zip:int>", "text"},
++ {"UnionType<array<string>, string,int>", "text"}
+ };
+
@Test
public void mapHiveTypeUnsupported() throws Exception {
@@@ -157,37 -152,6 +158,38 @@@
}
@Test
+ public void mapHiveTypeInvalidModifiers() throws Exception {
+ String badHiveType = "decimal(abc, xyz)";
+ hiveColumn = new FieldSchema("numericColumn", badHiveType, null);
+ try {
+ HiveUtilities.mapHiveType(hiveColumn);
+ fail("should fail with bad modifiers error");
+ } catch (UnsupportedTypeException e) {
+ String errorMsg = "HAWQ does not support type " + badHiveType + " (Field numericColumn), modifiers should be integers";
+ assertEquals(errorMsg, e.getMessage());
+ }
+ }
+
+ @Test
+ public void mapHiveTypeComplex() throws Exception {
+ /*
+ * array<dataType> -> text
+ * map<keyDataType, valueDataType> -> text
+ * struct<fieldName1:dataType, ..., fieldNameN:dataType> -> text
++ * uniontype<...> -> text
+ */
+ for (String[] line: complexTypes) {
+ String hiveType = line[0];
+ String expectedType = line[1];
+ hiveColumn = new FieldSchema("field" + hiveType, hiveType, null);
+ Metadata.Field result = HiveUtilities.mapHiveType(hiveColumn);
+ assertEquals("field" + hiveType, result.getName());
+ assertEquals(expectedType, result.getType().getTypeName());
+ assertNull(result.getModifiers());
+ }
+ }
+
+ @Test
public void parseTableQualifiedNameNoDbName() throws Exception {
String name = "orphan";
tblDesc = HiveUtilities.extractTableFromName(name);
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/b1db3991/src/bin/psql/describe.c
----------------------------------------------------------------------
[06/13] incubator-hawq git commit: HAWQ-683. Support for aggregate
javadoc
Posted by od...@apache.org.
HAWQ-683. Support for aggregate javadoc
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/f89fa9ca
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/f89fa9ca
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/f89fa9ca
Branch: refs/heads/HAWQ-703
Commit: f89fa9ca3c7b24afd519b14462d7cc40380fcc4c
Parents: e74109b
Author: Shivram Mani <sh...@gmail.com>
Authored: Fri Apr 22 16:12:40 2016 -0700
Committer: Shivram Mani <sh...@gmail.com>
Committed: Fri Apr 22 16:12:40 2016 -0700
----------------------------------------------------------------------
pxf/Makefile | 5 ++++-
pxf/build.gradle | 5 +++++
.../org/apache/hawq/pxf/api/package-info.java | 23 ++++++++++++++++++++
.../hawq/pxf/api/utilities/Utilities.java | 11 +++++-----
.../hawq/pxf/plugins/hbase/package-info.java | 23 ++++++++++++++++++++
.../hawq/pxf/plugins/hdfs/AvroResolver.java | 4 ++--
.../hawq/pxf/plugins/hdfs/package-info.java | 23 ++++++++++++++++++++
.../hawq/pxf/plugins/hive/package-info.java | 23 ++++++++++++++++++++
.../pxf/service/MetadataResponseFormatter.java | 1 +
.../apache/hawq/pxf/service/package-info.java | 23 ++++++++++++++++++++
.../pxf/service/utilities/ProtocolData.java | 2 +-
11 files changed, 134 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/Makefile
----------------------------------------------------------------------
diff --git a/pxf/Makefile b/pxf/Makefile
index 9f71478..7669772 100644
--- a/pxf/Makefile
+++ b/pxf/Makefile
@@ -39,6 +39,7 @@ help:
@echo " - - VENDOR=<vendor name> - add vendor name to created RPMs"
@echo " - tomcat - builds tomcat rpm from downloaded tarball"
@echo " - - LICENSE and VENDOR parameters can be used as well"
+ @echo " - doc - creates aggregate javadoc under docs"
all:
./gradlew clean release $(BUILD_PARAMS)
@@ -58,7 +59,9 @@ rpm:
clean:
./gradlew clean
+doc:
+ ./gradlew aggregateJavadoc
+
.PHONY: tomcat
tomcat:
./gradlew tomcatRpm $(BUILD_PARAMS)
-
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index ed7b9f0..3f3d31c 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -37,6 +37,7 @@ buildscript {
dependencies {
classpath "com.netflix.nebula:gradle-ospackage-plugin:2.2.6"
classpath "de.undercouch:gradle-download-task:2.1.0"
+ classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:2.2.+'
}
}
@@ -465,3 +466,7 @@ task tomcatRpm(type: Rpm) {
}
tomcatRpm.dependsOn tomcatGet
+
+
+buildDir = '.'
+apply plugin: 'nebula-aggregate-javadocs'
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/package-info.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/package-info.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/package-info.java
new file mode 100644
index 0000000..a8dbad3
--- /dev/null
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides PXF base classes and interfaces for all the PXF plugins.
+ */
+package org.apache.hawq.pxf.api;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
index 314583c..51326bc 100644
--- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
+++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/utilities/Utilities.java
@@ -19,13 +19,13 @@ package org.apache.hawq.pxf.api.utilities;
* under the License.
*/
-import java.lang.reflect.Constructor;
-import java.lang.reflect.InvocationTargetException;
-
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+
/**
* Utilities class exposes helper method for PXF classes
*/
@@ -53,8 +53,9 @@ public class Utilities {
try {
cls = Class.forName(className);
} catch (ClassNotFoundException e) {
- // in case the class name uses the old "com.pivotal.pxf" package
- // name, recommend using the new package "org.apache.hawq.pxf".
+ /* In case the class name uses the older and unsupported "com.pivotal.pxf"
+ * package name, recommend using the new package "org.apache.hawq.pxf"
+ */
if (className.startsWith("com.pivotal.pxf")) {
throw new Exception(
"Class "
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/package-info.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/package-info.java b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/package-info.java
new file mode 100644
index 0000000..93412f7
--- /dev/null
+++ b/pxf/pxf-hbase/src/main/java/org/apache/hawq/pxf/plugins/hbase/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides PXF HBase Plugin.
+ */
+package org.apache.hawq.pxf.plugins.hbase;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/AvroResolver.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/AvroResolver.java b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/AvroResolver.java
index 00ef830..9a6a1c9 100644
--- a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/AvroResolver.java
+++ b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/AvroResolver.java
@@ -151,8 +151,8 @@ public class AvroResolver extends Plugin implements ReadResolver {
/**
* The record can arrive from one out of two different sources: a sequence
* file or an AVRO file. If it comes from an AVRO file, then it was already
- * obtained as a {@link GenericRecord} when when it was fetched from the
- * file with the {@link AvroRecorReader} so in this case a cast is enough.
+ * obtained as a {@link GenericRecord} when it was fetched from the
+ * file so in this case a cast is enough.
* On the other hand, if the source is a sequence file, then the input
* parameter obj hides a bytes [] buffer which is in fact one Avro record
* serialized. Here, we build the Avro record from the flat buffer, using
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/package-info.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/package-info.java b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/package-info.java
new file mode 100644
index 0000000..00d0c46
--- /dev/null
+++ b/pxf/pxf-hdfs/src/main/java/org/apache/hawq/pxf/plugins/hdfs/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides PXF HDFS Plugin.
+ */
+package org.apache.hawq.pxf.plugins.hdfs;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/package-info.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/package-info.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/package-info.java
new file mode 100644
index 0000000..e2540d3
--- /dev/null
+++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides PXF Hive Plugin.
+ */
+package org.apache.hawq.pxf.plugins.hive;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
index 92d11de..55a6da5 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/MetadataResponseFormatter.java
@@ -42,6 +42,7 @@ public class MetadataResponseFormatter {
* Converts list of {@link Metadata} to JSON String format.
*
* @param metadataList list of metadata objects to convert
+ * @param path path string
* @return JSON formatted response
* @throws IOException if converting the data to JSON fails
*/
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/package-info.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/package-info.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/package-info.java
new file mode 100644
index 0000000..7084f86
--- /dev/null
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides PXF Rest API resources.
+ */
+package org.apache.hawq.pxf.service;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/f89fa9ca/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
index ec258fa..0337937 100644
--- a/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
+++ b/pxf/pxf-service/src/main/java/org/apache/hawq/pxf/service/utilities/ProtocolData.java
@@ -154,7 +154,7 @@ public class ProtocolData extends InputData {
* Constructs a ProtocolData. Parses X-GP-* configuration variables.
*
* @param paramsMap contains all query-specific parameters from Hawq
- * @param profile contains the profile name
+ * @param profileString contains the profile name
*/
public ProtocolData(Map<String, String> paramsMap, String profileString) {
requestParametersMap = paramsMap;
[04/13] incubator-hawq git commit: HAWQ-689. Create table with (oids)
in PBE should check whether arg of oids is NULL.
Posted by od...@apache.org.
HAWQ-689. Create table with (oids) in PBE should check whether arg of oids is NULL.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/76658bc1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/76658bc1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/76658bc1
Branch: refs/heads/HAWQ-703
Commit: 76658bc1ae4dea666ac6fe86e7fd3ae54474fe61
Parents: 95197ad
Author: hzhang2 <zh...@163.com>
Authored: Wed Apr 20 13:44:07 2016 +0800
Committer: hzhang2 <zh...@163.com>
Committed: Wed Apr 20 13:44:07 2016 +0800
----------------------------------------------------------------------
src/backend/executor/execMain.c | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/76658bc1/src/backend/executor/execMain.c
----------------------------------------------------------------------
diff --git a/src/backend/executor/execMain.c b/src/backend/executor/execMain.c
index 5e8c6b1..3a43cec 100644
--- a/src/backend/executor/execMain.c
+++ b/src/backend/executor/execMain.c
@@ -4333,7 +4333,7 @@ CreateIntoRel(QueryDesc *queryDesc)
if (!IsA(e, DefElem))
continue;
- if (!IsA(e->arg, String))
+ if (!e->arg || !IsA(e->arg, String))
continue;
if (pg_strcasecmp(e->defname, "appendonly"))
continue;
[08/13] incubator-hawq git commit: HAWQ-705. Fixed aggregation on
psql for Hive tables.
Posted by od...@apache.org.
HAWQ-705. Fixed aggregation on psql for Hive tables.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/395d790a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/395d790a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/395d790a
Branch: refs/heads/HAWQ-703
Commit: 395d790a87cec5e11a774572b0ed1ec97dd897f9
Parents: 4d44097
Author: Oleksandr Diachenko <od...@pivotal.io>
Authored: Fri Apr 22 16:44:53 2016 -0700
Committer: Oleksandr Diachenko <od...@pivotal.io>
Committed: Fri Apr 22 16:44:53 2016 -0700
----------------------------------------------------------------------
src/bin/psql/describe.c | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/395d790a/src/bin/psql/describe.c
----------------------------------------------------------------------
diff --git a/src/bin/psql/describe.c b/src/bin/psql/describe.c
index f1de41b..00cfe33 100644
--- a/src/bin/psql/describe.c
+++ b/src/bin/psql/describe.c
@@ -4307,8 +4307,10 @@ describePxfTable(const char *profile, const char *pattern, bool verbose)
/* First row for current table */
if (previous_itemname == NULL
+ || strlen(previous_itemname) != strlen(itemname)
|| strncmp(previous_itemname, itemname,
strlen(previous_itemname)) != 0
+ || strlen(previous_path) != strlen(path)
|| strncmp(previous_path, path,
strlen(previous_path)) != 0)
{
[07/13] incubator-hawq git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-hawq
Posted by od...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-hawq
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/4d440970
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/4d440970
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/4d440970
Branch: refs/heads/HAWQ-703
Commit: 4d44097085fd139002a255b1032082dc0b030414
Parents: f89fa9c 59f5432
Author: Shivram Mani <sh...@gmail.com>
Authored: Fri Apr 22 16:14:30 2016 -0700
Committer: Shivram Mani <sh...@gmail.com>
Committed: Fri Apr 22 16:14:30 2016 -0700
----------------------------------------------------------------------
depends/libyarn/src/CMakeLists.txt | 2 +-
.../libyarn/src/libyarnclient/LibYarnClient.cpp | 6 ++-
src/backend/executor/execMain.c | 2 +-
tools/bin/gpcheck | 13 ++++--
tools/doc/gpcheck_help | 44 +++++++++++++++++++-
5 files changed, 59 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
[11/13] incubator-hawq git commit: Fix Orca error message: Orca
supports Rollup #117665973
Posted by od...@apache.org.
Fix Orca error message: Orca supports Rollup #117665973
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/6209c42e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/6209c42e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/6209c42e
Branch: refs/heads/HAWQ-703
Commit: 6209c42e7206df44879f9dfe6ec84573203ff31b
Parents: 3be28ec
Author: Venkatesh (Venky) Raghavan <vr...@pivotal.io>
Authored: Tue Apr 26 12:04:49 2016 -0700
Committer: Ming LI <ml...@pivotal.io>
Committed: Wed Apr 27 18:28:12 2016 +0800
----------------------------------------------------------------------
src/backend/gpopt/translate/CTranslatorUtils.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/6209c42e/src/backend/gpopt/translate/CTranslatorUtils.cpp
----------------------------------------------------------------------
diff --git a/src/backend/gpopt/translate/CTranslatorUtils.cpp b/src/backend/gpopt/translate/CTranslatorUtils.cpp
index 2b12f1b..48e8a3b 100644
--- a/src/backend/gpopt/translate/CTranslatorUtils.cpp
+++ b/src/backend/gpopt/translate/CTranslatorUtils.cpp
@@ -1587,7 +1587,7 @@ CTranslatorUtils::PdrgpbsGroupBy
if (GROUPINGTYPE_CUBE == pgrcl->groupType)
{
- GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT(\u201cCube\u201d));
+ GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiQuery2DXLUnsupportedFeature, GPOS_WSZ_LIT("Cube"));
}
DrgPbs *pdrgpbs = GPOS_NEW(pmp) DrgPbs(pmp);
[09/13] incubator-hawq git commit: HAWQ-711. Integrate libhdfs3 and
libyarn makefile into hawq
Posted by od...@apache.org.
HAWQ-711. Integrate libhdfs3 and libyarn makefile into hawq
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/1b0e6e2d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/1b0e6e2d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/1b0e6e2d
Branch: refs/heads/HAWQ-703
Commit: 1b0e6e2d728b5f4f9da09ecc19dc32c87a9fdc81
Parents: 395d790
Author: Ming LI <ml...@pivotal.io>
Authored: Fri Apr 22 16:44:53 2016 -0700
Committer: Ming LI <ml...@pivotal.io>
Committed: Wed Apr 27 17:52:07 2016 +0800
----------------------------------------------------------------------
GNUmakefile.in | 8 ++++
configure | 66 ++++++++------------------------
configure.in | 5 ++-
depends/libhdfs3/.gitignore | 2 +
depends/libhdfs3/Makefile | 33 ++++++++++++++++
depends/libhdfs3/Makefile.global.in | 21 ++++++++++
depends/libyarn/.gitignore | 2 +
depends/libyarn/Makefile | 33 ++++++++++++++++
depends/libyarn/Makefile.global.in | 21 ++++++++++
9 files changed, 140 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/GNUmakefile.in
----------------------------------------------------------------------
diff --git a/GNUmakefile.in b/GNUmakefile.in
index f20ee2c..5523967 100644
--- a/GNUmakefile.in
+++ b/GNUmakefile.in
@@ -10,6 +10,8 @@ include $(top_builddir)/src/Makefile.global
all:
# $(MAKE) -C doc all
+ $(MAKE) -C depends/libhdfs3 all
+ $(MAKE) -C depends/libyarn all
$(MAKE) -C src all
$(MAKE) -C config all
$(MAKE) -C contrib/formatter_fixedwidth all
@@ -22,6 +24,8 @@ all:
install:
# $(MAKE) -C doc $@
+ $(MAKE) -C depends/libhdfs3 $@
+ $(MAKE) -C depends/libyarn $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
@@ -34,6 +38,8 @@ install:
installdirs uninstall:
# $(MAKE) -C doc $@
+ $(MAKE) -C depends/libhdfs3 $@
+ $(MAKE) -C depends/libyarn $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
@@ -53,6 +59,8 @@ distprep:
clean:
# $(MAKE) -C doc $@
$(MAKE) -C contrib $@
+ $(MAKE) -C depends/libhdfs3 $@
+ $(MAKE) -C depends/libyarn $@
$(MAKE) -C src $@
$(MAKE) -C config $@
$(MAKE) -C contrib/formatter_fixedwidth $@
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/configure
----------------------------------------------------------------------
diff --git a/configure b/configure
index 76842d4..5e65a48 100755
--- a/configure
+++ b/configure
@@ -2,7 +2,7 @@
# Guess values for system-dependent variables and create Makefiles.
# Generated by GNU Autoconf 2.69 for HAWQ 8.2.15.
#
-# Report bugs to https://issues.apache.org/jira/browse/HAWQ
+# Report bugs to <https://issues.apache.org/jira/browse/HAWQ>.
#
#
# Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc.
@@ -269,11 +269,10 @@ fi
$as_echo "$0: be upgraded to zsh 4.3.4 or later."
else
$as_echo "$0: Please tell bug-autoconf@gnu.org and
-$0: https://issues.apache.org/jira/browse/HAWQ
-$0: about your system, including any
-$0: error possibly output before this message. Then install
-$0: a modern shell, or manually run the script under such a
-$0: shell if you do have one."
+$0: https://issues.apache.org/jira/browse/HAWQ about your
+$0: system, including any error possibly output before this
+$0: message. Then install a modern shell, or manually run
+$0: the script under such a shell if you do have one."
fi
exit 1
fi
@@ -841,7 +840,6 @@ enable_cassert
enable_forceplan
enable_debugbreak
enable_debugntuplestore
-enable_mdverinstrumentation
enable_testutils
enable_snmp
enable_gphdfs
@@ -1516,7 +1514,6 @@ build with coverage testing instrumentation
--enable-forceplan enable forcing of plans via EXPLAIN FORCE (for debugging)
--enable-debugbreak enable debug_break and debug_break_n (for debugging)
--enable-ntuplestore enable debug_ntuplestore (for debugging)
- --enable-mdverinstrumentation enable metadata versioning instrumentation
--enable-testutils enable testing utilities
--enable-snmp enable snmp for MIB and alerts via TRAP/INFORM
--enable-gphdfs enables GPHDFS support
@@ -1579,7 +1576,7 @@ Some influential environment variables:
Use these variables to override the choices made by `configure' or to help
it to find libraries and programs with nonstandard names/locations.
-Report bugs to https://issues.apache.org/jira/browse/HAWQ
+Report bugs to <https://issues.apache.org/jira/browse/HAWQ>.
_ACEOF
ac_status=$?
fi
@@ -1849,9 +1846,9 @@ $as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;}
$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;}
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5
$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;}
-( $as_echo "## ------------------------------------ ##
+( $as_echo "## --------------------------------------------------------- ##
## Report this to https://issues.apache.org/jira/browse/HAWQ ##
-## ------------------------------------ ##"
+## --------------------------------------------------------- ##"
) | sed "s/^/$as_me: WARNING: /" >&2
;;
esac
@@ -2088,9 +2085,9 @@ $as_echo "$as_me: WARNING: $2: see the Autoconf documentation" >&2;}
$as_echo "$as_me: WARNING: $2: section \"Present But Cannot Be Compiled\"" >&2;}
{ $as_echo "$as_me:${as_lineno-$LINENO}: WARNING: $2: proceeding with the compiler's result" >&5
$as_echo "$as_me: WARNING: $2: proceeding with the compiler's result" >&2;}
-( $as_echo "## --------------------------------------------------- ##
+( $as_echo "## --------------------------------------------------------- ##
## Report this to https://issues.apache.org/jira/browse/HAWQ ##
-## -------------------------------------------------- ##"
+## --------------------------------------------------------- ##"
) | sed "s/^/$as_me: WARNING: /" >&2
;;
esac
@@ -5336,34 +5333,6 @@ fi
-# Enable metadata versioning instrumentation
-
-pgac_args="$pgac_args enable_mdverinstrumentation"
-
-# Check whether --enable-mdverinstrumentation was given.
-if test "${enable_mdverinstrumentation+set}" = set; then :
- enableval=$enable_mdverinstrumentation;
- case $enableval in
- yes)
-
-$as_echo "#define MD_VERSIONING_INSTRUMENTATION 1" >>confdefs.h
-
- ;;
- no)
- :
- ;;
- *)
- as_fn_error $? "no argument expected for --enable-mdverinstrumentation option" "$LINENO" 5
- ;;
- esac
-
-else
- enable_mdverinstrumentation=no
-
-fi
-
-
-
#
# Enable testing utilities
#
@@ -5532,9 +5501,6 @@ fi
-
-
-
#
# Include directories
#
@@ -11125,10 +11091,10 @@ fi
for ac_header in json-c/json.h
do :
- ac_fn_c_check_header_mongrel "$LINENO" "json-c/json.h" "ac_cv_header_json_json_h" "$ac_includes_default"
-if test "x$ac_cv_header_json_json_h" = xyes; then :
+ ac_fn_c_check_header_mongrel "$LINENO" "json-c/json.h" "ac_cv_header_json_c_json_h" "$ac_includes_default"
+if test "x$ac_cv_header_json_c_json_h" = xyes; then :
cat >>confdefs.h <<_ACEOF
-#define HAVE_JSON_JSON_H 1
+#define HAVE_JSON_C_JSON_H 1
_ACEOF
else
@@ -16540,7 +16506,7 @@ fi
-ac_config_files="$ac_config_files GNUmakefile src/VERSIONS.mk src/Makefile.global src/pl/pljava/src/java/Makefile.global"
+ac_config_files="$ac_config_files GNUmakefile src/VERSIONS.mk depends/libhdfs3/Makefile.global depends/libyarn/Makefile.global src/Makefile.global src/pl/pljava/src/java/Makefile.global"
ac_config_links="$ac_config_links src/backend/port/dynloader.c:src/backend/port/dynloader/${template}.c src/backend/port/pg_sema.c:${SEMA_IMPLEMENTATION} src/backend/port/pg_shmem.c:${SHMEM_IMPLEMENTATION} src/include/dynloader.h:src/backend/port/dynloader/${template}.h src/include/pg_config_os.h:src/include/port/${template}.h src/Makefile.port:src/makefiles/Makefile.${template}"
@@ -17128,7 +17094,7 @@ $config_links
Configuration commands:
$config_commands
-Report bugs to https://issues.apache.org/jira/browse/HAWQ"
+Report bugs to <https://issues.apache.org/jira/browse/HAWQ>."
_ACEOF
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
@@ -17260,6 +17226,8 @@ do
"src/backend/port/tas.s") CONFIG_LINKS="$CONFIG_LINKS src/backend/port/tas.s:src/backend/port/tas/${tas_file}" ;;
"GNUmakefile") CONFIG_FILES="$CONFIG_FILES GNUmakefile" ;;
"src/VERSIONS.mk") CONFIG_FILES="$CONFIG_FILES src/VERSIONS.mk" ;;
+ "depends/libhdfs3/Makefile.global") CONFIG_FILES="$CONFIG_FILES depends/libhdfs3/Makefile.global" ;;
+ "depends/libyarn/Makefile.global") CONFIG_FILES="$CONFIG_FILES depends/libyarn/Makefile.global" ;;
"src/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/Makefile.global" ;;
"src/pl/pljava/src/java/Makefile.global") CONFIG_FILES="$CONFIG_FILES src/pl/pljava/src/java/Makefile.global" ;;
"src/backend/port/dynloader.c") CONFIG_LINKS="$CONFIG_LINKS src/backend/port/dynloader.c:src/backend/port/dynloader/${template}.c" ;;
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/configure.in
----------------------------------------------------------------------
diff --git a/configure.in b/configure.in
index a5b322b..f1fdc02 100644
--- a/configure.in
+++ b/configure.in
@@ -17,7 +17,7 @@ dnl Read the Autoconf manual for details.
dnl
m4_pattern_forbid(^PGAC_)dnl to catch undefined macros
-AC_INIT([HAWQ], [8.2.15], [support@greenplum.com])
+AC_INIT([HAWQ], [8.2.15], [https://issues.apache.org/jira/browse/HAWQ])
AC_PREREQ(2.63)
dnl m4_if(m4_defn([m4_PACKAGE_VERSION]), [2.63], [], [m4_fatal([Autoconf version 2.63 is required.
@@ -310,6 +310,7 @@ elif test "$GCC" = yes; then
CFLAGS="-O0"
else
CFLAGS="-O3"
+ fi
else
# if the user selected debug mode, don't use -O
if test "$enable_debug" != yes; then
@@ -2055,7 +2056,7 @@ fi
AC_SUBST(vpath_build)
-AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk src/Makefile.global src/pl/pljava/src/java/Makefile.global])
+AC_CONFIG_FILES([GNUmakefile src/VERSIONS.mk depends/libhdfs3/Makefile.global depends/libyarn/Makefile.global src/Makefile.global src/pl/pljava/src/java/Makefile.global])
AC_CONFIG_LINKS([
src/backend/port/dynloader.c:src/backend/port/dynloader/${template}.c
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/depends/libhdfs3/.gitignore
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/.gitignore b/depends/libhdfs3/.gitignore
new file mode 100644
index 0000000..8341b2e
--- /dev/null
+++ b/depends/libhdfs3/.gitignore
@@ -0,0 +1,2 @@
+build/
+Makefile.global
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/depends/libhdfs3/Makefile
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/Makefile b/depends/libhdfs3/Makefile
new file mode 100644
index 0000000..e509659
--- /dev/null
+++ b/depends/libhdfs3/Makefile
@@ -0,0 +1,33 @@
+# -*-makefile-*-
+#------------------------------------------------------------------------------
+# A makefile that integrate building this module with hawq
+#------------------------------------------------------------------------------
+subdir = depends/libhdfs3
+top_builddir = ../../
+include Makefile.global
+
+PRE_CFG_ARG =
+# get argument for running ../boostrap
+ifeq ($(enable_debug), yes)
+ PRE_CFG_ARG += --enable-debug
+endif # enable_debug
+
+ifeq ($(enable_coverage), yes)
+ PRE_CFG_ARG += --enable-coverage
+endif # enable_coverage
+
+##########################################################################
+#
+.PHONY: all install make pre-config
+
+all : make
+
+install: make
+ cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8 install
+
+make: pre-config
+ cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8
+
+pre-config:
+ cd $(top_srcdir)/$(subdir)/ && mkdir -p build && cd build && ../bootstrap --prefix=$(prefix) $(PRE_CFG_ARG)
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/depends/libhdfs3/Makefile.global.in
----------------------------------------------------------------------
diff --git a/depends/libhdfs3/Makefile.global.in b/depends/libhdfs3/Makefile.global.in
new file mode 100644
index 0000000..12dd05c
--- /dev/null
+++ b/depends/libhdfs3/Makefile.global.in
@@ -0,0 +1,21 @@
+# -*-makefile-*-
+#------------------------------------------------------------------------------
+# A makefile that integrate building this module with hawq
+#------------------------------------------------------------------------------
+
+prefix := @prefix@
+enable_debug = @enable_debug@
+enable_coverage = @enable_coverage@
+
+# Support for VPATH builds
+vpath_build = @vpath_build@
+abs_top_srcdir = @abs_top_srcdir@
+
+ifneq ($(vpath_build),yes)
+top_srcdir = $(top_builddir)
+srcdir = .
+else # vpath_build = yes
+top_srcdir = $(abs_top_srcdir)
+srcdir = $(top_srcdir)/$(subdir)
+VPATH = $(srcdir)
+endif
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/depends/libyarn/.gitignore
----------------------------------------------------------------------
diff --git a/depends/libyarn/.gitignore b/depends/libyarn/.gitignore
new file mode 100644
index 0000000..8341b2e
--- /dev/null
+++ b/depends/libyarn/.gitignore
@@ -0,0 +1,2 @@
+build/
+Makefile.global
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/depends/libyarn/Makefile
----------------------------------------------------------------------
diff --git a/depends/libyarn/Makefile b/depends/libyarn/Makefile
new file mode 100644
index 0000000..3e17622
--- /dev/null
+++ b/depends/libyarn/Makefile
@@ -0,0 +1,33 @@
+# -*-makefile-*-
+#------------------------------------------------------------------------------
+# A makefile that integrate building this module with hawq
+#------------------------------------------------------------------------------
+subdir = depends/libyarn
+top_builddir = ../../
+include Makefile.global
+
+PRE_CFG_ARG =
+# get argument for running ../boostrap
+ifeq ($(enable_debug), yes)
+ PRE_CFG_ARG += --enable-debug
+endif # enable_debug
+
+ifeq ($(enable_coverage), yes)
+ PRE_CFG_ARG += --enable-coverage
+endif # enable_coverage
+
+##########################################################################
+#
+.PHONY: all install make pre-config
+
+all : make
+
+install: make
+ cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8 install
+
+make: pre-config
+ cd $(top_srcdir)/$(subdir)/build && $(MAKE) -j8
+
+pre-config:
+ cd $(top_srcdir)/$(subdir)/ && mkdir -p build && cd build && ../bootstrap --prefix=$(prefix) $(PRE_CFG_ARG)
+
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/1b0e6e2d/depends/libyarn/Makefile.global.in
----------------------------------------------------------------------
diff --git a/depends/libyarn/Makefile.global.in b/depends/libyarn/Makefile.global.in
new file mode 100644
index 0000000..12dd05c
--- /dev/null
+++ b/depends/libyarn/Makefile.global.in
@@ -0,0 +1,21 @@
+# -*-makefile-*-
+#------------------------------------------------------------------------------
+# A makefile that integrate building this module with hawq
+#------------------------------------------------------------------------------
+
+prefix := @prefix@
+enable_debug = @enable_debug@
+enable_coverage = @enable_coverage@
+
+# Support for VPATH builds
+vpath_build = @vpath_build@
+abs_top_srcdir = @abs_top_srcdir@
+
+ifneq ($(vpath_build),yes)
+top_srcdir = $(top_builddir)
+srcdir = .
+else # vpath_build = yes
+top_srcdir = $(abs_top_srcdir)
+srcdir = $(top_srcdir)/$(subdir)
+VPATH = $(srcdir)
+endif
[02/13] incubator-hawq git commit: HAWQ-668. hawq check should be
able to check yarn settings
Posted by od...@apache.org.
HAWQ-668. hawq check should be able to check yarn settings
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/e74109bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/e74109bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/e74109bf
Branch: refs/heads/HAWQ-703
Commit: e74109bf6645a24bbbd2ce37a32d01e981c670e5
Parents: 1469782
Author: rlei <rl...@pivotal.io>
Authored: Wed Apr 13 17:13:02 2016 +0800
Committer: rlei <rl...@pivotal.io>
Committed: Tue Apr 19 17:23:27 2016 +0800
----------------------------------------------------------------------
src/backend/utils/misc/etc/gpcheck.cnf | 48 +++-
tools/bin/gpcheck | 365 ++++++++++++++++++++++++++--
tools/bin/gppylib/gpcheckutil.py | 17 +-
tools/bin/hawqpylib/hawqlib.py | 18 ++
tools/sbin/gpcheck_hostdump | 50 +++-
5 files changed, 472 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e74109bf/src/backend/utils/misc/etc/gpcheck.cnf
----------------------------------------------------------------------
diff --git a/src/backend/utils/misc/etc/gpcheck.cnf b/src/backend/utils/misc/etc/gpcheck.cnf
index 9ccac0d..9d36de6 100644
--- a/src/backend/utils/misc/etc/gpcheck.cnf
+++ b/src/backend/utils/misc/etc/gpcheck.cnf
@@ -40,12 +40,11 @@ hard.nproc = 131072
diskusage.monitor.mounts = /
diskusage.monitor.usagemax = 90%
-[hdfs]
+[hdfs.base]
dfs.mem.namenode.heap = 40960
dfs.mem.datanode.heap = 6144
# in hdfs-site.xml
dfs.support.append = true
-dfs.client.enable.read.from.local = true
dfs.block.local-path-access.user = gpadmin
dfs.datanode.max.transfer.threads = 40960
dfs.client.socket-timeout = 300000000
@@ -54,5 +53,48 @@ dfs.namenode.handler.count = 60
ipc.server.handler.queue.size = 3300
dfs.datanode.handler.count = 60
ipc.client.connection.maxidletime = 3600000
-dfs.namenode.accesstime.precision = -1
+dfs.namenode.accesstime.precision = 0
+dfs.client.read.shortcircuit = true
+[hdfs.non]
+dfs.block.access.token.enable = FALSE
+
+[hdfs.ha]
+dfs.block.access.token.enable = FALSE
+
+[hdfs.kerberos]
+dfs.block.access.token.enable = TRUE
+dfs.datanode.data.dir.perm = 750
+
+[hdfs.ha.kerberos]
+dfs.block.access.token.enable = TRUE
+
+[yarn.base]
+yarn.resourcemanager.scheduler.class = org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler
+
+[yarn.non]
+
+[yarn.ha]
+
+[yarn.kerberos]
+hadoop.security.authentication = kerberos
+hadoop.proxyuser.yarn.groups = *
+hadoop.proxyuser.yarn.hosts = *
+hadoop.proxyuser.postgres.hosts = *
+hadoop.proxyuser.postgres.groups = *
+
+[yarn.ha.kerberos]
+hadoop.security.authentication = kerberos
+hadoop.proxyuser.yarn.groups = *
+hadoop.proxyuser.yarn.hosts = *
+hadoop.proxyuser.postgres.hosts = *
+hadoop.proxyuser.postgres.groups = *
+
+[hawq.base]
+dfs.client.read.shortcircuit = true
+
+[hawq.kerberos]
+hadoop.security.authentication = kerberos
+
+[hawq.yarn]
+hawq_global_rm_type = yarn
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e74109bf/tools/bin/gpcheck
----------------------------------------------------------------------
diff --git a/tools/bin/gpcheck b/tools/bin/gpcheck
index aefe499..1d0019c 100755
--- a/tools/bin/gpcheck
+++ b/tools/bin/gpcheck
@@ -26,8 +26,10 @@ try:
from gppylib.commands.unix import getLocalHostname, getUserName, SYSTEM
from gppylib.commands.base import WorkerPool, Command, REMOTE
from gppylib.gpcheckutil import HostType, hosttype_str
+ from hawqpylib.hawqlib import remote_ssh_output
from pgdb import DatabaseError
import pg
+ import stat
except ImportError, e:
sys.exit('Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e))
@@ -89,15 +91,29 @@ class GpCheckConfig:
self.hdfs_expected = { # default value for HDFS configuration
"dfs.mem.namenode.heap": 8192,
"dfs.mem.datanode.heap": 8192 }
+ self.hdfs_non_expected = {}
+ self.hdfs_ha_expected = {}
+ self.hdfs_kerberos_expected = {}
+ self.hdfs_ha_kerberos_expected = {}
+
+ self.yarn_expected = {}
+ self.yarn_non_expected = {}
+ self.yarn_ha_expected = {}
+ self.yarn_kerberos_expected = {}
+ self.yarn_ha_kerberos_expected = {}
+
+ self.hawq_expected = {}
+ self.hawq_kerberos_expected = {}
+ self.hawq_yarn_expected = {}
+
def readConfigFile(self, config_file):
parsed_list = self.parser.read(config_file)
if len(parsed_list) != 1:
raise GpCheckError("cannot open file!")
- for required_section in ("linux.sysctl", "hdfs"):
- if not self.parser.has_section(required_section):
- raise GpCheckError("require section '%s'" % required_section)
+ if not self.parser.has_section("linux.sysctl"):
+ raise GpCheckError("require section 'linux.sysctl'")
section = "global"
if self.parser.has_option(section, "configfile_version"):
@@ -136,15 +152,75 @@ class GpCheckConfig:
raise GpCheckError("Bad config entry value '%s' for 'diskusage.monitor.usagemax': %s" %
(self.diskusage_usagemax, e))
- section = 'hdfs'
- for opt in self.parser.options(section):
- self.hdfs_expected[opt] = self.parser.get(section, opt)
- try:
- self.hdfs_expected["dfs.mem.namenode.heap"] = int(self.hdfs_expected["dfs.mem.namenode.heap"])
- self.hdfs_expected["dfs.mem.datanode.heap"] = int(self.hdfs_expected["dfs.mem.datanode.heap"])
- except ValueError, e:
- raise GpCheckError("'dfs.mem.namenode.heap' or 'dfs.mem.namenode.heap' should be a number: %s" % e)
+ if not self.parser.has_section('hdfs.base'):
+ if not self.parser.has_section("hdfs"):
+ raise GpCheckError("require section 'hdfs'")
+
+ section = 'hdfs'
+ for opt in self.parser.options(section):
+ self.hdfs_expected[opt] = self.parser.get(section, opt)
+ try:
+ self.hdfs_expected["dfs.mem.namenode.heap"] = int(self.hdfs_expected["dfs.mem.namenode.heap"])
+ self.hdfs_expected["dfs.mem.datanode.heap"] = int(self.hdfs_expected["dfs.mem.datanode.heap"])
+ except ValueError, e:
+ raise GpCheckError("'dfs.mem.namenode.heap' or 'dfs.mem.namenode.heap' should be a number: %s" % e)
+ else:
+ section = 'hdfs.base'
+ for opt in self.parser.options(section):
+ self.hdfs_expected[opt] = self.parser.get(section, opt)
+ try:
+ self.hdfs_expected["dfs.mem.namenode.heap"] = int(self.hdfs_expected["dfs.mem.namenode.heap"])
+ self.hdfs_expected["dfs.mem.datanode.heap"] = int(self.hdfs_expected["dfs.mem.datanode.heap"])
+ except ValueError, e:
+ raise GpCheckError("'dfs.mem.namenode.heap' or 'dfs.mem.namenode.heap' should be a number: %s" % e)
+
+ section = 'hdfs.non'
+ for opt in self.parser.options(section):
+ self.hdfs_non_expected[opt] = self.parser.get(section, opt)
+
+ section = 'hdfs.ha'
+ for opt in self.parser.options(section):
+ self.hdfs_ha_expected[opt] = self.parser.get(section, opt)
+
+ section = 'hdfs.kerberos'
+ for opt in self.parser.options(section):
+ self.hdfs_kerberos_expected[opt] = self.parser.get(section, opt)
+
+ section = 'hdfs.ha.kerberos'
+ for opt in self.parser.options(section):
+ self.hdfs_ha_kerberos_expected[opt] = self.parser.get(section, opt)
+ section = 'yarn.base'
+ for opt in self.parser.options(section):
+ self.yarn_expected[opt] = self.parser.get(section, opt)
+
+ section = 'yarn.non'
+ for opt in self.parser.options(section):
+ self.yarn_non_expected[opt] = self.parser.get(section, opt)
+
+ section = 'yarn.ha'
+ for opt in self.parser.options(section):
+ self.yarn_ha_expected[opt] = self.parser.get(section, opt)
+
+ section = 'yarn.kerberos'
+ for opt in self.parser.options(section):
+ self.yarn_kerberos_expected[opt] = self.parser.get(section, opt)
+
+ section = 'yarn.ha.kerberos'
+ for opt in self.parser.options(section):
+ self.yarn_ha_kerberos_expected[opt] = self.parser.get(section, opt)
+
+ section = 'hawq.base'
+ for opt in self.parser.options(section):
+ self.hawq_expected[opt] = self.parser.get(section, opt)
+
+ section = 'hawq.kerberos'
+ for opt in self.parser.options(section):
+ self.hawq_kerberos_expected[opt] = self.parser.get(section, opt)
+
+ section = 'hawq.yarn'
+ for opt in self.parser.options(section):
+ self.hawq_yarn_expected[opt] = self.parser.get(section, opt)
###### Global Variables #############
logger = get_default_logger()
@@ -176,6 +252,16 @@ def checkPlatform():
raise GpCheckError("No tests exists for this platform in gpcheck")
+def parse_host_list_file(host_file):
+ host_list = list()
+ with open(host_file) as f:
+ hosts = f.readlines()
+ for host in hosts:
+ host = host.split("#",1)[0].strip()
+ if host:
+ host_list.append(host)
+ return host_list
+
def parseargs():
global options, GPHOME, HADOOP_HOME, GPCHECK_CONFIG_FILE
@@ -188,7 +274,12 @@ def parseargs():
parser.add_option('--zipin', type='string')
parser.add_option('--gphome', type='string')
# for HDFS xml and memory check
- parser.add_option('--hadoop', type='string')
+ parser.add_option('--hadoop', '--hadoop-home', type='string')
+ parser.add_option('--hdfs', action='store_true')
+ parser.add_option('--hdfs-ha', dest="hdfs_ha", action='store_true')
+ parser.add_option('--yarn', action='store_true')
+ parser.add_option('--yarn-ha', dest="yarn_ha", action='store_true')
+ parser.add_option('--kerberos', action='store_true')
parser.add_option('-c', '--config', type='string') # optional: gpcheck config file path
parser.add_option('-f', '--file', type='string') # host file, for testing a list of hosts
@@ -212,6 +303,10 @@ def parseargs():
if not HADOOP_HOME:
checkFailed(None, "utility will SKIP HDFS configuration check because HADOOP_HOME is not specified in environment variable or --hadoop")
+ if options.yarn and not HADOOP_HOME:
+ options.yarn = False
+ checkFailed(None, "utility will SKIP YARN configuration check because HADOOP_HOME is not specified in environment variable or --hadoop")
+
# params check
if not options.file and not options.host and not options.zipin:
raise GpCheckError(" --file or --host or --zipin must be specified")
@@ -242,6 +337,7 @@ def checkFailed(host, msg):
def getHDFSNamenodeHost():
core_site_file = os.path.join(HADOOP_HOME, "etc/hadoop/core-site.xml")
+ hdfs_site_file = os.path.join(HADOOP_HOME, "etc/hadoop/hdfs-site.xml")
logger.info("try to detect namenode from %s" % core_site_file)
# for processing property xml
@@ -255,12 +351,42 @@ def getHDFSNamenodeHost():
for node in xmldoc.getElementsByTagName('property'):
if getPropName(node) == 'fs.default.name' or getPropName(node) == 'fs.defaultFS':
fsurl = getPropValue(node).strip()
- namenode_addr = re.search(r"//([^:/]*)", fsurl).group(1)
+ namenode_list_alias = re.search(r"//([^:/]*)", fsurl).group(1)
+ if_ha_disabled = re.search(".*:[0-9]+$", fsurl)
+ if if_ha_disabled:
+ namenode_addr = namenode_list_alias
+ else:
+ namenode_addr = ''
break
# run hostname command on remote to get actual hostname
if namenode_addr == '':
- logger.error("cannot detect namenode from %s" % core_site_file)
+ ha_namenode_list = ''
+ default_namenode_alias = ''
+ with open(hdfs_site_file) as f:
+ xmldoc = minidom.parse(f)
+ for node in xmldoc.getElementsByTagName('property'):
+ if re.search('dfs.ha.namenodes.*', getPropName(node).strip()):
+ ha_namenode_list = getPropValue(node).strip()
+ default_namenode_alias = ha_namenode_list.split(',')[0].strip()
+ break
+
+ if ha_namenode_list == '':
+ logger.error("cannot detect namenode from %s" % core_site_file)
+ raise GpCheckError("cannot detect namenode from %s" % core_site_file)
+ #sys.exit(1)
+ else:
+ with open(hdfs_site_file) as f:
+ xmldoc = minidom.parse(f)
+ for node in xmldoc.getElementsByTagName('property'):
+ namenode_rpc_address = "dfs.namenode.rpc-address.%s.%s" % (namenode_list_alias,
+ default_namenode_alias)
+ if getPropName(node) == namenode_rpc_address:
+ default_namenode_rpc_address = getPropValue(node).strip()
+ namenode_addr = default_namenode_rpc_address.split(':')[0].strip()
+
+ if namenode_addr == '':
+ raise GpCheckError("cannot detect namenode from %s" % core_site_file)
else:
cmd = Command(namenode_addr, "hostname", REMOTE, namenode_addr)
pool.addCommand(cmd)
@@ -345,10 +471,12 @@ def runCollectionOnServers():
else:
raise GpCheckError("unsupported host type")
- cmd = "%s/sbin/gpcheck_hostdump %s" % (GPHOME, host_type_cl)
+ cmd = "%s/sbin/gpcheck_hostdump --hawq %s" % (GPHOME, host_type_cl)
cmd += " --sysctl %s" % ",".join(gpcheck_config.sysctl_expected.keys())
if HADOOP_HOME:
cmd += " --hadoop %s" % HADOOP_HOME
+ if options.yarn or options.yarn_ha:
+ cmd += " --yarn"
return cmd
try:
@@ -537,7 +665,7 @@ def testSolarisEtcUserAttr(host):
checkFailed(host.hostname, "/etc/user_attr is missing expected line '%s'" % line)
-def testHAWQ(host):
+def testHAWQGUC(host):
if not gpcheck_info.hawq_collected_ok:
return
@@ -567,7 +695,7 @@ def testHAWQ(host):
return
# check HAWQ master's memory size
- expected_vmemory_size = 1024
+ expected_vmemory_size = 8192
if guc_vmemsize_master != expected_vmemory_size:
checkFailed(host.hostname, "HAWQ master's %s GUC value is %s, expected %s" % (
HAWQ_GUC_MEMORY, guc_vmemsize_master, expected_vmemory_size))
@@ -582,7 +710,7 @@ def testHAWQ(host):
logger.warning("please change the expected data node memory 'dfs.mem.datanode.heap' in gpcheck.cnf file")
logger.warning("SKIP '%s' check" %(HAWQ_GUC_MEMORY))
return
- expect_vmemsize_per_segment = 1024
+ expect_vmemsize_per_segment = 8192
if guc_vmemsize_master != expect_vmemsize_per_segment:
checkFailed(host.hostname, "HAWQ segment's %s GUC value on this host is %s, expected %s" % (
HAWQ_GUC_MEMORY, guc_vmemsize_master, expect_vmemsize_per_segment))
@@ -602,6 +730,120 @@ def testDiskCapacity(host):
return
+def testHAWQconfig(host):
+ hawq = host.data.hawq
+ hdfs = host.data.hdfs
+ if hawq is None:
+ return # skip HAWQ test when hawq is None
+
+ if options.verbose:
+ logger.info("-- test HAWQ config")
+
+ if hawq.errormsg:
+ checkFailed(host.hostname, "collect HAWQ configuration error: %s" % hawq.errormsg)
+ return
+
+ datanode_list = parse_host_list_file("%s/etc/hadoop/slaves" % HADOOP_HOME)
+ is_datanode = False
+ if host.hostname in datanode_list:
+ is_datanode = True
+
+ expect_config = gpcheck_config.hawq_expected
+
+ if options.kerberos:
+ expect_config.update(gpcheck_config.hawq_kerberos_expected)
+
+ if options.yarn or options.yarn_ha:
+ expect_config.update(gpcheck_config.hawq_yarn_expected)
+
+ actual_config = hawq.site_config
+ hdfs_actual_config = hdfs.site_config
+
+ for exp_key, exp_val in expect_config.items():
+ if exp_key not in actual_config:
+ checkFailed(host.hostname, "HAWQ configuration missing: '%s' needs to be set to '%s'" % (exp_key, exp_val))
+
+ else:
+ actual_val = actual_config[exp_key]
+ et = (exp_key, exp_val, actual_val)
+
+ if exp_key == "dfs.block.local-path-access.user":
+ if exp_val not in actual_val.split(','):
+ checkFailed(host.hostname, "HDFS configuration: '%s' should include user '%s', actual value is '%s'" % et)
+ elif exp_key == "dfs.namenode.handler.count":
+ if int(exp_val) > int(actual_val):
+ checkFailed(host.hostname, "HDFS configuration: '%s' should be at least '%s', actual value is '%s'" % et)
+ else:
+ if exp_val != actual_val:
+ checkFailed(host.hostname, "HAWQ configuration: expected '%s' for '%s', actual value is '%s'" % et)
+
+ if not options.kerberos:
+ if 'hadoop.security.authentication' in actual_config:
+ if actual_config['hadoop.security.authentication'] != 'simple':
+ checkFailed(host.hostname, "HAWQ configuration: expected '%s' for '%s', actual value is '%s'" % ('simple', 'hadoop.security.authentication', actual_config[hadoop.security.authentication]))
+
+ if 'hadoop.security.authentication' in hdfs_actual_config:
+ if hdfs_actual_config['hadoop.security.authentication'] != 'simple':
+ checkFailed(host.hostname, "HAWQ configuration: expected '%s' for '%s', actual value is '%s'" % ('simple', 'hadoop.security.authentication', hdfs_actual_config[hadoop.security.authentication]))
+
+ if options.yarn or options.yarn_ha:
+ hawq_yarn_property_exist_list = ['hawq_rm_yarn_address', 'hawq_rm_yarn_scheduler_address', 'hawq_rm_yarn_app_name']
+ for item in hawq_yarn_property_exist_list:
+ if item in actual_config:
+ if not actual_config[item]:
+ checkFailed(host.hostname, "HAWQ configuration: yarn.resourcemanager.address is empty")
+ else:
+ checkFailed(host.hostname, "HAWQ configuration: yarn.resourcemanager.address not defined")
+
+ if 'dfs.client.read.shortcircuit' not in actual_config:
+ checkFailed(host.hostname, "HAWQ configuration dfs.client.read.shortcircuit not defined")
+
+ if 'dfs.client.read.shortcircuit' not in hdfs_actual_config:
+ checkFailed(host.hostname, "HAWQ configuration dfs.client.read.shortcircuit not defined")
+
+ if 'dfs.domain.socket.path' not in actual_config:
+ checkFailed(host.hostname, "HAWQ configuration dfs.domain.socket.path not defined")
+
+ if 'dfs.domain.socket.path' not in hdfs_actual_config:
+ checkFailed(host.hostname, "HDFS configuration dfs.domain.socket.path not defined")
+
+ if is_datanode and 'dfs.domain.socket.path' in actual_config and 'dfs.domain.socket.path' in hdfs_actual_config:
+ if actual_config['dfs.domain.socket.path'] != hdfs_actual_config['dfs.domain.socket.path']:
+ checkFailed(host.hostname, "HAWQ configuration: dfs.domain.socket.path expect to have the same value with HDFS configuration")
+ else:
+ cmd = "ls -l %s" % actual_config['dfs.domain.socket.path']
+ (result, output, errmsg) = remote_ssh_output(cmd, host.hostname, '')
+ if result == 0:
+ if output.split(' ')[0][7:9] != 'rw':
+ checkFailed(host.hostname, "HAWQ configuration dfs.domain.socket.path: %s should have R/W access for both hawq and HDFS on %s" % (actual_config['dfs.domain.socket.path'], host.hostname))
+ else:
+ checkFailed(host.hostname, "HAWQ configuration dfs.domain.socket.path: %s, does not exist on %s" % (actual_config['dfs.domain.socket.path'], host.hostname))
+
+ if 'output.replace-datanode-on-failure' in actual_config:
+ if len(datanode_list) < 4:
+ if actual_config['output.replace-datanode-on-failure'] == 'true':
+ checkFailed(host.hostname, "HAWQ configuration: output.replace-datanode-on-failure expect false, current is true")
+ else:
+ if actual_config['output.replace-datanode-on-failure'] == 'false':
+ checkFailed(host.hostname, "HAWQ configuration: output.replace-datanode-on-failure expect true, current is false")
+ else:
+ checkFailed(host.hostname, "HAWQ configuration: output.replace-datanode-on-failure not defined")
+
+
+def testDiskCapacity(host):
+ if options.verbose:
+ logger.info("-- test Disk Capacity")
+
+ for line in host.data.diskusage.lines:
+ if len(gpcheck_config.diskusage_mounts) == 0 or line.mount in gpcheck_config.diskusage_mounts:
+ actual_usage = int(line.used_percent[:-1])
+ if actual_usage > gpcheck_config.diskusage_usagemax:
+ checkFailed(host.hostname,
+ "potential disk full risk: %s mounted on %s has used %s space" % (
+ line.fs, line.mount, line.used_percent))
+ return
+
+
def testHDFSConfig(host):
hdfs = host.data.hdfs
if hdfs is None:
@@ -615,6 +857,30 @@ def testHDFSConfig(host):
return
expect_config = gpcheck_config.hdfs_expected
+
+ if not options.hdfs_ha and not options.kerberos:
+ expect_config.update(gpcheck_config.hdfs_non_expected)
+
+ if options.hdfs_ha and not options.kerberos:
+ expect_config.update(gpcheck_config.hdfs_ha_expected)
+
+ if options.kerberos and not options.hdfs_ha:
+ expect_config.update(gpcheck_config.hdfs_kerberos_expected)
+
+ if options.kerberos and options.hdfs_ha:
+ expect_config.update(gpcheck_config.hdfs_ha_kerberos_expected)
+
+
+ if options.yarn or options.yarn_ha:
+ expect_config.update(gpcheck_config.yarn_expected)
+ if not options.yarn_ha and not options.kerberos:
+ expect_config.update(gpcheck_config.yarn_non_expected)
+
+ if options.yarn_ha:
+ expect_config.update(gpcheck_config.yarn_ha_expected)
+ if options.kerberos:
+ expect_config.update(gpcheck_config.yarn_kerberos_expected)
+
actual_config = hdfs.site_config
actual_heap_size = hdfs.namenode_heap_size if host.is_namenode else hdfs.datanode_heap_size
@@ -658,6 +924,64 @@ def testHDFSConfig(host):
(actual_heap_size, expect_datanode_heap))
+ # Check if nodemanager direcotries exists
+ directory_check_list = []
+ datanode_list = parse_host_list_file("%s/etc/hadoop/slaves" % HADOOP_HOME)
+ is_datanode = False
+ if host.hostname in datanode_list:
+ is_datanode = True
+
+ if options.yarn or options.yarn_ha:
+ yarn_enabled = True
+ else:
+ yarn_enabled = False
+
+ if yarn_enabled and is_datanode:
+ if 'yarn.nodemanager.local-dirs' in actual_config:
+ directory_check_list += actual_config['yarn.nodemanager.local-dirs'].split(',')
+ else:
+ checkFailed(host.hostname, "YARN configuration: yarn.nodemanager.local-dirs not defined")
+
+ if 'yarn.nodemanager.log-dirs' in actual_config:
+ directory_check_list += actual_config['yarn.nodemanager.log-dirs'].split(',')
+ else:
+ checkFailed(host.hostname, "YARN configuration: yarn.nodemanager.log-dirs not defined")
+
+ for directory in directory_check_list:
+ cmd = "test -e %s" % directory
+ (result, output, errmsg) = remote_ssh_output(cmd, host.hostname, '')
+ if result != 0:
+ checkFailed(host.hostname, "YARN nodemanager directory %s does not exist" % directory)
+
+ # Check if resource manager property exists
+ if options.yarn:
+ yarn_property_exist_list = ['yarn.resourcemanager.address', 'yarn.resourcemanager.scheduler.address']
+
+ if options.yarn_ha:
+ yarn_property_exist_list = ['yarn.resourcemanager.address.rm1', 'yarn.resourcemanager.address.rm2', 'yarn.resourcemanager.scheduler.address.rm1', \
+ 'yarn.resourcemanager.scheduler.address.rm2']
+
+ if yarn_enabled:
+ for item in yarn_property_exist_list:
+ if item in actual_config:
+ if not actual_config[item]:
+ checkFailed(host.hostname, "YARN configuration: %s is empty" % item)
+ else:
+ checkFailed(host.hostname, "YARN configuration: %s not defined" % item)
+
+ # Check yarn kerberos properties
+ #yarn_kerberos_check_list = ['hadoop.proxyuser.yarn.groups', 'hadoop.proxyuser.yarn.hosts', 'hadoop.proxyuser.postgres.hosts', 'hadoop.proxyuser.postgres.groups']
+ if yarn_enabled and options.kerberos:
+ yarn_kerberos_check_list = ['yarn.nodemanager.keytab', 'yarn.nodemanager.principal','hadoop.proxyuser.postgres.groups', \
+ 'yarn.resourcemanager.keytab', 'yarn.resourcemanager.principal']
+ for item in yarn_kerberos_check_list:
+ if item in actual_config:
+ if not actual_config[item]:
+ checkFailed(host.hostname, "YARN configuration: %s is empty, expected non-empty" % item)
+ else:
+ checkFailed(host.hostname, "YARN configuration missing: %s" % item)
+
+
def testIOSchedulers(host):
if options.verbose:
logger.info("-- test IO scheduler")
@@ -774,6 +1098,8 @@ def testNtp(host):
def testGenericLinuxHost(host):
logger.info("test on host: %s" % host.hostname)
if host.is_namenode:
+ testHAWQGUC(host)
+ testHAWQconfig(host)
testHDFSConfig(host)
testDiskCapacity(host)
testSysctl(host)
@@ -782,7 +1108,8 @@ def testGenericLinuxHost(host):
testNtp(host)
else:
- testHAWQ(host)
+ testHAWQGUC(host)
+ testHAWQconfig(host)
testDiskCapacity(host)
testHDFSConfig(host)
testIOSchedulers(host)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e74109bf/tools/bin/gppylib/gpcheckutil.py
----------------------------------------------------------------------
diff --git a/tools/bin/gppylib/gpcheckutil.py b/tools/bin/gppylib/gpcheckutil.py
index 9956990..3419bb2 100755
--- a/tools/bin/gppylib/gpcheckutil.py
+++ b/tools/bin/gppylib/gpcheckutil.py
@@ -151,6 +151,20 @@ class hdfs:
return "============= HDFS ==========================\n" + output
+class hawq:
+ def __init__(self):
+ self.site_config = dict()
+ self.errormsg = None
+
+ def __str__(self):
+ if self.errormsg:
+ return "============= HAWQ ERROR ====================\n" + self.errormsg
+ else:
+ output = "HAWQ checks \n"
+ output += "\n".join(["%s = %s" % (k, self.site_config[k]) for k in sorted(self.site_config.iterkeys())])
+ return "============= HAWQ ==========================\n" + output
+
+
class diskusage_entry:
def __init__(self, fs, size, used, avail, used_percent, mount):
self.fs = fs
@@ -336,6 +350,7 @@ class GenericLinuxOutputData:
self.uname = None
self.machine = None
self.hdfs = None
+ self.hawq = None
self.diskusage = None
self.sysctl = None
self.limitsconf = None
@@ -346,7 +361,7 @@ class GenericLinuxOutputData:
def __str__(self):
applied_checks = filter(lambda x: x is not None,
- [ self.uname, self.machine, self.hdfs, self.diskusage, self.sysctl,
+ [ self.uname, self.machine, self.hdfs, self.hawq, self.diskusage, self.sysctl,
self.limitsconf, self.mounts, self.ioschedulers, self.blockdev, self.ntp ])
return "\n".join(map(str, applied_checks))
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e74109bf/tools/bin/hawqpylib/hawqlib.py
----------------------------------------------------------------------
diff --git a/tools/bin/hawqpylib/hawqlib.py b/tools/bin/hawqpylib/hawqlib.py
index ae0d852..c149ffc 100755
--- a/tools/bin/hawqpylib/hawqlib.py
+++ b/tools/bin/hawqpylib/hawqlib.py
@@ -131,6 +131,24 @@ def check_property_exist_xml(xml_file, property_name):
return property_exist, property_name, property_value
+def get_xml_values(xmlfile):
+ xml_dict = {}
+ with open(xmlfile) as f:
+ xmldoc = minidom.parse(f)
+
+ for node in xmldoc.getElementsByTagName('property'):
+ name = node.getElementsByTagName('name')[0].childNodes[0].data.encode('ascii')
+
+ try:
+ value = node.getElementsByTagName('value')[0].childNodes[0].data.encode('ascii')
+ except:
+ value = None
+
+ xml_dict[name] = value
+
+ return xml_dict
+
+
class HawqXMLParser:
def __init__(self, GPHOME):
self.GPHOME = GPHOME
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/e74109bf/tools/sbin/gpcheck_hostdump
----------------------------------------------------------------------
diff --git a/tools/sbin/gpcheck_hostdump b/tools/sbin/gpcheck_hostdump
index 7714cc3..28f074e 100755
--- a/tools/sbin/gpcheck_hostdump
+++ b/tools/sbin/gpcheck_hostdump
@@ -31,7 +31,7 @@ try:
from gppylib.gpparseopts import OptParser, OptChecker
from gppylib.gpcheckutil import ApplianceOutputData, GenericLinuxOutputData, GenericSolarisOutputData
from gppylib.gpcheckutil import chkconfig, omreport, grubconf, mounts, GpMount, GpMount, inittab, ntp
- from gppylib.gpcheckutil import securetty, ioschedulers, blockdev, bcu, rclocal, sysctl, limitsconf, limitsconf_entry, uname, connectemc, diskusage, diskusage_entry, hdfs, machine
+ from gppylib.gpcheckutil import securetty, ioschedulers, blockdev, bcu, rclocal, sysctl, limitsconf, limitsconf_entry, uname, connectemc, diskusage, diskusage_entry, hdfs, hawq, machine
from gppylib.gpcheckutil import solaris_etc_system, solaris_etc_project, solaris_etc_user_attr
except ImportError, e:
sys.exit('Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e))
@@ -413,15 +413,53 @@ def collectCPUandMemoryInfo():
return data
+def collectHAWQ():
+ if not options.hawq:
+ return None
+ data = hawq()
+ hawq_config_dir = os.environ.get('GPHOME')
+ if hawq_config_dir is None:
+ print "Please export GPHOME first, exit"
+ sys.exit(1)
+ hdfs_client_file = os.path.join(hawq_config_dir, "etc/hdfs-client.xml")
+ yarn_client_file = os.path.join(hawq_config_dir, "etc/yarn-client.xml")
+ hawq_site_file = os.path.join(hawq_config_dir, "etc/hawq-site.xml")
+
+ # collect HDFS site config
+ getPropName = lambda node: node.getElementsByTagName('name')[0].childNodes[0].data
+ getPropValue = lambda node: node.getElementsByTagName('value')[0].childNodes[0].data
+ hawq_config_file_list = [hdfs_client_file, hawq_site_file]
+ if options.yarn:
+ hawq_config_file_list.append(yarn_client_file)
+ for filename in hawq_config_file_list:
+ try:
+ with open(filename) as f:
+ xmldoc = minidom.parse(f)
+ for node in xmldoc.getElementsByTagName('property'):
+ try:
+ data.site_config[getPropName(node)] = getPropValue(node).strip()
+ except IndexError:
+ pass # the <value> tag may be empty, which causes IndexError in getPropValue
+
+ except Exception, e:
+ data.errormsg = "Failed to read HAWQ config file '%s': %s" % (filename, e)
+
+ return data
+
+
def collectHDFS():
if not options.hadoop:
return None
data = hdfs()
+ hawq_config_dir = os.environ.get('GPHOME')
+ if hawq_config_dir is None:
+ print "Please export GPHOME first, exit"
+ sys.exit(1)
hadoop_config_file = os.path.join(options.hadoop, "libexec/hadoop-config.sh")
hadoop_env_file = os.path.join(options.hadoop, "etc/hadoop/hadoop-env.sh")
hdfs_site_file = os.path.join(options.hadoop, "etc/hadoop/hdfs-site.xml")
+ yarn_site_file = os.path.join(options.hadoop, "etc/hadoop/yarn-site.xml")
core_site_file = os.path.join(options.hadoop, "etc/hadoop/core-site.xml")
- libhdfs3_site_file = os.environ.get("LIBHDFS3_CONF")
# collect java heap size config
p = subprocess.Popen(". %s; echo $JAVA_HEAP_MAX" % hadoop_config_file, shell = True,
@@ -457,7 +495,10 @@ def collectHDFS():
# collect HDFS site config
getPropName = lambda node: node.getElementsByTagName('name')[0].childNodes[0].data
getPropValue = lambda node: node.getElementsByTagName('value')[0].childNodes[0].data
- for filename in (hdfs_site_file, core_site_file, libhdfs3_site_file):
+ hdfs_config_file_list = [hdfs_site_file, core_site_file]
+ if options.yarn:
+ hdfs_config_file_list.append(yarn_site_file)
+ for filename in hdfs_config_file_list:
try:
with open(filename) as f:
xmldoc = minidom.parse(f)
@@ -804,6 +845,7 @@ def processGenericLinuxServer():
output = GenericLinuxOutputData()
output.hdfs = collectHDFS()
+ output.hawq = collectHAWQ()
output.uname = collectUname()
output.machine = collectCPUandMemoryInfo()
output.diskusage = collectDiskUsage()
@@ -844,6 +886,8 @@ def parseargs():
parser.remove_option('-h')
parser.add_option('-h', '-?', '--help', action='store_true')
parser.add_option('--hadoop', type='string')
+ parser.add_option('--hawq', action='store_true')
+ parser.add_option('--yarn', action='store_true')
parser.add_option('--sysctl', type='string')
parser.add_option('--appliance', action='store_true')
parser.add_option('--linux', action='store_true')