You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hawq.apache.org by es...@apache.org on 2017/02/03 09:00:45 UTC
[43/50] [abbrv] incubator-hawq git commit: HAWQ-1297. Make PXF
install ready from source
HAWQ-1297. Make PXF install ready from source
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/aac8868f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/aac8868f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/aac8868f
Branch: refs/heads/2.1.0.0-incubating
Commit: aac8868f8e3d8c0158a1a58b8593a713056099e4
Parents: d3983eb
Author: Shivram Mani <sh...@gmail.com>
Authored: Mon Jan 30 15:53:07 2017 -0800
Committer: Shivram Mani <sh...@gmail.com>
Committed: Mon Jan 30 15:53:07 2017 -0800
----------------------------------------------------------------------
pxf/Makefile | 23 +++-
pxf/build.gradle | 29 +++-
.../src/main/resources/pxf-private.classpath | 67 ++++++++++
pxf/pxf-service/src/scripts/pxf-env.sh | 6 +
pxf/pxf-service/src/scripts/pxf-service | 132 ++++++++++++++-----
5 files changed, 211 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/Makefile
----------------------------------------------------------------------
diff --git a/pxf/Makefile b/pxf/Makefile
index 7669772..81298d0 100644
--- a/pxf/Makefile
+++ b/pxf/Makefile
@@ -19,26 +19,36 @@
default: all
ifneq "$(HD)" ""
-BUILD_PARAMS= -Dhd=$(HD)
+ BUILD_PARAMS= -Dhd=$(HD)
+else
+ ifneq "$(PXF_HOME)" ""
+ BUILD_PARAMS= -DdeployPath=$(PXF_HOME)
+ else ifneq "$(GPHOME)" ""
+ BUILD_PARAMS= -DdeployPath="$(GPHOME)/pxf"
+ else
+ @echo "Cannot invoke install without configuring either PXF_HOME or GPHOME"
+ endif
endif
ifneq "$(LICENSE)" ""
-BUILD_PARAMS+= -Plicense="$(LICENSE)"
+ BUILD_PARAMS+= -Plicense="$(LICENSE)"
endif
+
ifneq "$(VENDOR)" ""
-BUILD_PARAMS+= -Pvendor="$(VENDOR)"
+ BUILD_PARAMS+= -Pvendor="$(VENDOR)"
endif
help:
@echo
- @echo "help it is then"
- @echo "Possible targets"
+ @echo"help it is then"
+ @echo "Possible targets"
@echo " - all (clean, build, unittest, jar, tar, rpm)"
@echo " - - HD=<phd|hdp> - set classpath to match hadoop distribution. default phd"
@echo " - - LICENSE=<license info> - add license info to created RPMs"
@echo " - - VENDOR=<vendor name> - add vendor name to created RPMs"
@echo " - tomcat - builds tomcat rpm from downloaded tarball"
@echo " - - LICENSE and VENDOR parameters can be used as well"
+ @echo " - deploy - setup PXF along with tomcat in the configured deployPath"
@echo " - doc - creates aggregate javadoc under docs"
all:
@@ -65,3 +75,6 @@ doc:
.PHONY: tomcat
tomcat:
./gradlew tomcatRpm $(BUILD_PARAMS)
+
+install:
+ ./gradlew install $(BUILD_PARAMS)
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/build.gradle
----------------------------------------------------------------------
diff --git a/pxf/build.gradle b/pxf/build.gradle
index 385bf08..3c6d591 100644
--- a/pxf/build.gradle
+++ b/pxf/build.gradle
@@ -473,8 +473,7 @@ task rpm(type: Copy, dependsOn: [subprojects.build, distSubprojects.buildRpm]) {
// tomcat
def tomcatName = "apache-tomcat-${tomcatVersion}"
-def tomcatTargetDir = "tomcat/build/"
-
+def tomcatTargetDir = "tomcat/build"
task tomcatGet << {
@@ -506,12 +505,12 @@ task tomcatGet << {
apply plugin: 'os-package'
task tomcatRpm(type: Rpm) {
- buildDir = 'tomcat/build/'
+ buildDir = "${tomcatTargetDir}"
// clean should not delete the downloaded tarball
// and RPM, so this is a bogus directory to delete instead.
clean {
- delete = 'tomcat/build/something'
+ delete = "${tomcatTargetDir}/something"
}
ospackage {
@@ -544,6 +543,26 @@ task tomcatRpm(type: Rpm) {
tomcatRpm.dependsOn tomcatGet
+def pxfTargetDir = System.properties['deployPath'] ?: "build/"
+
+task install(type: Copy, dependsOn: [subprojects.build, tomcatGet]) {
+ into "${pxfTargetDir}"
+ subprojects { subProject ->
+ from("${project.name}/build/libs") { into 'lib' }
+ }
+ from("pxf-service/src/scripts/pxf-service") {
+ into 'bin'
+ fileMode 0755
+ rename('pxf-service', 'pxf')
+ }
+
+ from("${tomcatTargetDir}/${tomcatName}") { into 'apache-tomcat' }
+ from("pxf-service/src/main/resources") { into 'conf' }
+ from("pxf-service/src/configs/pxf-site.xml") { into 'conf' }
+ from("pxf-service/src/scripts/pxf-env.sh") { into 'conf' }
+ from("pxf-service/src/configs/tomcat") { into 'tomcat-templates' }
+}
+
buildDir = '.'
-apply plugin: 'nebula-aggregate-javadocs'
\ No newline at end of file
+apply plugin: 'nebula-aggregate-javadocs'
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/pxf-service/src/main/resources/pxf-private.classpath
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/main/resources/pxf-private.classpath b/pxf/pxf-service/src/main/resources/pxf-private.classpath
new file mode 100644
index 0000000..48ac2f9
--- /dev/null
+++ b/pxf/pxf-service/src/main/resources/pxf-private.classpath
@@ -0,0 +1,67 @@
+##################################################################
+# This file contains the internal classpaths required to run PXF.
+# Edit to set the base paths according to your specific package layout
+# Adding new resources should be done using pxf-public.classpath file.
+##################################################################
+
+# PXF Configuration
+pxf/conf
+
+# Hadoop Configuration
+hadoop/etc/hadoop
+
+# Hive Configuration
+hive/conf
+
+# Hbase Configuration
+hbase/conf
+
+# PXF Libraries
+pxf/pxf-hbase-*[0-9].jar
+pxf/pxf-hdfs-*[0-9].jar
+pxf/pxf-hive-*[0-9].jar
+pxf/pxf-json-*[0-9].jar
+
+# Hadoop Libraries
+hadoop/share/hadoop/hdfs/hadoop-hdfs-*[0-9].jar
+hadoop/share/hadoop/mapreduce/hadoop-mapreduce-client-core-*[0-9].jar
+hadoop/share/hadoop/common/lib/hadoop-auth-*[0-9].jar
+hadoop/share/hadoop/common/hadoop-common-*[0-9].jar
+hadoop/share/hadoop/common/lib/asm-*[0-9].jar
+hadoop/share/hadoop/common/lib/avro-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-cli-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-codec-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-collections-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-configuration-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-io-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-lang-*[0-9].jar
+hadoop/share/hadoop/common/lib/commons-logging-*[0-9].jar
+hadoop/share/hadoop/common/lib/guava-*[0-9].jar
+hadoop/share/hadoop/common/lib/htrace-core-*[0-9]*.jar
+hadoop/share/hadoop/common/lib/jetty-*.jar
+hadoop/share/hadoop/common/lib/jackson-core-asl-*[0-9].jar
+hadoop/share/hadoop/common/lib/jackson-mapper-asl-*[0-9].jar
+hadoop/share/hadoop/common/lib/jersey-core-*[0-9].jar
+hadoop/share/hadoop/common/lib/jersey-server-*[0-9].jar
+hadoop/share/hadoop/common/lib/log4j-*[0-9].jar
+hadoop/share/hadoop/common/lib/protobuf-java-*[0-9].jar
+hadoop/share/hadoop/common/lib/slf4j-api-*[0-9].jar
+
+# Hive Libraries
+hive/lib/antlr-runtime-*[0-9].jar
+hive/lib/datanucleus-api-jdo-*[0-9].jar
+hive/lib/datanucleus-core-*[0-9].jar
+hive/lib/hive-exec-*[0-9].jar
+hive/lib/hive-metastore-*[0-9].jar
+hive/lib/jdo-api-*[0-9].jar
+hive/lib/libfb303-*[0-9].jar
+# when running on OSx, 1.0.5 or higher version is required
+hive/lib/snappy-java-*[0-9].jar
+
+# HBase Libraries
+hbase/lib/hbase-client-*[0-9].jar
+hbase/lib/hbase-common-*[0-9].jar
+hbase/lib/hbase-protocol-*[0-9].jar
+hbase/lib/htrace-core-*[0-9]*.jar
+hbase/lib/netty-*[0-9].Final.jar
+hbase/lib/zookeeper-*[0-9].jar
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/pxf-service/src/scripts/pxf-env.sh
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/scripts/pxf-env.sh b/pxf/pxf-service/src/scripts/pxf-env.sh
index 2ac80a9..ba9b368 100644
--- a/pxf/pxf-service/src/scripts/pxf-env.sh
+++ b/pxf/pxf-service/src/scripts/pxf-env.sh
@@ -28,3 +28,9 @@ export CATALINA_OUT=${PXF_LOGDIR}/catalina.out
# Path to Run directory
export PXF_RUNDIR=/var/run/pxf
+
+# Configured user
+export PXF_USER=pxf
+
+# Port
+export PXF_PORT=51200
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/aac8868f/pxf/pxf-service/src/scripts/pxf-service
----------------------------------------------------------------------
diff --git a/pxf/pxf-service/src/scripts/pxf-service b/pxf/pxf-service/src/scripts/pxf-service
index 7f62504..2d63310 100644
--- a/pxf/pxf-service/src/scripts/pxf-service
+++ b/pxf/pxf-service/src/scripts/pxf-service
@@ -20,18 +20,12 @@
# pxf-service start/stop/initialize/status the PXF instance
#
-pxf_root=/usr/lib/pxf
-env_script=/etc/pxf/conf/pxf-env.sh
-tomcat_root=/opt/apache-tomcat
-tomcat_templates=/opt/pxf/tomcat-templates
-instance_root=/var/pxf
-pxf_user=pxf
-instance_name=pxf-service
-instance_port=51200
-instance_owner=pxf:pxf
-
-curl=`which curl`
+if [ -z $PXF_HOME ]; then
+ env_script=/etc/pxf/conf/pxf-env.sh
+else
+ env_script=$PXF_HOME/conf/pxf-env.sh
+fi
# load pxf-env.sh script
if [ ! -f $env_script ]; then
@@ -40,6 +34,29 @@ else
source $env_script
fi
+pxf_user=$PXF_USER
+instance_port=$PXF_PORT
+instance_name=pxf-service
+
+if [ -z $PXF_HOME ]; then
+ # RPM based setup
+ pxf_root=/usr/lib/pxf
+ tomcat_root=/opt/apache-tomcat
+ tomcat_templates=/opt/pxf/tomcat-templates
+ instance_root=/var/pxf
+ instance_owner=$pxf_user:$pxf_user
+else
+ # OSS/Source code based setup
+ pxf_root=$PXF_HOME/lib
+ tomcat_root=$PXF_HOME/apache-tomcat
+ tomcat_templates=$PXF_HOME/tomcat-templates
+ instance_root=$PXF_HOME
+ instance_owner=$pxf_user
+fi
+
+curl=`which curl`
+
+
# validate JAVA_HOME
if [ ! -x $JAVA_HOME/bin/java ]; then
echo ERROR: \$JAVA_HOME is invalid
@@ -61,9 +78,8 @@ function createInstance()
return 1
fi
- chown $instance_owner -R $instance_root
+ chown -R $instance_owner $instance_root
chmod 700 $instance_root/$instance_name
-
return 0
}
@@ -97,7 +113,7 @@ function configureInstance()
# set pid
catalinaEnv=$instance_root/$instance_name/bin/setenv.sh
cat $catalinaEnv | \
- sed "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=$PXF_RUNDIR/catalina.pid|g" \
+ sed -e "s|^[[:blank:]]*CATALINA_PID=.*$|CATALINA_PID=$PXF_RUNDIR/catalina.pid|g" \
> ${catalinaEnv}.tmp
rm $catalinaEnv
mv ${catalinaEnv}.tmp $catalinaEnv
@@ -176,7 +192,7 @@ function checkWebapp()
# non zero otherwise
function instanceExists()
{
- if [ ! -d $instance_root ]; then
+ if [ ! -d "$instance_root/$instance_name" ]; then
return 1
fi
@@ -193,39 +209,83 @@ function doInit()
return 0
fi
- createInstance || return 1
+ createInstance || return 1
configureInstance || return 1
deployWebapp || return 1
}
+#
+# patchWebapp patches the webapp config files
+# patch applied only if PXF_HOME is defined
+#
+function patchWebapp()
+{
+ if [ -z $PXF_HOME ]; then
+ # webapp doesn't require patch
+ return 0
+ fi
+ pushd $instance_root/$instance_name/webapps || return 1
+ rm -rf pxf
+ mkdir pxf
+ cd pxf
+ unzip -q ../pxf.war
+ popd
+
+ context_file=$instance_root/$instance_name/webapps/pxf/META-INF/context.xml
+ cat $context_file | \
+ sed -e "s:classpathFiles=\"[a-zA-Z0-9\/\;.-]*\":classpathFiles=\"$PXF_HOME\/conf\/pxf-private.classpath\":" \
+ -e "s:secondaryClasspathFiles=\"[a-zA-Z0-9\/\;.-]*\":secondaryClasspathFiles=\"$PXF_HOME\/conf\/pxf-public.classpath\":" > context.xml.tmp
+ mv context.xml.tmp $context_file
+
+ web_file=$instance_root/$instance_name/webapps/pxf/WEB-INF/web.xml
+ cat $web_file | \
+ sed "s:<param-value>.*pxf-log4j.properties<\/param-value>:<param-value>$PXF_HOME\/conf\/pxf-log4j.properties<\/param-value>:" > web.xml.tmp
+ mv web.xml.tmp $web_file
+}
+
+function commandWebapp()
+{
+ command=$1
+ pushd $instance_root
+ su $pxf_user -c "$instance_root/$instance_name/bin/catalina.sh $command"
+ if [ $? -ne 0 ]; then
+ return 1
+ fi
+ popd
+}
+
#
-# doStartStop handles start/stop commands
-# commands are executed as the user $pxf_user
+# doStart handles start command
+# command is executed as the user $pxf_user
#
# after start, uses checkWebapp to verify the PXF webapp was loaded
# successfully
#
-function doStartStop()
+function doStart()
{
- command=$1
-
instanceExists
if [ $? -ne 0 ]; then
echo ERROR: cant find PXF instance, maybe call init?
return 1
fi
+ patchWebapp || return 1
+ commandWebapp start || return 1
+ checkWebapp 300 || return 1
+}
- pushd $instance_root
- su $pxf_user -c "$instance_root/$instance_name/bin/catalina.sh $command"
- if [ $? -ne 0 ]; then
- return 1
- fi
- popd
-
- if [ "$command" = "start" ]; then
- # try to connect for 5 minutes
- checkWebapp 300 || return 1
- fi
+#
+# doStart handles stop command
+# command is executed as the user $pxf_user
+#
+#
+function doStop()
+{
+ instanceExists
+ if [ $? -ne 0 ]; then
+ echo "ERROR: can't find PXF instance, maybe call init?"
+ return 1
+ fi
+ commandWebapp stop || return 1
}
function doStatus()
@@ -240,15 +300,15 @@ case "$command" in
doInit
;;
"start" )
- doStartStop $command
+ doStart
;;
"stop" )
- doStartStop $command
+ doStop
;;
"restart" )
- doStartStop stop
+ doStop
sleep 1s
- doStartStop start
+ doStart
;;
"status" )
doStatus