You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2013/02/14 20:39:58 UTC
svn commit: r1446306 - in /oozie/trunk: distro/src/main/bin/oozie-setup.sh
docs/src/site/twiki/AG_Install.twiki
docs/src/site/twiki/DG_QuickStart.twiki release-log.txt
tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
Author: tucu
Date: Thu Feb 14 19:39:58 2013
New Revision: 1446306
URL: http://svn.apache.org/r1446306
Log:
OOZIE-1054 Create script to properly upload sharelib to HDFS (bowenzhangusa via tucu)
Added:
oozie/trunk/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
Modified:
oozie/trunk/distro/src/main/bin/oozie-setup.sh
oozie/trunk/docs/src/site/twiki/AG_Install.twiki
oozie/trunk/docs/src/site/twiki/DG_QuickStart.twiki
oozie/trunk/release-log.txt
Modified: oozie/trunk/distro/src/main/bin/oozie-setup.sh
URL: http://svn.apache.org/viewvc/oozie/trunk/distro/src/main/bin/oozie-setup.sh?rev=1446306&r1=1446305&r2=1446306&view=diff
==============================================================================
--- oozie/trunk/distro/src/main/bin/oozie-setup.sh (original)
+++ oozie/trunk/distro/src/main/bin/oozie-setup.sh Thu Feb 14 19:39:58 2013
@@ -19,12 +19,24 @@
function printUsage() {
echo
- echo " Usage : oozie-setup.sh <OPTIONS>"
- echo " [-extjs EXTJS_PATH] (expanded or ZIP, to enable the Oozie webconsole)"
- echo " [-hadoop HADOOP_VERSION HADOOP_PATH] (Hadoop version [0.20.1|0.20.2|0.20.104|0.20.200|0.23.x|2.x]"
- echo " and Hadoop install dir)"
- echo " [-jars JARS_PATH] (multiple JAR path separated by ':')"
- echo " (without options does default setup, without the Oozie webconsole)"
+ echo " Usage : oozie-setup.sh <Command and OPTIONS>"
+ echo " prepare-war [-hadoop HADOOP_VERSION HADOOP_PATH] [-extjs EXTJS_PATH] [-jars JARS_PATH]"
+ echo " (prepare-war is to prepare war files for oozie)"
+ echo " (Hadoop version [0.20.1|0.20.2|0.20.104|0.20.200|0.23.x|2.x] and Hadoop install dir)"
+ echo " (EXTJS_PATH is expanded or ZIP, to enable the Oozie webconsole)"
+ echo " (JARS_PATH is multiple JAR path separated by ':')"
+ echo " sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] (create sharelib for oozie,"
+ echo " FS_URI is the fs.default.name"
+ echo " for hdfs uri; SHARED_LIBRARY, path to the"
+ echo " Oozie sharelib to install, it can be a tarball"
+ echo " or an expanded version of it. If ommited,"
+ echo " the Oozie sharelib tarball from the Oozie"
+ echo " installation directory will be used)"
+ echo " (action failes if sharelib is already installed"
+ echo " in HDFS)"
+ echo " sharelib upgrade -fs FS_URI [-locallib SHARED_LIBRARY] (upgrade existing sharelib, fails if there"
+ echo " is no existing sharelib installed in HDFS)"
+ echo " (without options prints this usage information)"
echo
echo " EXTJS can be downloaded from http://www.extjs.com/learn/Ext_Version_Archives"
echo
@@ -55,13 +67,33 @@ extjsHome=""
jarsPath=""
hadoopVersion=""
hadoopPath=""
+prepareWar=""
inputWar="${OOZIE_HOME}/oozie.war"
outputWar="${CATALINA_BASE}/webapps/oozie.war"
outputWarExpanded="${CATALINA_BASE}/webapps/oozie"
while [ $# -gt 0 ]
do
- if [ "$1" = "-extjs" ]; then
+ if [ "$1" = "sharelib" ]; then
+ shift
+ OOZIEFSCLI_OPTS="-Doozie.home.dir=${OOZIE_HOME}";
+ OOZIEFSCLI_OPTS="${OOZIEFSCLI_OPTS} -Doozie.config.dir=${OOZIE_CONFIG}";
+ OOZIEFSCLI_OPTS="${OOZIEFSCLI_OPTS} -Doozie.log.dir=${OOZIE_LOG}";
+ OOZIEFSCLI_OPTS="${OOZIEFSCLI_OPTS} -Doozie.data.dir=${OOZIE_DATA}";
+ OOZIEFSCLI_OPTS="${OOZIEFSCLI_OPTS} -Dderby.stream.error.file=${OOZIE_LOG}/derby.log"
+
+ OOZIECPPATH=""
+ OOZIECPPATH=${BASEDIR}/libtools/'*':${BASEDIR}/libext/'*'
+
+ if test -z ${JAVA_HOME}; then
+ JAVA_BIN=java
+ else
+ JAVA_BIN=${JAVA_HOME}/bin/java
+ fi
+
+ ${JAVA_BIN} ${OOZIEFSCLI_OPTS} -cp ${OOZIECPPATH} org.apache.oozie.tools.OozieSharelibCLI "${@}"
+ exit 0
+ elif [ "$1" = "-extjs" ]; then
shift
if [ $# -eq 0 ]; then
echo
@@ -102,6 +134,8 @@ do
shift
hadoopPath=$1
addHadoopJars=true
+ elif [ "$1" = "prepare-war" ]; then
+ prepareWar=true
else
printUsage
exit -1
@@ -116,49 +150,48 @@ if [ -e "${CATALINA_PID}" ]; then
exit -1
fi
-if [ -e "${outputWar}" ]; then
- chmod -f u+w ${outputWar}
- rm -rf ${outputWar}
-fi
-rm -rf ${outputWarExpanded}
-
echo
-# Adding extension JARs
-
-libext=${OOZIE_HOME}/libext
-if [ -d "${libext}" ]; then
- if [ `ls ${libext} | grep \.jar\$ | wc -c` != 0 ]; then
- for i in "${libext}/"*.jar; do
- echo "INFO: Adding extension: $i"
- jarsPath="${jarsPath}:$i"
- addJars="true"
- done
- fi
- if [ -f "${libext}/ext-2.2.zip" ]; then
- extjsHome=${libext}/ext-2.2.zip
- addExtjs=true
- fi
-# find war files (e.g., workflowgenerator) under /libext and deploy
- if [ `ls ${libext} | grep \.war\$ | wc -c` != 0 ]; then
- for i in "${libext}/"*.war; do
- echo "INFO: Deploying extention: $i"
- cp $i ${CATALINA_BASE}/webapps/
- done
+if [ "${addExtjs}${addJars}${addHadoopJars}${prepareWar}" == "" ]; then
+ echo "no arguments given"
+ printUsage
+ exit -1
+else
+ if [ -e "${outputWar}" ]; then
+ chmod -f u+w ${outputWar}
+ rm -rf ${outputWar}
+ fi
+ rm -rf ${outputWarExpanded}
+
+ # Adding extension JARs
+
+ libext=${OOZIE_HOME}/libext
+ if [ -d "${libext}" ]; then
+ if [ `ls ${libext} | grep \.jar\$ | wc -c` != 0 ]; then
+ for i in "${libext}/"*.jar; do
+ echo "INFO: Adding extension: $i"
+ jarsPath="${jarsPath}:$i"
+ addJars="true"
+ done
+ fi
+ if [ -f "${libext}/ext-2.2.zip" ]; then
+ extjsHome=${libext}/ext-2.2.zip
+ addExtjs=true
+ fi
+ # find war files (e.g., workflowgenerator) under /libext and deploy
+ if [ `ls ${libext} | grep \.war\$ | wc -c` != 0 ]; then
+ for i in "${libext}/"*.war; do
+ echo "INFO: Deploying extention: $i"
+ cp $i ${CATALINA_BASE}/webapps/
+ done
+ fi
fi
-fi
-
-if [ "${addExtjs}" == "" ]; then
- echo "INFO: Oozie webconsole disabled, ExtJS library not specified"
-fi
-if [ "${addExtjs}${addJars}${addHadoopJars}" == "" ]; then
- echo "INFO: Doing default installation"
- cp ${inputWar} ${outputWar}
-else
OPTIONS=""
if [ "${addExtjs}" != "" ]; then
OPTIONS="-extjs ${extjsHome}"
+ else
+ echo "INFO: Oozie webconsole disabled, ExtJS library not specified"
fi
if [ "${addJars}" != "" ]; then
OPTIONS="${OPTIONS} -jars ${jarsPath}"
@@ -172,9 +205,11 @@ else
if [ "$?" != "0" ]; then
exit -1
fi
-fi
-echo "INFO: Oozie is ready to be started"
+ echo
-echo
+ echo "INFO: Oozie is ready to be started"
+ echo
+
+fi
Modified: oozie/trunk/docs/src/site/twiki/AG_Install.twiki
URL: http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/AG_Install.twiki?rev=1446306&r1=1446305&r2=1446306&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/AG_Install.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/AG_Install.twiki Thu Feb 14 19:39:58 2013
@@ -54,11 +54,23 @@ The =oozie-setup.sh= script options are:
<verbatim>
Usage : oozie-setup.sh <OPTIONS>"
- [-extjs EXTJS_PATH] (expanded or ZIP, to enable the Oozie webconsole)"
- [-hadoop HADOOP_VERSION HADOOP_PATH] (Hadoop version [0.20.1|0.20.2|0.20.104|0.20.200]"
- and Hadoop install dir)"
- [-jars JARS_PATH] (multiple JAR path separated by ':')"
- (without options does default setup, without the Oozie webconsole)"
+ prepare-war [-hadoop HADOOP_VERSION HADOOP_PATH] [-extjs EXTJS_PATH] [-jars JARS_PATH]"
+ (prepare-war is to prepare war files for oozie)"
+ (Hadoop version [0.20.1|0.20.2|0.20.104|0.20.200|0.23.x|2.x] and Hadoop install dir)"
+ (EXTJS_PATH is expanded or ZIP, to enable the Oozie webconsole)"
+ (JARS_PATH is multiple JAR path separated by ':')"
+ sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] (create sharelib for oozie,"
+ FS_URI is the fs.default.name"
+ for hdfs uri; SHARED_LIBRARY, path to the"
+ Oozie sharelib to install, it can be a tarball"
+ or an expanded version of it. If ommited,"
+ the Oozie sharelib tarball from the Oozie"
+ installation directory will be used)"
+ (action failes if sharelib is already installed"
+ in HDFS)"
+ sharelib upgrade -fs FS_URI [-locallib SHARED_LIBRARY] (upgrade existing sharelib, fails if there"
+ is no existing sharelib installed in HDFS)"
+ (without options prints usage information)"
</verbatim>
If a directory =libext/= is present in Oozie installation directory, the =oozie-setup.sh= script
Modified: oozie/trunk/docs/src/site/twiki/DG_QuickStart.twiki
URL: http://svn.apache.org/viewvc/oozie/trunk/docs/src/site/twiki/DG_QuickStart.twiki?rev=1446306&r1=1446305&r2=1446306&view=diff
==============================================================================
--- oozie/trunk/docs/src/site/twiki/DG_QuickStart.twiki (original)
+++ oozie/trunk/docs/src/site/twiki/DG_QuickStart.twiki Thu Feb 14 19:39:58 2013
@@ -76,10 +76,19 @@ required Hadoop JARs from such version i
If using the ExtJS library copy the ZIP file to the *libext/* directory.
+A "sharelib create|upgrade -fs fs_default_name [-locallib sharelib]" command is available when running oozie-setup.sh
+for uploading new or upgrading existing sharelib into hdfs where the first argument is the default fs name
+and the second argument is the Oozie sharelib to install, it can be a tarball or the expanded version of it.
+If the second argument is omitted, the Oozie sharelib tarball from the Oozie installation directory will be used.
+
+"prepare-war" command is for creating war files for oozie.
+
Run the =oozie-setup.sh= script to configure Oozie with all the components added to the *libext/* directory.
<verbatim>
-$ bin/oozie-setup.sh
+$ bin/oozie-setup.sh prepare-war [-jars <PATHS>] [-extjs <PATH>]
+ sharelib create -fs <FS_URI> [-locallib <PATH>]
+ sharelib upgrade -fs <FS_URI> [-locallib <PATH>]
</verbatim>
Create the Oozie DB using the 'ooziedb.sh' command line tool:
Modified: oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/oozie/trunk/release-log.txt?rev=1446306&r1=1446305&r2=1446306&view=diff
==============================================================================
--- oozie/trunk/release-log.txt (original)
+++ oozie/trunk/release-log.txt Thu Feb 14 19:39:58 2013
@@ -1,5 +1,6 @@
-- Oozie 3.4.0 release (trunk - unreleased)
+OOZIE-1054 Create script to properly upload sharelib to HDFS (bowenzhangusa via tucu)
OOZIE-1096 Update wfgen README.txt to have the TLP mailing list (jun aoki via rkanter)
OOZIE-1015 HadoopAccessorService jobtracker validation should not have hardcoded conf key (mona)
OOZIE-1078 Help -> Documentation and Help -> Online Help should link to oozie.apache.org/ (jun via mohammad)
Added: oozie/trunk/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
URL: http://svn.apache.org/viewvc/oozie/trunk/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java?rev=1446306&view=auto
==============================================================================
--- oozie/trunk/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java (added)
+++ oozie/trunk/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java Thu Feb 14 19:39:58 2013
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.tools;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.filefilter.WildcardFileFilter;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.cli.CLIParser;
+import org.apache.commons.cli.Options;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.WorkflowAppService;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+import java.util.Collection;
+
+public class OozieSharelibCLI {
+ public static final String[] HELP_INFO = {
+ "",
+ "OozieSharelibCLI creates or upgrade sharelib for oozie",
+ };
+ public static final String HELP_CMD = "help";
+ public static final String CREATE_CMD = "create";
+ public static final String UPGRADE_CMD = "upgrade";
+ public static final String LIB_OPT = "locallib";
+ public static final String FS_OPT = "fs";
+ public static final String OOZIE_HOME = "oozie.home.dir";
+ private boolean used;
+
+ public static void main(String[] args) throws Exception{
+ System.exit(new OozieSharelibCLI().run(args));
+ }
+
+ public OozieSharelibCLI() {
+ used = false;
+ }
+
+ protected Options createUpgradeOptions(String subCommand){
+ Option sharelib = new Option(LIB_OPT, true, "Local share library directory");
+ Option uri = new Option(FS_OPT, true, "URI of the fileSystem to " + subCommand + " oozie share library");
+ Options options = new Options();
+ options.addOption(sharelib);
+ options.addOption(uri);
+ return options;
+ }
+
+ public synchronized int run(String[] args) throws Exception{
+ if (used) {
+ throw new IllegalStateException("CLI instance already used");
+ }
+
+ used = true;
+
+ CLIParser parser = new CLIParser("oozie-setup.sh", HELP_INFO);
+ String oozieHome = System.getProperty(OOZIE_HOME);
+ parser.addCommand(HELP_CMD, "", "display usage for all commands or specified command", new Options(), false);
+ parser.addCommand(CREATE_CMD, "", "create oozie sharelib", createUpgradeOptions(CREATE_CMD), false);
+ parser.addCommand(UPGRADE_CMD, "", "upgrade oozie sharelib", createUpgradeOptions(UPGRADE_CMD), false);
+
+ try {
+ final CLIParser.Command command = parser.parse(args);
+ String sharelibAction = command.getName();
+
+ if (sharelibAction.equals(HELP_CMD)){
+ parser.showHelp(command.getCommandLine());
+ return 0;
+ }
+
+ if (!command.getCommandLine().hasOption(FS_OPT)){
+ throw new Exception("-fs option must be specified");
+ }
+
+ File srcFile = null;
+
+ //Check whether user provided locallib
+ if (command.getCommandLine().hasOption(LIB_OPT)){
+ srcFile = new File(command.getCommandLine().getOptionValue(LIB_OPT));
+ }
+ else {
+ //Since user did not provide locallib, find the default one under oozie home dir
+ Collection<File> files =
+ FileUtils.listFiles(new File(oozieHome), new WildcardFileFilter("oozie-sharelib*.tar.gz"), null);
+
+ if (files.size() > 1){
+ throw new IOException("more than one sharelib tar found at " + oozieHome);
+ }
+
+ if (files.isEmpty()){
+ throw new IOException("default sharelib tar not found in oozie home dir: " + oozieHome);
+ }
+
+ srcFile = files.iterator().next();
+ }
+
+ File temp = File.createTempFile("oozie", ".dir");
+ temp.delete();
+ temp.mkdir();
+ temp.deleteOnExit();
+
+ //Check whether the lib is a tar file or folder
+ if (!srcFile.isDirectory()){
+ FileUtil.unTar(srcFile, temp);
+ srcFile = new File(temp.toString() + "/share/lib");
+ }
+ else {
+ //Get the lib directory since it's a folder
+ srcFile = new File(srcFile, "lib");
+ }
+
+ String hdfsUri = command.getCommandLine().getOptionValue(FS_OPT);
+ Path srcPath = new Path(srcFile.toString());
+
+ Services services = new Services();
+ services.getConf().set(Services.CONF_SERVICE_CLASSES, "org.apache.oozie.service.LiteWorkflowAppService, org.apache.oozie.service.HadoopAccessorService");
+ services.init();
+ WorkflowAppService lwas = services.get(WorkflowAppService.class);
+ HadoopAccessorService has = services.get(HadoopAccessorService.class);
+ Path dstPath = lwas.getSystemLibPath();
+ System.out.println("the destination path for sharelib is: " + dstPath);
+
+ URI uri = new Path(hdfsUri).toUri();
+ Configuration fsConf = has.createJobConf(uri.getAuthority());
+ FileSystem fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
+
+
+ if (!srcFile.exists()){
+ throw new IOException(srcPath + " cannot be found");
+ }
+
+ if (fs.exists(dstPath)){
+ if (sharelibAction.equals(CREATE_CMD)){
+ throw new IllegalStateException("share lib already exists");
+ }
+
+ fs.delete(dstPath, true);
+ }
+ else {
+ if (sharelibAction.equals(UPGRADE_CMD)){
+ throw new IllegalStateException("original share lib doesn't exist");
+ }
+ }
+
+ fs.copyFromLocalFile(false, srcPath, dstPath);
+ services.destroy();
+ FileUtils.deleteDirectory(temp);
+
+ return 0;
+ }
+ catch (ParseException ex) {
+ System.err.println("Invalid sub-command: " + ex.getMessage());
+ System.err.println();
+ System.err.println(parser.shortHelp());
+ return 1;
+ }
+ catch (Exception ex) {
+ System.err.println();
+ System.err.println("Error: " + ex.getMessage());
+ System.err.println();
+ System.err.println("Stack trace for the error was (for debug purposes):");
+ System.err.println("--------------------------------------");
+ ex.printStackTrace(System.err);
+ System.err.println("--------------------------------------");
+ System.err.println();
+ return 1;
+ }
+ }
+}