You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by tu...@apache.org on 2011/12/02 01:36:08 UTC
svn commit: r1209322 - in /incubator/oozie/trunk: ./
client/src/main/java/org/apache/oozie/cli/ client/src/main/resources/ core/
core/src/main/conf/ core/src/main/java/org/apache/oozie/action/hadoop/
core/src/test/java/org/apache/oozie/action/hadoop/ d...
Author: tucu
Date: Fri Dec 2 00:36:04 2011
New Revision: 1209322
URL: http://svn.apache.org/viewvc?rev=1209322&view=rev
Log:
OOZIE-156. Add support for a SQOOP action. (tucu)
Added:
incubator/oozie/trunk/client/src/main/resources/sqoop-action-0.2.xsd
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
incubator/oozie/trunk/docs/src/site/twiki/DG_SqoopActionExtension.twiki
incubator/oozie/trunk/examples/src/main/apps/sqoop/
incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/
incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.properties
incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.script
incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/job.properties
incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/workflow.xml
incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.properties
incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.script
incubator/oozie/trunk/examples/src/main/apps/sqoop/job.properties
incubator/oozie/trunk/examples/src/main/apps/sqoop/workflow.xml
incubator/oozie/trunk/sharelib/sqoop/
incubator/oozie/trunk/sharelib/sqoop/pom.xml
Modified:
incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
incubator/oozie/trunk/core/pom.xml
incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki
incubator/oozie/trunk/docs/src/site/twiki/index.twiki
incubator/oozie/trunk/examples/pom.xml
incubator/oozie/trunk/pom.xml
incubator/oozie/trunk/release-log.txt
incubator/oozie/trunk/sharelib/pom.xml
incubator/oozie/trunk/src/main/assemblies/sharelib.xml
Modified: incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java (original)
+++ incubator/oozie/trunk/client/src/main/java/org/apache/oozie/cli/OozieCLI.java Fri Dec 2 00:36:04 2011
@@ -1242,6 +1242,8 @@ public class OozieCLI {
"oozie-sla-0.1.xsd")));
sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
"hive-action-0.2.xsd")));
+ sources.add(new StreamSource(Thread.currentThread().getContextClassLoader().getResourceAsStream(
+ "sqoop-action-0.2.xsd")));
SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI);
Schema schema = factory.newSchema(sources.toArray(new StreamSource[sources.size()]));
Validator validator = schema.newValidator();
Added: incubator/oozie/trunk/client/src/main/resources/sqoop-action-0.2.xsd
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/client/src/main/resources/sqoop-action-0.2.xsd?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/client/src/main/resources/sqoop-action-0.2.xsd (added)
+++ incubator/oozie/trunk/client/src/main/resources/sqoop-action-0.2.xsd Fri Dec 2 00:36:04 2011
@@ -0,0 +1,70 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:sqoop="uri:oozie:sqoop-action:0.2" elementFormDefault="qualified"
+ targetNamespace="uri:oozie:sqoop-action:0.2">
+
+ <xs:element name="sqoop" type="sqoop:ACTION"/>
+
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="prepare" type="sqoop:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="configuration" type="sqoop:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:choice>
+ <xs:element name="command" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="arg" type="xs:string" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:choice>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="PREPARE">
+ <xs:sequence>
+ <xs:element name="delete" type="sqoop:DELETE" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="mkdir" type="sqoop:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+
+ <xs:complexType name="DELETE">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+
+ <xs:complexType name="MKDIR">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+
+</xs:schema>
Modified: incubator/oozie/trunk/core/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/pom.xml?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/core/pom.xml (original)
+++ incubator/oozie/trunk/core/pom.xml Fri Dec 2 00:36:04 2011
@@ -199,6 +199,12 @@
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-sqoop</artifactId>
+ <scope>provided</scope>
+ </dependency>
+
</dependencies>
<build>
@@ -604,6 +610,34 @@
</dependency>
</dependencies>
</profile>
+
+ <!-- Forcing Hadoop 0.20.2-cdh3u2 version because Sqoop -->
+ <!-- uses Hadoop API that is avail 0.21 onwards and the -->
+ <!-- Hadoop CDH version is the only stable version that -->
+ <!-- currently has such API (SQOOP-384) and published -->
+ <!-- in public Maven repos. This is a temporary fix -->
+ <profile>
+ <id>testSqoop</id>
+ <activation>
+ <activeByDefault>false</activeByDefault>
+ <property>
+ <name>testSqoop</name>
+ </property>
+ </activation>
+ <properties>
+ <hadoop.version>0.20.2-cdh3u2</hadoop.version>
+ </properties>
+ <repositories>
+ <repository>
+ <id>cdh.releases.repo</id>
+ <url>https://repository.cloudera.com/content/groups/cdh-releases-rcs</url>
+ <name>CDH Releases Repository</name>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+ </profile>
</profiles>
</project>
Modified: incubator/oozie/trunk/core/src/main/conf/oozie-site.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/conf/oozie-site.xml?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/conf/oozie-site.xml (original)
+++ incubator/oozie/trunk/core/src/main/conf/oozie-site.xml Fri Dec 2 00:36:04 2011
@@ -29,13 +29,14 @@
<name>oozie.service.ActionService.executor.ext.classes</name>
<value>
org.apache.oozie.action.email.EmailActionExecutor,
- org.apache.oozie.action.hadoop.HiveActionExecutor
+ org.apache.oozie.action.hadoop.HiveActionExecutor,
+ org.apache.oozie.action.hadoop.SqoopActionExecutor
</value>
</property>
<property>
<name>oozie.service.SchemaService.wf.ext.schemas</name>
- <value>email-action-0.1.xsd,hive-action-0.2.xsd</value>
+ <value>email-action-0.1.xsd,hive-action-0.2.xsd,sqoop-action-0.2.xsd</value>
</property>
-->
@@ -83,7 +84,7 @@
<property>
<name>oozie.service.CallableQueueService.queue.size</name>
- <value>1000</value>
+ <value>10000</value>
<description>Max callable queue size</description>
</property>
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/MapReduceMain.java Fri Dec 2 00:36:04 2011
@@ -132,6 +132,9 @@ public class MapReduceMain extends Launc
String[] values = new String[conf.getInt(key + ".size", 0)];
for (int i = 0; i < values.length; i++) {
values[i] = conf.get(key + "." + i);
+ if (values[i] == null) {
+ values[i] = "";
+ }
}
return values;
}
Added: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java (added)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopActionExecutor.java Fri Dec 2 00:36:04 2011
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.IOException;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.action.ActionExecutorException;
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
+import org.jdom.Element;
+import org.jdom.JDOMException;
+import org.jdom.Namespace;
+
+public class SqoopActionExecutor extends JavaActionExecutor {
+
+
+ public SqoopActionExecutor() {
+ super("sqoop");
+ }
+
+ @Override
+ protected List<Class> getLauncherClasses() {
+ List<Class> classes = super.getLauncherClasses();
+ classes.add(LauncherMain.class);
+ classes.add(MapReduceMain.class);
+ classes.add(HiveMain.class);
+ classes.add(SqoopMain.class);
+ return classes;
+ }
+
+ @Override
+ protected String getLauncherMain(Configuration launcherConf, Element actionXml) {
+ return launcherConf.get(LauncherMapper.CONF_OOZIE_ACTION_MAIN_CLASS, SqoopMain.class.getName());
+ }
+
+ @Override
+ Configuration setupLauncherConf(Configuration conf, Element actionXml, Path appPath, Context context)
+ throws ActionExecutorException {
+ super.setupLauncherConf(conf, actionXml, appPath, context);
+
+ HiveActionExecutor hiveAE = new HiveActionExecutor();
+ hiveAE.setupHiveDefault(conf, appPath, actionXml);
+
+ return conf;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ Configuration setupActionConf(Configuration actionConf, Context context, Element actionXml, Path appPath)
+ throws ActionExecutorException {
+ super.setupActionConf(actionConf, context, actionXml, appPath);
+ Namespace ns = actionXml.getNamespace();
+
+ try {
+ Element e = actionXml.getChild("configuration", ns);
+ if (e != null) {
+ String strConf = XmlUtils.prettyPrint(e).toString();
+ XConfiguration inlineConf = new XConfiguration(new StringReader(strConf));
+ checkForDisallowedProps(inlineConf, "inline configuration");
+ XConfiguration.copy(inlineConf, actionConf);
+ }
+ } catch (IOException ex) {
+ throw convertException(ex);
+ }
+
+ String[] args;
+ if (actionXml.getChild("command", ns) != null) {
+ String command = actionXml.getChild("command", ns).getTextTrim();
+ StringTokenizer st = new StringTokenizer(command, " ");
+ List<String> l = new ArrayList<String>();
+ while (st.hasMoreTokens()) {
+ l.add(st.nextToken());
+ }
+ args = l.toArray(new String[l.size()]);
+ }
+ else {
+ List<Element> eArgs = (List<Element>) actionXml.getChildren("arg", ns);
+ args = new String[eArgs.size()];
+ for (int i = 0; i < eArgs.size(); i++) {
+ args[i] = eArgs.get(i).getTextTrim();
+ }
+ }
+
+ SqoopMain.setSqoopCommand(actionConf, args);
+ return actionConf;
+ }
+
+ @Override
+ protected boolean getCaptureOutput(WorkflowAction action) throws JDOMException {
+ return true;
+ }
+
+
+ /**
+ * Return the sharelib postfix for the action.
+ *
+ * @param context executor context.
+ * @param actionXml the action XML.
+ * @return the action sharelib post fix, this implementation returns <code>hive</code>.
+ */
+ protected String getShareLibPostFix(Context context, Element actionXml) {
+ return "sqoop";
+ }
+
+}
Added: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java (added)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/action/hadoop/SqoopMain.java Fri Dec 2 00:36:04 2011
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.log4j.PropertyConfigurator;
+import org.apache.sqoop.Sqoop;
+
+public class SqoopMain extends LauncherMain {
+
+ private static final String SQOOP_ARGS = "oozie.sqoop.args";
+
+ public static final String SQOOP_SITE_CONF = "sqoop-site.xml";
+
+ private static final String JOB_ID_LOG_PREFIX = "Job complete: ";
+ private static final String SQOOP_LOG4J_PROPS = "sqoop-log4j.properties";
+
+ public static void main(String[] args) throws Exception {
+ run(SqoopMain.class, args);
+ }
+
+ private static Configuration initActionConf() {
+ // loading action conf prepared by Oozie
+ Configuration sqoopConf = new Configuration(false);
+
+ String actionXml = System.getProperty("oozie.action.conf.xml");
+
+ if (actionXml == null) {
+ throw new RuntimeException("Missing Java System Property [oozie.action.conf.xml]");
+ }
+ if (!new File(actionXml).exists()) {
+ throw new RuntimeException("Action Configuration XML file [" + actionXml + "] does not exist");
+ }
+
+ sqoopConf.addResource(new Path("file:///", actionXml));
+
+ String delegationToken = System.getenv("HADOOP_TOKEN_FILE_LOCATION");
+ if (delegationToken != null) {
+ sqoopConf.set("mapreduce.job.credentials.binary", delegationToken);
+ System.out.println("------------------------");
+ System.out.println("Setting env property for mapreduce.job.credentials.binary to: " + delegationToken);
+ System.out.println("------------------------");
+ System.setProperty("mapreduce.job.credentials.binary", delegationToken);
+ } else {
+ System.out.println("Non-Kerberos execution");
+ }
+
+ return sqoopConf;
+ }
+
+ public static Configuration setUpSqoopSite() throws Exception {
+ Configuration sqoopConf = initActionConf();
+
+ // Write the action configuration out to sqoop-site.xml
+ OutputStream os = new FileOutputStream(SQOOP_SITE_CONF);
+ try {
+ sqoopConf.writeXml(os);
+ }
+ finally {
+ os.close();
+ }
+
+ System.out.println();
+ System.out.println("Sqoop Configuration Properties:");
+ System.out.println("------------------------");
+ for (Map.Entry<String, String> entry : sqoopConf) {
+ System.out.println(entry.getKey() + "=" + entry.getValue());
+ }
+ System.out.flush();
+ System.out.println("------------------------");
+ System.out.println();
+ return sqoopConf;
+ }
+
+ public static String setUpSqoopLog4J(Configuration sqoopConf) throws IOException {
+ //Logfile to capture job IDs
+ String hadoopJobId = System.getProperty("oozie.launcher.job.id");
+ if (hadoopJobId == null) {
+ throw new RuntimeException("Launcher Hadoop Job ID system property not set");
+ }
+
+ String logFile = new File("sqoop-oozie-" + hadoopJobId + ".log").getAbsolutePath();
+
+ Properties hadoopProps = new Properties();
+
+ // Preparing log4j configuration
+ URL log4jFile = Thread.currentThread().getContextClassLoader().getResource("log4j.properties");
+ if (log4jFile != null) {
+ // getting hadoop log4j configuration
+ hadoopProps.load(log4jFile.openStream());
+ }
+
+ String logLevel = sqoopConf.get("oozie.sqoop.log.level", "INFO");
+
+ hadoopProps.setProperty("log4j.logger.org.apache.sqoop", logLevel + ", A");
+ hadoopProps.setProperty("log4j.appender.A", "org.apache.log4j.ConsoleAppender");
+ hadoopProps.setProperty("log4j.appender.A.layout", "org.apache.log4j.PatternLayout");
+ hadoopProps.setProperty("log4j.appender.A.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
+
+ hadoopProps.setProperty("log4j.appender.jobid", "org.apache.log4j.FileAppender");
+ hadoopProps.setProperty("log4j.appender.jobid.file", logFile);
+ hadoopProps.setProperty("log4j.appender.jobid.layout", "org.apache.log4j.PatternLayout");
+ hadoopProps.setProperty("log4j.appender.jobid.layout.ConversionPattern", "%-4r [%t] %-5p %c %x - %m%n");
+ hadoopProps.setProperty("log4j.logger.org.apache.hadoop.mapred", "INFO, jobid");
+
+ String localProps = new File(SQOOP_LOG4J_PROPS).getAbsolutePath();
+ OutputStream os1 = new FileOutputStream(localProps);
+ try {
+ hadoopProps.store(os1, "");
+ }
+ finally {
+ os1.close();
+ }
+
+ PropertyConfigurator.configure(SQOOP_LOG4J_PROPS);
+
+ return logFile;
+ }
+
+ protected void run(String[] args) throws Exception {
+ System.out.println();
+ System.out.println("Oozie Sqoop action configuration");
+ System.out.println("=================================================================");
+
+ Configuration sqoopConf = setUpSqoopSite();
+ String logFile = setUpSqoopLog4J(sqoopConf);
+
+ String[] sqoopArgs = MapReduceMain.getStrings(sqoopConf, SQOOP_ARGS);
+ if (sqoopArgs == null) {
+ throw new RuntimeException("Action Configuration does not have [" + SQOOP_ARGS + "] property");
+ }
+
+ System.out.println("Sqoop command arguments :");
+ for (String arg : sqoopArgs) {
+ System.out.println(" " + arg);
+ }
+
+ System.out.println("=================================================================");
+ System.out.println();
+ System.out.println(">>> Invoking Sqoop command line now >>>");
+ System.out.println();
+ System.out.flush();
+
+ try {
+ runSqoopJob(sqoopArgs);
+ }
+ catch (SecurityException ex) {
+ if (LauncherSecurityManager.getExitInvoked()) {
+ if (LauncherSecurityManager.getExitCode() != 0) {
+ throw ex;
+ }
+ }
+ }
+
+ System.out.println();
+ System.out.println("<<< Invocation of Sqoop command completed <<<");
+ System.out.println();
+
+ // harvesting and recording Hadoop Job IDs
+ Properties jobIds = HiveMain.getHadoopJobIds(logFile, JOB_ID_LOG_PREFIX);
+ File file = new File(System.getProperty("oozie.action.output.properties"));
+ OutputStream os = new FileOutputStream(file);
+ try {
+ jobIds.store(os, "");
+ }
+ finally {
+ os.close();
+ }
+ System.out.println(" Hadoop Job IDs executed by Sqoop: " + jobIds.getProperty("hadoopJobs"));
+ System.out.println();
+ }
+
+ protected void runSqoopJob(String[] args) throws Exception {
+ // running as from the command line
+ Sqoop.main(args);
+ }
+
+ public static void setSqoopCommand(Configuration conf, String[] args) {
+ MapReduceMain.setStrings(conf, SQOOP_ARGS, args);
+ }
+}
Added: incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java (added)
+++ incubator/oozie/trunk/core/src/test/java/org/apache/oozie/action/hadoop/TestSqoopActionExecutor.java Fri Dec 2 00:36:04 2011
@@ -0,0 +1,337 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.oozie.action.hadoop;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapred.RunningJob;
+import org.apache.oozie.WorkflowActionBean;
+import org.apache.oozie.WorkflowJobBean;
+import org.apache.oozie.client.OozieClient;
+import org.apache.oozie.client.WorkflowAction;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.util.ClassUtils;
+import org.apache.oozie.util.IOUtils;
+import org.apache.oozie.util.XConfiguration;
+import org.apache.oozie.util.XmlUtils;
+import org.apache.sqoop.Sqoop;
+import org.jdom.Element;
+import org.jdom.Namespace;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.Statement;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+public class TestSqoopActionExecutor extends ActionExecutorTestCase {
+
+ private static final String SQOOP_COMMAND = "import --connect {0} --table TT --target-dir {1} -m 1";
+
+ private static final String SQOOP_ACTION_COMMAND_XML =
+ "<sqoop xmlns=\"uri:oozie:sqoop-action:0.1\">" +
+ "<job-tracker>{0}</job-tracker>" +
+ "<name-node>{1}</name-node>" +
+ "<configuration>" +
+ "<property>" +
+ "<name>{2}</name>" +
+ "<value>{3}</value>" +
+ "</property>" +
+ "</configuration>" +
+ "<command>{4}</command>" +
+ "</sqoop>";
+
+ private static final String SQOOP_ACTION_ARGS_XML =
+ "<sqoop xmlns=\"uri:oozie:sqoop-action:0.1\">" +
+ "<job-tracker>{0}</job-tracker>" +
+ "<name-node>{1}</name-node>" +
+ "<configuration>" +
+ "<property>" +
+ "<name>oozie.sqoop.log.level</name>" +
+ "<value>INFO</value>" +
+ "</property>" +
+ "</configuration>" +
+ "<arg>import</arg>" +
+ "<arg>--connect</arg>" +
+ "<arg>{2}</arg>" +
+ "<arg>--username</arg>" +
+ "<arg>sa</arg>" +
+ "<arg>--password</arg>" +
+ "<arg></arg>" +
+ "<arg>--verbose</arg>" +
+ "<arg>--query</arg>" +
+ "<arg>{3}</arg>" +
+ "<arg>--target-dir</arg>" +
+ "<arg>{4}</arg>" +
+ "<arg>--split-by</arg>" +
+ "<arg>I</arg>" +
+ "</sqoop>";
+
+ protected void setSystemProps() {
+ super.setSystemProps();
+ setSystemProperty("oozie.service.ActionService.executor.ext.classes", SqoopActionExecutor.class.getName());
+ }
+
+ public void testSetupMethods() throws Exception {
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+ assertEquals("sqoop", ae.getType());
+ }
+
+ public void testLauncherJar() throws Exception {
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+ Path jar = new Path(ae.getOozieRuntimeDir(), ae.getLauncherJarName());
+ assertTrue(new File(jar.toString()).exists());
+ }
+
+ private String getDbFile() {
+ return "db.hsqldb";
+ }
+
+ private String getDbPath() {
+ return new File(getTestCaseDir(), getDbFile()).getAbsolutePath();
+ }
+
+ private String getLocalJdbcUri() {
+ return "jdbc:hsqldb:file:" + getDbPath() + ";shutdown=true";
+ }
+
+ private String getActionJdbcUri() {
+ return "jdbc:hsqldb:file:" + getDbFile();
+ }
+
+ private String getSqoopOutputDir() {
+ return new Path(getFsTestCaseDir(), "output").toString();
+ }
+
+ private String getActionXml() {
+ String command = MessageFormat.format(SQOOP_COMMAND, getActionJdbcUri(), getSqoopOutputDir());
+ return MessageFormat.format(SQOOP_ACTION_COMMAND_XML, getJobTrackerUri(), getNameNodeUri(),
+ "dummy", "dummyValue", command);
+ }
+
+ private String getActionXmlFreeFromQuery() {
+ String query = "select TT.I, TT.S from TT where $CONDITIONS";
+ return MessageFormat.format(SQOOP_ACTION_ARGS_XML, getJobTrackerUri(), getNameNodeUri(),
+ getActionJdbcUri(), query, getSqoopOutputDir());
+ }
+
+ private void createDB() throws Exception {
+ Class.forName("org.hsqldb.jdbcDriver");
+ Connection conn = DriverManager.getConnection(getLocalJdbcUri(), "sa", "");
+ Statement st = conn.createStatement();
+ st.executeUpdate("CREATE TABLE TT (I INTEGER PRIMARY KEY, S VARCHAR(256))");
+ st.executeUpdate("INSERT INTO TT (I, S) VALUES (1, 'a')");
+ st.executeUpdate("INSERT INTO TT (I, S) VALUES (2, 'a')");
+ st.executeUpdate("INSERT INTO TT (I, S) VALUES (3, 'a')");
+ st.close();
+ conn.close();
+ }
+
+ public void testSqoopAction() throws Exception {
+ createDB();
+
+ Context context = createContext(getActionXml());
+ final RunningJob launcherJob = submitAction(context);
+ String launcherId = context.getAction().getExternalId();
+ waitFor(120 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return launcherJob.isComplete();
+ }
+ });
+ assertTrue(launcherJob.isSuccessful());
+
+ assertFalse(LauncherMapper.hasIdSwap(launcherJob));
+
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+ ae.check(context, context.getAction());
+ assertTrue(launcherId.equals(context.getAction().getExternalId()));
+ assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
+ assertNotNull(context.getAction().getData());
+ ae.end(context, context.getAction());
+ assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
+ FileSystem fs = getFileSystem();
+ BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(new Path(getSqoopOutputDir(), "part-m-00000"))));
+ int count = 0;
+ String line = br.readLine();
+ while (line != null) {
+ assertTrue(line.contains("a"));
+ count++;
+ line = br.readLine();
+ }
+ br.close();
+ assertEquals(3, count);
+
+ assertNotNull(context.getAction().getData());
+ Properties outputData = new Properties();
+ outputData.load(new StringReader(context.getAction().getData()));
+ assertTrue(outputData.containsKey("hadoopJobs"));
+ assertTrue(outputData.getProperty("hadoopJobs").trim().length() > 0);
+ }
+
+ public void testSqoopActionFreeFormQuery() throws Exception {
+ createDB();
+
+ Context context = createContext(getActionXmlFreeFromQuery());
+ final RunningJob launcherJob = submitAction(context);
+ String launcherId = context.getAction().getExternalId();
+ waitFor(120 * 1000, new Predicate() {
+ public boolean evaluate() throws Exception {
+ return launcherJob.isComplete();
+ }
+ });
+ assertTrue(launcherJob.isSuccessful());
+
+ assertFalse(LauncherMapper.hasIdSwap(launcherJob));
+
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+ ae.check(context, context.getAction());
+ assertTrue(launcherId.equals(context.getAction().getExternalId()));
+ assertEquals("SUCCEEDED", context.getAction().getExternalStatus());
+ assertNotNull(context.getAction().getData());
+ ae.end(context, context.getAction());
+ assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus());
+ FileSystem fs = getFileSystem();
+ FileStatus[] parts = fs.listStatus(new Path(getSqoopOutputDir()), new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ return path.getName().startsWith("part-");
+ }
+ });
+ int count = 0;
+ for (FileStatus part : parts) {
+ BufferedReader br = new BufferedReader(new InputStreamReader(fs.open(part.getPath())));
+ String line = br.readLine();
+ while (line != null) {
+ assertTrue(line.contains("a"));
+ count++;
+ line = br.readLine();
+ }
+ br.close();
+ }
+ assertEquals(3, count);
+
+ assertNotNull(context.getAction().getData());
+ Properties outputData = new Properties();
+ outputData.load(new StringReader(context.getAction().getData()));
+ assertTrue(outputData.containsKey("hadoopJobs"));
+ assertTrue(outputData.getProperty("hadoopJobs").trim().length() > 0);
+ }
+
+
+ private RunningJob submitAction(Context context) throws Exception {
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+
+ WorkflowAction action = context.getAction();
+
+ ae.prepareActionDir(getFileSystem(), context);
+ ae.submitLauncher(getFileSystem(), context, action);
+
+ String jobId = action.getExternalId();
+ String jobTracker = action.getTrackerUri();
+ String consoleUrl = action.getConsoleUrl();
+ assertNotNull(jobId);
+ assertNotNull(jobTracker);
+ assertNotNull(consoleUrl);
+ Element e = XmlUtils.parseXml(action.getConf());
+ Namespace ns = Namespace.getNamespace("uri:oozie:sqoop-action:0.1");
+ XConfiguration conf = new XConfiguration(
+ new StringReader(XmlUtils.prettyPrint(e.getChild("configuration", ns)).toString()));
+ conf.set("mapred.job.tracker", e.getChildTextTrim("job-tracker", ns));
+ conf.set("fs.default.name", e.getChildTextTrim("name-node", ns));
+ conf.set("user.name", context.getProtoActionConf().get("user.name"));
+ conf.set("group.name", getTestGroup());
+ injectKerberosInfo(conf);
+ JobConf jobConf = new JobConf(conf);
+ String user = jobConf.get("user.name");
+ String group = jobConf.get("group.name");
+ JobClient jobClient = Services.get().get(HadoopAccessorService.class).createJobClient(user, group, jobConf);
+ final RunningJob runningJob = jobClient.getJob(JobID.forName(jobId));
+ assertNotNull(runningJob);
+ return runningJob;
+ }
+
+ private String copyJar(String targetFile, Class<?> anyContainedClass)
+ throws Exception {
+ String file = ClassUtils.findContainingJar(anyContainedClass);
+ Path targetPath = new Path(getAppPath(), targetFile);
+ FileSystem fs = getFileSystem();
+ InputStream is = new FileInputStream(file);
+ OutputStream os = fs.create(new Path(getAppPath(), targetPath));
+ IOUtils.copyStream(is, os);
+ return targetPath.toString();
+ }
+
+ private Context createContext(String actionXml) throws Exception {
+ List<String> jars = new ArrayList<String>();
+ SqoopActionExecutor ae = new SqoopActionExecutor();
+
+ jars.add(copyJar("lib/commons-io-x.jar", org.apache.commons.io.IOUtils.class));
+ jars.add(copyJar("lib/hsqldb-x.jar", org.hsqldb.jdbcDriver.class));
+ jars.add(copyJar("lib/sqoop-x.jar", Sqoop.class));
+
+ XConfiguration protoConf = new XConfiguration();
+ protoConf.set(WorkflowAppService.HADOOP_USER, getTestUser());
+ protoConf.set(WorkflowAppService.HADOOP_UGI, getTestUser() + "," + getTestGroup());
+ protoConf.set(OozieClient.GROUP_NAME, getTestGroup());
+ injectKerberosInfo(protoConf);
+
+ //Making DB file to show up in the DistributeCache
+ jars.addAll(copyDbToHdfs());
+
+ protoConf.setStrings(WorkflowAppService.APP_LIB_PATH_LIST, jars.toArray(new String[jars.size()]));
+
+ WorkflowJobBean wf = createBaseWorkflow(protoConf, "sqoop-action");
+ WorkflowActionBean action = (WorkflowActionBean) wf.getActions().get(0);
+ action.setType(ae.getType());
+ action.setConf(actionXml);
+
+ return new Context(wf, action);
+ }
+
+ private List<String> copyDbToHdfs() throws Exception {
+ List<String> files = new ArrayList<String>();
+ String[] exts = {".script", ".properties"};
+ for (String ext : exts) {
+ String file = getDbPath() + ext;
+ String name = getDbFile() + ext;
+ Path targetPath = new Path(getAppPath(), name);
+ FileSystem fs = getFileSystem();
+ InputStream is = new FileInputStream(file);
+ OutputStream os = fs.create(new Path(getAppPath(), targetPath));
+ IOUtils.copyStream(is, os);
+ files.add(targetPath.toString() + "#" + name);
+ }
+ return files;
+ }
+}
Added: incubator/oozie/trunk/docs/src/site/twiki/DG_SqoopActionExtension.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/DG_SqoopActionExtension.twiki?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/DG_SqoopActionExtension.twiki (added)
+++ incubator/oozie/trunk/docs/src/site/twiki/DG_SqoopActionExtension.twiki Fri Dec 2 00:36:04 2011
@@ -0,0 +1,243 @@
+<noautolink>
+
+[[index][::Go back to Oozie Documentation Index::]]
+
+-----
+
+---+!! Oozie Sqoop Action Extension
+
+%TOC%
+
+---++ Sqoop Action
+
+*IMPORTANT:* The Sqoop action requires Apache Hadoop 0.23.
+
+The =sqoop= action runs a Sqoop job.
+
+The workflow job will wait until the Sqoop job completes before
+continuing to the next action.
+
+To run the Sqoop job, you have to configure the =sqoop= action with the
+=job-tracker=, =name-node= and Sqoop =command= or =arg= elements as
+well as configuration.
+
+A =sqoop= action can be configured to create or delete HDFS directories
+before starting the Sqoop job.
+
+Sqoop configuration can be specified with a file, using the =job-xml=
+element, and inline, using the =configuration= elements.
+
+Oozie EL expressions can be used in the inline configuration. Property
+values specified in the =configuration= element override values specified
+in the =job-xml= file.
+
+Note that Hadoop =mapred.job.tracker= and =fs.default.name= properties
+must not be present in the inline configuration.
+
+As with Hadoop =map-reduce= jobs, it is possible to add files and
+archives in order to make them available to the Sqoop job. Refer to the
+[WorkflowFunctionalSpec#FilesAchives][Adding Files and Archives for the Job]
+section for more information about this feature.
+
+*Syntax:*
+
+<verbatim>
+<workflow-app name="[WF-DEF-NAME]" xmlns="uri:oozie:workflow:0.1">
+ ...
+ <action name="[NODE-NAME]">
+ <sqoop xmlns="uri:oozie:sqoop-action:0.2">
+ <job-tracker>[JOB-TRACKER]</job-tracker>
+ <name-node>[NAME-NODE]</name-node>
+ <prepare>
+ <delete path="[PATH]"/>
+ ...
+ <mkdir path="[PATH]"/>
+ ...
+ </prepare>
+ <configuration>
+ <property>
+ <name>[PROPERTY-NAME]</name>
+ <value>[PROPERTY-VALUE]</value>
+ </property>
+ ...
+ </configuration>
+ <command>[SQOOP-COMMAND]</command>
+ <arg>[SQOOP-ARGUMENT]</arg>
+ ...
+ <file>[FILE-PATH]</file>
+ ...
+ <archive>[FILE-PATH]</archive>
+ ...
+ </sqoop>
+ <ok to="[NODE-NAME]"/>
+ <error to="[NODE-NAME]"/>
+ </action>
+ ...
+</workflow-app>
+</verbatim>
+
+The =prepare= element, if present, indicates a list of paths to delete
+or create before starting the job. Specified paths must start with =hdfs://HOST:PORT=.
+
+The =job-xml= element, if present, specifies a file containing configuration
+for the Sqoop job.
+
+The =configuration= element, if present, contains configuration
+properties that are passed to the Sqoop job.
+
+*Sqoop command*
+
+The Sqoop command can be specified either using the =command= element or multiple =arg=
+elements.
+
+When using the =command= element, Oozie will split the command on every space
+into multiple arguments.
+
+When using the =arg= elements, Oozie will pass each argument value as an argument to Sqoop.
+
+The =arg= variant should be used when there are spaces within a single argument.
+
+Consult the Sqoop documentation for a complete list of valid Sqoop commands.
+
+All the above elements can be parameterized (templatized) using EL
+expressions.
+
+*Examples:*
+
+Using the =command= element:
+
+<verbatim>
+<workflow-app name="sample-wf" xmlns="uri:oozie:workflow:0.1">
+ ...
+ <action name="myfirsthivejob">
+ <sqoop xmlns="uri:oozie:sqoop-action:0.2">
+ <job-traker>foo:9001</job-tracker>
+ <name-node>bar:9000</name-node>
+ <prepare>
+ <delete path="${jobOutput}"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ </property>
+ </configuration>
+ <command>import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir hdfs://localhost:9000/user/tucu/foo -m 1</command>
+ </sqoop>
+ <ok to="myotherjob"/>
+ <error to="errorcleanup"/>
+ </action>
+ ...
+</workflow-app>
+</verbatim>
+
+The same Sqoop action using =arg= elements:
+
+<verbatim>
+<workflow-app name="sample-wf" xmlns="uri:oozie:workflow:0.1">
+ ...
+ <action name="myfirsthivejob">
+ <sqoop xmlns="uri:oozie:sqoop-action:0.2">
+ <job-traker>foo:9001</job-tracker>
+ <name-node>bar:9000</name-node>
+ <prepare>
+ <delete path="${jobOutput}"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.compress.map.output</name>
+ <value>true</value>
+ </property>
+ </configuration>
+ <arg>import</arg>
+ <arg>--connect</arg>
+ <arg>jdbc:hsqldb:file:db.hsqldb</arg>
+ <arg>--table</arg>
+ <arg>TT</arg>
+ <arg>--target-dir</arg>
+ <arg>hdfs://localhost:9000/user/tucu/foo</arg>
+ <arg>-m</arg>
+ <arg>1</arg>
+ </sqoop>
+ <ok to="myotherjob"/>
+ <error to="errorcleanup"/>
+ </action>
+ ...
+</workflow-app>
+</verbatim>
+
+NOTE: The =arg= elements syntax, while more verbose, allows to have spaces in a single argument, something useful when
+using free from queries.
+
+---+++ Sqoop Action Logging
+
+Sqoop action logs are redirected to the Oozie Launcher map-reduce job task STDOUT/STDERR that runs Sqoop.
+
+From Oozie web-console, from the Sqoop action pop up using the 'Console URL' link, it is possible
+to navigate to the Oozie Launcher map-reduce job task logs via the Hadoop job-tracker web-console.
+
+The logging level of the Sqoop action can set in the Sqoop action configuration using the
+property =oozie.sqoop.log.level=. The default value is =INFO=.
+
+---++ Appendix, Sqoop XML-Schema
+
+---+++ AE.A Appendix A, Sqoop XML-Schema
+
+<verbatim>
+<xs:schema xmlns:xs="http://www.w3.org/2001/XMLSchema"
+ xmlns:sqoop="uri:oozie:sqoop-action:0.2" elementFormDefault="qualified"
+ targetNamespace="uri:oozie:sqoop-action:0.2">
+
+ <xs:element name="sqoop" type="sqoop:ACTION"/>
+.
+ <xs:complexType name="ACTION">
+ <xs:sequence>
+ <xs:element name="job-tracker" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="name-node" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="prepare" type="sqoop:PREPARE" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="job-xml" type="xs:string" minOccurs="0" maxOccurs="1"/>
+ <xs:element name="configuration" type="sqoop:CONFIGURATION" minOccurs="0" maxOccurs="1"/>
+ <xs:choice>
+ <xs:element name="command" type="xs:string" minOccurs="1" maxOccurs="1"/>
+ <xs:element name="arg" type="xs:string" minOccurs="1" maxOccurs="unbounded"/>
+ </xs:choice>
+ <xs:element name="file" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="archive" type="xs:string" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="CONFIGURATION">
+ <xs:sequence>
+ <xs:element name="property" minOccurs="1" maxOccurs="unbounded">
+ <xs:complexType>
+ <xs:sequence>
+ <xs:element name="name" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="value" minOccurs="1" maxOccurs="1" type="xs:string"/>
+ <xs:element name="description" minOccurs="0" maxOccurs="1" type="xs:string"/>
+ </xs:sequence>
+ </xs:complexType>
+ </xs:element>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="PREPARE">
+ <xs:sequence>
+ <xs:element name="delete" type="sqoop:DELETE" minOccurs="0" maxOccurs="unbounded"/>
+ <xs:element name="mkdir" type="sqoop:MKDIR" minOccurs="0" maxOccurs="unbounded"/>
+ </xs:sequence>
+ </xs:complexType>
+.
+ <xs:complexType name="DELETE">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+.
+ <xs:complexType name="MKDIR">
+ <xs:attribute name="path" type="xs:string" use="required"/>
+ </xs:complexType>
+.
+</xs:schema>
+</verbatim>
+
+[[index][::Go back to Oozie Documentation Index::]]
+
+</noautolink>
Modified: incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/ENG_Building.twiki Fri Dec 2 00:36:04 2011
@@ -199,6 +199,21 @@ must be used together with the name of t
$ mvn -DtestHive -Dtest=TestHiveActionExecutor,TestHiveMain
</verbatim>
+---+++ Testing Sqoop Action
+
+Apache Sqoop 1.5.0 requires functionality only available in Hadoop 0.21 and onwards. Currently, the only
+stable version of Hadoop implementing those API (SQOOP-384) that is being published by to public Maven
+repositories are Hadoop CDH versions. This profile activates the use of that Hadoop version for testing
+purposes. *This is a temporary fix*.
+
+<verbatim>
+$ mvn -DtestSqoop -Dtest=TestSqoopActionExecutor
+</verbatim>
+
+<verbatim>
+$ mvn -DtestHive -Dtest=TestHiveActionExecutor,TestHiveMain
+</verbatim>
+
---++ Building an Oozie Distribution
An Oozie distribution bundles an embedded Tomcat server. The Oozie distro module downloads Tomcat TAR.GZ from Apache
Modified: incubator/oozie/trunk/docs/src/site/twiki/index.twiki
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/docs/src/site/twiki/index.twiki?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/docs/src/site/twiki/index.twiki (original)
+++ incubator/oozie/trunk/docs/src/site/twiki/index.twiki Fri Dec 2 00:36:04 2011
@@ -46,15 +46,16 @@ Enough reading already? Follow the steps
* [[DG_WorkflowReRun][Workflow Re-runs Explained]]
* [[DG_UsingHadoopKerberos][Using a Hadoop cluster with Kerberos Authentication]]
- * [[DG_CustomActionExecutor][Writing a Custom Action Executor]]
* [[./client/apidocs/index.html][Oozie Client Javadocs]]
* [[./core/apidocs/index.html][Oozie Core Javadocs]]
* [[WebServicesAPI][Oozie Web Services API]]
----+++ Actions Extension
+---+++ Action Extensions
* [[DG_EmailActionExtension][Email Action]]
* [[DG_HiveActionExtension][Hive Action]]
+ * [[DG_SqoopActionExtension][Sqoop Action]]
+ * [[DG_CustomActionExecutor][Writing a Custom Action Executor]]
---++ Administrator Documentation
Modified: incubator/oozie/trunk/examples/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/pom.xml?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/examples/pom.xml (original)
+++ incubator/oozie/trunk/examples/pom.xml Fri Dec 2 00:36:04 2011
@@ -91,6 +91,11 @@
<scope>compile</scope>
</dependency>
+ <dependency>
+ <groupId>hsqldb</groupId>
+ <artifactId>hsqldb</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.properties
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.properties?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.properties (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.properties Fri Dec 2 00:36:04 2011
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#HSQL Database Engine 1.8.0.5
+#Tue Oct 05 11:20:19 SGT 2010
+hsqldb.script_format=0
+runtime.gc_interval=0
+sql.enforce_strict_size=false
+hsqldb.cache_size_scale=8
+readonly=false
+hsqldb.nio_data_file=true
+hsqldb.cache_scale=14
+version=1.8.0
+hsqldb.default_table_type=memory
+hsqldb.cache_file_scale=1
+hsqldb.log_size=200
+modified=no
+hsqldb.cache_version=1.7.0
+hsqldb.original_version=1.8.0
+hsqldb.compatible_version=1.8.0
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.script
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.script?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.script (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/db.hsqldb.script Fri Dec 2 00:36:04 2011
@@ -0,0 +1,9 @@
+CREATE SCHEMA PUBLIC AUTHORIZATION DBA
+CREATE MEMORY TABLE TT(I INTEGER NOT NULL PRIMARY KEY,S VARCHAR(256))
+CREATE USER SA PASSWORD ""
+GRANT DBA TO SA
+SET WRITE_DELAY 10
+SET SCHEMA PUBLIC
+INSERT INTO TT VALUES(1,'a')
+INSERT INTO TT VALUES(2,'a')
+INSERT INTO TT VALUES(3,'a')
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/job.properties
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/job.properties?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/job.properties (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/job.properties Fri Dec 2 00:36:04 2011
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+examplesRoot=examples
+
+oozie.use.system.libpath=true
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/sqoop-freeform
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/workflow.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/workflow.xml?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/workflow.xml (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop-freeform/workflow.xml Fri Dec 2 00:36:04 2011
@@ -0,0 +1,61 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.2" name="sqoop-freeform-wf">
+ <start to="sqoop-freeform-node"/>
+
+ <action name="sqoop-freeform-node">
+ <sqoop xmlns="uri:oozie:sqoop-action:0.2">
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <prepare>
+ <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/sqoop-freeform"/>
+ <mkdir path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ </configuration>
+ <arg>import</arg>
+ <arg>--connect</arg>
+ <arg>jdbc:hsqldb:file:db.hsqldb</arg>
+ <arg>--username</arg>
+ <arg>sa</arg>
+ <arg>--password</arg>
+ <arg></arg>
+ <arg>--verbose</arg>
+ <arg>--query</arg>
+ <arg>select TT.I, TT.S from TT where $CONDITIONS</arg>
+ <arg>--target-dir</arg>
+ <arg>/user/${wf:user()}/${examplesRoot}/output-data/sqoop-freeform</arg>
+ <arg>-m</arg>
+ <arg>1</arg>
+ <file>db.hsqldb.properties#db.hsqldb.properties</file>
+ <file>db.hsqldb.script#db.hsqldb.script</file>
+ </sqoop>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+
+ <kill name="fail">
+ <message>Sqoop free form failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.properties
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.properties?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.properties (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.properties Fri Dec 2 00:36:04 2011
@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#HSQL Database Engine 1.8.0.5
+#Tue Oct 05 11:20:19 SGT 2010
+hsqldb.script_format=0
+runtime.gc_interval=0
+sql.enforce_strict_size=false
+hsqldb.cache_size_scale=8
+readonly=false
+hsqldb.nio_data_file=true
+hsqldb.cache_scale=14
+version=1.8.0
+hsqldb.default_table_type=memory
+hsqldb.cache_file_scale=1
+hsqldb.log_size=200
+modified=no
+hsqldb.cache_version=1.7.0
+hsqldb.original_version=1.8.0
+hsqldb.compatible_version=1.8.0
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.script
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.script?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.script (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop/db.hsqldb.script Fri Dec 2 00:36:04 2011
@@ -0,0 +1,9 @@
+CREATE SCHEMA PUBLIC AUTHORIZATION DBA
+CREATE MEMORY TABLE TT(I INTEGER NOT NULL PRIMARY KEY,S VARCHAR(256))
+CREATE USER SA PASSWORD ""
+GRANT DBA TO SA
+SET WRITE_DELAY 10
+SET SCHEMA PUBLIC
+INSERT INTO TT VALUES(1,'a')
+INSERT INTO TT VALUES(2,'a')
+INSERT INTO TT VALUES(3,'a')
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop/job.properties
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop/job.properties?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop/job.properties (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop/job.properties Fri Dec 2 00:36:04 2011
@@ -0,0 +1,26 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+nameNode=hdfs://localhost:8020
+jobTracker=localhost:8021
+queueName=default
+examplesRoot=examples
+
+oozie.use.system.libpath=true
+
+oozie.wf.application.path=${nameNode}/user/${user.name}/${examplesRoot}/apps/sqoop
Added: incubator/oozie/trunk/examples/src/main/apps/sqoop/workflow.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/examples/src/main/apps/sqoop/workflow.xml?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/examples/src/main/apps/sqoop/workflow.xml (added)
+++ incubator/oozie/trunk/examples/src/main/apps/sqoop/workflow.xml Fri Dec 2 00:36:04 2011
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<workflow-app xmlns="uri:oozie:workflow:0.2" name="sqoop-wf">
+ <start to="sqoop-node"/>
+
+ <action name="sqoop-node">
+ <sqoop xmlns="uri:oozie:sqoop-action:0.2">
+ <job-tracker>${jobTracker}</job-tracker>
+ <name-node>${nameNode}</name-node>
+ <prepare>
+ <delete path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data/sqoop"/>
+ <mkdir path="${nameNode}/user/${wf:user()}/${examplesRoot}/output-data"/>
+ </prepare>
+ <configuration>
+ <property>
+ <name>mapred.job.queue.name</name>
+ <value>${queueName}</value>
+ </property>
+ </configuration>
+ <command>import --connect jdbc:hsqldb:file:db.hsqldb --table TT --target-dir /user/${wf:user()}/${examplesRoot}/output-data/sqoop -m 1</command>
+ <file>db.hsqldb.properties#db.hsqldb.properties</file>
+ <file>db.hsqldb.script#db.hsqldb.script</file>
+ </sqoop>
+ <ok to="end"/>
+ <error to="fail"/>
+ </action>
+
+ <kill name="fail">
+ <message>Sqoop failed, error message[${wf:errorMessage(wf:lastErrorNode())}]</message>
+ </kill>
+ <end name="end"/>
+</workflow-app>
Modified: incubator/oozie/trunk/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/pom.xml?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/pom.xml (original)
+++ incubator/oozie/trunk/pom.xml Fri Dec 2 00:36:04 2011
@@ -199,6 +199,11 @@
</dependency>
<dependency>
<groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-sharelib-sqoop</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.oozie</groupId>
<artifactId>oozie-docs</artifactId>
<version>3.2.0-SNAPSHOT</version>
<type>war</type>
@@ -385,8 +390,13 @@
<dependency>
<groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.5.8</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
- <version>1.4.3</version>
+ <version>1.5.8</version>
</dependency>
<dependency>
@@ -541,6 +551,18 @@
</dependency>
<dependency>
+ <groupId>org.apache.sqoop</groupId>
+ <artifactId>sqoop</artifactId>
+ <version>1.5.0-incubating-SNAPSHOT</version>
+ </dependency>
+
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>1.4</version>
+ </dependency>
+
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
<version>0.23.0-SNAPSHOT</version>
@@ -696,8 +718,11 @@
<exclude>**/${test.exclude}.java</exclude>
<exclude>${test.exclude.pattern}</exclude>
- <!-- See 'testHive' profile in core/pom.xml -->
+ <!-- See 'testHive' profile in core/pom.xml and the Building doc-->
<exclude>**/TestHive*.java</exclude>
+
+ <!-- See 'testSqoop' profile in core/pom.xml and the Building doc-->
+ <exclude>**/TestSqoop*.java</exclude>
</excludes>
</configuration>
</plugin>
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Fri Dec 2 00:36:04 2011
@@ -1,5 +1,6 @@
-- Oozie 3.2.0 release
+OOZIE-156. Add support for a SQOOP action. (tucu)
OOZIE-77. Oozie should support Kerberos authentication on its HTTP REST API. (tucu)
OOZIE-622. Remove system sharelib tests from TestLiteWorkflowAppService. (tucu)
OOZIE-68 Add Hive action. (tucu)
Modified: incubator/oozie/trunk/sharelib/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/pom.xml?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/sharelib/pom.xml (original)
+++ incubator/oozie/trunk/sharelib/pom.xml Fri Dec 2 00:36:04 2011
@@ -32,6 +32,7 @@
<module>streaming</module>
<module>pig</module>
<module>hive</module>
+ <module>sqoop</module>
</modules>
<build>
Added: incubator/oozie/trunk/sharelib/sqoop/pom.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/sharelib/sqoop/pom.xml?rev=1209322&view=auto
==============================================================================
--- incubator/oozie/trunk/sharelib/sqoop/pom.xml (added)
+++ incubator/oozie/trunk/sharelib/sqoop/pom.xml Fri Dec 2 00:36:04 2011
@@ -0,0 +1,207 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.oozie</groupId>
+ <artifactId>oozie-main</artifactId>
+ <version>3.2.0-SNAPSHOT</version>
+ <relativePath>../..</relativePath>
+ </parent>
+ <artifactId>oozie-sharelib-sqoop</artifactId>
+ <description>Oozie Share Lib Sqoop</description>
+ <name>Oozie Share Lib Sqoop</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <sharelib.action.postfix>sqoop</sharelib.action.postfix>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.sqoop</groupId>
+ <artifactId>sqoop</artifactId>
+ <scope>compile</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-codec</groupId>
+ <artifactId>commons-codec</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>hsqldb</groupId>
+ <artifactId>hsqldb</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </exclusion>
+
+
+ <exclusion>
+ <groupId>org.apache.thrift</groupId>
+ <artifactId>thrift</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>servlet-api-2.5</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-compiler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jruby</groupId>
+ <artifactId>jruby-complete</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-json</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-server</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.ws.rs</groupId>
+ <artifactId>jsr311-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-core-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-test</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.protobuf</groupId>
+ <artifactId>protobuf-java</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>stax</groupId>
+ <artifactId>stax-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <resources>
+ <resource>
+ <directory>src/main/resources</directory>
+ <filtering>true</filtering>
+ </resource>
+ </resources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <configuration>
+ <skip>true</skip>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <finalName>partial-sharelib</finalName>
+ <appendAssemblyId>false</appendAssemblyId>
+ <descriptors>
+ <descriptor>../../src/main/assemblies/partial-sharelib.xml</descriptor>
+ </descriptors>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
+
Modified: incubator/oozie/trunk/src/main/assemblies/sharelib.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/src/main/assemblies/sharelib.xml?rev=1209322&r1=1209321&r2=1209322&view=diff
==============================================================================
--- incubator/oozie/trunk/src/main/assemblies/sharelib.xml (original)
+++ incubator/oozie/trunk/src/main/assemblies/sharelib.xml Fri Dec 2 00:36:04 2011
@@ -43,6 +43,10 @@
<directory>${basedir}/hive/target/partial-sharelib</directory>
<outputDirectory>/</outputDirectory>
</fileSet>
+ <fileSet>
+ <directory>${basedir}/sqoop/target/partial-sharelib</directory>
+ <outputDirectory>/</outputDirectory>
+ </fileSet>
</fileSets>
</assembly>